From d8c52d6c408a172f1210c77df3e3a9629ea68dc6 Mon Sep 17 00:00:00 2001
From: nasr <nsrddyn@gmail.com>
Date: Thu, 5 Mar 2026 22:42:55 +0000
Subject: feature(main): helper functions for lexing

refactor(main): helper script for testing to gitignore
---
 source/base/base_string.h   |  9 ++++++
 source/engine/engine.c      |  9 +++---
 source/lexer/lexer.c        | 74 +++++++++++++++++++++++++++++++++++++++++++--
 source/storage/csv_reader.c |  8 -----
 source/storage/csv_reader.h |  8 ++---
 5 files changed, 89 insertions(+), 19 deletions(-)

(limited to 'source')

diff --git a/source/base/base_string.h b/source/base/base_string.h
index 64a3162..189b38a 100644
--- a/source/base/base_string.h
+++ b/source/base/base_string.h
@@ -47,4 +47,13 @@ string8_append_char(string8 *buf, u8 c)
     buf->size += 1;
 }
 
+read_only global_variable
+string8 nil_string =
+{
+
+    .data = NULL,
+    .size = 0,
+
+};
+
 #endif /* BASE_STRING_H */
diff --git a/source/engine/engine.c b/source/engine/engine.c
index ada4ecb..05c143c 100644
--- a/source/engine/engine.c
+++ b/source/engine/engine.c
@@ -1,6 +1,8 @@
 #define BASE_UNITY
 #include "../base/base_include.h"
 
+#include <stdio.h>
+
 #include "../lexer/lexer.h"
 #include "../lexer/lexer.c"
 
@@ -13,17 +15,14 @@
 #include "../storage/csv_reader.h"
 #include "../storage/csv_reader.c"
 
-#if 1
-#include <stdio.h>
-#endif
-
 
 int main(int c, char **v)
 {
     if(c < 2) return -999;
 
     string8 buffer = load_file(v[1]);
-    read_csv(buffer);
+    // read_csv(buffer);
+    tokenize_csv(buffer);
 
 
     // for(;;)
diff --git a/source/lexer/lexer.c b/source/lexer/lexer.c
index 8182c5a..1c7ab38 100644
--- a/source/lexer/lexer.c
+++ b/source/lexer/lexer.c
@@ -1,10 +1,80 @@
+internal b32
+is_alpha(u8 point)
+{
+  return ((point >= 'a' && point <= 'z') ||
+          (point >= 'A' && point <= 'Z') ||
+          (point == '_'));
+}
+
+internal b32
+is_digit(u8 point)
+{
+  return (point >= '0' && point <= '9');
+}
+
+internal b32
+is_alpha_num(u8 point)
+{
+  return (is_alpha(point) || is_digit(point));
+}
+
+internal b32
+is_whitespace(u8 point)
+{
+  return (point == '\n' || point == '\r' ||
+          point == ' ' || point == '\t');
+}
+
+internal b32
+is_delimiter(u8 point)
+{
+
+    return (point == ',');
+
+}
+
 internal token *
 tokenize_csv(string8 buffer)
 {
+    i32 count = 0;
+    string8 **tokens = PushArray(arena, string8 *, buffer.size / 10);
+
     if(buffer.size < 0) return NULL;
     for(i32 index = 0;
-         buffer.data[index] != '\0'
-         ;)
+         buffer.data[index] != '\0';
+         ++index)
+    {
+        string8 tokens = {0};
+
+        u8 point = buffer.data[index];
+        if(is_whitespace(point)) continue;
+
+        u8 *start = &buffer.data;
+
+        if(is_delimiter(point))
+        {
+
+
+        }
+
+        u8 *end = start - 1;
+
+        unused(start);
+        unused(end);
+
+        switch (point)
+        {
+
+            default:
+                {
+                    printf("point: %c\n", point);
+                    count++;
+                }
+        }
+
+    }
+
+    printf("%d", count);
 
     return NULL;
 }
diff --git a/source/storage/csv_reader.c b/source/storage/csv_reader.c
index a06e9c4..2fcbe04 100644
--- a/source/storage/csv_reader.c
+++ b/source/storage/csv_reader.c
@@ -1,15 +1,7 @@
-#define STD_TEST
-#if defined(STD_TEST)
-#include <stdio.h>
-#endif
-
 internal void
 read_csv(string8 buffer)
 {
-#if defined(STD_TEST)
     printf("\nsize:%lu\ndata %s\n", buffer.size, buffer.data);
-#endif
 
 }
 
-
diff --git a/source/storage/csv_reader.h b/source/storage/csv_reader.h
index 711499f..36e07a4 100644
--- a/source/storage/csv_reader.h
+++ b/source/storage/csv_reader.h
@@ -23,8 +23,8 @@ struct csv_table
 read_only global_variable
 csv_row  nil_csv_row =
 {
-    .fields = {NULL,  0},
-    .count = 0,
+    .fields     = &nil_string,
+    .count      = 0,
 };
 
 
@@ -32,8 +32,8 @@ csv_row  nil_csv_row =
 read_only global_variable
 csv_table nil_csv_table =
 {
-    .string8     = {NULL, 0},
-    .csv_row     = &nil_csv_row,
+    .headers     = &nil_string,
+    .rows        = &nil_csv_row,
     .col_count   = 0,
     .row_count   = 0,
 };
-- 
cgit v1.3