From 71ced998122c357bc62e54d9bb4e124c88acf94b Mon Sep 17 00:00:00 2001
From: nasr <nsrddyn@gmail.com>
Date: Sun, 8 Mar 2026 21:01:43 +0000
Subject: refactor(main): worked on string handling in C and other stuff

---
 source/lexer.c | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 94 insertions(+)
 create mode 100644 source/lexer.c

(limited to 'source/lexer.c')

diff --git a/source/lexer.c b/source/lexer.c
new file mode 100644
index 0000000..c84a831
--- /dev/null
+++ b/source/lexer.c
@@ -0,0 +1,94 @@
+// the lexer acts as a table builder from a csv  file
+// and parsing indivudal rows and columns
+// the next step would be building a the b-tree
+internal b32
+is_alpha(u8 point)
+{
+    return ((point >= 'a' && point <= 'z') || (point >= 'A' && point <= 'Z') || (point == '_'));
+}
+
+internal b32
+is_digit(u8 point)
+{
+    return (point >= '0' && point <= '9');
+}
+
+internal b32
+is_alpha_num(u8 point)
+{
+    return (is_alpha(point) || is_digit(point));
+}
+
+internal b32
+is_whitespace(u8 point)
+{
+    return (point == '\n' || point == '\r' || point == ' ' || point == '\t');
+}
+
+internal b32
+is_delimiter(u8 point)
+{
+    return (point == ',');
+}
+
+internal token *
+tokenize_csv(string8 buffer, mem_arena *arena)
+{
+    s32 count = 0;
+    string8 **tokens = PushString(arena, buffer.size);
+
+    b32 FL = TRUE;
+
+    if(buffer.size < 0) return NULL;
+    for(s32 index = 0;
+        buffer.data[index] != '\0';
+        ++index)
+    {
+        csv_row *row = PushStruct(arena, csv_row);
+        token *tok = PushStruct(arena, token);
+
+        u8 point = buffer.data[index];
+
+        u8 *start = buffer.data;
+        u8 *end = NULL;
+
+        unused(row);
+
+        switch(point)
+        {
+            case('\n'):
+                {
+
+                    if(FL)
+                    {
+                        FL = FALSE;
+                        tok->flags |= END_FL;
+                    }
+
+                    break;
+
+                }
+
+            case(','):
+                {
+                    end = start - 1;
+                    break;
+                }
+
+            default:
+                {
+                    printf("point: %c\n", point);
+                    count++;
+                    break;
+                }
+        }
+
+        token->lexeme = String8Cast(start, end - start);
+
+        *tokens = token;
+        ++tokens;
+
+
+        return NULL;
+    }
+}
-- 
cgit v1.3