source/lexer/lexer.c


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80

internal b32
is_alpha(u8 point)
{
  return ((point >= 'a' && point <= 'z') ||
          (point >= 'A' && point <= 'Z') ||
          (point == '_'));
}

internal b32
is_digit(u8 point)
{
  return (point >= '0' && point <= '9');
}

internal b32
is_alpha_num(u8 point)
{
  return (is_alpha(point) || is_digit(point));
}

internal b32
is_whitespace(u8 point)
{
  return (point == '\n' || point == '\r' ||
          point == ' ' || point == '\t');
}

internal b32
is_delimiter(u8 point)
{

    return (point == ',');

}

internal token *
tokenize_csv(string8 buffer)
{
    i32 count = 0;
    string8 **tokens = PushArray(arena, string8 *, buffer.size / 10);

    if(buffer.size < 0) return NULL;
    for(i32 index = 0;
         buffer.data[index] != '\0';
         ++index)
    {
        string8 tokens = {0};

        u8 point = buffer.data[index];
        if(is_whitespace(point)) continue;

        u8 *start = &buffer.data;

        if(is_delimiter(point))
        {


        }

        u8 *end = start - 1;

        unused(start);
        unused(end);

        switch (point)
        {

            default:
                {
                    printf("point: %c\n", point);
                    count++;
                }
        }

    }

    printf("%d", count);

    return NULL;
}