1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
|
#ifndef EDITOR_LEXER_H
#define EDITOR_LEXER_H
enum token_type
{
TokenUndefined = 256,
TokenIdentifier,
TokenIdentifierAssignmentValue,
TokenValue,
TokenString,
TokenNumber,
TokenDoubleEqual,
TokenGreaterEqual,
TokenLesserEqual,
TokenParam,
TokenFunc,
TokenReturn,
TokenIf,
TokenElse,
TokenFor,
TokenWhile,
TokenBreak,
TokenContinue,
TokenExpression,
TokenFuncBody,
TokenUnwantedChild,
TokenNewLine,
TokenRightShift,
TokenLeftShift,
TokenStar,
};
typedef enum token_type token_type;
// TODO(nasr): implement a global tokenizer that holds the current position of the lexical position.
// useful for the case we dont want to relex everything or something
// what we currently do is save the position in the function
// so this gets reset every single time.
// so we relex everything
//
// this was the plan
// but could be interesting the moment we dont do any incremental parsing
// not necesarrly for the same reason Sean Barret or Casey Muratori do it because
// that gets handler using the editor->App[index]
typedef struct Tokenizer Tokenizer;
struct Tokenizer
{
s32 Line;
s32 Column;
};
enum token_flags
{
FlagNone = (0),
FlagConstant = (1 << 0),
FlagGlobal = (1 << 1),
FlagsValue = (1 << 2),
FlagDefinition = (1 << 3),
FlagComparison = (1 << 4),
FlagDeprecated = (1 << 5),
FlagDirty = (1 << 6),
};
typedef enum token_flags token_flags;
typedef struct token token;
struct token
{
str8 Lexeme;
token_type Type;
token_flags Flags;
u64 ByteOffset;
s32 Column;
s32 Line;
str8 MetaData;
};
typedef struct token_node token_node;
struct token_node
{
token_node *Next;
token_node *Previous;
token *Token;
};
typedef struct token_list token_list;
struct token_list
{
token_node *Root;
token_node *Current;
};
typedef struct lexer lexer;
struct lexer
{
u8 *Text;
u64 TextCount;
u8 *EndOfFile;
u8 *UndefinedTokens;
};
global_variable const rune Delimiters[] =
{
'{',
'}',
'(',
')',
'[',
']',
';',
};
read_only global_variable token nil_token =
{
.Lexeme = {NULL, 0},
.Type = TokenUndefined,
.Flags = FlagNone,
.ByteOffset = 0,
.Column = 0,
.Line = 0,
};
read_only global_variable token_node nil_token_node =
{
.Next = &nil_token_node,
.Previous = &nil_token_node,
.Token = NULL,
};
#endif // EDITOR_LEXER_H
|
