refactor(main): worked on string handling in C and other stuff

1 files changed, 32 insertions, 0 deletions

diff --git a/source/lexer.h b/source/lexer.h
new file mode 100644
index 0000000..7bafc0d
--- /dev/null
+++ b/source/lexer.h
@@ -0,0 +1,32 @@
+#ifndef ENGINE_LEXER_H
+#define ENGINE_LEXER_H
+
+typedef enum token_flags token_flags;
+enum token_flags 
+{
+    START_FL    = 1 << 1,
+    END_FL      = 1 << 2,
+};
+
+
+typedef enum token_type token_type;
+enum token_type
+{
+    // first 255 tokens  for ascii characters
+    TOKEN_UNDEFINED = 255,
+    TOKEN_IDENTIFIER,
+    TOKEN_VALUE,
+
+};
+
+typedef struct token token;
+struct token
+{
+    string8 lexeme;
+    token_type type;
+    token_flags flags;
+};
+
+
+
+#endif /* ENGINE_LEXER_H */