- Added string strip methods lstrip and rstrip

- Fixed the tokenizer to chomp whitespace from left and right of tokens
- Fixed the tokenizer so it returns reserved symbols not just constants and expressions
- Added some tests for the basic tokenizer and parser
- Started working on structures to allow the basic interpreter to store lines in memory

tests/basic_tokenizer.c

@@ -6,12 +6,12 @@
 char *_tokenize(char *ptr, char *token);
 char *_token_get(void);
 
-#define assert_lvalue(str, lval, ret_null, ret_neq) \
+#define assert_token_value(str, val, ret_null, ret_neq) \
     ptr = _tokenize(str, BASIC_TOKENIZER_TOKENS); \
     value = _token_get(); \
     if ( ptr == NULL ) return ret_null; \
-    rc = strcmp(value, lval); \
-    printf("(value) == (lval) ? : (%s) == (%s) %d\n", value, lval, rc); \
+    rc = strcmp(value, val); \
+    printf("(%s) => (value) == (val) ? : (%s) == (%s) %d\n", str, value, val, rc); \
     if ( rc != 0 ) return ret_neq;
 
 
@@ -21,9 +21,10 @@ int main(void)
     char *value = NULL;
     int rc = 0;
 
-    assert_lvalue("1+1", "1", 1, 2);
-    assert_lvalue("1 + 1", "1", 2, 3);
-    assert_lvalue("10 + 10", "10", 4, 5);
-
+    assert_token_value("1+1", "1", 1, 2);
+    assert_token_value("1 + 1", "1", 2, 3);
+    assert_token_value("10 + 10", "10", 4, 5);
+    assert_token_value("1+ 2", "1", 6, 7)
+    assert_token_value("+ 2", "+", 8, 9)
     return 0;
 }