8 files changed, 384 insertions, 327 deletions

diff --git a/include/lexer/ast.h b/include/lexer/ast.h
index b823c09..3434aa3 100644
--- a/include/lexer/ast.h
+++ b/include/lexer/ast.h
@@ -22,138 +22,4 @@ public:
     size_t right = string::npos;
 };
 
-string T_lexer::final_trim(size_t _begin, size_t _end)
-{
-    string s;
-    for (size_t i = _begin; i < _end; i++)
-    {
-        if (token[i].token_type != TEXT)
-            continue;
-        s += token[i].text + " ";
-    }
-    s = s.substr(0, s.size()-1); // eat final space
-    return s;
-}
-
-size_t T_lexer::alloc_ast()
-{
-    ast.emplace_back();
-    return ast_index++;
-}
-
-int T_lexer::get_pivot_order(enum_token_type x)
-{
-    string s = symbol_enum2string(x);
-    if (s != "")
-    {
-        return symbol_property[s].pivot_priority;
-    }
-    // not found
-    return -1;
-}
-
-bool T_lexer::is_operator(enum_token_type x)
-{
-    switch(x)
-    {
-        case LOGIC_AND:
-        case LOGIC_OR:
-        case PIPE:
-        case ASYNC:
-            return true;
-        default:
-            return false;
-    }
-}
-
-// region [begin, end)
-void T_lexer::parse(size_t cmd_begin, size_t cmd_end, size_t ast_vec)
-{
-    int current_depth = 0;
-    int current_order = -1;
-    int max_depth = -1;
-    size_t found_pos = string::npos;
-
-    for (size_t i = cmd_begin; i < cmd_end; i++)
-    {
-        if (token[i].bracket_depth > max_depth)
-            max_depth = token[i].bracket_depth;
-    }
-
-
-    while (found_pos == string::npos)
-    {
-        // scan from right to left
-        for (size_t i = cmd_end - 1; i >= cmd_begin && i < cmd_end; i--)
-        {
-            // if same depth
-            // and higher order number
-            if (token[i].bracket_depth == current_depth 
-                && is_operator(token[i].token_type) 
-                && get_pivot_order(token[i].token_type) > current_order)
-            {
-                current_order =  get_pivot_order(token[i].token_type);
-                found_pos = i;
-                println("Found pos: {} order: {} depth: {}", i, current_order, current_depth);
-            }
-        }
-        current_depth++;
-        if (current_depth > max_depth)
-        {
-            break;
-        }
-    }
-
-    if (found_pos == string::npos)
-    {
-        println("Smallest unit");
-        ast[ast_vec].token_type = TEXT;
-        ast[ast_vec].command_text = final_trim(cmd_begin, cmd_end);
-        return;
-    }
-
-    // left child
-
-    string token_str = symbol_enum2string(token[found_pos].token_type);
-
-    // miss left
-    if (symbol_property[token_str].left == EXIST && cmd_begin == found_pos)
-    {
-        println("parse: missing left child");
-        exit(-1); // TODO
-    }
-
-    // miss right
-    if (symbol_property[token_str].right == EXIST && cmd_end-1 == found_pos)
-    {
-        println("parse: missing right child");
-        exit(-1); // TODO
-    }
-
-    // should not exist left
-    if (symbol_property[token_str].left == NOT_EXIST && cmd_begin != found_pos)
-    {
-        println("parse: left child should NOT exist");
-        exit(-1); // TODO
-    }
-
-    // should not exist right
-    if (symbol_property[token_str].right == NOT_EXIST && cmd_end-1 != found_pos)
-    {
-        println("parse: right child should NOT exist");
-        exit(-1); // TODO
-    }
-
-    size_t l = alloc_ast();
-    size_t r = alloc_ast();
-    ast[ast_vec].left = l;
-    ast[ast_vec].right = r;
-    ast[ast_vec].token_type = token[found_pos].token_type;
-
-    println("Parse sub: {}, {}, {}", cmd_begin, found_pos, cmd_end);
-
-    parse(cmd_begin, found_pos, l);
-    parse(found_pos+1, cmd_end, r);
-}
-
 }
diff --git a/include/lexer/lexer_class.h b/include/lexer/lexer_class.h
index 881ef47..04b2e1c 100644
--- a/include/lexer/lexer_class.h
+++ b/include/lexer/lexer_class.h
@@ -2,6 +2,7 @@
 
 #include <string>
 #include <unordered_map>
+#include <vector>
 
 #include "lexer/enum_type.h"
 
diff --git a/include/lexer/symbol_property.h b/include/lexer/symbol_property.h
index 5fc0bd6..53120e0 100644
--- a/include/lexer/symbol_property.h
+++ b/include/lexer/symbol_property.h
@@ -2,6 +2,7 @@
 
 #include <unordered_map>
 #include <string>
+
 #include "lexer/enum_type.h"
 
 using namespace std;
@@ -24,7 +25,7 @@ public:
     enum_token_type enum_type = TEXT;
 };
 
-unordered_map<string, T_property> symbol_property = {
+inline unordered_map<string, T_property> symbol_property = {
     // non-operator
     {"\'",   {.enum_type = SINGLE_QUOTATION}},
     {"\"",   {.enum_type = SINGLE_QUOTATION}},
@@ -41,21 +42,8 @@ unordered_map<string, T_property> symbol_property = {
 
 };
 
-string symbol_enum2string(enum_token_type e)
-{
-    for (auto &c : symbol_property)
-    {
-        // pair<string, T_property>
-        
-        if (c.second.enum_type == e)
-            return c.first;
-    }
-    return "";
-}
+string symbol_enum2string(enum_token_type e);
 
-bool symbol_contains_enum_type(enum_token_type e)
-{
-    return (symbol_enum2string(e) == "") ? false : true;
-}
+bool symbol_contains_enum_type(enum_token_type e);
     
 }
diff --git a/include/lexer/token.h b/include/lexer/token.h
index f2de46b..93bdc22 100644
--- a/include/lexer/token.h
+++ b/include/lexer/token.h
@@ -22,178 +22,4 @@ public:
     int bracket_depth = 0;
 };
 
-string T_lexer::trim(const string &s, const string &trim_target) // trim_target = " \t"
-{
-    size_t b = s.find_first_not_of(trim_target);
-    if (b == string::npos)
-        return "";
-    size_t e = s.find_last_not_of(trim_target);
-    return s.substr(b, e - b + 1);
-}
-
-
-bool T_lexer::is_single_symbol(char c)
-{
-    string s = " ";
-    s[0] = c;
-    if (symbol_property.contains(s))
-        return true;
-    return false;
-}
-
-void T_lexer::bracket_depth_changer(char c, int &bracket_depth)
-{
-    if (c == '(')
-    {
-        bracket_depth++;
-        println("changed depth: {}", bracket_depth);
-        return;
-    }
-    if (c == ')')
-    {
-        bracket_depth--;
-        println("changed depth: {}", bracket_depth);
-        if (bracket_depth < 0)
-        {
-            cerr << "Ooooops bracket unmatched, quitting" << endl;
-            exit(-1); // TODO: std::expected or what, rather than quitting
-        }
-        return;
-    }
-}
-
-
-bool T_lexer::token_continue(const string &s, char next)
-{
-    println("token_continue: receiving {} {}", s, next);
-    // s is empty, so next can be part of the token
-    if (s.size() == 0) return true;
-
-    // text -> text/symbol
-    if (!is_single_symbol(s[s.size()-1]))
-    {
-        // text -> symbol
-        if (is_single_symbol(next))
-            return false;
-
-        // text still
-        return true;
-    }
-
-    // symbol -> symbol
-    T_property tmp;
-    if (symbol_property.contains(s))
-    {
-        tmp = symbol_property[s];
-        if (tmp.continuable)
-        {
-            string prev_and_next = s + next;
-            if (symbol_property.contains(prev_and_next))
-                return true;
-            return false;
-        }
-        return false;
-    }
-    else
-    {
-        println("symbol not found in map");
-        return false;
-    }
-}
-
-// handle [prev, current)
-void T_lexer::token_push(const string &push_s, int bracket_depth, bool force_text) // force_text = false
-{
-    T_token tmp;
-    tmp.bracket_depth = bracket_depth;
-    string trimmed = trim(push_s);
-    if (trimmed == "")   return;
-
-    println("finding {} in map", trimmed);
-
-    if (symbol_property.contains(trimmed) && !force_text) // symbol
-    {
-        println("token_push: adding symbol");
-        tmp.token_type = symbol_property[trimmed].enum_type;
-    } 
-    else
-    {
-        println("token_push: adding text");
-        if (force_text)
-            trimmed = trim(trimmed, "\'\"");
-        tmp.token_type = TEXT;
-        tmp.text = trimmed;
-    }
-    token.push_back(tmp);
-}
-
-int T_lexer::tokenizer(const string &s)
-{
-    if (s.size() == 0) return -1;
-
-    size_t len = s.size();
-    size_t token_vector_index = 0;
-    string prev_str = "";
-    bool force_text = false;
-    int bracket_depth = 0;
-    guard_type guard = GUARD_OFF;
-
-
-    for (size_t i = 0; i < len; i++)
-    {
-
-        char c = s[i];
-
-        prev_str += c;
-        
-        println("---");
-        println("prev_str .{}. next char {}", prev_str, s[i+1]);
-
-        if (c == '\'' || c == '\"')
-        {
-            println("quotation");
-
-            // turn on GUARD
-            if (guard == GUARD_OFF)
-            {
-                guard = (c == '\'') ? GUARD_STRONG : GUARD_WEAK;
-                println("set guard {}", (c == '\'') ? "GUARD_STRONG" : "GUARD_WEAK");
-                continue;
-            }
-            // turn off
-            else if (guard == GUARD_STRONG && c == '\'')
-            {
-                guard = GUARD_OFF;
-                println("quitting strong guard");
-                force_text = true;
-            }
-            //turn off
-            else if (guard == GUARD_WEAK && c == '\"')
-            {
-                guard = GUARD_OFF;
-                println("quitting weak guard");
-                force_text = true;
-            } else
-                continue;
-        }
-        // not quotation mark
-        else if ((guard == GUARD_STRONG || guard == GUARD_WEAK))
-        {
-            println("tokenizer: guarded, continue");
-            continue;
-        }
-
-        bracket_depth_changer(c, bracket_depth);
-
-        if ((i+1 == len) || !token_continue(prev_str, s[i+1]))
-        {
-            token_push(prev_str, bracket_depth, force_text);
-            force_text = false;
-            prev_str = "";
-            continue;
-        }
-    }
-    return 0;
-}
-
 }
diff --git a/module-test/lexer_test.cc b/module-test/lexer_test.cc
index ffdf7b2..b5656c7 100644
--- a/module-test/lexer_test.cc
+++ b/module-test/lexer_test.cc
@@ -1,5 +1,13 @@
 // test lexer
-// $ g++ module-test/ast.cc -std=c++23 -Iinclude -Ithird_party/magic_enum/include 
+/* 
+$ 
+g++ \
+    module-test/lexer_test.cc \
+    src/lexer/token.cc \
+    src/lexer/ast.cc \
+    src/lexer/symbol_property.cc \
+    -std=c++23 -Iinclude -Ithird_party/magic_enum/include 
+*/
 
 #include <iostream>
 #include <string>
@@ -21,8 +29,6 @@ using namespace ov4;
 constexpr int MAXLINE = 1024;
 
 
-
-
 int main()
 {
     int test_case = 13;
diff --git a/src/lexer/ast.cc b/src/lexer/ast.cc
new file mode 100644
index 0000000..9b18738
--- /dev/null
+++ b/src/lexer/ast.cc
@@ -0,0 +1,151 @@
+#include "lexer/ast.h"
+
+#include <string>
+#include <iostream>
+#include <print>
+
+#include "lexer/lexer_class.h"
+#include "lexer/enum_type.h"
+#include "lexer/token.h"
+#include "lexer/symbol_property.h"
+
+using namespace std;
+
+namespace ov4
+{
+
+size_t T_lexer::alloc_ast()
+{
+    ast.emplace_back();
+    return ast_index++;
+}
+
+string T_lexer::final_trim(size_t _begin, size_t _end)
+{
+    string s;
+    for (size_t i = _begin; i < _end; i++)
+    {
+        if (token[i].token_type != TEXT)
+            continue;
+        s += token[i].text + " ";
+    }
+    s = s.substr(0, s.size()-1); // eat final space
+    return s;
+}
+
+int T_lexer::get_pivot_order(enum_token_type x)
+{
+    string s = symbol_enum2string(x);
+    if (s != "")
+    {
+        return symbol_property[s].pivot_priority;
+    }
+    // not found
+    return -1;
+}
+
+bool T_lexer::is_operator(enum_token_type x)
+{
+    switch(x)
+    {
+        case LOGIC_AND:
+        case LOGIC_OR:
+        case PIPE:
+        case ASYNC:
+            return true;
+        default:
+            return false;
+    }
+}
+
+// region [begin, end)
+void T_lexer::parse(size_t cmd_begin, size_t cmd_end, size_t ast_vec)
+{
+    int current_depth = 0;
+    int current_order = -1;
+    int max_depth = -1;
+    size_t found_pos = string::npos;
+
+    for (size_t i = cmd_begin; i < cmd_end; i++)
+    {
+        if (token[i].bracket_depth > max_depth)
+            max_depth = token[i].bracket_depth;
+    }
+
+
+    while (found_pos == string::npos)
+    {
+        // scan from right to left
+        for (size_t i = cmd_end - 1; i >= cmd_begin && i < cmd_end; i--)
+        {
+            // if same depth
+            // and higher order number
+            if (token[i].bracket_depth == current_depth 
+                && is_operator(token[i].token_type) 
+                && get_pivot_order(token[i].token_type) > current_order)
+            {
+                current_order =  get_pivot_order(token[i].token_type);
+                found_pos = i;
+                println("Found pos: {} order: {} depth: {}", i, current_order, current_depth);
+            }
+        }
+        current_depth++;
+        if (current_depth > max_depth)
+        {
+            break;
+        }
+    }
+
+    if (found_pos == string::npos)
+    {
+        println("Smallest unit");
+        ast[ast_vec].token_type = TEXT;
+        ast[ast_vec].command_text = final_trim(cmd_begin, cmd_end);
+        return;
+    }
+
+    // left child
+
+    string token_str = symbol_enum2string(token[found_pos].token_type);
+
+    // miss left
+    if (symbol_property[token_str].left == EXIST && cmd_begin == found_pos)
+    {
+        println("parse: missing left child");
+        exit(-1); // TODO
+    }
+
+    // miss right
+    if (symbol_property[token_str].right == EXIST && cmd_end-1 == found_pos)
+    {
+        println("parse: missing right child");
+        exit(-1); // TODO
+    }
+
+    // should not exist left
+    if (symbol_property[token_str].left == NOT_EXIST && cmd_begin != found_pos)
+    {
+        println("parse: left child should NOT exist");
+        exit(-1); // TODO
+    }
+
+    // should not exist right
+    if (symbol_property[token_str].right == NOT_EXIST && cmd_end-1 != found_pos)
+    {
+        println("parse: right child should NOT exist");
+        exit(-1); // TODO
+    }
+
+    size_t l = alloc_ast();
+    size_t r = alloc_ast();
+    ast[ast_vec].left = l;
+    ast[ast_vec].right = r;
+    ast[ast_vec].token_type = token[found_pos].token_type;
+
+    println("Parse sub: {}, {}, {}", cmd_begin, found_pos, cmd_end);
+
+    parse(cmd_begin, found_pos, l);
+    parse(found_pos+1, cmd_end, r);
+}
+
+}
diff --git a/src/lexer/symbol_property.cc b/src/lexer/symbol_property.cc
new file mode 100644
index 0000000..c78dd61
--- /dev/null
+++ b/src/lexer/symbol_property.cc
@@ -0,0 +1,29 @@
+#include "lexer/symbol_property.h"
+
+#include <string>
+
+#include "lexer/enum_type.h"
+
+using namespace std;
+
+namespace ov4
+{
+
+string symbol_enum2string(enum_token_type e)
+{
+    for (auto &c : symbol_property)
+    {
+        // pair<string, T_property>
+        
+        if (c.second.enum_type == e)
+            return c.first;
+    }
+    return "";
+}
+
+bool symbol_contains_enum_type(enum_token_type e)
+{
+    return (symbol_enum2string(e) == "") ? false : true;
+}
+
+}
diff --git a/src/lexer/token.cc b/src/lexer/token.cc
new file mode 100644
index 0000000..2b88b7e
--- /dev/null
+++ b/src/lexer/token.cc
@@ -0,0 +1,190 @@
+#include "lexer/token.h"
+
+#include <string>
+#include <iostream>
+#include <print>
+
+#include "lexer/lexer_class.h"
+#include "lexer/enum_type.h"
+#include "lexer/symbol_property.h"
+
+using namespace std; // remove this
+
+namespace ov4
+{
+
+string T_lexer::trim(const string &s, const string &trim_target) // trim_target = " \t"
+{
+    size_t b = s.find_first_not_of(trim_target);
+    if (b == string::npos)
+        return "";
+    size_t e = s.find_last_not_of(trim_target);
+    return s.substr(b, e - b + 1);
+}
+
+
+bool T_lexer::is_single_symbol(char c)
+{
+    string s = " ";
+    s[0] = c;
+    if (symbol_property.contains(s))
+        return true;
+    return false;
+}
+
+void T_lexer::bracket_depth_changer(char c, int &bracket_depth)
+{
+    if (c == '(')
+    {
+        bracket_depth++;
+        println("changed depth: {}", bracket_depth);
+        return;
+    }
+    if (c == ')')
+    {
+        bracket_depth--;
+        println("changed depth: {}", bracket_depth);
+        if (bracket_depth < 0)
+        {
+            cerr << "Ooooops bracket unmatched, quitting" << endl;
+            exit(-1); // TODO: std::expected or what, rather than quitting
+        }
+        return;
+    }
+}
+
+
+bool T_lexer::token_continue(const string &s, char next)
+{
+    println("token_continue: receiving {} {}", s, next);
+    // s is empty, so next can be part of the token
+    if (s.size() == 0) return true;
+
+    // text -> text/symbol
+    if (!is_single_symbol(s[s.size()-1]))
+    {
+        // text -> symbol
+        if (is_single_symbol(next))
+            return false;
+
+        // text still
+        return true;
+    }
+
+    // symbol -> symbol
+    T_property tmp;
+    if (symbol_property.contains(s))
+    {
+        tmp = symbol_property[s];
+        if (tmp.continuable)
+        {
+            string prev_and_next = s + next;
+            if (symbol_property.contains(prev_and_next))
+                return true;
+            return false;
+        }
+        return false;
+    }
+    else
+    {
+        println("symbol not found in map");
+        return false;
+    }
+}
+
+// handle [prev, current)
+void T_lexer::token_push(const string &push_s, int bracket_depth, bool force_text) // force_text = false
+{
+    T_token tmp;
+    tmp.bracket_depth = bracket_depth;
+    string trimmed = trim(push_s);
+    if (trimmed == "")   return;
+
+    println("finding {} in map", trimmed);
+
+    if (symbol_property.contains(trimmed) && !force_text) // symbol
+    {
+        println("token_push: adding symbol");
+        tmp.token_type = symbol_property[trimmed].enum_type;
+    } 
+    else
+    {
+        println("token_push: adding text");
+        if (force_text)
+            trimmed = trim(trimmed, "\'\"");
+        tmp.token_type = TEXT;
+        tmp.text = trimmed;
+    }
+    token.push_back(tmp);
+}
+
+int T_lexer::tokenizer(const string &s)
+{
+    if (s.size() == 0) return -1;
+
+    size_t len = s.size();
+    size_t token_vector_index = 0;
+    string prev_str = "";
+    bool force_text = false;
+    int bracket_depth = 0;
+    guard_type guard = GUARD_OFF;
+
+
+    for (size_t i = 0; i < len; i++)
+    {
+
+        char c = s[i];
+
+        prev_str += c;
+        
+        println("---");
+        println("prev_str .{}. next char {}", prev_str, s[i+1]);
+
+        if (c == '\'' || c == '\"')
+        {
+            println("quotation");
+
+            // turn on GUARD
+            if (guard == GUARD_OFF)
+            {
+                guard = (c == '\'') ? GUARD_STRONG : GUARD_WEAK;
+                println("set guard {}", (c == '\'') ? "GUARD_STRONG" : "GUARD_WEAK");
+                continue;
+            }
+            // turn off
+            else if (guard == GUARD_STRONG && c == '\'')
+            {
+                guard = GUARD_OFF;
+                println("quitting strong guard");
+                force_text = true;
+            }
+            //turn off
+            else if (guard == GUARD_WEAK && c == '\"')
+            {
+                guard = GUARD_OFF;
+                println("quitting weak guard");
+                force_text = true;
+            } else
+                continue;
+        }
+        // not quotation mark
+        else if ((guard == GUARD_STRONG || guard == GUARD_WEAK))
+        {
+            println("tokenizer: guarded, continue");
+            continue;
+        }
+
+        bracket_depth_changer(c, bracket_depth);
+
+        if ((i+1 == len) || !token_continue(prev_str, s[i+1]))
+        {
+            token_push(prev_str, bracket_depth, force_text);
+            force_text = false;
+            prev_str = "";
+            continue;
+        }
+    }
+    return 0;
+}
+
+}