aboutsummaryrefslogtreecommitdiff
path: root/include/lexer
diff options
context:
space:
mode:
Diffstat (limited to 'include/lexer')
-rw-r--r--include/lexer/ast.h134
-rw-r--r--include/lexer/lexer_class.h1
-rw-r--r--include/lexer/symbol_property.h20
-rw-r--r--include/lexer/token.h174
4 files changed, 5 insertions, 324 deletions
diff --git a/include/lexer/ast.h b/include/lexer/ast.h
index b823c09..3434aa3 100644
--- a/include/lexer/ast.h
+++ b/include/lexer/ast.h
@@ -22,138 +22,4 @@ public:
size_t right = string::npos;
};
-string T_lexer::final_trim(size_t _begin, size_t _end)
-{
- string s;
- for (size_t i = _begin; i < _end; i++)
- {
- if (token[i].token_type != TEXT)
- continue;
- s += token[i].text + " ";
- }
- s = s.substr(0, s.size()-1); // eat final space
- return s;
-}
-
-size_t T_lexer::alloc_ast()
-{
- ast.emplace_back();
- return ast_index++;
-}
-
-int T_lexer::get_pivot_order(enum_token_type x)
-{
- string s = symbol_enum2string(x);
- if (s != "")
- {
- return symbol_property[s].pivot_priority;
- }
- // not found
- return -1;
-}
-
-bool T_lexer::is_operator(enum_token_type x)
-{
- switch(x)
- {
- case LOGIC_AND:
- case LOGIC_OR:
- case PIPE:
- case ASYNC:
- return true;
- default:
- return false;
- }
-}
-
-// region [begin, end)
-void T_lexer::parse(size_t cmd_begin, size_t cmd_end, size_t ast_vec)
-{
- int current_depth = 0;
- int current_order = -1;
- int max_depth = -1;
- size_t found_pos = string::npos;
-
- for (size_t i = cmd_begin; i < cmd_end; i++)
- {
- if (token[i].bracket_depth > max_depth)
- max_depth = token[i].bracket_depth;
- }
-
-
- while (found_pos == string::npos)
- {
- // scan from right to left
- for (size_t i = cmd_end - 1; i >= cmd_begin && i < cmd_end; i--)
- {
- // if same depth
- // and higher order number
- if (token[i].bracket_depth == current_depth
- && is_operator(token[i].token_type)
- && get_pivot_order(token[i].token_type) > current_order)
- {
- current_order = get_pivot_order(token[i].token_type);
- found_pos = i;
- println("Found pos: {} order: {} depth: {}", i, current_order, current_depth);
- }
- }
- current_depth++;
- if (current_depth > max_depth)
- {
- break;
- }
- }
-
- if (found_pos == string::npos)
- {
- println("Smallest unit");
- ast[ast_vec].token_type = TEXT;
- ast[ast_vec].command_text = final_trim(cmd_begin, cmd_end);
- return;
- }
-
- // left child
-
- string token_str = symbol_enum2string(token[found_pos].token_type);
-
- // miss left
- if (symbol_property[token_str].left == EXIST && cmd_begin == found_pos)
- {
- println("parse: missing left child");
- exit(-1); // TODO
- }
-
- // miss right
- if (symbol_property[token_str].right == EXIST && cmd_end-1 == found_pos)
- {
- println("parse: missing right child");
- exit(-1); // TODO
- }
-
- // should not exist left
- if (symbol_property[token_str].left == NOT_EXIST && cmd_begin != found_pos)
- {
- println("parse: left child should NOT exist");
- exit(-1); // TODO
- }
-
- // should not exist right
- if (symbol_property[token_str].right == NOT_EXIST && cmd_end-1 != found_pos)
- {
- println("parse: right child should NOT exist");
- exit(-1); // TODO
- }
-
- size_t l = alloc_ast();
- size_t r = alloc_ast();
- ast[ast_vec].left = l;
- ast[ast_vec].right = r;
- ast[ast_vec].token_type = token[found_pos].token_type;
-
- println("Parse sub: {}, {}, {}", cmd_begin, found_pos, cmd_end);
-
- parse(cmd_begin, found_pos, l);
- parse(found_pos+1, cmd_end, r);
-}
-
}
diff --git a/include/lexer/lexer_class.h b/include/lexer/lexer_class.h
index 881ef47..04b2e1c 100644
--- a/include/lexer/lexer_class.h
+++ b/include/lexer/lexer_class.h
@@ -2,6 +2,7 @@
#include <string>
#include <unordered_map>
+#include <vector>
#include "lexer/enum_type.h"
diff --git a/include/lexer/symbol_property.h b/include/lexer/symbol_property.h
index 5fc0bd6..53120e0 100644
--- a/include/lexer/symbol_property.h
+++ b/include/lexer/symbol_property.h
@@ -2,6 +2,7 @@
#include <unordered_map>
#include <string>
+
#include "lexer/enum_type.h"
using namespace std;
@@ -24,7 +25,7 @@ public:
enum_token_type enum_type = TEXT;
};
-unordered_map<string, T_property> symbol_property = {
+inline unordered_map<string, T_property> symbol_property = {
// non-operator
{"\'", {.enum_type = SINGLE_QUOTATION}},
{"\"", {.enum_type = SINGLE_QUOTATION}},
@@ -41,21 +42,8 @@ unordered_map<string, T_property> symbol_property = {
};
-string symbol_enum2string(enum_token_type e)
-{
- for (auto &c : symbol_property)
- {
- // pair<string, T_property>
-
- if (c.second.enum_type == e)
- return c.first;
- }
- return "";
-}
+string symbol_enum2string(enum_token_type e);
-bool symbol_contains_enum_type(enum_token_type e)
-{
- return (symbol_enum2string(e) == "") ? false : true;
-}
+bool symbol_contains_enum_type(enum_token_type e);
}
diff --git a/include/lexer/token.h b/include/lexer/token.h
index f2de46b..93bdc22 100644
--- a/include/lexer/token.h
+++ b/include/lexer/token.h
@@ -22,178 +22,4 @@ public:
int bracket_depth = 0;
};
-string T_lexer::trim(const string &s, const string &trim_target) // trim_target = " \t"
-{
- size_t b = s.find_first_not_of(trim_target);
- if (b == string::npos)
- return "";
- size_t e = s.find_last_not_of(trim_target);
- return s.substr(b, e - b + 1);
-}
-
-
-bool T_lexer::is_single_symbol(char c)
-{
- string s = " ";
- s[0] = c;
- if (symbol_property.contains(s))
- return true;
- return false;
-}
-
-void T_lexer::bracket_depth_changer(char c, int &bracket_depth)
-{
- if (c == '(')
- {
- bracket_depth++;
- println("changed depth: {}", bracket_depth);
- return;
- }
- if (c == ')')
- {
- bracket_depth--;
- println("changed depth: {}", bracket_depth);
- if (bracket_depth < 0)
- {
- cerr << "Ooooops bracket unmatched, quitting" << endl;
- exit(-1); // TODO: std::expected or what, rather than quitting
- }
- return;
- }
-}
-
-
-bool T_lexer::token_continue(const string &s, char next)
-{
- println("token_continue: receiving {} {}", s, next);
- // s is empty, so next can be part of the token
- if (s.size() == 0) return true;
-
- // text -> text/symbol
- if (!is_single_symbol(s[s.size()-1]))
- {
- // text -> symbol
- if (is_single_symbol(next))
- return false;
-
- // text still
- return true;
- }
-
- // symbol -> symbol
- T_property tmp;
- if (symbol_property.contains(s))
- {
- tmp = symbol_property[s];
- if (tmp.continuable)
- {
- string prev_and_next = s + next;
- if (symbol_property.contains(prev_and_next))
- return true;
- return false;
- }
- return false;
- }
- else
- {
- println("symbol not found in map");
- return false;
- }
-}
-
-// handle [prev, current)
-void T_lexer::token_push(const string &push_s, int bracket_depth, bool force_text) // force_text = false
-{
- T_token tmp;
- tmp.bracket_depth = bracket_depth;
- string trimmed = trim(push_s);
- if (trimmed == "") return;
-
- println("finding {} in map", trimmed);
-
- if (symbol_property.contains(trimmed) && !force_text) // symbol
- {
- println("token_push: adding symbol");
- tmp.token_type = symbol_property[trimmed].enum_type;
- }
- else
- {
- println("token_push: adding text");
- if (force_text)
- trimmed = trim(trimmed, "\'\"");
- tmp.token_type = TEXT;
- tmp.text = trimmed;
- }
- token.push_back(tmp);
-}
-
-int T_lexer::tokenizer(const string &s)
-{
- if (s.size() == 0) return -1;
-
- size_t len = s.size();
- size_t token_vector_index = 0;
- string prev_str = "";
- bool force_text = false;
- int bracket_depth = 0;
- guard_type guard = GUARD_OFF;
-
-
- for (size_t i = 0; i < len; i++)
- {
-
- char c = s[i];
-
- prev_str += c;
-
- println("---");
- println("prev_str .{}. next char {}", prev_str, s[i+1]);
-
- if (c == '\'' || c == '\"')
- {
- println("quotation");
-
- // turn on GUARD
- if (guard == GUARD_OFF)
- {
- guard = (c == '\'') ? GUARD_STRONG : GUARD_WEAK;
- println("set guard {}", (c == '\'') ? "GUARD_STRONG" : "GUARD_WEAK");
- continue;
- }
- // turn off
- else if (guard == GUARD_STRONG && c == '\'')
- {
- guard = GUARD_OFF;
- println("quitting strong guard");
- force_text = true;
- }
- //turn off
- else if (guard == GUARD_WEAK && c == '\"')
- {
- guard = GUARD_OFF;
- println("quitting weak guard");
- force_text = true;
- } else
- continue;
- }
- // not quotation mark
- else if ((guard == GUARD_STRONG || guard == GUARD_WEAK))
- {
- println("tokenizer: guarded, continue");
- continue;
- }
-
- bracket_depth_changer(c, bracket_depth);
-
- if ((i+1 == len) || !token_continue(prev_str, s[i+1]))
- {
- token_push(prev_str, bracket_depth, force_text);
- force_text = false;
- prev_str = "";
- continue;
- }
- }
- return 0;
-}
-
}
generated by cgit v1.3.1-4-g156e (git 2.54.0) at 2026-11-14 22:49:30 +0000