From 7ec58e8db1096e7916f99b84011ddb13a010da75 Mon Sep 17 00:00:00 2001 From: _ov4 Date: Wed, 9 Sep 2026 10:57:22 +0000 Subject: separate compilation --- src/lexer/ast.cc | 151 ++++++++++++++++++++++++++++++++++ src/lexer/symbol_property.cc | 29 +++++++ src/lexer/token.cc | 190 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 370 insertions(+) create mode 100644 src/lexer/ast.cc create mode 100644 src/lexer/symbol_property.cc create mode 100644 src/lexer/token.cc (limited to 'src') diff --git a/src/lexer/ast.cc b/src/lexer/ast.cc new file mode 100644 index 0000000..9b18738 --- /dev/null +++ b/src/lexer/ast.cc @@ -0,0 +1,151 @@ +#include "lexer/ast.h" + +#include +#include +#include + +#include "lexer/lexer_class.h" +#include "lexer/enum_type.h" +#include "lexer/token.h" +#include "lexer/symbol_property.h" + +using namespace std; + +namespace ov4 +{ + +size_t T_lexer::alloc_ast() +{ + ast.emplace_back(); + return ast_index++; +} + +string T_lexer::final_trim(size_t _begin, size_t _end) +{ + string s; + for (size_t i = _begin; i < _end; i++) + { + if (token[i].token_type != TEXT) + continue; + s += token[i].text + " "; + } + s = s.substr(0, s.size()-1); // eat final space + return s; +} + +int T_lexer::get_pivot_order(enum_token_type x) +{ + string s = symbol_enum2string(x); + if (s != "") + { + return symbol_property[s].pivot_priority; + } + // not found + return -1; +} + +bool T_lexer::is_operator(enum_token_type x) +{ + switch(x) + { + case LOGIC_AND: + case LOGIC_OR: + case PIPE: + case ASYNC: + return true; + default: + return false; + } +} + +// region [begin, end) +void T_lexer::parse(size_t cmd_begin, size_t cmd_end, size_t ast_vec) +{ + int current_depth = 0; + int current_order = -1; + int max_depth = -1; + size_t found_pos = string::npos; + + for (size_t i = cmd_begin; i < cmd_end; i++) + { + if (token[i].bracket_depth > max_depth) + max_depth = token[i].bracket_depth; + } + + + while (found_pos == string::npos) + { + // scan from right to left + for (size_t i = cmd_end - 1; i >= cmd_begin && i < cmd_end; i--) + { + // if same depth + // and higher order number + if (token[i].bracket_depth == current_depth + && is_operator(token[i].token_type) + && get_pivot_order(token[i].token_type) > current_order) + { + current_order = get_pivot_order(token[i].token_type); + found_pos = i; + println("Found pos: {} order: {} depth: {}", i, current_order, current_depth); + } + } + current_depth++; + if (current_depth > max_depth) + { + break; + } + } + + if (found_pos == string::npos) + { + println("Smallest unit"); + ast[ast_vec].token_type = TEXT; + ast[ast_vec].command_text = final_trim(cmd_begin, cmd_end); + return; + } + + // left child + + string token_str = symbol_enum2string(token[found_pos].token_type); + + // miss left + if (symbol_property[token_str].left == EXIST && cmd_begin == found_pos) + { + println("parse: missing left child"); + exit(-1); // TODO + } + + // miss right + if (symbol_property[token_str].right == EXIST && cmd_end-1 == found_pos) + { + println("parse: missing right child"); + exit(-1); // TODO + } + + // should not exist left + if (symbol_property[token_str].left == NOT_EXIST && cmd_begin != found_pos) + { + println("parse: left child should NOT exist"); + exit(-1); // TODO + } + + // should not exist right + if (symbol_property[token_str].right == NOT_EXIST && cmd_end-1 != found_pos) + { + println("parse: right child should NOT exist"); + exit(-1); // TODO + } + + size_t l = alloc_ast(); + size_t r = alloc_ast(); + ast[ast_vec].left = l; + ast[ast_vec].right = r; + ast[ast_vec].token_type = token[found_pos].token_type; + + println("Parse sub: {}, {}, {}", cmd_begin, found_pos, cmd_end); + + parse(cmd_begin, found_pos, l); + parse(found_pos+1, cmd_end, r); +} + +} diff --git a/src/lexer/symbol_property.cc b/src/lexer/symbol_property.cc new file mode 100644 index 0000000..c78dd61 --- /dev/null +++ b/src/lexer/symbol_property.cc @@ -0,0 +1,29 @@ +#include "lexer/symbol_property.h" + +#include + +#include "lexer/enum_type.h" + +using namespace std; + +namespace ov4 +{ + +string symbol_enum2string(enum_token_type e) +{ + for (auto &c : symbol_property) + { + // pair + + if (c.second.enum_type == e) + return c.first; + } + return ""; +} + +bool symbol_contains_enum_type(enum_token_type e) +{ + return (symbol_enum2string(e) == "") ? false : true; +} + +} diff --git a/src/lexer/token.cc b/src/lexer/token.cc new file mode 100644 index 0000000..2b88b7e --- /dev/null +++ b/src/lexer/token.cc @@ -0,0 +1,190 @@ +#include "lexer/token.h" + +#include +#include +#include + +#include "lexer/lexer_class.h" +#include "lexer/enum_type.h" +#include "lexer/symbol_property.h" + +using namespace std; // remove this + +namespace ov4 +{ + +string T_lexer::trim(const string &s, const string &trim_target) // trim_target = " \t" +{ + size_t b = s.find_first_not_of(trim_target); + if (b == string::npos) + return ""; + size_t e = s.find_last_not_of(trim_target); + return s.substr(b, e - b + 1); +} + + +bool T_lexer::is_single_symbol(char c) +{ + string s = " "; + s[0] = c; + if (symbol_property.contains(s)) + return true; + return false; +} + +void T_lexer::bracket_depth_changer(char c, int &bracket_depth) +{ + if (c == '(') + { + bracket_depth++; + println("changed depth: {}", bracket_depth); + return; + } + if (c == ')') + { + bracket_depth--; + println("changed depth: {}", bracket_depth); + if (bracket_depth < 0) + { + cerr << "Ooooops bracket unmatched, quitting" << endl; + exit(-1); // TODO: std::expected or what, rather than quitting + } + return; + } +} + + +bool T_lexer::token_continue(const string &s, char next) +{ + println("token_continue: receiving {} {}", s, next); + // s is empty, so next can be part of the token + if (s.size() == 0) return true; + + // text -> text/symbol + if (!is_single_symbol(s[s.size()-1])) + { + // text -> symbol + if (is_single_symbol(next)) + return false; + + // text still + return true; + } + + // symbol -> symbol + T_property tmp; + if (symbol_property.contains(s)) + { + tmp = symbol_property[s]; + if (tmp.continuable) + { + string prev_and_next = s + next; + if (symbol_property.contains(prev_and_next)) + return true; + return false; + } + return false; + } + else + { + println("symbol not found in map"); + return false; + } +} + +// handle [prev, current) +void T_lexer::token_push(const string &push_s, int bracket_depth, bool force_text) // force_text = false +{ + T_token tmp; + tmp.bracket_depth = bracket_depth; + string trimmed = trim(push_s); + if (trimmed == "") return; + + println("finding {} in map", trimmed); + + if (symbol_property.contains(trimmed) && !force_text) // symbol + { + println("token_push: adding symbol"); + tmp.token_type = symbol_property[trimmed].enum_type; + } + else + { + println("token_push: adding text"); + if (force_text) + trimmed = trim(trimmed, "\'\""); + tmp.token_type = TEXT; + tmp.text = trimmed; + } + token.push_back(tmp); +} + +int T_lexer::tokenizer(const string &s) +{ + if (s.size() == 0) return -1; + + size_t len = s.size(); + size_t token_vector_index = 0; + string prev_str = ""; + bool force_text = false; + int bracket_depth = 0; + guard_type guard = GUARD_OFF; + + + for (size_t i = 0; i < len; i++) + { + + char c = s[i]; + + prev_str += c; + + println("---"); + println("prev_str .{}. next char {}", prev_str, s[i+1]); + + if (c == '\'' || c == '\"') + { + println("quotation"); + + // turn on GUARD + if (guard == GUARD_OFF) + { + guard = (c == '\'') ? GUARD_STRONG : GUARD_WEAK; + println("set guard {}", (c == '\'') ? "GUARD_STRONG" : "GUARD_WEAK"); + continue; + } + // turn off + else if (guard == GUARD_STRONG && c == '\'') + { + guard = GUARD_OFF; + println("quitting strong guard"); + force_text = true; + } + //turn off + else if (guard == GUARD_WEAK && c == '\"') + { + guard = GUARD_OFF; + println("quitting weak guard"); + force_text = true; + } else + continue; + } + // not quotation mark + else if ((guard == GUARD_STRONG || guard == GUARD_WEAK)) + { + println("tokenizer: guarded, continue"); + continue; + } + + bracket_depth_changer(c, bracket_depth); + + if ((i+1 == len) || !token_continue(prev_str, s[i+1])) + { + token_push(prev_str, bracket_depth, force_text); + force_text = false; + prev_str = ""; + continue; + } + } + return 0; +} + +} -- cgit v1.3.1-4-g156e