From 1c630655b2beb0f4b84fb7e1bc034ba62520be37 Mon Sep 17 00:00:00 2001 From: _ov4 Date: Mon, 21 Sep 2026 04:50:06 +0000 Subject: fix name typo, and remove outdated parse --- include/all.h | 1 - include/eval.h | 2 +- include/lang_analysis/ast.h | 30 +++++ include/lang_analysis/enum_type.h | 35 ++++++ include/lang_analysis/lexer_class.h | 42 +++++++ include/lang_analysis/parseline.h | 10 ++ include/lang_analysis/symbol_property.h | 48 ++++++++ include/lang_analysis/token.h | 25 ++++ include/lexer/ast.h | 30 ----- include/lexer/enum_type.h | 35 ------ include/lexer/lexer_class.h | 42 ------- include/lexer/parseline.h | 10 -- include/lexer/symbol_property.h | 48 -------- include/lexer/token.h | 25 ---- include/parse.h | 13 -- module-test/lexer_test.cc | 12 +- module-test/parseline_test.cc | 6 +- src/eval.cc | 12 +- src/lang_analysis/ast.cc | 161 +++++++++++++++++++++++++ src/lang_analysis/parseline.cc | 80 +++++++++++++ src/lang_analysis/symbol_property.cc | 29 +++++ src/lang_analysis/token.cc | 204 ++++++++++++++++++++++++++++++++ src/lexer/ast.cc | 161 ------------------------- src/lexer/parseline.cc | 80 ------------- src/lexer/symbol_property.cc | 29 ----- src/lexer/token.cc | 204 -------------------------------- src/parse.cc | 72 ----------- 27 files changed, 680 insertions(+), 766 deletions(-) create mode 100644 include/lang_analysis/ast.h create mode 100644 include/lang_analysis/enum_type.h create mode 100644 include/lang_analysis/lexer_class.h create mode 100644 include/lang_analysis/parseline.h create mode 100644 include/lang_analysis/symbol_property.h create mode 100644 include/lang_analysis/token.h delete mode 100644 include/lexer/ast.h delete mode 100644 include/lexer/enum_type.h delete mode 100644 include/lexer/lexer_class.h delete mode 100644 include/lexer/parseline.h delete mode 100644 include/lexer/symbol_property.h delete mode 100644 include/lexer/token.h delete mode 100644 include/parse.h create mode 100644 src/lang_analysis/ast.cc create mode 100644 src/lang_analysis/parseline.cc create mode 100644 src/lang_analysis/symbol_property.cc create mode 100644 src/lang_analysis/token.cc delete mode 100644 src/lexer/ast.cc delete mode 100644 src/lexer/parseline.cc delete mode 100644 src/lexer/symbol_property.cc delete mode 100644 src/lexer/token.cc delete mode 100644 src/parse.cc diff --git a/include/all.h b/include/all.h index 8c0d6f9..742eb00 100644 --- a/include/all.h +++ b/include/all.h @@ -12,6 +12,5 @@ #include "handler.h" #include "init.h" #include "job.h" -#include "parse.h" #include "path.h" #include "option.h" diff --git a/include/eval.h b/include/eval.h index 0466cb4..360de66 100644 --- a/include/eval.h +++ b/include/eval.h @@ -1,7 +1,7 @@ #pragma once #include "all.h" -#include "lexer/lexer_class.h" +#include "lang_analysis/lexer_class.h" namespace ov4 { diff --git a/include/lang_analysis/ast.h b/include/lang_analysis/ast.h new file mode 100644 index 0000000..9e5a717 --- /dev/null +++ b/include/lang_analysis/ast.h @@ -0,0 +1,30 @@ +#pragma once + +#include + +#include "lang_analysis/lexer_class.h" +#include "lang_analysis/enum_type.h" +#include "lang_analysis/token.h" + +using namespace std; // remove this + +namespace ov4 +{ + +class T_lexer::T_ast +{ +public: + T_ast() = default; + + enum_token_type token_type; + string command_text; + size_t left = string::npos; + size_t right = string::npos; + + // if this is marked as `subshell` + // it will fork a subshell + // then the subshell will handler the rest + bool subshell; +}; + +} diff --git a/include/lang_analysis/enum_type.h b/include/lang_analysis/enum_type.h new file mode 100644 index 0000000..bfc7cdc --- /dev/null +++ b/include/lang_analysis/enum_type.h @@ -0,0 +1,35 @@ +#pragma once + +namespace ov4 +{ + +enum enum_token_type +{ + TEXT, // foo + SINGLE_QUOTATION, // ' + DOUBLE_QUOTATION, // " + LOGIC_AND, // && + LOGIC_OR, // || + ASYNC, // & + PIPE, // | + SEMICOLON, // ; + EXPAND, // $ + LEFT_BRACKET, // ( + RIGHT_BRACKET, // ) +}; + +enum guard_type +{ + GUARD_OFF, + GUARD_WEAK, + GUARD_STRONG, +}; + +enum class child_existence_type +{ + NOT_EXIST, + OPTIONAL, + EXIST, +}; + +} diff --git a/include/lang_analysis/lexer_class.h b/include/lang_analysis/lexer_class.h new file mode 100644 index 0000000..4100b97 --- /dev/null +++ b/include/lang_analysis/lexer_class.h @@ -0,0 +1,42 @@ +#pragma once + +#include +#include +#include + +#include "lang_analysis/enum_type.h" + +using namespace std; + +namespace ov4 +{ + +class T_lexer +{ +public: + + // token.h + class T_token; + string trim(const string &s, const string &trim_target = " \t"); + bool is_single_symbol(char c); + void bracket_depth_changer(char c, int &bracket_depth); + bool token_continue(const string &s, char next); + void token_push(const string &push_s, int bracket_depth, bool force_text = false); + int tokenizer(const string &s); + + // ast.h + class T_ast; + string final_trim(size_t _begin, size_t _end); + size_t alloc_ast(); + int get_pivot_order(enum_token_type x); + bool is_operator(enum_token_type x); + void parse(size_t cmd_begin, size_t cmd_end, size_t ast_vec); + + // shared + std::vector ast; + std::vector token; + size_t ast_index = 0; + +}; + +} diff --git a/include/lang_analysis/parseline.h b/include/lang_analysis/parseline.h new file mode 100644 index 0000000..e52b50e --- /dev/null +++ b/include/lang_analysis/parseline.h @@ -0,0 +1,10 @@ +#pragma once + +#include + +namespace ov4 +{ + +char **parseline(const std::string &s); + +} diff --git a/include/lang_analysis/symbol_property.h b/include/lang_analysis/symbol_property.h new file mode 100644 index 0000000..f21b145 --- /dev/null +++ b/include/lang_analysis/symbol_property.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include + +#include "lang_analysis/enum_type.h" + +using namespace std; + +namespace ov4 +{ + +class T_property +{ +public: + bool continuable = true; + + // -1 means it's NOT operator + // any positive values mean it's operator + int pivot_priority = -1; + + // require: MUST + // accept: optional, but if it doesn't accept, but there is indeed a child, then error + child_existence_type left; + child_existence_type right; + + enum_token_type enum_type = TEXT; +}; + +inline const unordered_map symbol_property = { + + // bracket + {"(", {.continuable = false, .enum_type = LEFT_BRACKET}}, + {")", {.continuable = false, .enum_type = RIGHT_BRACKET}}, + + // operator + {"&", {.continuable = true, .pivot_priority = 2000, .left = child_existence_type::EXIST, .right = child_existence_type::NOT_EXIST, .enum_type = ASYNC}}, + {"|", {.continuable = true, .pivot_priority = 100, .left = child_existence_type::EXIST, .right = child_existence_type::EXIST, .enum_type = PIPE}}, + {"&&", {.continuable = true, .pivot_priority = 1000, .left = child_existence_type::EXIST, .right = child_existence_type::EXIST, .enum_type = LOGIC_AND}}, + {"||", {.continuable = true, .pivot_priority = 1000, .left = child_existence_type::EXIST, .right = child_existence_type::EXIST, .enum_type = LOGIC_OR}}, + +}; + +string symbol_enum2string(enum_token_type e); + +bool symbol_contains_enum_type(enum_token_type e); + +} diff --git a/include/lang_analysis/token.h b/include/lang_analysis/token.h new file mode 100644 index 0000000..508d142 --- /dev/null +++ b/include/lang_analysis/token.h @@ -0,0 +1,25 @@ +#pragma once + +#include + +#include "lang_analysis/lexer_class.h" +#include "lang_analysis/enum_type.h" +#include "lang_analysis/symbol_property.h" + +using namespace std; // remove this + +namespace ov4 +{ + +class T_lexer::T_token +{ +public: + T_token() = default; + + enum_token_type token_type; + string text; + guard_type guard = GUARD_OFF; + int bracket_depth = 0; +}; + +} diff --git a/include/lexer/ast.h b/include/lexer/ast.h deleted file mode 100644 index 871afaf..0000000 --- a/include/lexer/ast.h +++ /dev/null @@ -1,30 +0,0 @@ -#pragma once - -#include - -#include "lexer/lexer_class.h" -#include "lexer/enum_type.h" -#include "lexer/token.h" - -using namespace std; // remove this - -namespace ov4 -{ - -class T_lexer::T_ast -{ -public: - T_ast() = default; - - enum_token_type token_type; - string command_text; - size_t left = string::npos; - size_t right = string::npos; - - // if this is marked as `subshell` - // it will fork a subshell - // then the subshell will handler the rest - bool subshell; -}; - -} diff --git a/include/lexer/enum_type.h b/include/lexer/enum_type.h deleted file mode 100644 index bfc7cdc..0000000 --- a/include/lexer/enum_type.h +++ /dev/null @@ -1,35 +0,0 @@ -#pragma once - -namespace ov4 -{ - -enum enum_token_type -{ - TEXT, // foo - SINGLE_QUOTATION, // ' - DOUBLE_QUOTATION, // " - LOGIC_AND, // && - LOGIC_OR, // || - ASYNC, // & - PIPE, // | - SEMICOLON, // ; - EXPAND, // $ - LEFT_BRACKET, // ( - RIGHT_BRACKET, // ) -}; - -enum guard_type -{ - GUARD_OFF, - GUARD_WEAK, - GUARD_STRONG, -}; - -enum class child_existence_type -{ - NOT_EXIST, - OPTIONAL, - EXIST, -}; - -} diff --git a/include/lexer/lexer_class.h b/include/lexer/lexer_class.h deleted file mode 100644 index 04b2e1c..0000000 --- a/include/lexer/lexer_class.h +++ /dev/null @@ -1,42 +0,0 @@ -#pragma once - -#include -#include -#include - -#include "lexer/enum_type.h" - -using namespace std; - -namespace ov4 -{ - -class T_lexer -{ -public: - - // token.h - class T_token; - string trim(const string &s, const string &trim_target = " \t"); - bool is_single_symbol(char c); - void bracket_depth_changer(char c, int &bracket_depth); - bool token_continue(const string &s, char next); - void token_push(const string &push_s, int bracket_depth, bool force_text = false); - int tokenizer(const string &s); - - // ast.h - class T_ast; - string final_trim(size_t _begin, size_t _end); - size_t alloc_ast(); - int get_pivot_order(enum_token_type x); - bool is_operator(enum_token_type x); - void parse(size_t cmd_begin, size_t cmd_end, size_t ast_vec); - - // shared - std::vector ast; - std::vector token; - size_t ast_index = 0; - -}; - -} diff --git a/include/lexer/parseline.h b/include/lexer/parseline.h deleted file mode 100644 index e52b50e..0000000 --- a/include/lexer/parseline.h +++ /dev/null @@ -1,10 +0,0 @@ -#pragma once - -#include - -namespace ov4 -{ - -char **parseline(const std::string &s); - -} diff --git a/include/lexer/symbol_property.h b/include/lexer/symbol_property.h deleted file mode 100644 index c933605..0000000 --- a/include/lexer/symbol_property.h +++ /dev/null @@ -1,48 +0,0 @@ -#pragma once - -#include -#include - -#include "lexer/enum_type.h" - -using namespace std; - -namespace ov4 -{ - -class T_property -{ -public: - bool continuable = true; - - // -1 means it's NOT operator - // any positive values mean it's operator - int pivot_priority = -1; - - // require: MUST - // accept: optional, but if it doesn't accept, but there is indeed a child, then error - child_existence_type left; - child_existence_type right; - - enum_token_type enum_type = TEXT; -}; - -inline const unordered_map symbol_property = { - - // bracket - {"(", {.continuable = false, .enum_type = LEFT_BRACKET}}, - {")", {.continuable = false, .enum_type = RIGHT_BRACKET}}, - - // operator - {"&", {.continuable = true, .pivot_priority = 2000, .left = child_existence_type::EXIST, .right = child_existence_type::NOT_EXIST, .enum_type = ASYNC}}, - {"|", {.continuable = true, .pivot_priority = 100, .left = child_existence_type::EXIST, .right = child_existence_type::EXIST, .enum_type = PIPE}}, - {"&&", {.continuable = true, .pivot_priority = 1000, .left = child_existence_type::EXIST, .right = child_existence_type::EXIST, .enum_type = LOGIC_AND}}, - {"||", {.continuable = true, .pivot_priority = 1000, .left = child_existence_type::EXIST, .right = child_existence_type::EXIST, .enum_type = LOGIC_OR}}, - -}; - -string symbol_enum2string(enum_token_type e); - -bool symbol_contains_enum_type(enum_token_type e); - -} diff --git a/include/lexer/token.h b/include/lexer/token.h deleted file mode 100644 index 93bdc22..0000000 --- a/include/lexer/token.h +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once - -#include - -#include "lexer/lexer_class.h" -#include "lexer/enum_type.h" -#include "lexer/symbol_property.h" - -using namespace std; // remove this - -namespace ov4 -{ - -class T_lexer::T_token -{ -public: - T_token() = default; - - enum_token_type token_type; - string text; - guard_type guard = GUARD_OFF; - int bracket_depth = 0; -}; - -} diff --git a/include/parse.h b/include/parse.h deleted file mode 100644 index 8fdbe12..0000000 --- a/include/parse.h +++ /dev/null @@ -1,13 +0,0 @@ -#include -#include -#include -#include - -#include "global.h" - -namespace ov4 -{ - -int parseline(const char *cmdline, char **argv); - -} diff --git a/module-test/lexer_test.cc b/module-test/lexer_test.cc index cbc55d7..b563f74 100644 --- a/module-test/lexer_test.cc +++ b/module-test/lexer_test.cc @@ -3,9 +3,9 @@ $ g++ \ module-test/lexer_test.cc \ - src/lexer/token.cc \ - src/lexer/ast.cc \ - src/lexer/symbol_property.cc \ + src/lang_analysis/token.cc \ + src/lang_analysis/ast.cc \ + src/lang_analysis/symbol_property.cc \ src/util/string.cc \ -std=c++23 -Iinclude -Ithird_party/magic_enum/include \ -O0 -ggdb3 -fno-omit-frame-pointer -fno-inline -D_GLIBCXX_ASSERTIONS \ @@ -23,9 +23,9 @@ g++ \ #include "magic_enum/magic_enum.hpp" -#include "lexer/ast.h" -#include "lexer/enum_type.h" -#include "lexer/token.h" +#include "lang_analysis/ast.h" +#include "lang_analysis/enum_type.h" +#include "lang_analysis/token.h" #include "global.h" #include "error.h" #include "util/string.h" diff --git a/module-test/parseline_test.cc b/module-test/parseline_test.cc index 047be48..e9aae88 100644 --- a/module-test/parseline_test.cc +++ b/module-test/parseline_test.cc @@ -1,9 +1,9 @@ /* - * test lexer/parseline + * test lang_analysis/parseline $ g++ \ module-test/parseline_test.cc \ - src/lexer/parseline.cc \ + src/lang_analysis/parseline.cc \ src/util/string.cc \ -std=c++23 -Iinclude \ -O0 -ggdb3 -fno-omit-frame-pointer -fno-inline -D_GLIBCXX_ASSERTIONS \ @@ -13,7 +13,7 @@ g++ \ #include #include -#include "lexer/parseline.h" +#include "lang_analysis/parseline.h" using namespace std; diff --git a/src/eval.cc b/src/eval.cc index c188105..9db9b9d 100644 --- a/src/eval.cc +++ b/src/eval.cc @@ -16,12 +16,12 @@ #include "magic_enum/magic_enum.hpp" #include "error.h" -#include "lexer/lexer_class.h" -#include "lexer/ast.h" -#include "lexer/token.h" -#include "lexer/enum_type.h" -#include "lexer/symbol_property.h" -#include "lexer/parseline.h" +#include "lang_analysis/lexer_class.h" +#include "lang_analysis/ast.h" +#include "lang_analysis/token.h" +#include "lang_analysis/enum_type.h" +#include "lang_analysis/symbol_property.h" +#include "lang_analysis/parseline.h" #include "handler.h" using namespace std; diff --git a/src/lang_analysis/ast.cc b/src/lang_analysis/ast.cc new file mode 100644 index 0000000..ce931bc --- /dev/null +++ b/src/lang_analysis/ast.cc @@ -0,0 +1,161 @@ +#include "lang_analysis/ast.h" + +#include +#include +#include + +#include "lang_analysis/lexer_class.h" +#include "lang_analysis/enum_type.h" +#include "lang_analysis/token.h" +#include "lang_analysis/symbol_property.h" +#include "util/io.h" +#include "error.h" +#include "magic_enum/magic_enum.hpp" + +using namespace std; + +namespace ov4 +{ + +size_t T_lexer::alloc_ast() +{ + ast.emplace_back(); + return ast_index++; +} + +string T_lexer::final_trim(size_t _begin, size_t _end) +{ + string s; + for (size_t i = _begin; i < _end; i++) + { + if (token[i].token_type != TEXT) + continue; + s += token[i].text + " "; + } + s = s.substr(0, s.size()-1); // eat final space + return s; +} + +int T_lexer::get_pivot_order(enum_token_type x) +{ + string s = symbol_enum2string(x); + if (s != "") + { + return symbol_property.at(s).pivot_priority; + } + // not found + return -1; +} + +bool T_lexer::is_operator(enum_token_type x) +{ + string s = symbol_enum2string(x); + if (s != "") + return (symbol_property.at(s).pivot_priority >= 0) ? true : false; + return false; +} + +// region [begin, end) +void T_lexer::parse(size_t cmd_begin, size_t cmd_end, size_t ast_vec) +{ + int current_depth = 0; + int current_order = -1; + int max_depth = -1; + size_t found_pos = string::npos; + + // if it's fully inside bracket + // then it's subshell + if (cmd_begin != cmd_end) + loggerln("parse: left token type: {}, right: {}", + magic_enum::enum_name(token[cmd_begin].token_type), + magic_enum::enum_name(token[cmd_end-1].token_type)); + if (cmd_begin != cmd_end && token[cmd_begin].token_type == LEFT_BRACKET && token[cmd_end-1].token_type == RIGHT_BRACKET) + { + ast[ast_vec].subshell = true; + loggerln("parse: detect subshell {}", ast_vec); + + // eat () subshell token + cmd_begin++; + cmd_end--; + } + else + { + ast[ast_vec].subshell = false; + } + + for (size_t i = cmd_begin; i < cmd_end; i++) + { + if (token[i].bracket_depth > max_depth) + max_depth = token[i].bracket_depth; + } + + + while (found_pos == string::npos) + { + // scan from right to left + for (size_t i = cmd_end - 1; i >= cmd_begin && i < cmd_end; i--) + { + // if same depth + // and higher order number + if (token[i].bracket_depth == current_depth + && is_operator(token[i].token_type) + && get_pivot_order(token[i].token_type) > current_order) + { + current_order = get_pivot_order(token[i].token_type); + found_pos = i; + loggerln("Found pos: {} order: {} depth: {}", i, current_order, current_depth); + } + } + current_depth++; + if (current_depth > max_depth) + { + break; + } + } + + if (found_pos == string::npos) + { + loggerln("Smallest unit"); + ast[ast_vec].token_type = TEXT; + ast[ast_vec].command_text = final_trim(cmd_begin, cmd_end); + return; + } + + string token_str = symbol_enum2string(token[found_pos].token_type); + + // miss left + if (symbol_property.at(token_str).left == child_existence_type::EXIST && cmd_begin == found_pos) + throw T_error{error_code::AST_MISS_LEFT_CHILD, "parse: missing left child"}; + // miss right + if (symbol_property.at(token_str).right == child_existence_type::EXIST && cmd_end-1 == found_pos) + throw T_error{error_code::AST_MISS_RIGHT_CHILD, "parse: missing right child"}; + // should not exist left + if (symbol_property.at(token_str).left == child_existence_type::NOT_EXIST && cmd_begin != found_pos) + throw T_error{error_code::AST_EXIST_LEFT_CHILLD, "parse: left child should NOT exist"}; + // should not exist right + if (symbol_property.at(token_str).right == child_existence_type::NOT_EXIST && cmd_end-1 != found_pos) + throw T_error{error_code::AST_EXIST_LEFT_CHILLD, "parse: right child should NOT exist"}; + + + ast[ast_vec].token_type = token[found_pos].token_type; + loggerln("Parse sub: {}, {}, {}", cmd_begin, found_pos, cmd_end); + + if (symbol_property.at(token_str).left == child_existence_type::EXIST + || symbol_property.at(token_str).left == child_existence_type::OPTIONAL) + { + size_t l = alloc_ast(); + ast[ast_vec].left = l; + parse(cmd_begin, found_pos, l); + } + + if (symbol_property.at(token_str).right == child_existence_type::EXIST + || symbol_property.at(token_str).right == child_existence_type::OPTIONAL) + { + size_t r = alloc_ast(); + ast[ast_vec].right = r; + parse(found_pos+1, cmd_end, r); + } + +} + +} diff --git a/src/lang_analysis/parseline.cc b/src/lang_analysis/parseline.cc new file mode 100644 index 0000000..d71cf6f --- /dev/null +++ b/src/lang_analysis/parseline.cc @@ -0,0 +1,80 @@ +#include "lang_analysis/parseline.h" + +#include + +#include "lang_analysis/enum_type.h" +#include "util/string.h" + +using namespace std; + +namespace ov4 +{ + +char **parseline(const std::string &s) +{ + guard_type guard = GUARD_OFF; + vector v; + size_t len = s.size(); + string prev_str = ""; + char c; + + for (size_t i = 0; i < len; i++) + { + + c = s[i]; + if (c == ' ' && guard == GUARD_OFF) continue; + + if (c == '\'' || c == '\"') + { + + // turn on GUARD + if (guard == GUARD_OFF) + { + guard = (c == '\'') ? GUARD_STRONG : GUARD_WEAK; + if (i+1 == len) + { + v.push_back(prev_str); + prev_str = ""; + } + continue; + } + // turn off + else if (guard == GUARD_STRONG && c == '\'') + { + guard = GUARD_OFF; + if (i+1 == len) + { + v.push_back(prev_str); + prev_str = ""; + } + continue; + } + // turn off + else if (guard == GUARD_WEAK && c == '\"') + { + guard = GUARD_OFF; + if (i+1 == len) + { + v.push_back(prev_str); + prev_str = ""; + } + continue; + } + // if this (single/double) quotation mark is guarded + // it should be saved + } + + prev_str += c; + + if ((i+1 >= len) || (s[i+1] == ' ' && guard == GUARD_OFF)) + { + v.push_back(prev_str); + prev_str = ""; + continue; + } + } + + return string2argv_alloc(v); +} + +} diff --git a/src/lang_analysis/symbol_property.cc b/src/lang_analysis/symbol_property.cc new file mode 100644 index 0000000..bb5015c --- /dev/null +++ b/src/lang_analysis/symbol_property.cc @@ -0,0 +1,29 @@ +#include "lang_analysis/symbol_property.h" + +#include + +#include "lang_analysis/enum_type.h" + +using namespace std; + +namespace ov4 +{ + +string symbol_enum2string(enum_token_type e) +{ + for (auto &c : symbol_property) + { + // pair + + if (c.second.enum_type == e) + return c.first; + } + return ""; +} + +bool symbol_contains_enum_type(enum_token_type e) +{ + return (symbol_enum2string(e) == "") ? false : true; +} + +} diff --git a/src/lang_analysis/token.cc b/src/lang_analysis/token.cc new file mode 100644 index 0000000..f210e57 --- /dev/null +++ b/src/lang_analysis/token.cc @@ -0,0 +1,204 @@ +#include "lang_analysis/token.h" + +#include +#include +#include + +#include "lang_analysis/lexer_class.h" +#include "lang_analysis/enum_type.h" +#include "lang_analysis/symbol_property.h" +#include "util/io.h" +#include "error.h" + +using namespace std; // remove this + +constexpr bool TOKEN_DEBUG = false; + +namespace ov4 +{ + +/* + * remove leading chars + * by default remove " \t" + */ +string T_lexer::trim(const string &s, const string &trim_target) // trim_target = " \t" +{ + size_t b = s.find_first_not_of(trim_target); + if (b == string::npos) + return ""; + size_t e = s.find_last_not_of(trim_target); + return s.substr(b, e - b + 1); +} + + +bool T_lexer::is_single_symbol(char c) +{ + string s = " "; + s[0] = c; + // quotation marks are treated as text + if (symbol_property.contains(s) && c != '\'' && c != '\"') + return true; + return false; +} + +void T_lexer::bracket_depth_changer(char c, int &bracket_depth) +{ + if (c == '(') + { + bracket_depth++; + if constexpr (TOKEN_DEBUG) loggerln("changed depth: {}", bracket_depth); + return; + } + if (c == ')') + { + bracket_depth--; + loggerln("changed depth: {}", bracket_depth); + if (bracket_depth < 0) + { + throw T_error{error_code::TOKEN_UNMATCHED_BRACKET, "tokenizer: unmatched bracket(s)"}; + } + return; + } +} + +/* + * determine whether `s` and `next` can be combined to form a token + */ +bool T_lexer::token_continue(const string &s, char next) +{ + if constexpr (TOKEN_DEBUG) loggerln("token_continue: receiving {} {}", s, next); + // s is empty, so next can be part of the token + if (s.size() == 0) return true; + + // text -> text/symbol + if (!is_single_symbol(s[s.size()-1])) + { + // text -> symbol + if (is_single_symbol(next)) + return false; + + // text still + return true; + } + + // symbol -> symbol + T_property tmp; + if (symbol_property.contains(s)) + { + tmp = symbol_property.at(s); + if (tmp.continuable) + { + string prev_and_next = s + next; + if (symbol_property.contains(prev_and_next)) + return true; + return false; + } + return false; + } + else + { + if constexpr (TOKEN_DEBUG) loggerln("symbol not found in map"); + return false; + } +} + +// handle [prev, current) +void T_lexer::token_push(const string &push_s, int bracket_depth, bool force_text) // force_text = false +{ + T_token tmp; + tmp.bracket_depth = bracket_depth; + string trimmed = trim(push_s); + if (trimmed == "") return; + + if constexpr (TOKEN_DEBUG) loggerln("finding {} in map", trimmed); + + if (symbol_property.contains(trimmed) && !force_text) // symbol + { + if constexpr (TOKEN_DEBUG) loggerln("token_push: adding symbol"); + tmp.token_type = symbol_property.at(trimmed).enum_type; + } + else + { + if constexpr (TOKEN_DEBUG) loggerln("token_push: adding text"); + // if (force_text) + // trimmed = trim(trimmed, "\'\""); + tmp.token_type = TEXT; + tmp.text = trimmed; + } + token.push_back(tmp); +} + +int T_lexer::tokenizer(const string &s) +{ + if (s.size() == 0) return -1; + + size_t len = s.size(); + size_t token_vector_index = 0; + string prev_str = ""; + bool force_text = false; + int bracket_depth = 0; + guard_type guard = GUARD_OFF; + + + for (size_t i = 0; i < len; i++) + { + + char c = s[i]; + + prev_str += c; + + if constexpr (TOKEN_DEBUG) loggerln("---"); + if constexpr (TOKEN_DEBUG) loggerln("prev_str .{}. next char {}", prev_str, s[i+1]); + + if (c == '\'' || c == '\"') + { + if constexpr (TOKEN_DEBUG) loggerln("quotation"); + + // turn on GUARD + if (guard == GUARD_OFF) + { + guard = (c == '\'') ? GUARD_STRONG : GUARD_WEAK; + if constexpr (TOKEN_DEBUG) loggerln("set guard {}", (c == '\'') ? "GUARD_STRONG" : "GUARD_WEAK"); + continue; + } + // turn off + else if (guard == GUARD_STRONG && c == '\'') + { + guard = GUARD_OFF; + if constexpr (TOKEN_DEBUG) loggerln("quitting strong guard"); + force_text = true; + } + //turn off + else if (guard == GUARD_WEAK && c == '\"') + { + guard = GUARD_OFF; + if constexpr (TOKEN_DEBUG) loggerln("quitting weak guard"); + force_text = true; + } else + continue; + } + // not quotation mark + else if ((guard == GUARD_STRONG || guard == GUARD_WEAK)) + { + if constexpr (TOKEN_DEBUG) loggerln("tokenizer: guarded, continue"); + continue; + } + + bracket_depth_changer(c, bracket_depth); + + if ((i+1 == len) || !token_continue(prev_str, s[i+1])) + { + token_push(prev_str, bracket_depth, force_text); + force_text = false; + prev_str = ""; + continue; + } + } + + if (bracket_depth != 0) + throw T_error{error_code::TOKEN_UNMATCHED_BRACKET, "tokenizer: unmatched bracket(s)"}; + + return 0; +} + +} diff --git a/src/lexer/ast.cc b/src/lexer/ast.cc deleted file mode 100644 index da96c6f..0000000 --- a/src/lexer/ast.cc +++ /dev/null @@ -1,161 +0,0 @@ -#include "lexer/ast.h" - -#include -#include -#include - -#include "lexer/lexer_class.h" -#include "lexer/enum_type.h" -#include "lexer/token.h" -#include "lexer/symbol_property.h" -#include "util/io.h" -#include "error.h" -#include "magic_enum/magic_enum.hpp" - -using namespace std; - -namespace ov4 -{ - -size_t T_lexer::alloc_ast() -{ - ast.emplace_back(); - return ast_index++; -} - -string T_lexer::final_trim(size_t _begin, size_t _end) -{ - string s; - for (size_t i = _begin; i < _end; i++) - { - if (token[i].token_type != TEXT) - continue; - s += token[i].text + " "; - } - s = s.substr(0, s.size()-1); // eat final space - return s; -} - -int T_lexer::get_pivot_order(enum_token_type x) -{ - string s = symbol_enum2string(x); - if (s != "") - { - return symbol_property.at(s).pivot_priority; - } - // not found - return -1; -} - -bool T_lexer::is_operator(enum_token_type x) -{ - string s = symbol_enum2string(x); - if (s != "") - return (symbol_property.at(s).pivot_priority >= 0) ? true : false; - return false; -} - -// region [begin, end) -void T_lexer::parse(size_t cmd_begin, size_t cmd_end, size_t ast_vec) -{ - int current_depth = 0; - int current_order = -1; - int max_depth = -1; - size_t found_pos = string::npos; - - // if it's fully inside bracket - // then it's subshell - if (cmd_begin != cmd_end) - loggerln("parse: left token type: {}, right: {}", - magic_enum::enum_name(token[cmd_begin].token_type), - magic_enum::enum_name(token[cmd_end-1].token_type)); - if (cmd_begin != cmd_end && token[cmd_begin].token_type == LEFT_BRACKET && token[cmd_end-1].token_type == RIGHT_BRACKET) - { - ast[ast_vec].subshell = true; - loggerln("parse: detect subshell {}", ast_vec); - - // eat () subshell token - cmd_begin++; - cmd_end--; - } - else - { - ast[ast_vec].subshell = false; - } - - for (size_t i = cmd_begin; i < cmd_end; i++) - { - if (token[i].bracket_depth > max_depth) - max_depth = token[i].bracket_depth; - } - - - while (found_pos == string::npos) - { - // scan from right to left - for (size_t i = cmd_end - 1; i >= cmd_begin && i < cmd_end; i--) - { - // if same depth - // and higher order number - if (token[i].bracket_depth == current_depth - && is_operator(token[i].token_type) - && get_pivot_order(token[i].token_type) > current_order) - { - current_order = get_pivot_order(token[i].token_type); - found_pos = i; - loggerln("Found pos: {} order: {} depth: {}", i, current_order, current_depth); - } - } - current_depth++; - if (current_depth > max_depth) - { - break; - } - } - - if (found_pos == string::npos) - { - loggerln("Smallest unit"); - ast[ast_vec].token_type = TEXT; - ast[ast_vec].command_text = final_trim(cmd_begin, cmd_end); - return; - } - - string token_str = symbol_enum2string(token[found_pos].token_type); - - // miss left - if (symbol_property.at(token_str).left == child_existence_type::EXIST && cmd_begin == found_pos) - throw T_error{error_code::AST_MISS_LEFT_CHILD, "parse: missing left child"}; - // miss right - if (symbol_property.at(token_str).right == child_existence_type::EXIST && cmd_end-1 == found_pos) - throw T_error{error_code::AST_MISS_RIGHT_CHILD, "parse: missing right child"}; - // should not exist left - if (symbol_property.at(token_str).left == child_existence_type::NOT_EXIST && cmd_begin != found_pos) - throw T_error{error_code::AST_EXIST_LEFT_CHILLD, "parse: left child should NOT exist"}; - // should not exist right - if (symbol_property.at(token_str).right == child_existence_type::NOT_EXIST && cmd_end-1 != found_pos) - throw T_error{error_code::AST_EXIST_LEFT_CHILLD, "parse: right child should NOT exist"}; - - - ast[ast_vec].token_type = token[found_pos].token_type; - loggerln("Parse sub: {}, {}, {}", cmd_begin, found_pos, cmd_end); - - if (symbol_property.at(token_str).left == child_existence_type::EXIST - || symbol_property.at(token_str).left == child_existence_type::OPTIONAL) - { - size_t l = alloc_ast(); - ast[ast_vec].left = l; - parse(cmd_begin, found_pos, l); - } - - if (symbol_property.at(token_str).right == child_existence_type::EXIST - || symbol_property.at(token_str).right == child_existence_type::OPTIONAL) - { - size_t r = alloc_ast(); - ast[ast_vec].right = r; - parse(found_pos+1, cmd_end, r); - } - -} - -} diff --git a/src/lexer/parseline.cc b/src/lexer/parseline.cc deleted file mode 100644 index 064f876..0000000 --- a/src/lexer/parseline.cc +++ /dev/null @@ -1,80 +0,0 @@ -#include "lexer/parseline.h" - -#include - -#include "lexer/enum_type.h" -#include "util/string.h" - -using namespace std; - -namespace ov4 -{ - -char **parseline(const std::string &s) -{ - guard_type guard = GUARD_OFF; - vector v; - size_t len = s.size(); - string prev_str = ""; - char c; - - for (size_t i = 0; i < len; i++) - { - - c = s[i]; - if (c == ' ' && guard == GUARD_OFF) continue; - - if (c == '\'' || c == '\"') - { - - // turn on GUARD - if (guard == GUARD_OFF) - { - guard = (c == '\'') ? GUARD_STRONG : GUARD_WEAK; - if (i+1 == len) - { - v.push_back(prev_str); - prev_str = ""; - } - continue; - } - // turn off - else if (guard == GUARD_STRONG && c == '\'') - { - guard = GUARD_OFF; - if (i+1 == len) - { - v.push_back(prev_str); - prev_str = ""; - } - continue; - } - // turn off - else if (guard == GUARD_WEAK && c == '\"') - { - guard = GUARD_OFF; - if (i+1 == len) - { - v.push_back(prev_str); - prev_str = ""; - } - continue; - } - // if this (single/double) quotation mark is guarded - // it should be saved - } - - prev_str += c; - - if ((i+1 >= len) || (s[i+1] == ' ' && guard == GUARD_OFF)) - { - v.push_back(prev_str); - prev_str = ""; - continue; - } - } - - return string2argv_alloc(v); -} - -} diff --git a/src/lexer/symbol_property.cc b/src/lexer/symbol_property.cc deleted file mode 100644 index c78dd61..0000000 --- a/src/lexer/symbol_property.cc +++ /dev/null @@ -1,29 +0,0 @@ -#include "lexer/symbol_property.h" - -#include - -#include "lexer/enum_type.h" - -using namespace std; - -namespace ov4 -{ - -string symbol_enum2string(enum_token_type e) -{ - for (auto &c : symbol_property) - { - // pair - - if (c.second.enum_type == e) - return c.first; - } - return ""; -} - -bool symbol_contains_enum_type(enum_token_type e) -{ - return (symbol_enum2string(e) == "") ? false : true; -} - -} diff --git a/src/lexer/token.cc b/src/lexer/token.cc deleted file mode 100644 index ec69090..0000000 --- a/src/lexer/token.cc +++ /dev/null @@ -1,204 +0,0 @@ -#include "lexer/token.h" - -#include -#include -#include - -#include "lexer/lexer_class.h" -#include "lexer/enum_type.h" -#include "lexer/symbol_property.h" -#include "util/io.h" -#include "error.h" - -using namespace std; // remove this - -constexpr bool TOKEN_DEBUG = false; - -namespace ov4 -{ - -/* - * remove leading chars - * by default remove " \t" - */ -string T_lexer::trim(const string &s, const string &trim_target) // trim_target = " \t" -{ - size_t b = s.find_first_not_of(trim_target); - if (b == string::npos) - return ""; - size_t e = s.find_last_not_of(trim_target); - return s.substr(b, e - b + 1); -} - - -bool T_lexer::is_single_symbol(char c) -{ - string s = " "; - s[0] = c; - // quotation marks are treated as text - if (symbol_property.contains(s) && c != '\'' && c != '\"') - return true; - return false; -} - -void T_lexer::bracket_depth_changer(char c, int &bracket_depth) -{ - if (c == '(') - { - bracket_depth++; - if constexpr (TOKEN_DEBUG) loggerln("changed depth: {}", bracket_depth); - return; - } - if (c == ')') - { - bracket_depth--; - loggerln("changed depth: {}", bracket_depth); - if (bracket_depth < 0) - { - throw T_error{error_code::TOKEN_UNMATCHED_BRACKET, "tokenizer: unmatched bracket(s)"}; - } - return; - } -} - -/* - * determine whether `s` and `next` can be combined to form a token - */ -bool T_lexer::token_continue(const string &s, char next) -{ - if constexpr (TOKEN_DEBUG) loggerln("token_continue: receiving {} {}", s, next); - // s is empty, so next can be part of the token - if (s.size() == 0) return true; - - // text -> text/symbol - if (!is_single_symbol(s[s.size()-1])) - { - // text -> symbol - if (is_single_symbol(next)) - return false; - - // text still - return true; - } - - // symbol -> symbol - T_property tmp; - if (symbol_property.contains(s)) - { - tmp = symbol_property.at(s); - if (tmp.continuable) - { - string prev_and_next = s + next; - if (symbol_property.contains(prev_and_next)) - return true; - return false; - } - return false; - } - else - { - if constexpr (TOKEN_DEBUG) loggerln("symbol not found in map"); - return false; - } -} - -// handle [prev, current) -void T_lexer::token_push(const string &push_s, int bracket_depth, bool force_text) // force_text = false -{ - T_token tmp; - tmp.bracket_depth = bracket_depth; - string trimmed = trim(push_s); - if (trimmed == "") return; - - if constexpr (TOKEN_DEBUG) loggerln("finding {} in map", trimmed); - - if (symbol_property.contains(trimmed) && !force_text) // symbol - { - if constexpr (TOKEN_DEBUG) loggerln("token_push: adding symbol"); - tmp.token_type = symbol_property.at(trimmed).enum_type; - } - else - { - if constexpr (TOKEN_DEBUG) loggerln("token_push: adding text"); - // if (force_text) - // trimmed = trim(trimmed, "\'\""); - tmp.token_type = TEXT; - tmp.text = trimmed; - } - token.push_back(tmp); -} - -int T_lexer::tokenizer(const string &s) -{ - if (s.size() == 0) return -1; - - size_t len = s.size(); - size_t token_vector_index = 0; - string prev_str = ""; - bool force_text = false; - int bracket_depth = 0; - guard_type guard = GUARD_OFF; - - - for (size_t i = 0; i < len; i++) - { - - char c = s[i]; - - prev_str += c; - - if constexpr (TOKEN_DEBUG) loggerln("---"); - if constexpr (TOKEN_DEBUG) loggerln("prev_str .{}. next char {}", prev_str, s[i+1]); - - if (c == '\'' || c == '\"') - { - if constexpr (TOKEN_DEBUG) loggerln("quotation"); - - // turn on GUARD - if (guard == GUARD_OFF) - { - guard = (c == '\'') ? GUARD_STRONG : GUARD_WEAK; - if constexpr (TOKEN_DEBUG) loggerln("set guard {}", (c == '\'') ? "GUARD_STRONG" : "GUARD_WEAK"); - continue; - } - // turn off - else if (guard == GUARD_STRONG && c == '\'') - { - guard = GUARD_OFF; - if constexpr (TOKEN_DEBUG) loggerln("quitting strong guard"); - force_text = true; - } - //turn off - else if (guard == GUARD_WEAK && c == '\"') - { - guard = GUARD_OFF; - if constexpr (TOKEN_DEBUG) loggerln("quitting weak guard"); - force_text = true; - } else - continue; - } - // not quotation mark - else if ((guard == GUARD_STRONG || guard == GUARD_WEAK)) - { - if constexpr (TOKEN_DEBUG) loggerln("tokenizer: guarded, continue"); - continue; - } - - bracket_depth_changer(c, bracket_depth); - - if ((i+1 == len) || !token_continue(prev_str, s[i+1])) - { - token_push(prev_str, bracket_depth, force_text); - force_text = false; - prev_str = ""; - continue; - } - } - - if (bracket_depth != 0) - throw T_error{error_code::TOKEN_UNMATCHED_BRACKET, "tokenizer: unmatched bracket(s)"}; - - return 0; -} - -} diff --git a/src/parse.cc b/src/parse.cc deleted file mode 100644 index f61913a..0000000 --- a/src/parse.cc +++ /dev/null @@ -1,72 +0,0 @@ -#include "parse.h" - -namespace ov4 -{ - -/* - * parseline - Parse the command line and build the argv array. - * - * Characters enclosed in single quotes are treated as a single - * argument. Return true if the user has requested a BG job, false if - * the user has requested a FG job. - */ -int parseline(const char *cmdline, char **argv) -{ - static char array[MAXLINE]; /* holds local copy of command line */ - char *buf = array; /* ptr that traverses command line */ - char *delim; /* points to first space delimiter */ - int argc; /* number of args */ - int bg; /* background job? */ - - strcpy(buf, cmdline); - if (buf[strlen(buf)-1] == '\n') - buf[strlen(buf)-1] = ' '; /* replace trailing '\n' with space */ - else - { - // if buffer is not large enough - // might be UB - // TODO: fix UB - buf[strlen(buf)] = ' '; - buf[strlen(buf)+1] = '\0'; - } - while (*buf && (*buf == ' ')) /* ignore leading spaces */ - buf++; - - /* Build the argv list */ - argc = 0; - if (*buf == '\'') { - buf++; - delim = strchr(buf, '\''); - } - else { - delim = strchr(buf, ' '); - } - - while (delim) { - argv[argc++] = buf; - *delim = '\0'; - buf = delim + 1; - while (*buf && (*buf == ' ')) /* ignore spaces */ - buf++; - - if (*buf == '\'') { - buf++; - delim = strchr(buf, '\''); - } - else { - delim = strchr(buf, ' '); - } - } - argv[argc] = NULL; - - if (argc == 0) /* ignore blank line */ - return 1; - - /* should the job run in the background? */ - if ((bg = (*argv[argc-1] == '&')) != 0) { - argv[--argc] = NULL; - } - return bg; -} - -} -- cgit v1.3.1-4-g156e