diff options
| author | _ov4 <ov4@ov4.org> | 2026-09-21 04:50:06 +0000 |
|---|---|---|
| committer | _ov4 <ov4@ov4.org> | 2026-09-21 04:50:06 +0000 |
| commit | 1c630655b2beb0f4b84fb7e1bc034ba62520be37 (patch) | |
| tree | e90e70c425813df29c8bbd5c1e84ef22f4e4a176 /src/lexer | |
| parent | 9f5936bf64deb625a27259d778937e281ca19cea (diff) | |
fix name typo, and remove outdated parse
Diffstat (limited to 'src/lexer')
| -rw-r--r-- | src/lexer/ast.cc | 161 | ||||
| -rw-r--r-- | src/lexer/parseline.cc | 80 | ||||
| -rw-r--r-- | src/lexer/symbol_property.cc | 29 | ||||
| -rw-r--r-- | src/lexer/token.cc | 204 |
4 files changed, 0 insertions, 474 deletions
diff --git a/src/lexer/ast.cc b/src/lexer/ast.cc deleted file mode 100644 index da96c6f..0000000 --- a/src/lexer/ast.cc +++ /dev/null @@ -1,161 +0,0 @@ -#include "lexer/ast.h" - -#include <string> -#include <iostream> -#include <print> - -#include "lexer/lexer_class.h" -#include "lexer/enum_type.h" -#include "lexer/token.h" -#include "lexer/symbol_property.h" -#include "util/io.h" -#include "error.h" -#include "magic_enum/magic_enum.hpp" - -using namespace std; - -namespace ov4 -{ - -size_t T_lexer::alloc_ast() -{ - ast.emplace_back(); - return ast_index++; -} - -string T_lexer::final_trim(size_t _begin, size_t _end) -{ - string s; - for (size_t i = _begin; i < _end; i++) - { - if (token[i].token_type != TEXT) - continue; - s += token[i].text + " "; - } - s = s.substr(0, s.size()-1); // eat final space - return s; -} - -int T_lexer::get_pivot_order(enum_token_type x) -{ - string s = symbol_enum2string(x); - if (s != "") - { - return symbol_property.at(s).pivot_priority; - } - // not found - return -1; -} - -bool T_lexer::is_operator(enum_token_type x) -{ - string s = symbol_enum2string(x); - if (s != "") - return (symbol_property.at(s).pivot_priority >= 0) ? true : false; - return false; -} - -// region [begin, end) -void T_lexer::parse(size_t cmd_begin, size_t cmd_end, size_t ast_vec) -{ - int current_depth = 0; - int current_order = -1; - int max_depth = -1; - size_t found_pos = string::npos; - - // if it's fully inside bracket - // then it's subshell - if (cmd_begin != cmd_end) - loggerln("parse: left token type: {}, right: {}", - magic_enum::enum_name(token[cmd_begin].token_type), - magic_enum::enum_name(token[cmd_end-1].token_type)); - if (cmd_begin != cmd_end && token[cmd_begin].token_type == LEFT_BRACKET && token[cmd_end-1].token_type == RIGHT_BRACKET) - { - ast[ast_vec].subshell = true; - loggerln("parse: detect subshell {}", ast_vec); - - // eat () subshell token - cmd_begin++; - cmd_end--; - } - else - { - ast[ast_vec].subshell = false; - } - - for (size_t i = cmd_begin; i < cmd_end; i++) - { - if (token[i].bracket_depth > max_depth) - max_depth = token[i].bracket_depth; - } - - - while (found_pos == string::npos) - { - // scan from right to left - for (size_t i = cmd_end - 1; i >= cmd_begin && i < cmd_end; i--) - { - // if same depth - // and higher order number - if (token[i].bracket_depth == current_depth - && is_operator(token[i].token_type) - && get_pivot_order(token[i].token_type) > current_order) - { - current_order = get_pivot_order(token[i].token_type); - found_pos = i; - loggerln("Found pos: {} order: {} depth: {}", i, current_order, current_depth); - } - } - current_depth++; - if (current_depth > max_depth) - { - break; - } - } - - if (found_pos == string::npos) - { - loggerln("Smallest unit"); - ast[ast_vec].token_type = TEXT; - ast[ast_vec].command_text = final_trim(cmd_begin, cmd_end); - return; - } - - string token_str = symbol_enum2string(token[found_pos].token_type); - - // miss left - if (symbol_property.at(token_str).left == child_existence_type::EXIST && cmd_begin == found_pos) - throw T_error{error_code::AST_MISS_LEFT_CHILD, "parse: missing left child"}; - // miss right - if (symbol_property.at(token_str).right == child_existence_type::EXIST && cmd_end-1 == found_pos) - throw T_error{error_code::AST_MISS_RIGHT_CHILD, "parse: missing right child"}; - // should not exist left - if (symbol_property.at(token_str).left == child_existence_type::NOT_EXIST && cmd_begin != found_pos) - throw T_error{error_code::AST_EXIST_LEFT_CHILLD, "parse: left child should NOT exist"}; - // should not exist right - if (symbol_property.at(token_str).right == child_existence_type::NOT_EXIST && cmd_end-1 != found_pos) - throw T_error{error_code::AST_EXIST_LEFT_CHILLD, "parse: right child should NOT exist"}; - - - ast[ast_vec].token_type = token[found_pos].token_type; - loggerln("Parse sub: {}, {}, {}", cmd_begin, found_pos, cmd_end); - - if (symbol_property.at(token_str).left == child_existence_type::EXIST - || symbol_property.at(token_str).left == child_existence_type::OPTIONAL) - { - size_t l = alloc_ast(); - ast[ast_vec].left = l; - parse(cmd_begin, found_pos, l); - } - - if (symbol_property.at(token_str).right == child_existence_type::EXIST - || symbol_property.at(token_str).right == child_existence_type::OPTIONAL) - { - size_t r = alloc_ast(); - ast[ast_vec].right = r; - parse(found_pos+1, cmd_end, r); - } - -} - -} diff --git a/src/lexer/parseline.cc b/src/lexer/parseline.cc deleted file mode 100644 index 064f876..0000000 --- a/src/lexer/parseline.cc +++ /dev/null @@ -1,80 +0,0 @@ -#include "lexer/parseline.h" - -#include <vector> - -#include "lexer/enum_type.h" -#include "util/string.h" - -using namespace std; - -namespace ov4 -{ - -char **parseline(const std::string &s) -{ - guard_type guard = GUARD_OFF; - vector<string> v; - size_t len = s.size(); - string prev_str = ""; - char c; - - for (size_t i = 0; i < len; i++) - { - - c = s[i]; - if (c == ' ' && guard == GUARD_OFF) continue; - - if (c == '\'' || c == '\"') - { - - // turn on GUARD - if (guard == GUARD_OFF) - { - guard = (c == '\'') ? GUARD_STRONG : GUARD_WEAK; - if (i+1 == len) - { - v.push_back(prev_str); - prev_str = ""; - } - continue; - } - // turn off - else if (guard == GUARD_STRONG && c == '\'') - { - guard = GUARD_OFF; - if (i+1 == len) - { - v.push_back(prev_str); - prev_str = ""; - } - continue; - } - // turn off - else if (guard == GUARD_WEAK && c == '\"') - { - guard = GUARD_OFF; - if (i+1 == len) - { - v.push_back(prev_str); - prev_str = ""; - } - continue; - } - // if this (single/double) quotation mark is guarded - // it should be saved - } - - prev_str += c; - - if ((i+1 >= len) || (s[i+1] == ' ' && guard == GUARD_OFF)) - { - v.push_back(prev_str); - prev_str = ""; - continue; - } - } - - return string2argv_alloc(v); -} - -} diff --git a/src/lexer/symbol_property.cc b/src/lexer/symbol_property.cc deleted file mode 100644 index c78dd61..0000000 --- a/src/lexer/symbol_property.cc +++ /dev/null @@ -1,29 +0,0 @@ -#include "lexer/symbol_property.h" - -#include <string> - -#include "lexer/enum_type.h" - -using namespace std; - -namespace ov4 -{ - -string symbol_enum2string(enum_token_type e) -{ - for (auto &c : symbol_property) - { - // pair<string, T_property> - - if (c.second.enum_type == e) - return c.first; - } - return ""; -} - -bool symbol_contains_enum_type(enum_token_type e) -{ - return (symbol_enum2string(e) == "") ? false : true; -} - -} diff --git a/src/lexer/token.cc b/src/lexer/token.cc deleted file mode 100644 index ec69090..0000000 --- a/src/lexer/token.cc +++ /dev/null @@ -1,204 +0,0 @@ -#include "lexer/token.h" - -#include <string> -#include <iostream> -#include <print> - -#include "lexer/lexer_class.h" -#include "lexer/enum_type.h" -#include "lexer/symbol_property.h" -#include "util/io.h" -#include "error.h" - -using namespace std; // remove this - -constexpr bool TOKEN_DEBUG = false; - -namespace ov4 -{ - -/* - * remove leading chars - * by default remove " \t" - */ -string T_lexer::trim(const string &s, const string &trim_target) // trim_target = " \t" -{ - size_t b = s.find_first_not_of(trim_target); - if (b == string::npos) - return ""; - size_t e = s.find_last_not_of(trim_target); - return s.substr(b, e - b + 1); -} - - -bool T_lexer::is_single_symbol(char c) -{ - string s = " "; - s[0] = c; - // quotation marks are treated as text - if (symbol_property.contains(s) && c != '\'' && c != '\"') - return true; - return false; -} - -void T_lexer::bracket_depth_changer(char c, int &bracket_depth) -{ - if (c == '(') - { - bracket_depth++; - if constexpr (TOKEN_DEBUG) loggerln("changed depth: {}", bracket_depth); - return; - } - if (c == ')') - { - bracket_depth--; - loggerln("changed depth: {}", bracket_depth); - if (bracket_depth < 0) - { - throw T_error{error_code::TOKEN_UNMATCHED_BRACKET, "tokenizer: unmatched bracket(s)"}; - } - return; - } -} - -/* - * determine whether `s` and `next` can be combined to form a token - */ -bool T_lexer::token_continue(const string &s, char next) -{ - if constexpr (TOKEN_DEBUG) loggerln("token_continue: receiving {} {}", s, next); - // s is empty, so next can be part of the token - if (s.size() == 0) return true; - - // text -> text/symbol - if (!is_single_symbol(s[s.size()-1])) - { - // text -> symbol - if (is_single_symbol(next)) - return false; - - // text still - return true; - } - - // symbol -> symbol - T_property tmp; - if (symbol_property.contains(s)) - { - tmp = symbol_property.at(s); - if (tmp.continuable) - { - string prev_and_next = s + next; - if (symbol_property.contains(prev_and_next)) - return true; - return false; - } - return false; - } - else - { - if constexpr (TOKEN_DEBUG) loggerln("symbol not found in map"); - return false; - } -} - -// handle [prev, current) -void T_lexer::token_push(const string &push_s, int bracket_depth, bool force_text) // force_text = false -{ - T_token tmp; - tmp.bracket_depth = bracket_depth; - string trimmed = trim(push_s); - if (trimmed == "") return; - - if constexpr (TOKEN_DEBUG) loggerln("finding {} in map", trimmed); - - if (symbol_property.contains(trimmed) && !force_text) // symbol - { - if constexpr (TOKEN_DEBUG) loggerln("token_push: adding symbol"); - tmp.token_type = symbol_property.at(trimmed).enum_type; - } - else - { - if constexpr (TOKEN_DEBUG) loggerln("token_push: adding text"); - // if (force_text) - // trimmed = trim(trimmed, "\'\""); - tmp.token_type = TEXT; - tmp.text = trimmed; - } - token.push_back(tmp); -} - -int T_lexer::tokenizer(const string &s) -{ - if (s.size() == 0) return -1; - - size_t len = s.size(); - size_t token_vector_index = 0; - string prev_str = ""; - bool force_text = false; - int bracket_depth = 0; - guard_type guard = GUARD_OFF; - - - for (size_t i = 0; i < len; i++) - { - - char c = s[i]; - - prev_str += c; - - if constexpr (TOKEN_DEBUG) loggerln("---"); - if constexpr (TOKEN_DEBUG) loggerln("prev_str .{}. next char {}", prev_str, s[i+1]); - - if (c == '\'' || c == '\"') - { - if constexpr (TOKEN_DEBUG) loggerln("quotation"); - - // turn on GUARD - if (guard == GUARD_OFF) - { - guard = (c == '\'') ? GUARD_STRONG : GUARD_WEAK; - if constexpr (TOKEN_DEBUG) loggerln("set guard {}", (c == '\'') ? "GUARD_STRONG" : "GUARD_WEAK"); - continue; - } - // turn off - else if (guard == GUARD_STRONG && c == '\'') - { - guard = GUARD_OFF; - if constexpr (TOKEN_DEBUG) loggerln("quitting strong guard"); - force_text = true; - } - //turn off - else if (guard == GUARD_WEAK && c == '\"') - { - guard = GUARD_OFF; - if constexpr (TOKEN_DEBUG) loggerln("quitting weak guard"); - force_text = true; - } else - continue; - } - // not quotation mark - else if ((guard == GUARD_STRONG || guard == GUARD_WEAK)) - { - if constexpr (TOKEN_DEBUG) loggerln("tokenizer: guarded, continue"); - continue; - } - - bracket_depth_changer(c, bracket_depth); - - if ((i+1 == len) || !token_continue(prev_str, s[i+1])) - { - token_push(prev_str, bracket_depth, force_text); - force_text = false; - prev_str = ""; - continue; - } - } - - if (bracket_depth != 0) - throw T_error{error_code::TOKEN_UNMATCHED_BRACKET, "tokenizer: unmatched bracket(s)"}; - - return 0; -} - -} |
