aboutsummaryrefslogtreecommitdiff
path: root/src/lang_analysis
diff options
context:
space:
mode:
Diffstat (limited to 'src/lang_analysis')
-rw-r--r--src/lang_analysis/ast.cc161
-rw-r--r--src/lang_analysis/parseline.cc80
-rw-r--r--src/lang_analysis/symbol_property.cc29
-rw-r--r--src/lang_analysis/token.cc204
4 files changed, 474 insertions, 0 deletions
diff --git a/src/lang_analysis/ast.cc b/src/lang_analysis/ast.cc
new file mode 100644
index 0000000..ce931bc
--- /dev/null
+++ b/src/lang_analysis/ast.cc
@@ -0,0 +1,161 @@
+#include "lang_analysis/ast.h"
+
+#include <string>
+#include <iostream>
+#include <print>
+
+#include "lang_analysis/lexer_class.h"
+#include "lang_analysis/enum_type.h"
+#include "lang_analysis/token.h"
+#include "lang_analysis/symbol_property.h"
+#include "util/io.h"
+#include "error.h"
+#include "magic_enum/magic_enum.hpp"
+
+using namespace std;
+
+namespace ov4
+{
+
+size_t T_lexer::alloc_ast()
+{
+ ast.emplace_back();
+ return ast_index++;
+}
+
+string T_lexer::final_trim(size_t _begin, size_t _end)
+{
+ string s;
+ for (size_t i = _begin; i < _end; i++)
+ {
+ if (token[i].token_type != TEXT)
+ continue;
+ s += token[i].text + " ";
+ }
+ s = s.substr(0, s.size()-1); // eat final space
+ return s;
+}
+
+int T_lexer::get_pivot_order(enum_token_type x)
+{
+ string s = symbol_enum2string(x);
+ if (s != "")
+ {
+ return symbol_property.at(s).pivot_priority;
+ }
+ // not found
+ return -1;
+}
+
+bool T_lexer::is_operator(enum_token_type x)
+{
+ string s = symbol_enum2string(x);
+ if (s != "")
+ return (symbol_property.at(s).pivot_priority >= 0) ? true : false;
+ return false;
+}
+
+// region [begin, end)
+void T_lexer::parse(size_t cmd_begin, size_t cmd_end, size_t ast_vec)
+{
+ int current_depth = 0;
+ int current_order = -1;
+ int max_depth = -1;
+ size_t found_pos = string::npos;
+
+ // if it's fully inside bracket
+ // then it's subshell
+ if (cmd_begin != cmd_end)
+ loggerln("parse: left token type: {}, right: {}",
+ magic_enum::enum_name(token[cmd_begin].token_type),
+ magic_enum::enum_name(token[cmd_end-1].token_type));
+ if (cmd_begin != cmd_end && token[cmd_begin].token_type == LEFT_BRACKET && token[cmd_end-1].token_type == RIGHT_BRACKET)
+ {
+ ast[ast_vec].subshell = true;
+ loggerln("parse: detect subshell {}", ast_vec);
+
+ // eat () subshell token
+ cmd_begin++;
+ cmd_end--;
+ }
+ else
+ {
+ ast[ast_vec].subshell = false;
+ }
+
+ for (size_t i = cmd_begin; i < cmd_end; i++)
+ {
+ if (token[i].bracket_depth > max_depth)
+ max_depth = token[i].bracket_depth;
+ }
+
+
+ while (found_pos == string::npos)
+ {
+ // scan from right to left
+ for (size_t i = cmd_end - 1; i >= cmd_begin && i < cmd_end; i--)
+ {
+ // if same depth
+ // and higher order number
+ if (token[i].bracket_depth == current_depth
+ && is_operator(token[i].token_type)
+ && get_pivot_order(token[i].token_type) > current_order)
+ {
+ current_order = get_pivot_order(token[i].token_type);
+ found_pos = i;
+ loggerln("Found pos: {} order: {} depth: {}", i, current_order, current_depth);
+ }
+ }
+ current_depth++;
+ if (current_depth > max_depth)
+ {
+ break;
+ }
+ }
+
+ if (found_pos == string::npos)
+ {
+ loggerln("Smallest unit");
+ ast[ast_vec].token_type = TEXT;
+ ast[ast_vec].command_text = final_trim(cmd_begin, cmd_end);
+ return;
+ }
+
+ string token_str = symbol_enum2string(token[found_pos].token_type);
+
+ // miss left
+ if (symbol_property.at(token_str).left == child_existence_type::EXIST && cmd_begin == found_pos)
+ throw T_error{error_code::AST_MISS_LEFT_CHILD, "parse: missing left child"};
+ // miss right
+ if (symbol_property.at(token_str).right == child_existence_type::EXIST && cmd_end-1 == found_pos)
+ throw T_error{error_code::AST_MISS_RIGHT_CHILD, "parse: missing right child"};
+ // should not exist left
+ if (symbol_property.at(token_str).left == child_existence_type::NOT_EXIST && cmd_begin != found_pos)
+ throw T_error{error_code::AST_EXIST_LEFT_CHILLD, "parse: left child should NOT exist"};
+ // should not exist right
+ if (symbol_property.at(token_str).right == child_existence_type::NOT_EXIST && cmd_end-1 != found_pos)
+ throw T_error{error_code::AST_EXIST_LEFT_CHILLD, "parse: right child should NOT exist"};
+
+
+ ast[ast_vec].token_type = token[found_pos].token_type;
+ loggerln("Parse sub: {}, {}, {}", cmd_begin, found_pos, cmd_end);
+
+ if (symbol_property.at(token_str).left == child_existence_type::EXIST
+ || symbol_property.at(token_str).left == child_existence_type::OPTIONAL)
+ {
+ size_t l = alloc_ast();
+ ast[ast_vec].left = l;
+ parse(cmd_begin, found_pos, l);
+ }
+
+ if (symbol_property.at(token_str).right == child_existence_type::EXIST
+ || symbol_property.at(token_str).right == child_existence_type::OPTIONAL)
+ {
+ size_t r = alloc_ast();
+ ast[ast_vec].right = r;
+ parse(found_pos+1, cmd_end, r);
+ }
+
+}
+
+}
diff --git a/src/lang_analysis/parseline.cc b/src/lang_analysis/parseline.cc
new file mode 100644
index 0000000..d71cf6f
--- /dev/null
+++ b/src/lang_analysis/parseline.cc
@@ -0,0 +1,80 @@
+#include "lang_analysis/parseline.h"
+
+#include <vector>
+
+#include "lang_analysis/enum_type.h"
+#include "util/string.h"
+
+using namespace std;
+
+namespace ov4
+{
+
+char **parseline(const std::string &s)
+{
+ guard_type guard = GUARD_OFF;
+ vector<string> v;
+ size_t len = s.size();
+ string prev_str = "";
+ char c;
+
+ for (size_t i = 0; i < len; i++)
+ {
+
+ c = s[i];
+ if (c == ' ' && guard == GUARD_OFF) continue;
+
+ if (c == '\'' || c == '\"')
+ {
+
+ // turn on GUARD
+ if (guard == GUARD_OFF)
+ {
+ guard = (c == '\'') ? GUARD_STRONG : GUARD_WEAK;
+ if (i+1 == len)
+ {
+ v.push_back(prev_str);
+ prev_str = "";
+ }
+ continue;
+ }
+ // turn off
+ else if (guard == GUARD_STRONG && c == '\'')
+ {
+ guard = GUARD_OFF;
+ if (i+1 == len)
+ {
+ v.push_back(prev_str);
+ prev_str = "";
+ }
+ continue;
+ }
+ // turn off
+ else if (guard == GUARD_WEAK && c == '\"')
+ {
+ guard = GUARD_OFF;
+ if (i+1 == len)
+ {
+ v.push_back(prev_str);
+ prev_str = "";
+ }
+ continue;
+ }
+ // if this (single/double) quotation mark is guarded
+ // it should be saved
+ }
+
+ prev_str += c;
+
+ if ((i+1 >= len) || (s[i+1] == ' ' && guard == GUARD_OFF))
+ {
+ v.push_back(prev_str);
+ prev_str = "";
+ continue;
+ }
+ }
+
+ return string2argv_alloc(v);
+}
+
+}
diff --git a/src/lang_analysis/symbol_property.cc b/src/lang_analysis/symbol_property.cc
new file mode 100644
index 0000000..bb5015c
--- /dev/null
+++ b/src/lang_analysis/symbol_property.cc
@@ -0,0 +1,29 @@
+#include "lang_analysis/symbol_property.h"
+
+#include <string>
+
+#include "lang_analysis/enum_type.h"
+
+using namespace std;
+
+namespace ov4
+{
+
+string symbol_enum2string(enum_token_type e)
+{
+ for (auto &c : symbol_property)
+ {
+ // pair<string, T_property>
+
+ if (c.second.enum_type == e)
+ return c.first;
+ }
+ return "";
+}
+
+bool symbol_contains_enum_type(enum_token_type e)
+{
+ return (symbol_enum2string(e) == "") ? false : true;
+}
+
+}
diff --git a/src/lang_analysis/token.cc b/src/lang_analysis/token.cc
new file mode 100644
index 0000000..f210e57
--- /dev/null
+++ b/src/lang_analysis/token.cc
@@ -0,0 +1,204 @@
+#include "lang_analysis/token.h"
+
+#include <string>
+#include <iostream>
+#include <print>
+
+#include "lang_analysis/lexer_class.h"
+#include "lang_analysis/enum_type.h"
+#include "lang_analysis/symbol_property.h"
+#include "util/io.h"
+#include "error.h"
+
+using namespace std; // remove this
+
+constexpr bool TOKEN_DEBUG = false;
+
+namespace ov4
+{
+
+/*
+ * remove leading chars
+ * by default remove " \t"
+ */
+string T_lexer::trim(const string &s, const string &trim_target) // trim_target = " \t"
+{
+ size_t b = s.find_first_not_of(trim_target);
+ if (b == string::npos)
+ return "";
+ size_t e = s.find_last_not_of(trim_target);
+ return s.substr(b, e - b + 1);
+}
+
+
+bool T_lexer::is_single_symbol(char c)
+{
+ string s = " ";
+ s[0] = c;
+ // quotation marks are treated as text
+ if (symbol_property.contains(s) && c != '\'' && c != '\"')
+ return true;
+ return false;
+}
+
+void T_lexer::bracket_depth_changer(char c, int &bracket_depth)
+{
+ if (c == '(')
+ {
+ bracket_depth++;
+ if constexpr (TOKEN_DEBUG) loggerln("changed depth: {}", bracket_depth);
+ return;
+ }
+ if (c == ')')
+ {
+ bracket_depth--;
+ loggerln("changed depth: {}", bracket_depth);
+ if (bracket_depth < 0)
+ {
+ throw T_error{error_code::TOKEN_UNMATCHED_BRACKET, "tokenizer: unmatched bracket(s)"};
+ }
+ return;
+ }
+}
+
+/*
+ * determine whether `s` and `next` can be combined to form a token
+ */
+bool T_lexer::token_continue(const string &s, char next)
+{
+ if constexpr (TOKEN_DEBUG) loggerln("token_continue: receiving {} {}", s, next);
+ // s is empty, so next can be part of the token
+ if (s.size() == 0) return true;
+
+ // text -> text/symbol
+ if (!is_single_symbol(s[s.size()-1]))
+ {
+ // text -> symbol
+ if (is_single_symbol(next))
+ return false;
+
+ // text still
+ return true;
+ }
+
+ // symbol -> symbol
+ T_property tmp;
+ if (symbol_property.contains(s))
+ {
+ tmp = symbol_property.at(s);
+ if (tmp.continuable)
+ {
+ string prev_and_next = s + next;
+ if (symbol_property.contains(prev_and_next))
+ return true;
+ return false;
+ }
+ return false;
+ }
+ else
+ {
+ if constexpr (TOKEN_DEBUG) loggerln("symbol not found in map");
+ return false;
+ }
+}
+
+// handle [prev, current)
+void T_lexer::token_push(const string &push_s, int bracket_depth, bool force_text) // force_text = false
+{
+ T_token tmp;
+ tmp.bracket_depth = bracket_depth;
+ string trimmed = trim(push_s);
+ if (trimmed == "") return;
+
+ if constexpr (TOKEN_DEBUG) loggerln("finding {} in map", trimmed);
+
+ if (symbol_property.contains(trimmed) && !force_text) // symbol
+ {
+ if constexpr (TOKEN_DEBUG) loggerln("token_push: adding symbol");
+ tmp.token_type = symbol_property.at(trimmed).enum_type;
+ }
+ else
+ {
+ if constexpr (TOKEN_DEBUG) loggerln("token_push: adding text");
+ // if (force_text)
+ // trimmed = trim(trimmed, "\'\"");
+ tmp.token_type = TEXT;
+ tmp.text = trimmed;
+ }
+ token.push_back(tmp);
+}
+
+int T_lexer::tokenizer(const string &s)
+{
+ if (s.size() == 0) return -1;
+
+ size_t len = s.size();
+ size_t token_vector_index = 0;
+ string prev_str = "";
+ bool force_text = false;
+ int bracket_depth = 0;
+ guard_type guard = GUARD_OFF;
+
+
+ for (size_t i = 0; i < len; i++)
+ {
+
+ char c = s[i];
+
+ prev_str += c;
+
+ if constexpr (TOKEN_DEBUG) loggerln("---");
+ if constexpr (TOKEN_DEBUG) loggerln("prev_str .{}. next char {}", prev_str, s[i+1]);
+
+ if (c == '\'' || c == '\"')
+ {
+ if constexpr (TOKEN_DEBUG) loggerln("quotation");
+
+ // turn on GUARD
+ if (guard == GUARD_OFF)
+ {
+ guard = (c == '\'') ? GUARD_STRONG : GUARD_WEAK;
+ if constexpr (TOKEN_DEBUG) loggerln("set guard {}", (c == '\'') ? "GUARD_STRONG" : "GUARD_WEAK");
+ continue;
+ }
+ // turn off
+ else if (guard == GUARD_STRONG && c == '\'')
+ {
+ guard = GUARD_OFF;
+ if constexpr (TOKEN_DEBUG) loggerln("quitting strong guard");
+ force_text = true;
+ }
+ //turn off
+ else if (guard == GUARD_WEAK && c == '\"')
+ {
+ guard = GUARD_OFF;
+ if constexpr (TOKEN_DEBUG) loggerln("quitting weak guard");
+ force_text = true;
+ } else
+ continue;
+ }
+ // not quotation mark
+ else if ((guard == GUARD_STRONG || guard == GUARD_WEAK))
+ {
+ if constexpr (TOKEN_DEBUG) loggerln("tokenizer: guarded, continue");
+ continue;
+ }
+
+ bracket_depth_changer(c, bracket_depth);
+
+ if ((i+1 == len) || !token_continue(prev_str, s[i+1]))
+ {
+ token_push(prev_str, bracket_depth, force_text);
+ force_text = false;
+ prev_str = "";
+ continue;
+ }
+ }
+
+ if (bracket_depth != 0)
+ throw T_error{error_code::TOKEN_UNMATCHED_BRACKET, "tokenizer: unmatched bracket(s)"};
+
+ return 0;
+}
+
+}
generated by cgit v1.3.1-4-g156e (git 2.54.0) at 2026-11-14 22:34:29 +0000