aboutsummaryrefslogtreecommitdiff
path: root/src/lexer
diff options
context:
space:
mode:
Diffstat (limited to 'src/lexer')
-rw-r--r--src/lexer/ast.cc161
-rw-r--r--src/lexer/parseline.cc80
-rw-r--r--src/lexer/symbol_property.cc29
-rw-r--r--src/lexer/token.cc204
4 files changed, 0 insertions, 474 deletions
diff --git a/src/lexer/ast.cc b/src/lexer/ast.cc
deleted file mode 100644
index da96c6f..0000000
--- a/src/lexer/ast.cc
+++ /dev/null
@@ -1,161 +0,0 @@
-#include "lexer/ast.h"
-
-#include <string>
-#include <iostream>
-#include <print>
-
-#include "lexer/lexer_class.h"
-#include "lexer/enum_type.h"
-#include "lexer/token.h"
-#include "lexer/symbol_property.h"
-#include "util/io.h"
-#include "error.h"
-#include "magic_enum/magic_enum.hpp"
-
-using namespace std;
-
-namespace ov4
-{
-
-size_t T_lexer::alloc_ast()
-{
- ast.emplace_back();
- return ast_index++;
-}
-
-string T_lexer::final_trim(size_t _begin, size_t _end)
-{
- string s;
- for (size_t i = _begin; i < _end; i++)
- {
- if (token[i].token_type != TEXT)
- continue;
- s += token[i].text + " ";
- }
- s = s.substr(0, s.size()-1); // eat final space
- return s;
-}
-
-int T_lexer::get_pivot_order(enum_token_type x)
-{
- string s = symbol_enum2string(x);
- if (s != "")
- {
- return symbol_property.at(s).pivot_priority;
- }
- // not found
- return -1;
-}
-
-bool T_lexer::is_operator(enum_token_type x)
-{
- string s = symbol_enum2string(x);
- if (s != "")
- return (symbol_property.at(s).pivot_priority >= 0) ? true : false;
- return false;
-}
-
-// region [begin, end)
-void T_lexer::parse(size_t cmd_begin, size_t cmd_end, size_t ast_vec)
-{
- int current_depth = 0;
- int current_order = -1;
- int max_depth = -1;
- size_t found_pos = string::npos;
-
- // if it's fully inside bracket
- // then it's subshell
- if (cmd_begin != cmd_end)
- loggerln("parse: left token type: {}, right: {}",
- magic_enum::enum_name(token[cmd_begin].token_type),
- magic_enum::enum_name(token[cmd_end-1].token_type));
- if (cmd_begin != cmd_end && token[cmd_begin].token_type == LEFT_BRACKET && token[cmd_end-1].token_type == RIGHT_BRACKET)
- {
- ast[ast_vec].subshell = true;
- loggerln("parse: detect subshell {}", ast_vec);
-
- // eat () subshell token
- cmd_begin++;
- cmd_end--;
- }
- else
- {
- ast[ast_vec].subshell = false;
- }
-
- for (size_t i = cmd_begin; i < cmd_end; i++)
- {
- if (token[i].bracket_depth > max_depth)
- max_depth = token[i].bracket_depth;
- }
-
-
- while (found_pos == string::npos)
- {
- // scan from right to left
- for (size_t i = cmd_end - 1; i >= cmd_begin && i < cmd_end; i--)
- {
- // if same depth
- // and higher order number
- if (token[i].bracket_depth == current_depth
- && is_operator(token[i].token_type)
- && get_pivot_order(token[i].token_type) > current_order)
- {
- current_order = get_pivot_order(token[i].token_type);
- found_pos = i;
- loggerln("Found pos: {} order: {} depth: {}", i, current_order, current_depth);
- }
- }
- current_depth++;
- if (current_depth > max_depth)
- {
- break;
- }
- }
-
- if (found_pos == string::npos)
- {
- loggerln("Smallest unit");
- ast[ast_vec].token_type = TEXT;
- ast[ast_vec].command_text = final_trim(cmd_begin, cmd_end);
- return;
- }
-
- string token_str = symbol_enum2string(token[found_pos].token_type);
-
- // miss left
- if (symbol_property.at(token_str).left == child_existence_type::EXIST && cmd_begin == found_pos)
- throw T_error{error_code::AST_MISS_LEFT_CHILD, "parse: missing left child"};
- // miss right
- if (symbol_property.at(token_str).right == child_existence_type::EXIST && cmd_end-1 == found_pos)
- throw T_error{error_code::AST_MISS_RIGHT_CHILD, "parse: missing right child"};
- // should not exist left
- if (symbol_property.at(token_str).left == child_existence_type::NOT_EXIST && cmd_begin != found_pos)
- throw T_error{error_code::AST_EXIST_LEFT_CHILLD, "parse: left child should NOT exist"};
- // should not exist right
- if (symbol_property.at(token_str).right == child_existence_type::NOT_EXIST && cmd_end-1 != found_pos)
- throw T_error{error_code::AST_EXIST_LEFT_CHILLD, "parse: right child should NOT exist"};
-
-
- ast[ast_vec].token_type = token[found_pos].token_type;
- loggerln("Parse sub: {}, {}, {}", cmd_begin, found_pos, cmd_end);
-
- if (symbol_property.at(token_str).left == child_existence_type::EXIST
- || symbol_property.at(token_str).left == child_existence_type::OPTIONAL)
- {
- size_t l = alloc_ast();
- ast[ast_vec].left = l;
- parse(cmd_begin, found_pos, l);
- }
-
- if (symbol_property.at(token_str).right == child_existence_type::EXIST
- || symbol_property.at(token_str).right == child_existence_type::OPTIONAL)
- {
- size_t r = alloc_ast();
- ast[ast_vec].right = r;
- parse(found_pos+1, cmd_end, r);
- }
-
-}
-
-}
diff --git a/src/lexer/parseline.cc b/src/lexer/parseline.cc
deleted file mode 100644
index 064f876..0000000
--- a/src/lexer/parseline.cc
+++ /dev/null
@@ -1,80 +0,0 @@
-#include "lexer/parseline.h"
-
-#include <vector>
-
-#include "lexer/enum_type.h"
-#include "util/string.h"
-
-using namespace std;
-
-namespace ov4
-{
-
-char **parseline(const std::string &s)
-{
- guard_type guard = GUARD_OFF;
- vector<string> v;
- size_t len = s.size();
- string prev_str = "";
- char c;
-
- for (size_t i = 0; i < len; i++)
- {
-
- c = s[i];
- if (c == ' ' && guard == GUARD_OFF) continue;
-
- if (c == '\'' || c == '\"')
- {
-
- // turn on GUARD
- if (guard == GUARD_OFF)
- {
- guard = (c == '\'') ? GUARD_STRONG : GUARD_WEAK;
- if (i+1 == len)
- {
- v.push_back(prev_str);
- prev_str = "";
- }
- continue;
- }
- // turn off
- else if (guard == GUARD_STRONG && c == '\'')
- {
- guard = GUARD_OFF;
- if (i+1 == len)
- {
- v.push_back(prev_str);
- prev_str = "";
- }
- continue;
- }
- // turn off
- else if (guard == GUARD_WEAK && c == '\"')
- {
- guard = GUARD_OFF;
- if (i+1 == len)
- {
- v.push_back(prev_str);
- prev_str = "";
- }
- continue;
- }
- // if this (single/double) quotation mark is guarded
- // it should be saved
- }
-
- prev_str += c;
-
- if ((i+1 >= len) || (s[i+1] == ' ' && guard == GUARD_OFF))
- {
- v.push_back(prev_str);
- prev_str = "";
- continue;
- }
- }
-
- return string2argv_alloc(v);
-}
-
-}
diff --git a/src/lexer/symbol_property.cc b/src/lexer/symbol_property.cc
deleted file mode 100644
index c78dd61..0000000
--- a/src/lexer/symbol_property.cc
+++ /dev/null
@@ -1,29 +0,0 @@
-#include "lexer/symbol_property.h"
-
-#include <string>
-
-#include "lexer/enum_type.h"
-
-using namespace std;
-
-namespace ov4
-{
-
-string symbol_enum2string(enum_token_type e)
-{
- for (auto &c : symbol_property)
- {
- // pair<string, T_property>
-
- if (c.second.enum_type == e)
- return c.first;
- }
- return "";
-}
-
-bool symbol_contains_enum_type(enum_token_type e)
-{
- return (symbol_enum2string(e) == "") ? false : true;
-}
-
-}
diff --git a/src/lexer/token.cc b/src/lexer/token.cc
deleted file mode 100644
index ec69090..0000000
--- a/src/lexer/token.cc
+++ /dev/null
@@ -1,204 +0,0 @@
-#include "lexer/token.h"
-
-#include <string>
-#include <iostream>
-#include <print>
-
-#include "lexer/lexer_class.h"
-#include "lexer/enum_type.h"
-#include "lexer/symbol_property.h"
-#include "util/io.h"
-#include "error.h"
-
-using namespace std; // remove this
-
-constexpr bool TOKEN_DEBUG = false;
-
-namespace ov4
-{
-
-/*
- * remove leading chars
- * by default remove " \t"
- */
-string T_lexer::trim(const string &s, const string &trim_target) // trim_target = " \t"
-{
- size_t b = s.find_first_not_of(trim_target);
- if (b == string::npos)
- return "";
- size_t e = s.find_last_not_of(trim_target);
- return s.substr(b, e - b + 1);
-}
-
-
-bool T_lexer::is_single_symbol(char c)
-{
- string s = " ";
- s[0] = c;
- // quotation marks are treated as text
- if (symbol_property.contains(s) && c != '\'' && c != '\"')
- return true;
- return false;
-}
-
-void T_lexer::bracket_depth_changer(char c, int &bracket_depth)
-{
- if (c == '(')
- {
- bracket_depth++;
- if constexpr (TOKEN_DEBUG) loggerln("changed depth: {}", bracket_depth);
- return;
- }
- if (c == ')')
- {
- bracket_depth--;
- loggerln("changed depth: {}", bracket_depth);
- if (bracket_depth < 0)
- {
- throw T_error{error_code::TOKEN_UNMATCHED_BRACKET, "tokenizer: unmatched bracket(s)"};
- }
- return;
- }
-}
-
-/*
- * determine whether `s` and `next` can be combined to form a token
- */
-bool T_lexer::token_continue(const string &s, char next)
-{
- if constexpr (TOKEN_DEBUG) loggerln("token_continue: receiving {} {}", s, next);
- // s is empty, so next can be part of the token
- if (s.size() == 0) return true;
-
- // text -> text/symbol
- if (!is_single_symbol(s[s.size()-1]))
- {
- // text -> symbol
- if (is_single_symbol(next))
- return false;
-
- // text still
- return true;
- }
-
- // symbol -> symbol
- T_property tmp;
- if (symbol_property.contains(s))
- {
- tmp = symbol_property.at(s);
- if (tmp.continuable)
- {
- string prev_and_next = s + next;
- if (symbol_property.contains(prev_and_next))
- return true;
- return false;
- }
- return false;
- }
- else
- {
- if constexpr (TOKEN_DEBUG) loggerln("symbol not found in map");
- return false;
- }
-}
-
-// handle [prev, current)
-void T_lexer::token_push(const string &push_s, int bracket_depth, bool force_text) // force_text = false
-{
- T_token tmp;
- tmp.bracket_depth = bracket_depth;
- string trimmed = trim(push_s);
- if (trimmed == "") return;
-
- if constexpr (TOKEN_DEBUG) loggerln("finding {} in map", trimmed);
-
- if (symbol_property.contains(trimmed) && !force_text) // symbol
- {
- if constexpr (TOKEN_DEBUG) loggerln("token_push: adding symbol");
- tmp.token_type = symbol_property.at(trimmed).enum_type;
- }
- else
- {
- if constexpr (TOKEN_DEBUG) loggerln("token_push: adding text");
- // if (force_text)
- // trimmed = trim(trimmed, "\'\"");
- tmp.token_type = TEXT;
- tmp.text = trimmed;
- }
- token.push_back(tmp);
-}
-
-int T_lexer::tokenizer(const string &s)
-{
- if (s.size() == 0) return -1;
-
- size_t len = s.size();
- size_t token_vector_index = 0;
- string prev_str = "";
- bool force_text = false;
- int bracket_depth = 0;
- guard_type guard = GUARD_OFF;
-
-
- for (size_t i = 0; i < len; i++)
- {
-
- char c = s[i];
-
- prev_str += c;
-
- if constexpr (TOKEN_DEBUG) loggerln("---");
- if constexpr (TOKEN_DEBUG) loggerln("prev_str .{}. next char {}", prev_str, s[i+1]);
-
- if (c == '\'' || c == '\"')
- {
- if constexpr (TOKEN_DEBUG) loggerln("quotation");
-
- // turn on GUARD
- if (guard == GUARD_OFF)
- {
- guard = (c == '\'') ? GUARD_STRONG : GUARD_WEAK;
- if constexpr (TOKEN_DEBUG) loggerln("set guard {}", (c == '\'') ? "GUARD_STRONG" : "GUARD_WEAK");
- continue;
- }
- // turn off
- else if (guard == GUARD_STRONG && c == '\'')
- {
- guard = GUARD_OFF;
- if constexpr (TOKEN_DEBUG) loggerln("quitting strong guard");
- force_text = true;
- }
- //turn off
- else if (guard == GUARD_WEAK && c == '\"')
- {
- guard = GUARD_OFF;
- if constexpr (TOKEN_DEBUG) loggerln("quitting weak guard");
- force_text = true;
- } else
- continue;
- }
- // not quotation mark
- else if ((guard == GUARD_STRONG || guard == GUARD_WEAK))
- {
- if constexpr (TOKEN_DEBUG) loggerln("tokenizer: guarded, continue");
- continue;
- }
-
- bracket_depth_changer(c, bracket_depth);
-
- if ((i+1 == len) || !token_continue(prev_str, s[i+1]))
- {
- token_push(prev_str, bracket_depth, force_text);
- force_text = false;
- prev_str = "";
- continue;
- }
- }
-
- if (bracket_depth != 0)
- throw T_error{error_code::TOKEN_UNMATCHED_BRACKET, "tokenizer: unmatched bracket(s)"};
-
- return 0;
-}
-
-}
generated by cgit v1.3.1-4-g156e (git 2.54.0) at 2026-11-14 23:09:50 +0000