From 16a3288af9d214d11feb0c1c3bdb7ecaca2bfdd0 Mon Sep 17 00:00:00 2001 From: _ov4 Date: Sat, 5 Sep 2026 14:45:12 +0000 Subject: refactor very simple tokenizer --- include/parser/ast.h | 56 +++++++++++++ include/parser/enum_type.h | 28 +++++++ include/parser/shared.h | 19 +++++ include/parser/token.h | 193 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 296 insertions(+) create mode 100644 include/parser/ast.h create mode 100644 include/parser/enum_type.h create mode 100644 include/parser/shared.h create mode 100644 include/parser/token.h (limited to 'include/parser') diff --git a/include/parser/ast.h b/include/parser/ast.h new file mode 100644 index 0000000..151afb0 --- /dev/null +++ b/include/parser/ast.h @@ -0,0 +1,56 @@ +#pragma once + +#include + +#include "parser/enum_type.h" +#include "parser/shared.h" + +using namespace std; // remove this + +namespace ov4 +{ + +class T_ast +{ +public: + T_ast() = default; + + enum_token_type token_type; + string command_text; + size_t left; + size_t right; +}; + +size_t alloc_ast() +{ + ast.emplace_back(); + return ast_index++; +} + +int get_pivot_order(enum_token_type x) +{ + switch(x) + { + case ASYNC: + case SEMICOLON: + return 1000; + + case LOGIC_AND: + case LOGIC_OR: + return 500; + + case PIPE: + return 100; + + default: + return 0; + } + return 0; +} + +void parse(size_t cmd_begin, size_t cmd_end, size_t ast_vec) +{ + ; +} + +} diff --git a/include/parser/enum_type.h b/include/parser/enum_type.h new file mode 100644 index 0000000..9de3dc7 --- /dev/null +++ b/include/parser/enum_type.h @@ -0,0 +1,28 @@ +#pragma once + +namespace ov4 +{ + +enum enum_token_type +{ + TEXT, // foo + SINGLE_QUOTATION, // ' + DOUBLE_QUOTATION, // " + LOGIC_AND, // && + LOGIC_OR, // || + ASYNC, // & + PIPE, // | + SEMICOLON, // ; + EXPAND, // $ + LEFT_BRACKET, // ( + RIGHT_BRACKET, // ) +}; + +enum guard_type +{ + GUARD_OFF, + GUARD_WEAK, + GUARD_STRONG, +}; + +} diff --git a/include/parser/shared.h b/include/parser/shared.h new file mode 100644 index 0000000..01f1483 --- /dev/null +++ b/include/parser/shared.h @@ -0,0 +1,19 @@ +#pragma once + +#include +#include + +#include "parser/enum_type.h" + +namespace ov4 +{ + +class T_ast; +class T_token; + +inline size_t ast_index = 0; + +inline std::vector ast; +inline std::vector token; + +} diff --git a/include/parser/token.h b/include/parser/token.h new file mode 100644 index 0000000..1896b3f --- /dev/null +++ b/include/parser/token.h @@ -0,0 +1,193 @@ +#pragma once + +#include + +#include "parser/enum_type.h" +#include "parser/shared.h" + +using namespace std; // remove this + +namespace ov4 +{ + +class T_token +{ +public: + T_token() = default; + + enum_token_type token_type; + string text; + guard_type guard = GUARD_OFF; + int bracket_depth = 0; +}; + +bool is_symbol(char c) +{ + if (c == '&' || c == '|' || c == '(' || c == ')') + return true; + return false; +} + +bool is_symbol_same(string x, string y) +{ + string tmp = ""; + for (size_t i = 0; i < x.size(); i++) + { + if (x[i] != ' ') + tmp += x[i]; + } + return (tmp == y); +} + +void bracket_depth_changer(char c, int &bracket_depth) +{ + if (c == '(') + { + bracket_depth++; + println("changed depth: {}", bracket_depth); + return; + } + if (c == ')') + { + bracket_depth--; + println("changed depth: {}", bracket_depth); + return; + } +} + +// handle [prev, current) +void token_push(string s, size_t current, size_t prev, int bracket_depth, bool is_symbol_prev) +{ + T_token tmp; + string sub = s.substr(prev, current - prev); + cout << "substr " << sub << endl; + + tmp.bracket_depth = bracket_depth; + if (is_symbol_prev) + { + if (is_symbol_same(sub, "&&")) + { + cout << "&& symbol" << endl; + tmp.token_type = LOGIC_AND; + } + else if (is_symbol_same(sub, "||")) + { + cout << "|| symbol" << endl; + tmp.token_type = LOGIC_OR; + } + else if (is_symbol_same(sub, "&")) + { + cout << "& symbol" << endl; + tmp.token_type = ASYNC; + } + else if (is_symbol_same(sub, "|")) + { + cout << "| symbol" << endl; + tmp.token_type = PIPE; + } + else if (is_symbol_same(sub, "(")) + { + cout << "( symbol" << endl; + tmp.token_type = LEFT_BRACKET; + } + else if (is_symbol_same(sub, ")")) + { + cout << ") symbol" << endl; + tmp.token_type = RIGHT_BRACKET; + } + else + { + cerr << "ERROR: is symbol but not symbol" << endl; + return; + } + token.push_back(tmp); + return; + } + + tmp.token_type = TEXT; + tmp.text = sub; + token.push_back(tmp); +} + +int tokenizer(string s) +{ + if (s.size() == 0) return -1; + + size_t len = s.size(); + size_t prev = 0; + size_t token_vector_index = 0; + bool is_in_symbol; + int bracket_depth = 0; + + if (is_symbol(s[0])) + is_in_symbol = true; + else + is_in_symbol = false; + for (size_t i = 0; i < len; i++) + { + println("---"); + println("prev {} current char {}", prev, s[i]); + + + // turn from symbol to text + if (is_in_symbol && !is_symbol(s[i])) + { + cout << s[i] <<" symbol -> text" << endl; + is_in_symbol = false; + token_push(s, i, prev, bracket_depth, true); + prev = i; + bracket_depth_changer(s[i], bracket_depth); + continue; + } + + // turn from text to symbol + if (!is_in_symbol && is_symbol(s[i])) + { + cout << s[i] << " text -> symbol" << endl; + is_in_symbol = true; + token_push(s, i, prev, bracket_depth, false); + prev = i; + bracket_depth_changer(s[i], bracket_depth); + continue; + } + + bracket_depth_changer(s[i], bracket_depth); + + // text continue + if (!is_in_symbol && !is_symbol(s[i])) + { + cout << s[i] <<" still text" << endl; + continue; + } + + // symbol continue + if (is_in_symbol && is_symbol(s[i])) + { + cout << s[i] <<" still symbol" << endl; + continue; + } + } + + if (bracket_depth != 0) + { + cerr << "Brackets are not paired" << endl; + return -1; + } + + // final round text + if (!is_symbol(s[len - 1])) + { + cout << s[len - 1] << " final text" << endl; + token_push(s, len, prev, 0, false); + } + + // final round symbol + if (is_symbol(s[len - 1])) + { + cout << s[len - 1] << " final symbol" << endl; + token_push(s, len, prev, 0, true); + } + return 0; +} + +} -- cgit v1.3.1-4-g156e