1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
|
#pragma once
#include <string>
#include "lexer/enum_type.h"
#include "lexer/shared.h"
#include "lexer/symbol_property.h"
using namespace std; // remove this
namespace ov4
{
class T_token
{
public:
T_token() = default;
enum_token_type token_type;
string text;
guard_type guard = GUARD_OFF;
int bracket_depth = 0;
};
string trim_space(const string &s)
{
size_t b = s.find_first_not_of(" \t");
if (b == string::npos)
return "";
size_t e = s.find_last_not_of(" \t");
return s.substr(b, e - b + 1);
}
bool is_single_symbol(char c)
{
string s = " ";
s[0] = c;
if (symbol_property.contains(s))
return true;
return false;
}
void bracket_depth_changer(char c, int &bracket_depth)
{
if (c == '(')
{
bracket_depth++;
println("changed depth: {}", bracket_depth);
return;
}
if (c == ')')
{
bracket_depth--;
println("changed depth: {}", bracket_depth);
if (bracket_depth < 0)
{
cerr << "Ooooops bracket unmatched, quitting" << endl;
exit(-1); // TODO: std::expected or what, rather than quitting
}
return;
}
}
bool token_continue(string s, char next)
{
println("token_continue: receiving {} {}", s, next);
// s is empty, so next can be part of the token
if (s.size() == 0) return true;
// text -> text/symbol
if (!is_single_symbol(s[s.size()-1]))
{
// text -> symbol
if (is_single_symbol(next))
return false;
// text still
return true;
}
// symbol -> symbol
T_property tmp;
if (symbol_property.contains(s))
{
tmp = symbol_property[s];
if (tmp.continuable)
{
string prev_and_next = s + next;
if (symbol_property.contains(prev_and_next))
return true;
return false;
}
return false;
}
else
{
println("symbol not found in map");
return false;
}
}
// handle [prev, current)
void token_push(string push_s, int bracket_depth)
{
T_token tmp;
tmp.bracket_depth = bracket_depth;
string trimmed = trim_space(push_s);
if (trimmed == "") return;
println("finding {} in map", trimmed);
if (symbol_property.contains(trimmed)) // symbol
{
tmp.token_type = symbol_property[trimmed].enum_type;
}
else
{
tmp.token_type = TEXT;
tmp.text = trimmed;
}
token.push_back(tmp);
}
int tokenizer(string s)
{
if (s.size() == 0) return -1;
size_t len = s.size();
size_t prev = 0;
size_t token_vector_index = 0;
string prev_str = "";
bool is_in_symbol;
int bracket_depth = 0;
for (size_t i = 0; i < len; i++)
{
println("---");
println("prev {} current char {}", prev, s[i]);
prev_str += s[i];
bracket_depth_changer(s[i], bracket_depth);
if ((i+1 == len) || !token_continue(prev_str, s[i+1]))
{
token_push(prev_str, bracket_depth);
prev_str = "";
continue;
}
}
return 0;
}
}
|
