Skip to content

Commit 7a8520a

Browse files
TuringKiTuringKi
committed
Parser: Add Context for parse_functions.
1 parent 2ca476f commit 7a8520a

File tree

8 files changed

+115
-82
lines changed

8 files changed

+115
-82
lines changed

‎gram/create_parse_functions.py

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ def create_serial_parse_function(serial_idx, key, v, tag_name="TagName"):
4343

4444
content_init = ""
4545
content_type = f"using parse_func_type = SerialParseFunctions<"
46-
content_func_def = f"""SerialParseFunctions serial_funcs("{name}_serial",
46+
content_func_def = f"""SerialParseFunctions serial_funcs(context_,"{name}_serial",
4747
ParseFunctionInputs(false ,calling_depth(), output.last_token_, output.cur_token_),"""
4848
for idx, s_ in enumerate(v):
4949
assert isinstance(s_, str)
@@ -63,7 +63,7 @@ def create_serial_parse_function(serial_idx, key, v, tag_name="TagName"):
6363
break
6464
content_type += f"""ParseFunction,"""
6565
content_func_def += f"""ParseFunction::
66-
create_single_token_check({opt_str},calling_depth() + 1,
66+
create_single_token_check(context_, {opt_str},calling_depth() + 1,
6767
token::details::TokenKind::{sp_tokens[s]},
6868
diag::DiagKind::{k.replace("-","_")}_expect_{sp_tokens[s]}),"""
6969

@@ -74,7 +74,7 @@ def create_serial_parse_function(serial_idx, key, v, tag_name="TagName"):
7474
else:
7575
assert s in gram_tree
7676
content_type += f"{camel_case(s)},"
77-
content_func_def += f"{camel_case(s)}({opt_str},calling_depth() + 1),"
77+
content_func_def += f"{camel_case(s)}(context_, {opt_str},calling_depth() + 1),"
7878
content_func_def = content_func_def[:-1]
7979
content_type = content_type[:-1]
8080
comments = "// \t"
@@ -102,7 +102,7 @@ def create_serial_in_parallel_function(s_v, k, flg, idx_str, tidx):
102102
tag_name = f""" {name}::kTag """
103103
assert isinstance(s_v, list)
104104
contents_type = f"SerialParseFunctions<"
105-
contents = f"""SerialParseFunctions("{name}_{idx_str}_serial_{tidx}", ParseFunctionInputs(
105+
contents = f"""SerialParseFunctions(context_, "{name}_{idx_str}_serial_{tidx}", ParseFunctionInputs(
106106
false,calling_depth() + 1),"""
107107
for s_ in s_v:
108108
assert isinstance(s_, str)
@@ -117,7 +117,7 @@ def create_serial_in_parallel_function(s_v, k, flg, idx_str, tidx):
117117
break
118118
contents_type += f""" ParseFunction,"""
119119
contents += f"""ParseFunction::
120-
create_single_token_check({opt_str},calling_depth() + 2,
120+
create_single_token_check(context_, {opt_str},calling_depth() + 2,
121121
token::details::TokenKind::{sp_tokens[s]},
122122
diag::DiagKind::{k.replace("-","_")}_expect_{sp_tokens[s]}),"""
123123

@@ -128,7 +128,7 @@ def create_serial_in_parallel_function(s_v, k, flg, idx_str, tidx):
128128
else:
129129
assert s in gram_tree
130130
contents_type += f"{camel_case(s)},"
131-
contents += f"{camel_case(s)}({opt_str}, calling_depth() + 2),"
131+
contents += f"{camel_case(s)}(context_, {opt_str}, calling_depth() + 2),"
132132
contents = contents[:-1]
133133
contents_type = contents_type[:-1]
134134
contents_type += ">,"
@@ -145,11 +145,11 @@ def create_parallel_function_normal(v, k, idx, offset, out_name="output", type_r
145145
sub_str = f"""recursive_{idx}"""
146146
contents_type = f"""RecursiveParseFunctions<"""
147147
content_func_def = f"""
148-
parallel_funcs_{idx}("{name}_parallel_{idx}",ParseFunctionInputs(false,calling_depth(), output.last_token_,output.cur_token_),
148+
parallel_funcs_{idx}(context_,"{name}_parallel_{idx}",ParseFunctionInputs(false,calling_depth(), output.last_token_,output.cur_token_),
149149
"""
150150
if type_recursive:
151151
content_func_def = f"""
152-
recursive_funcs_{idx}("{name}_recursive", ParseFunctionInputs(false,calling_depth(), output.last_token_,output.cur_token_),
152+
recursive_funcs_{idx}(context_,"{name}_recursive", ParseFunctionInputs(false,calling_depth(), output.last_token_,output.cur_token_),
153153
"""
154154
flg = True
155155
for ii, s_v in enumerate(v):
@@ -397,11 +397,11 @@ class NAME : public ParseFunction<N> { \\
397397
}\\
398398
~NAME() = default; \\
399399
template <typename... Params> \\
400-
explicit NAME(bool opt, Params... params) : base(#NAME,opt, params...) {} \\
400+
explicit NAME(Context* context, bool opt, Params... params) : base(context, #NAME,opt, params...) {} \\
401401
template <typename... Params> \\
402-
explicit NAME(Params... params) : base(#NAME,params...) {} \\
403-
explicit NAME(const ParseFunctionInputs& param) \\
404-
: base(#NAME,param) {} \\
402+
explicit NAME(Context* context,Params... params) : base(context,#NAME,params...) {} \\
403+
explicit NAME(Context* context,const ParseFunctionInputs& param) \\
404+
: base(context,#NAME,param) {} \\
405405
ParseFunctionOutputs operator()() override; \\
406406
};
407407

‎include/parse_function/parallel_serial_recursive_function.h

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,10 @@ class ParallelParseFunctions : public ParseFunction<NumElements> {
3535

3636
public:
3737
using base = ParseFunction<NumElements>;
38-
explicit ParallelParseFunctions(const char* kName,
38+
explicit ParallelParseFunctions(Context* context, const char* kName,
3939
const ParseFunctionInputs& param,
4040
ParseFuncs&&... funcs)
41-
: base(kName, param), parse_functions_(funcs...) {}
41+
: base(context, kName, param), parse_functions_(funcs...) {}
4242
ParseFunctionOutputs operator()() override;
4343
void reset() override;
4444
ParseFunctionKind kind() override {
@@ -55,10 +55,10 @@ class SerialParseFunctions : public ParseFunction<1> {
5555

5656
public:
5757
using base = ParseFunction<1>;
58-
explicit SerialParseFunctions(const char* kName,
58+
explicit SerialParseFunctions(Context* context, const char* kName,
5959
const ParseFunctionInputs& param,
6060
ParseFuncs&&... funcs)
61-
: base(kName, param), parse_functions_(funcs...) {}
61+
: base(context, kName, param), parse_functions_(funcs...) {}
6262
ParseFunctionOutputs operator()() override;
6363
void reset() override;
6464
ParseFunctionKind kind() override {
@@ -78,10 +78,10 @@ class RecursiveParseFunctions : public ParseFunction<1> {
7878
using working_list_type = basic::Bitset<kNFuncs>;
7979
using working_list_stack_type = std::stack<working_list_type>;
8080
using base = ParseFunction<1>;
81-
explicit RecursiveParseFunctions(const char* kName,
81+
explicit RecursiveParseFunctions(Context* context, const char* kName,
8282
const ParseFunctionInputs& param,
8383
ParseFuncs&&... funcs)
84-
: base(kName, param), parse_functions_(funcs...) {}
84+
: base(context, kName, param), parse_functions_(funcs...) {}
8585
ParseFunctionOutputs operator()() override;
8686
ParseFunctionKind kind() override {
8787
static constexpr ParseFunctionKind kKind = ParseFunctionKind::kUnknown;

‎include/parse_function/serial_function_impl.h

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -113,16 +113,8 @@ ParseFunctionOutputs SerialParseFunctions<ParseFuncs...>::operator()() {
113113
}
114114

115115
auto output(path_stack.top());
116-
auto* node_ptr = Tree::instance().append(local_output.node_);
117-
node_ptr->kind_ = func.kKind;
118-
if (func.kKind == ParseFunctionKind::kExpectedToken) {
119-
node_ptr->expected_token_kind_ = func.expected_token_kind();
120-
}
121116
output.concat(std::move(local_output), func.opt());
122117
path_stack.push(output);
123-
if (output.work_) {
124-
node.sub_nodes_.append(node_ptr);
125-
}
126118
if ((running_sub_func_idx == running_sub_func_size &&
127119
output.work_) ||
128120
!func.valid()) {

‎include/parser.h

Lines changed: 46 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,26 @@ namespace lps::parser {
3838

3939
namespace details {
4040

41+
class Context {
42+
public:
43+
friend class ContextTrait;
44+
using executed_func_type = std::function<void(Context*)>;
45+
void with(const executed_func_type& func) { func(this); }
46+
token::TokenLists& token_lists() { return token_lists_; }
47+
48+
private:
49+
token::TokenLists token_lists_;
50+
};
51+
52+
class ContextTrait {
53+
public:
54+
explicit ContextTrait(Context* context) : context_(context) {}
55+
Context* context() { return context_; }
56+
57+
protected:
58+
Context* context_;
59+
};
60+
4161
enum class ParseFunctionKind : uint16_t {
4262
kUnknown = 0,
4363
kExpectedToken = 1,
@@ -218,9 +238,10 @@ struct ParseFunctionInputs {
218238
};
219239

220240
template <size_t NumElements = 1>
221-
class ParseFunction {
241+
class ParseFunction : public ContextTrait {
222242
public:
223243
using type = ParseFunction<NumElements>;
244+
using context_trait_type = ContextTrait;
224245
using output_type = ParseFunctionOutputs;
225246
using custom_func_type = std::function<output_type(type*)>;
226247
using bitset_type = basic::Bitset<NumElements>;
@@ -233,27 +254,33 @@ class ParseFunction {
233254
}
234255

235256
template <typename... Params>
236-
explicit ParseFunction(const char* kName, Params... params)
237-
: kName_(kName), params_(params...) {}
257+
explicit ParseFunction(Context* context, const char* kName, Params... params)
258+
: context_trait_type(context), kName_(kName), params_(params...) {}
238259
template <typename... Params>
239-
explicit ParseFunction(const char* kName, custom_func_type func,
260+
explicit ParseFunction(Context* context, const char* kName,
261+
custom_func_type func,
240262
token::details::TokenKind expected_token_kind,
241263
Params... params)
242-
: kName_(kName),
264+
: context_trait_type(context),
265+
kName_(kName),
243266
custom_func_(func),
244267
params_(params...),
245268
expected_token_kind_(expected_token_kind) {}
246-
explicit ParseFunction(const char* kName, ParseFunctionInputs param,
269+
explicit ParseFunction(Context* context, const char* kName,
270+
ParseFunctionInputs param,
247271
token::details::TokenKind expected_token_kind,
248272
custom_func_type func)
249-
: kName_(kName),
273+
: context_trait_type(context),
274+
kName_(kName),
250275
params_(std::move(param)),
251276
custom_func_(func),
252277
expected_token_kind_(expected_token_kind) {}
253-
explicit ParseFunction(const char* kName, const ParseFunctionInputs& param)
254-
: kName_(kName), params_(param) {}
255-
explicit ParseFunction(const char* kName, ParseFunctionInputs&& param)
256-
: kName_(kName), params_(std::move(param)) {}
278+
explicit ParseFunction(Context* context, const char* kName,
279+
const ParseFunctionInputs& param)
280+
: context_trait_type(context), kName_(kName), params_(param) {}
281+
explicit ParseFunction(Context* context, const char* kName,
282+
ParseFunctionInputs&& param)
283+
: context_trait_type(context), kName_(kName), params_(std::move(param)) {}
257284

258285
virtual output_type operator()() {
259286
if (custom_func_) {
@@ -292,7 +319,8 @@ class ParseFunction {
292319
return !status;
293320
}
294321

295-
static type create_single_token_check(bool opt, size_t calling_depth,
322+
static type create_single_token_check(Context* context, bool opt,
323+
size_t calling_depth,
296324
token::details::TokenKind token_kind,
297325
diag::DiagKind diag_kind) {
298326
typename type::custom_func_type z([token_kind, diag_kind](type* func) {
@@ -305,9 +333,9 @@ class ParseFunction {
305333
}
306334
token::Token next_tok;
307335
bool flg_use_lexer = true;
308-
if (token::TokenLists::instance().has(output.cur_token_)) {
336+
if (func->context()->token_lists().has(output.cur_token_)) {
309337
const auto* tok_ptr =
310-
token::TokenLists::instance().at(output.cur_token_).next();
338+
func->context()->token_lists().at(output.cur_token_).next();
311339
if (tok_ptr) {
312340
flg_use_lexer = false;
313341
next_tok = *tok_ptr;
@@ -324,7 +352,7 @@ class ParseFunction {
324352
lps_assert("create_single_token_check",
325353
next_tok.kind() != token::details::TokenKind::unknown);
326354

327-
token::TokenLists::instance().append(
355+
func->context()->token_lists().append(
328356
next_tok, token::TokenLists::Info::create(output.cur_token_));
329357
}
330358

@@ -346,14 +374,14 @@ class ParseFunction {
346374
output.cur_token_ = next_tok;
347375
if (next_tok.kind() != token::details::TokenKind::unknown) {
348376
output.token_list_infos_.append(
349-
&(token::TokenLists::instance().at(next_tok)));
377+
&(func->context()->token_lists().at(next_tok)));
350378
}
351379
}
352380

353381
return output;
354382
});
355-
return type(token::details::kMap.at(token_kind), z, token_kind, opt,
356-
calling_depth);
383+
return type(context, token::details::kMap.at(token_kind), z, token_kind,
384+
opt, calling_depth);
357385
}
358386

359387
protected:

‎include/src.h

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -197,10 +197,7 @@ class TokenLists {
197197
};
198198

199199
using ele_type = token::archived_type;
200-
static TokenLists& instance() {
201-
static TokenLists lists;
202-
return lists;
203-
}
200+
204201
bool has(uint32_t file_id, uint64_t offset) const {
205202
if (lists_.contains(file_id)) {
206203
return lists_.at(file_id).contains(offset);
@@ -228,11 +225,16 @@ class TokenLists {
228225
auto info = Info::create(tok);
229226
return lists_.at(info.file_id_).at(info.offset_);
230227
}
228+
const ele_type& at(const Token& tok) const {
229+
lps_assert(kTag, has(tok));
230+
auto info = Info::create(tok);
231+
return lists_.at(info.file_id_).at(info.offset_);
232+
}
231233

232234
const ele_type& last(const Token& tok) const {
233235
lps_assert(kTag, has(tok));
234236
auto info = Info::create(tok);
235-
const auto* ptr = token::TokenLists::instance().at(tok).last();
237+
const auto* ptr = at(tok).last();
236238
if (!ptr) {
237239
static const ele_type kEmptyTok;
238240
return kEmptyTok;
@@ -243,7 +245,7 @@ class TokenLists {
243245
const ele_type& next(const Token& tok) const {
244246
lps_assert(kTag, has(tok));
245247
auto info = Info::create(tok);
246-
const auto* ptr = token::TokenLists::instance().at(tok).next();
248+
const auto* ptr = at(tok).next();
247249
if (!ptr) {
248250
static const ele_type kEmptyTok;
249251
return kEmptyTok;

‎src/constant_expression.cc

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -31,18 +31,22 @@ bool Preprocessing::lex_conditional_expression(
3131
token::Token& tok) {
3232

3333
{
34-
parser::details::ParseFunctionInputs params;
35-
params.opt_ = false;
36-
token::Token next_tok;
37-
lexer::Lexer lexer(first_ptr.file_id(), first_ptr.pos());
38-
lexer.lex(next_tok);
39-
if (next_tok.kind() != token::details::TokenKind::unknown) {
40-
token::TokenLists::instance().append(next_tok);
41-
}
42-
params.cur_token_ = next_tok;
43-
parser::details::ConditionalExpression func(params);
44-
auto output = func();
45-
int dummy = -1;
34+
35+
parser::details::Context context;
36+
context.with([first_ptr](parser::details::Context* context) {
37+
parser::details::ParseFunctionInputs params;
38+
params.opt_ = false;
39+
token::Token next_tok;
40+
lexer::Lexer lexer(first_ptr.file_id(), first_ptr.pos());
41+
lexer.lex(next_tok);
42+
if (next_tok.kind() != token::details::TokenKind::unknown) {
43+
context->token_lists().append(next_tok);
44+
}
45+
params.cur_token_ = next_tok;
46+
parser::details::ConditionalExpression func(context, params);
47+
auto output = func();
48+
int dummy = -1;
49+
});
4650
}
4751

4852
return false;

‎src/parser.cc

Lines changed: 22 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -34,27 +34,30 @@ namespace lps::parser {
3434
void Parser::parse(uint32_t file_id) {
3535
auto content = src::Manager::instance().ref_of_char_file(file_id);
3636
if (!content.empty()) {
37-
details::ParseFunctionInputs params;
38-
params.opt_ = false;
39-
token::Token next_tok;
40-
lexer::Lexer lexer(file_id, 0);
41-
lexer.lex(next_tok);
42-
params.cur_token_ = next_tok;
43-
if (next_tok.kind() != token::details::TokenKind::unknown) {
44-
token::TokenLists::instance().append(next_tok);
45-
}
46-
details::TranslationUnit func(params);
47-
auto output = func();
48-
for (const auto& a : output.diag_inputs_) {
49-
diag::doing(a.main_token_, a.kind_, a.context_tokens_);
50-
}
51-
if (output.work_) {
52-
for (const auto& node : output.node_.sub_nodes_) {
53-
if (node->kind_ != details::ParseFunctionKind::kUnknown) {
54-
// todo(@mxlol233): handle the recursive tree nodes...
37+
details::Context context;
38+
context.with([file_id](details::Context* context) {
39+
details::ParseFunctionInputs params;
40+
params.opt_ = false;
41+
token::Token next_tok;
42+
lexer::Lexer lexer(file_id, 0);
43+
lexer.lex(next_tok);
44+
params.cur_token_ = next_tok;
45+
if (next_tok.kind() != token::details::TokenKind::unknown) {
46+
context->token_lists().append(next_tok);
47+
}
48+
details::TranslationUnit func(context, params);
49+
auto output = func();
50+
for (const auto& a : output.diag_inputs_) {
51+
diag::doing(a.main_token_, a.kind_, a.context_tokens_);
52+
}
53+
if (output.work_) {
54+
for (const auto& node : output.node_.sub_nodes_) {
55+
if (node->kind_ != details::ParseFunctionKind::kUnknown) {
56+
// todo(@mxlol233): handle the recursive tree nodes...
57+
}
5558
}
5659
}
57-
}
60+
});
5861
}
5962
}
6063

0 commit comments

Comments
 (0)