Skip to content

Commit fbbef99

Browse files
TuringKiTuringKi
committed
[WIP] Parser: Let ParseFunctionOutputs records Line.
1 parent 7bb1d89 commit fbbef99

13 files changed

+315
-208
lines changed

‎gram/create_parse_functions.py

Lines changed: 29 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,8 @@ def create_serial_parse_function(serial_idx, key, v, tag_name="TagName"):
4242
tag_name = f""" {name}::kTag """
4343

4444
content_init = ""
45-
content_type = f"using parse_func_type = SerialParseFunctions<"
46-
content_func_def = f"""SerialParseFunctions serial_funcs(context_,"{name}_serial",
45+
content_type = f"using parse_func_type = SerialParseFunctions<ParseFunctionKind::k{name},"
46+
content_func_def = f"""parse_func_type serial_funcs(context_,"{name}_serial",
4747
ParseFunctionInputs(false ,calling_depth(), output.last_token_, output.cur_token_),"""
4848
for idx, s_ in enumerate(v):
4949
assert isinstance(s_, str)
@@ -83,6 +83,7 @@ def create_serial_parse_function(serial_idx, key, v, tag_name="TagName"):
8383
comments = comments[:-2]
8484

8585
content_op = f"""
86+
{content_type + ">;"}
8687
{content_func_def});
8788
static_assert(base::kNumberOfElements == 1);
8889
serial_funcs.executed_mask( decltype(serial_funcs)::base::bitset_type(this->executed_mask_.value() ) );
@@ -101,8 +102,8 @@ def create_serial_in_parallel_function(s_v, k, flg, idx_str, tidx):
101102
name = camel_case(k)
102103
tag_name = f""" {name}::kTag """
103104
assert isinstance(s_v, list)
104-
contents_type = f"SerialParseFunctions<"
105-
contents = f"""SerialParseFunctions(context_, "{name}_{idx_str}_serial_{tidx}", ParseFunctionInputs(
105+
contents_type = f"SerialParseFunctions<ParseFunctionKind::k{name},"
106+
contents = f"""(context_, "{name}_{idx_str}_serial_{tidx}", ParseFunctionInputs(
106107
false,calling_depth() + 1),"""
107108
for s_ in s_v:
108109
assert isinstance(s_, str)
@@ -117,7 +118,7 @@ def create_serial_in_parallel_function(s_v, k, flg, idx_str, tidx):
117118
break
118119
contents_type += f""" ParseFunction,"""
119120
contents += f"""ParseFunction::
120-
create_single_token_check(context_, {opt_str},calling_depth() + 2,
121+
create_single_token_check(context_, {opt_str},calling_depth() + 1,
121122
token::details::TokenKind::{sp_tokens[s]},
122123
diag::DiagKind::{k.replace("-","_")}_expect_{sp_tokens[s]}),"""
123124

@@ -128,9 +129,10 @@ def create_serial_in_parallel_function(s_v, k, flg, idx_str, tidx):
128129
else:
129130
assert s in gram_tree
130131
contents_type += f"{camel_case(s)},"
131-
contents += f"{camel_case(s)}(context_, {opt_str}, calling_depth() + 2),"
132+
contents += f"{camel_case(s)}(context_, {opt_str}, calling_depth() + 1),"
132133
contents = contents[:-1]
133134
contents_type = contents_type[:-1]
135+
contents = contents_type + ">" + contents
134136
contents_type += ">,"
135137
contents += "),"
136138
return contents, contents_type, flg
@@ -139,11 +141,11 @@ def create_serial_in_parallel_function(s_v, k, flg, idx_str, tidx):
139141
def create_parallel_function_normal(v, k, idx, offset, out_name="output", type_recursive=False):
140142
name = camel_case(k)
141143
tag_name = f""" {name}::kTag """
142-
contents_type = f"""ParallelParseFunctions<{len(v)}, """
144+
contents_type = f"""ParallelParseFunctions<ParseFunctionKind::k{name},{len(v)}, """
143145
sub_str = f"""parallel_{idx}"""
144146
if type_recursive:
145147
sub_str = f"""recursive_{idx}"""
146-
contents_type = f"""RecursiveParseFunctions<"""
148+
contents_type = f"""RecursiveParseFunctions<ParseFunctionKind::k{name},"""
147149
content_func_def = f"""
148150
parallel_funcs_{idx}(context_,"{name}_parallel_{idx}",ParseFunctionInputs(false,calling_depth(), output.last_token_,output.cur_token_),
149151
"""
@@ -234,10 +236,25 @@ def create_parallel_function(serial_idx, key, v):
234236
""" + "\n}\n" + f"""
235237
diag::infos() << basic::str::from({tag_name}, "into recursive. \\n");
236238
{content_r_type} {content_r_func_def}
237-
{content_r_op}""" + """
238-
if(!output.work_){
239+
{content_r_op}""" + f"""
240+
if(!output.work_){{
239241
return non_recursive_output;
240-
}
242+
}}
243+
const auto* p_start = &context_->token_lists().at(start_token);
244+
const auto* p_end = &context_->token_lists().at(output.cur_token_);
245+
Line line{{
246+
p_start,
247+
p_end,
248+
this->kind(),
249+
token::details::TokenKind::unknown,
250+
token::TokenLists::len(p_start, p_end),
251+
this->calling_depth(),
252+
Line::segments_type()
253+
}};
254+
line.segments_.append(non_recursive_output.line_);
255+
line.segments_.append(output.line_);
256+
output.line_ = context_->paint(line);
257+
241258
return output;
242259
243260
"""
@@ -458,6 +475,7 @@ class NAME : public ParseFunction<N> { \\
458475
if (!this->valid()) {
459476
return output;
460477
}
478+
auto start_token = output.cur_token_;
461479
___CONTENT_OP___
462480
}
463481
"""

‎include/ast.h

Lines changed: 83 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,4 +22,86 @@
2222
*/
2323
#pragma once
2424

25-
namespace lsp::ast {}
25+
#include <array>
26+
#include "basic/map.h"
27+
#include "basic/vec.h"
28+
#include "token.h"
29+
30+
namespace lps::parser::details {
31+
32+
enum class ParseFunctionKind : uint16_t {
33+
kUnknown = 0,
34+
kExpectedToken = 1,
35+
#define PARSE_FUNC(FUNC) FUNC,
36+
#include "parse_function/kinds.def"
37+
kNum,
38+
};
39+
40+
namespace kind {
41+
42+
static constexpr std::array<std::pair<ParseFunctionKind, const char*>,
43+
static_cast<uint16_t>(ParseFunctionKind::kNum)>
44+
kLists = {{
45+
#define PARSE_FUNC(X) {ParseFunctionKind::X, #X},
46+
#include "parse_function/kinds.def"
47+
}};
48+
49+
static constexpr lps::basic::map::Map<ParseFunctionKind, const char*,
50+
static_cast<uint16_t>(
51+
ParseFunctionKind::kNum)>
52+
kMap{kLists};
53+
54+
} // namespace kind
55+
56+
inline std::ostream& operator<<(std::ostream& s, ParseFunctionKind kind) {
57+
s << kind::kMap.at(kind);
58+
return s;
59+
}
60+
61+
struct Line {
62+
using segments_type = basic::Vector<2, const Line*>;
63+
Line() = default;
64+
Line(const token::Token* start, const token::Token* end,
65+
ParseFunctionKind kind, token::details::TokenKind token_kind, size_t len,
66+
size_t calling_depth, segments_type&& segments)
67+
: start_(start),
68+
end_(end),
69+
kind_(kind),
70+
token_kind_(token_kind),
71+
len_(len),
72+
calling_depth_(calling_depth),
73+
segments_(std::move(segments)) {}
74+
const token::Token* start_{nullptr};
75+
const token::Token* end_{nullptr};
76+
ParseFunctionKind kind_{ParseFunctionKind::kUnknown};
77+
token::details::TokenKind token_kind_{token::details::TokenKind::unknown};
78+
size_t len_{0};
79+
size_t calling_depth_{0};
80+
segments_type segments_;
81+
};
82+
83+
inline bool operator==(const Line& a, const Line& b) {
84+
return a.start_ == b.start_ && a.end_ == b.end_ && a.kind_ == b.kind_ &&
85+
a.token_kind_ == b.token_kind_ && a.len_ == b.len_ &&
86+
a.calling_depth_ == b.calling_depth_ && a.segments_ == b.segments_;
87+
}
88+
89+
class Tree {
90+
public:
91+
struct Node {
92+
using sub_nodes_type = basic::Vector<4, Node*>;
93+
sub_nodes_type children_;
94+
Line line_;
95+
};
96+
Node* append(const Node& n) {
97+
nodes_.append(n);
98+
return &nodes_.back();
99+
}
100+
Node& root() { return root_; }
101+
102+
private:
103+
Node root_;
104+
basic::Vector<8, Node> nodes_;
105+
};
106+
107+
} // namespace lps::parser::details

‎include/basic/vec.h

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,12 @@ class Vector : public vec::details::Impl<N, T> {
270270
SET();
271271
}
272272

273+
Vector(size_t N_hat, const ele_type& val) : base_type("Vector") {
274+
for (size_t i = 0; i < N_hat; i++) {
275+
this->append(val);
276+
}
277+
}
278+
273279
explicit Vector(const Vector& v) : base_type(v.tag_) {
274280
for (const T& a : v) {
275281
this->append(a);
@@ -290,6 +296,30 @@ class Vector : public vec::details::Impl<N, T> {
290296
return *this;
291297
}
292298

299+
bool operator==(const Vector& v) const {
300+
if (this->size() != v.size()) {
301+
return false;
302+
}
303+
for (size_t i = 0; i < this->size_; ++i) {
304+
if (this->first_[i] != v.first_[i]) {
305+
return false;
306+
}
307+
}
308+
return true;
309+
}
310+
311+
bool operator!=(const Vector& v) const {
312+
if (this->size() != v.size()) {
313+
return true;
314+
}
315+
for (size_t i = 0; i < this->size_; ++i) {
316+
if (this->first_[i] != v.first_[i]) {
317+
return true;
318+
}
319+
}
320+
return false;
321+
}
322+
293323
#undef SET
294324
explicit Vector(const T (&d)[N]) : base_type(d) {}
295325
Vector& operator=(const T (&d)[N]) {

‎include/parse_function/parallel_function_impl.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,9 @@
2929

3030
namespace lps::parser::details {
3131

32-
template <size_t NumElements, typename... ParseFuncs>
32+
template <ParseFunctionKind Kind, size_t NumElements, typename... ParseFuncs>
3333
ParseFunctionOutputs
34-
ParallelParseFunctions<NumElements, ParseFuncs...>::operator()() {
34+
ParallelParseFunctions<Kind, NumElements, ParseFuncs...>::operator()() {
3535
lps_assert("ParallelParseFunctions", this->ok_to_try());
3636
auto output = base::operator()();
3737
if (!this->valid()) {
@@ -96,8 +96,8 @@ ParallelParseFunctions<NumElements, ParseFuncs...>::operator()() {
9696
return output;
9797
}
9898

99-
template <size_t NumElements, typename... ParseFuncs>
100-
void ParallelParseFunctions<NumElements, ParseFuncs...>::reset() {
99+
template <ParseFunctionKind Kind, size_t NumElements, typename... ParseFuncs>
100+
void ParallelParseFunctions<Kind, NumElements, ParseFuncs...>::reset() {
101101
base::reset();
102102
std::apply(
103103
[](ParseFuncs&... funcs) {

‎include/parse_function/parallel_serial_recursive_function.h

Lines changed: 6 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030

3131
namespace lps::parser::details {
3232

33-
template <size_t NumElements, typename... ParseFuncs>
33+
template <ParseFunctionKind Kind, size_t NumElements, typename... ParseFuncs>
3434
class ParallelParseFunctions : public ParseFunction<NumElements> {
3535

3636
public:
@@ -41,16 +41,13 @@ class ParallelParseFunctions : public ParseFunction<NumElements> {
4141
: base(context, kName, param), parse_functions_(funcs...) {}
4242
ParseFunctionOutputs operator()() override;
4343
void reset() override;
44-
ParseFunctionKind kind() override {
45-
static constexpr ParseFunctionKind kKind = ParseFunctionKind::kUnknown;
46-
return kKind;
47-
}
44+
ParseFunctionKind kind() override { return Kind; }
4845

4946
protected:
5047
std::tuple<ParseFuncs...> parse_functions_;
5148
};
5249

53-
template <typename... ParseFuncs>
50+
template <ParseFunctionKind Kind, typename... ParseFuncs>
5451
class SerialParseFunctions : public ParseFunction<1> {
5552

5653
public:
@@ -61,16 +58,13 @@ class SerialParseFunctions : public ParseFunction<1> {
6158
: base(context, kName, param), parse_functions_(funcs...) {}
6259
ParseFunctionOutputs operator()() override;
6360
void reset() override;
64-
ParseFunctionKind kind() override {
65-
static constexpr ParseFunctionKind kKind = ParseFunctionKind::kUnknown;
66-
return kKind;
67-
}
61+
ParseFunctionKind kind() override { return Kind; }
6862

6963
protected:
7064
std::tuple<ParseFuncs...> parse_functions_;
7165
};
7266

73-
template <typename... ParseFuncs>
67+
template <ParseFunctionKind Kind, typename... ParseFuncs>
7468
class RecursiveParseFunctions : public ParseFunction<1> {
7569

7670
public:
@@ -83,10 +77,7 @@ class RecursiveParseFunctions : public ParseFunction<1> {
8377
ParseFuncs&&... funcs)
8478
: base(context, kName, param), parse_functions_(funcs...) {}
8579
ParseFunctionOutputs operator()() override;
86-
ParseFunctionKind kind() override {
87-
static constexpr ParseFunctionKind kKind = ParseFunctionKind::kUnknown;
88-
return kKind;
89-
}
80+
ParseFunctionKind kind() override { return Kind; }
9081

9182
protected:
9283
void execute(ParseFunctionOutputs&, working_list_type& executed_mask);

‎include/parse_function/recursive_function_impl.h

Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,9 @@
3030

3131
namespace lps::parser::details {
3232

33-
template <typename... ParseFuncs>
34-
ParseFunctionOutputs RecursiveParseFunctions<ParseFuncs...>::operator()() {
33+
template <ParseFunctionKind Kind, typename... ParseFuncs>
34+
ParseFunctionOutputs
35+
RecursiveParseFunctions<Kind, ParseFuncs...>::operator()() {
3536
lps_assert("RecursiveParseFunctions", this->ok_to_try());
3637
auto output = base::operator()();
3738
this->executed_mask_.set();
@@ -41,6 +42,9 @@ ParseFunctionOutputs RecursiveParseFunctions<ParseFuncs...>::operator()() {
4142
uint32_t recursive_depth = 0;
4243
working_list_type executed_mask;
4344
auto tmp_output = output;
45+
46+
Line::segments_type saved_lines;
47+
4448
do {
4549

4650
execute(tmp_output, executed_mask);
@@ -62,18 +66,32 @@ ParseFunctionOutputs RecursiveParseFunctions<ParseFuncs...>::operator()() {
6266
executed_mask.reset();
6367
++recursive_depth;
6468
output = tmp_output;
69+
saved_lines.append(tmp_output.line_);
6570
} while (true);
6671

72+
const auto* p_start = &context_->token_lists().at(this->cur_token());
73+
const auto* p_end = &context_->token_lists().at(output.cur_token_);
74+
Line line{
75+
p_start,
76+
p_end,
77+
this->kind(),
78+
token::details::TokenKind::unknown,
79+
token::TokenLists::len(p_start, p_end),
80+
this->calling_depth(),
81+
std::move(saved_lines),
82+
};
83+
output.line_ = context_->paint(line);
84+
6785
return output;
6886
}
69-
template <typename... ParseFuncs>
70-
bool RecursiveParseFunctions<ParseFuncs...>::regret(
87+
template <ParseFunctionKind Kind, typename... ParseFuncs>
88+
bool RecursiveParseFunctions<Kind, ParseFuncs...>::regret(
7189
const working_list_type& executed_mask) {
7290
return !executed_mask.all();
7391
}
7492

75-
template <typename... ParseFuncs>
76-
void RecursiveParseFunctions<ParseFuncs...>::reset() {
93+
template <ParseFunctionKind Kind, typename... ParseFuncs>
94+
void RecursiveParseFunctions<Kind, ParseFuncs...>::reset() {
7795
std::apply(
7896
[](ParseFuncs&... funcs) {
7997
(
@@ -85,8 +103,8 @@ void RecursiveParseFunctions<ParseFuncs...>::reset() {
85103
parse_functions_);
86104
}
87105

88-
template <typename... ParseFuncs>
89-
void RecursiveParseFunctions<ParseFuncs...>::execute(
106+
template <ParseFunctionKind Kind, typename... ParseFuncs>
107+
void RecursiveParseFunctions<Kind, ParseFuncs...>::execute(
90108
ParseFunctionOutputs& output, working_list_type& executed_mask) {
91109
bool flg_continue = true;
92110
uint32_t running_sub_func_idx = 0;

0 commit comments

Comments
 (0)