Skip to content

Commit c1b376f

Browse files
TuringKiTuringKi
committed
Parser: Make constant_expression works.
1 parent 8fe100b commit c1b376f

File tree

14 files changed

+488
-160
lines changed

14 files changed

+488
-160
lines changed

‎gram/config.py‎

Lines changed: 37 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,19 @@
185185
"`false`":"kw_false",
186186
"`true`":"kw_true",
187187
"`nullptr`":"kw_nullptr",
188+
"`binary-literal`":"binary_literal",
189+
"`floating-point-literal`":"floating_point_literal",
190+
"`character-literal`":"char_literal",
191+
"`string-literal`":"string_literal",
192+
"`integer-literal`":"integer_literal",
193+
"`decimal-literal`":"decimal_literal",
194+
"`octal-literal`":"octal_literal",
195+
"`hexadecimal-literal`":"hexadecimal_literal",
196+
"`raw-string`":"raw_string",
197+
"`user-defined-character-literal`":"user_defined_char_literal",
198+
"`user-defined-string-literal`":"user_defined_string_literal",
199+
"`user-defined-floating-point-literal`":"user_defined_floating_point_literal",
200+
"`user-defined-integer-literal`":"user_defined_integer_literal",
188201
}
189202

190203
gram_tree = {}
@@ -654,7 +667,7 @@
654667

655668
gram_tree["static_assert-declaration"] = [
656669
["`static_assert`", "`(`", "constant-expression", "`)`", "`;`"],
657-
["`static_assert`", "`(`", "constant-expression", "`,`", "string-literal", "`)`", "`;`"]
670+
["`static_assert`", "`(`", "constant-expression", "`,`", "`string-literal`", "`)`", "`;`"]
658671
]
659672

660673
gram_tree["empty-declaration"] = [
@@ -1056,16 +1069,16 @@
10561069
]
10571070

10581071
gram_tree["asm-declaration"] = [
1059-
"attribute-specifier-seq[opt]", "`asm`", "`(`", "string-literal", "`)`", "`;`",
1072+
"attribute-specifier-seq[opt]", "`asm`", "`(`", "`string-literal`", "`)`", "`;`",
10601073
]
10611074

10621075
gram_tree["linkage-specification"] = [
1063-
["`extern`", "string-literal", "`{`", "declaration-seq[opt]", "`}`"],
1064-
["`extern`", "string-literal", "declaration"],
1076+
["`extern`", "`string-literal`", "`{`", "declaration-seq[opt]", "`}`"],
1077+
["`extern`", "`string-literal`", "declaration"],
10651078
]
10661079

10671080
gram_tree["attribute-specifier-seq"] = [
1068-
"attribute-specifier", "attribute-specifier-seq[opt]",
1081+
"attribute-specifier", "attribute-specifier-seq[opt]",
10691082
]
10701083

10711084
gram_tree["attribute-specifier"] = [
@@ -1310,8 +1323,8 @@
13101323
]
13111324

13121325
gram_tree["literal-operator-id"] = [
1313-
["`operator`", "string-literal", "`identifier`"],
1314-
["`operator`", "user-defined-string-literal"],
1326+
["`operator`", "`string-literal`", "`identifier`"],
1327+
["`operator`", "`user-defined-string-literal`"],
13151328
]
13161329

13171330
gram_tree["template-declaration"] = [
@@ -1562,7 +1575,7 @@
15621575
]
15631576

15641577
gram_tree["header-name-tokens"] = [
1565-
["string-literal"],
1578+
["`string-literal`"],
15661579
["`<`", "h-pp-tokens", "`>`"],
15671580
]
15681581

@@ -1605,10 +1618,10 @@
16051618
["`export`"],
16061619
["`identifier`"],
16071620
["pp-number"],
1608-
["character-literal"],
1609-
["user-defined-character-literal"],
1610-
["string-literal"],
1611-
["user-defined-string-literal"],
1621+
["`character-literal`"],
1622+
["`user-defined-character-literal`"],
1623+
["`string-literal`"],
1624+
["`user-defined-string-literal`"],
16121625
["preprocessing-op-or-punc"],
16131626
# each non-whitespace character that cannot be one of the above
16141627
]
@@ -1698,43 +1711,19 @@
16981711
]
16991712

17001713
gram_tree["literal"] = [
1701-
["integer-literal"],
1702-
["character-literal"],
1703-
["floating-point-literal"],
1704-
["string-literal"],
1714+
["`integer-literal`"],
1715+
["`binary-literal`"],
1716+
["`octal-literal`"],
1717+
["`decimal-literal`"],
1718+
["`hexadecimal-literal`"],
1719+
["`character-literal`"],
1720+
["`floating-point-literal`"],
1721+
["`string-literal`"],
17051722
["boolean-literal"],
17061723
["pointer-literal"],
17071724
["user-defined-literal"],
17081725
]
17091726

1710-
gram_tree["integer-literal"] = [
1711-
["binary-literal", "integer-suffix[opt]"],
1712-
["octal-literal", "integer-suffix[opt]"],
1713-
["decimal-literal", "integer-suffix[opt]"],
1714-
["hexadecimal-literal", "integer-suffix[opt]"],
1715-
]
1716-
1717-
gram_tree["binary-literal"] = [
1718-
["`0b`", "binary-digit"],
1719-
["`0B`", "`binary-digit`"],
1720-
["binary-literal", "`'`"],
1721-
["binary-literal", "`'`[opt]", "binary-digit"],
1722-
]
1723-
1724-
gram_tree["octal-literal"] = [
1725-
["`0`"],
1726-
["octal-literal", "`'`[opt]", "octal-digit"],
1727-
]
1728-
1729-
gram_tree["decimal-literal"] = [
1730-
["nonzero-digit"],
1731-
["decimal-literal", "`'`[opt]", "digit"],
1732-
]
1733-
1734-
gram_tree["hexadecimal-literal"] = [
1735-
"hexadecimal-prefix", "hexadecimal-digit-sequence",
1736-
]
1737-
17381727
gram_tree["binary-digit"] = [
17391728
["`0`"],
17401729
["`1`"],
@@ -1810,10 +1799,6 @@
18101799
["`ll`"], ["`LL`"],
18111800
]
18121801

1813-
gram_tree["character-literal"] = [
1814-
"encoding-prefix[opt]", "`'`","c-char-sequence", "`'`",
1815-
]
1816-
18171802
gram_tree["encoding-prefix"] = [
18181803
["`u8`"],["`u`"], ["`U`"], ["`L`"],
18191804
]
@@ -1851,11 +1836,6 @@
18511836
["hexadecimal-escape-sequence", "hexadecimal-digit"],
18521837
]
18531838

1854-
gram_tree["floating-point-literal"] = [
1855-
["decimal-floating-point-literal"],
1856-
["hexadecimal-floating-point-literal"],
1857-
]
1858-
18591839
gram_tree["decimal-floating-point-literal"] = [
18601840
["fractional-constant", "exponent-part[opt]", "floating-point-suffix[opt]"],
18611841
["digit-sequence", "exponent-part[opt]", "floating-point-suffix[opt]"],
@@ -1899,11 +1879,6 @@
18991879
["`f`"], ["`l`"], ["`F`"], ["`L`"],
19001880
]
19011881

1902-
gram_tree["string-literal"] = [
1903-
["encoding-prefix[opt]", '`"`', "s-char-sequence[opt]", '`"`'],
1904-
["encoding-prefix[opt]", '`R`', "raw-string"],
1905-
]
1906-
19071882
gram_tree["s-char-sequence"] = [
19081883
["s-char"],
19091884
["s-char-sequence", "s-char"],
@@ -1915,9 +1890,6 @@
19151890
["universal-character-name"],
19161891
]
19171892

1918-
gram_tree["raw-string"] = [
1919-
'`"`', "d-char-sequence[opt]", "`(`", "r-char-sequence[opt]" ,"`)`", "d-char-sequence[opt]", '`"`',
1920-
]
19211893

19221894
gram_tree["r-char-sequence"] = [
19231895
["r-char"],
@@ -1950,33 +1922,12 @@
19501922
]
19511923

19521924
gram_tree["user-defined-literal"] = [
1953-
["user-defined-integer-literal"],
1954-
["user-defined-floating-point-literal"],
1955-
["user-defined-string-literal"],
1956-
["user-defined-character-literal"],
1957-
]
1958-
1959-
gram_tree["user-defined-integer-literal"] = [
1960-
["decimal-literal", "ud-suffix"],
1961-
["octal-literal", "ud-suffix"],
1962-
["hexadecimal-literal", "ud-suffix"],
1963-
["binary-literal", "ud-suffix"],
1964-
]
1965-
1966-
gram_tree["user-defined-floating-point-literal"] = [
1967-
["fractional-constant", "exponent-part[opt]", "ud-suffix"],
1968-
["digit-sequence", "exponent-part", "ud-suffix"],
1969-
["hexadecimal-prefix", "hexadecimal-fractional-constant", "binary-exponent-part", "ud-suffix"],
1970-
["hexadecimal-prefix", "hexadecimal-digit-sequence", "binary-exponent-part", "ud-suffix"],
1925+
["`user-defined-integer-literal`"],
1926+
["`user-defined-floating-point-literal`"],
1927+
["`user-defined-string-literal`"],
1928+
["`user-defined-character-literal`"],
19711929
]
19721930

1973-
gram_tree["user-defined-string-literal"] = [
1974-
"string-literal", "ud-suffix",
1975-
]
1976-
1977-
gram_tree["user-defined-character-literal"] = [
1978-
"character-literal", "ud-suffix",
1979-
]
19801931

19811932
gram_tree["ud-suffix"] = [
19821933
"`identifier`",

‎gram/create_parse_functions.py‎

Lines changed: 27 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ def create_serial_parse_function(serial_idx, key, v, tag_name="TagName"):
4343

4444
content_init = ""
4545
content_type = f"using parse_func_type = SerialParseFunctions<"
46-
content_func_def = f"""SerialParseFunctions serial_funcs(
46+
content_func_def = f"""SerialParseFunctions serial_funcs("{name}_serial",
4747
ParseFunctionInputs(false ,calling_depth(), output.last_token_, output.cur_token_),"""
4848
for idx, s_ in enumerate(v):
4949
assert isinstance(s_, str)
@@ -87,7 +87,7 @@ def create_serial_parse_function(serial_idx, key, v, tag_name="TagName"):
8787
static_assert(base::kNumberOfElements == 1);
8888
serial_funcs.executed_mask( decltype(serial_funcs)::base::bitset_type(this->executed_mask_.value() ) );
8989
output = serial_funcs();
90-
this->executed_mask(base::bitset_type(serial_funcs.executed_mask().value()));
90+
this->executed_mask_.set(serial_funcs.executed_mask().value());
9191
return output;
9292
"""
9393
if not flg:
@@ -97,12 +97,12 @@ def create_serial_parse_function(serial_idx, key, v, tag_name="TagName"):
9797
return content_op, content_init, content_type + ">;", "", comments
9898

9999

100-
def create_serial_in_parallel_function(s_v, k, flg, tidx):
100+
def create_serial_in_parallel_function(s_v, k, flg, idx_str, tidx):
101101
name = camel_case(k)
102102
tag_name = f""" {name}::kTag """
103103
assert isinstance(s_v, list)
104104
contents_type = f"SerialParseFunctions<"
105-
contents = f"""SerialParseFunctions(ParseFunctionInputs(
105+
contents = f"""SerialParseFunctions("{name}_{idx_str}_serial_{tidx}", ParseFunctionInputs(
106106
false,calling_depth() + 1),"""
107107
for s_ in s_v:
108108
assert isinstance(s_, str)
@@ -136,30 +136,41 @@ def create_serial_in_parallel_function(s_v, k, flg, tidx):
136136
return contents, contents_type, flg
137137

138138

139-
def create_parallel_function_normal(v, k, idx, offset, out_name="output"):
139+
def create_parallel_function_normal(v, k, idx, offset, out_name="output", type_recursive=False):
140140
name = camel_case(k)
141141
tag_name = f""" {name}::kTag """
142142
contents_type = f"""ParallelParseFunctions<{len(v)}, """
143+
sub_str = f"""parallel_{idx}"""
144+
if type_recursive:
145+
sub_str = f"""recursive_{idx}"""
146+
contents_type = f"""RecursiveParseFunctions<"""
143147
content_func_def = f"""
144-
parallel_funcs_{idx}(ParseFunctionInputs(false,calling_depth(), output.last_token_,output.cur_token_),
148+
parallel_funcs_{idx}("{name}_parallel_{idx}",ParseFunctionInputs(false,calling_depth(), output.last_token_,output.cur_token_),
149+
"""
150+
if type_recursive:
151+
content_func_def = f"""
152+
recursive_funcs_{idx}("{name}_recursive", ParseFunctionInputs(false,calling_depth(), output.last_token_,output.cur_token_),
145153
"""
146154
flg = True
147155
for ii, s_v in enumerate(v):
148156
contents_, contents_type_, flg = create_serial_in_parallel_function(
149-
s_v, k, flg, ii)
157+
s_v, k, flg, sub_str, ii)
150158
if not flg:
151159
break
152160
content_func_def += contents_
153161
contents_type += contents_type_
154162
content_func_def = content_func_def[:-1] + ");"
155163
contents_type = contents_type[:-1] + "> "
164+
156165
content_op = f"""
157166
static_assert(base::kNumberOfElements >= {len(v)});
158167
using parallel_funcs_bitset_map = decltype(parallel_funcs_{idx})::base::bitset_type;
159-
parallel_funcs_{idx}.executed_mask( parallel_funcs_bitset_map(base::bitset_type::range<{offset}, {offset + len(v)}>(this->executed_mask_.value({offset})) ));
168+
parallel_funcs_{idx}.executed_mask( parallel_funcs_bitset_map(this->executed_mask_.value({offset})) );
160169
{out_name} = parallel_funcs_{idx}();
161-
this->executed_mask(base::bitset_type(parallel_funcs_{idx}.executed_mask().value() ,{offset}));
170+
this->executed_mask_.set(parallel_funcs_{idx}.executed_mask().value() ,{offset});
162171
"""
172+
if type_recursive:
173+
content_op = f"""{out_name} = recursive_funcs_{idx}();"""
163174
content_init = ""
164175
return content_op, content_init, contents_type, content_func_def, flg
165176

@@ -182,7 +193,8 @@ def create_parallel_function(serial_idx, key, v):
182193
if not flg_has_recursive:
183194
non_recursive_eles.append(s_v)
184195
else:
185-
recursive_eles.append(s_v)
196+
assert (len(s_v) > 1)
197+
recursive_eles.append(s_v[1:])
186198

187199
assert (len(non_recursive_eles) > 0)
188200

@@ -205,7 +217,7 @@ def create_parallel_function(serial_idx, key, v):
205217
content_non_op = f"unreachable({tag_name});return output;"
206218
return content_non_op, "", "", "", flg
207219
content_r_op, content_r_init, content_r_type, content_r_func_def, flg = create_parallel_function_normal(
208-
recursive_eles, k, 1, len(non_recursive_eles))
220+
recursive_eles, k, 1, len(non_recursive_eles), "output", True)
209221
if not flg:
210222
content_r_op = f"unreachable({tag_name});return output;"
211223
return content_r_op, "", "", "", flg
@@ -382,11 +394,11 @@ class NAME : public ParseFunction<N> { \\
382394
constexpr static ParseFunctionKind kind = ParseFunctionKind::TYPE; \\
383395
~NAME() = default; \\
384396
template <typename... Params> \\
385-
explicit NAME(bool opt, Params... params) : base(opt, params...) {} \\
397+
explicit NAME(bool opt, Params... params) : base(#NAME,opt, params...) {} \\
386398
template <typename... Params> \\
387-
explicit NAME(Params... params) : base(params...) {} \\
399+
explicit NAME(Params... params) : base(#NAME,params...) {} \\
388400
explicit NAME(const ParseFunctionInputs& param) \\
389-
: base(param) {} \\
401+
: base(#NAME,param) {} \\
390402
ParseFunctionOutputs operator()() override; \\
391403
};
392404
@@ -423,6 +435,7 @@ class NAME : public ParseFunction<N> { \\
423435
#include "diag.h"
424436
#include "parse_function/function.h"
425437
#include "parse_function/parallel_function_impl.h"
438+
#include "parse_function/recursive_function_impl.h"
426439
#include "parse_function/serial_function_impl.h"
427440
#include "token.h"
428441

‎include/basic/bitset.h‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,9 @@ class Bitset {
7171
size_t m = std::min(N, N1);
7272
if (start_idx > 0) {
7373
lps_assert(kTag, N1 < N);
74-
value_ = value_.to_ullong() || (other.to_ullong() << start_idx);
74+
for (int i = 0; i < N1; i++) {
75+
value_[start_idx + i] = other[i];
76+
}
7577
} else {
7678

7779
for (int i = 0; i < m; i++) {
@@ -108,7 +110,7 @@ class Bitset {
108110
return a;
109111
}
110112

111-
bool all() {
113+
[[nodiscard]] bool all() const {
112114
size_t cnt = 0;
113115
for (int i = start_idx_; i < start_idx_ + len_; i++) {
114116
if (value_[i]) {

‎include/diag/kinds_lex.def‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,3 +71,9 @@ DIAG(expected_r_paren_in_the_define_with_parameters, "expected `)`.", "",
7171
Level::kError)
7272
DIAG(expected_vertws_after_slash_in_preprocessing, "expected `\\n` after `\\`.",
7373
"", Level::kError)
74+
DIAG(expected_directive_command_after_hash,
75+
"expected directive command after `#`.", "", Level::kError)
76+
DIAG(expected_constant_expression_after_if_directive,
77+
"expected `constant-expression` after `#if`.", "", Level::kError)
78+
DIAG(expected_new_line_after_if_constant_expression_directive,
79+
"expected `\\n` after `#if constant-expression`.", "", Level::kError)

‎include/lex/base.h‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,10 +151,22 @@ class Base {
151151
return {c, sz};
152152
}
153153

154-
static inline void horzws_skipping(ptr_type& ptr) {
154+
static inline bool horzws_skipping(ptr_type& ptr) {
155155
if (basic::str::ascii::is::HorzWs(*ptr)) {
156156
ptr.horzws_skipping();
157157
}
158+
if (*ptr == '\\') {
159+
if (!basic::str::ascii::is::VertWs(*(ptr + 1))) {
160+
diag(ptr, ptr + 1,
161+
diag::DiagKind::expected_vertws_after_slash_in_preprocessing);
162+
throw basic::vfile::Eof();
163+
return false;
164+
}
165+
++ptr;
166+
++ptr;
167+
horzws_skipping(ptr);
168+
}
169+
return true;
158170
}
159171
static inline CharSize advance(ptr_type& ptr) {
160172
uint32_t sz = 0;

0 commit comments

Comments
 (0)