Parser: Make constant_expression works.

Author: TuringKi

gram/config.py

@@ -185,6 +185,19 @@
     "`false`":"kw_false",
     "`true`":"kw_true",
     "`nullptr`":"kw_nullptr",
+    "`binary-literal`":"binary_literal",
+    "`floating-point-literal`":"floating_point_literal",
+    "`character-literal`":"char_literal",
+    "`string-literal`":"string_literal",
+    "`integer-literal`":"integer_literal",
+    "`decimal-literal`":"decimal_literal",
+    "`octal-literal`":"octal_literal",
+    "`hexadecimal-literal`":"hexadecimal_literal",
+    "`raw-string`":"raw_string",
+    "`user-defined-character-literal`":"user_defined_char_literal",
+    "`user-defined-string-literal`":"user_defined_string_literal",
+    "`user-defined-floating-point-literal`":"user_defined_floating_point_literal",
+    "`user-defined-integer-literal`":"user_defined_integer_literal",
 }
 
 gram_tree = {}
@@ -654,7 +667,7 @@
 
 gram_tree["static_assert-declaration"] = [
     ["`static_assert`", "`(`", "constant-expression", "`)`", "`;`"],
-    ["`static_assert`", "`(`", "constant-expression", "`,`", "string-literal", "`)`", "`;`"]
+    ["`static_assert`", "`(`", "constant-expression", "`,`", "`string-literal`", "`)`", "`;`"]
 ]
 
 gram_tree["empty-declaration"] = [
@@ -1056,16 +1069,16 @@
 ]
 
 gram_tree["asm-declaration"] = [
-    "attribute-specifier-seq[opt]", "`asm`", "`(`", "string-literal", "`)`", "`;`",
+    "attribute-specifier-seq[opt]", "`asm`", "`(`", "`string-literal`", "`)`", "`;`",
 ]
 
 gram_tree["linkage-specification"] = [
-    ["`extern`", "string-literal", "`{`", "declaration-seq[opt]", "`}`"],
-    ["`extern`", "string-literal", "declaration"],
+    ["`extern`", "`string-literal`", "`{`", "declaration-seq[opt]", "`}`"],
+    ["`extern`", "`string-literal`", "declaration"],
 ]
 
 gram_tree["attribute-specifier-seq"] = [
-    "attribute-specifier", "attribute-specifier-seq[opt]", 
+    "attribute-specifier", "attribute-specifier-seq[opt]",
 ]
 
 gram_tree["attribute-specifier"] = [
@@ -1310,8 +1323,8 @@
 ]
 
 gram_tree["literal-operator-id"] = [
-    ["`operator`", "string-literal", "`identifier`"],
-    ["`operator`", "user-defined-string-literal"],
+    ["`operator`", "`string-literal`", "`identifier`"],
+    ["`operator`", "`user-defined-string-literal`"],
 ]
 
 gram_tree["template-declaration"] = [
@@ -1562,7 +1575,7 @@
 ]
 
 gram_tree["header-name-tokens"] = [
-    ["string-literal"],
+    ["`string-literal`"],
     ["`<`", "h-pp-tokens", "`>`"],
 ]
 
@@ -1605,10 +1618,10 @@
     ["`export`"],
     ["`identifier`"],
     ["pp-number"],
-    ["character-literal"],
-    ["user-defined-character-literal"],
-    ["string-literal"],
-    ["user-defined-string-literal"],
+    ["`character-literal`"],
+    ["`user-defined-character-literal`"],
+    ["`string-literal`"],
+    ["`user-defined-string-literal`"],
     ["preprocessing-op-or-punc"],
     # each non-whitespace character that cannot be one of the above
 ]
@@ -1698,43 +1711,19 @@
 ]
 
 gram_tree["literal"] = [
-    ["integer-literal"],
-    ["character-literal"],
-    ["floating-point-literal"],
-    ["string-literal"],
+    ["`integer-literal`"],
+    ["`binary-literal`"],
+    ["`octal-literal`"],
+    ["`decimal-literal`"],
+    ["`hexadecimal-literal`"],
+    ["`character-literal`"],
+    ["`floating-point-literal`"],
+    ["`string-literal`"],
     ["boolean-literal"],
     ["pointer-literal"],
     ["user-defined-literal"],
 ]
 
-gram_tree["integer-literal"] = [
-    ["binary-literal", "integer-suffix[opt]"],
-    ["octal-literal", "integer-suffix[opt]"],
-    ["decimal-literal", "integer-suffix[opt]"],
-    ["hexadecimal-literal", "integer-suffix[opt]"],
-]
-
-gram_tree["binary-literal"] = [
-    ["`0b`", "binary-digit"],
-    ["`0B`", "`binary-digit`"],
-    ["binary-literal", "`'`"],
-    ["binary-literal", "`'`[opt]", "binary-digit"],
-]
-
-gram_tree["octal-literal"] = [
-    ["`0`"],
-    ["octal-literal", "`'`[opt]", "octal-digit"],
-]
-
-gram_tree["decimal-literal"] = [
-    ["nonzero-digit"],
-    ["decimal-literal", "`'`[opt]", "digit"],
-]
-
-gram_tree["hexadecimal-literal"] = [
-    "hexadecimal-prefix", "hexadecimal-digit-sequence",
-]
-
 gram_tree["binary-digit"] = [
     ["`0`"],
     ["`1`"],
@@ -1810,10 +1799,6 @@
     ["`ll`"], ["`LL`"],
 ]
 
-gram_tree["character-literal"] = [
-    "encoding-prefix[opt]", "`'`","c-char-sequence", "`'`",
-]
-
 gram_tree["encoding-prefix"] = [
     ["`u8`"],["`u`"], ["`U`"], ["`L`"],
 ]
@@ -1851,11 +1836,6 @@
     ["hexadecimal-escape-sequence", "hexadecimal-digit"],
 ]
 
-gram_tree["floating-point-literal"] = [
-    ["decimal-floating-point-literal"],
-    ["hexadecimal-floating-point-literal"],
-]
-
 gram_tree["decimal-floating-point-literal"] = [
     ["fractional-constant", "exponent-part[opt]", "floating-point-suffix[opt]"],
     ["digit-sequence", "exponent-part[opt]", "floating-point-suffix[opt]"],
@@ -1899,11 +1879,6 @@
     ["`f`"], ["`l`"], ["`F`"], ["`L`"],
 ]
 
-gram_tree["string-literal"] = [
-    ["encoding-prefix[opt]", '`"`', "s-char-sequence[opt]", '`"`'],
-    ["encoding-prefix[opt]", '`R`', "raw-string"],
-]
-
 gram_tree["s-char-sequence"] = [
     ["s-char"],
     ["s-char-sequence", "s-char"],
@@ -1915,9 +1890,6 @@
     ["universal-character-name"],
 ]
 
-gram_tree["raw-string"] = [
-    '`"`', "d-char-sequence[opt]", "`(`", "r-char-sequence[opt]"  ,"`)`", "d-char-sequence[opt]", '`"`',
-]
 
 gram_tree["r-char-sequence"] = [
     ["r-char"],
@@ -1950,33 +1922,12 @@
 ]
 
 gram_tree["user-defined-literal"] = [
-    ["user-defined-integer-literal"],
-    ["user-defined-floating-point-literal"],
-    ["user-defined-string-literal"],
-    ["user-defined-character-literal"],
-]
-
-gram_tree["user-defined-integer-literal"] = [
-    ["decimal-literal", "ud-suffix"],
-    ["octal-literal", "ud-suffix"],
-    ["hexadecimal-literal", "ud-suffix"],
-    ["binary-literal", "ud-suffix"],
-]
-
-gram_tree["user-defined-floating-point-literal"] = [
-    ["fractional-constant", "exponent-part[opt]", "ud-suffix"],
-    ["digit-sequence", "exponent-part", "ud-suffix"],
-    ["hexadecimal-prefix", "hexadecimal-fractional-constant", "binary-exponent-part", "ud-suffix"],
-    ["hexadecimal-prefix", "hexadecimal-digit-sequence", "binary-exponent-part", "ud-suffix"],
+    ["`user-defined-integer-literal`"],
+    ["`user-defined-floating-point-literal`"],
+    ["`user-defined-string-literal`"],
+    ["`user-defined-character-literal`"],
 ]
 
-gram_tree["user-defined-string-literal"] = [
-    "string-literal", "ud-suffix",
-]
-
-gram_tree["user-defined-character-literal"] = [
-    "character-literal", "ud-suffix",
-]
 
 gram_tree["ud-suffix"] = [
     "`identifier`",

gram/create_parse_functions.py

@@ -43,7 +43,7 @@ def create_serial_parse_function(serial_idx, key, v, tag_name="TagName"):
 
     content_init = ""
     content_type = f"using parse_func_type = SerialParseFunctions<"
-    content_func_def = f"""SerialParseFunctions serial_funcs(
+    content_func_def = f"""SerialParseFunctions serial_funcs("{name}_serial",
         ParseFunctionInputs(false ,calling_depth(), output.last_token_, output.cur_token_),"""
     for idx, s_ in enumerate(v):
         assert isinstance(s_, str)
@@ -87,7 +87,7 @@ def create_serial_parse_function(serial_idx, key, v, tag_name="TagName"):
         static_assert(base::kNumberOfElements == 1);
      serial_funcs.executed_mask( decltype(serial_funcs)::base::bitset_type(this->executed_mask_.value() ) );
     output = serial_funcs();
-    this->executed_mask(base::bitset_type(serial_funcs.executed_mask().value()));
+    this->executed_mask_.set(serial_funcs.executed_mask().value());
     return output;
     """
     if not flg:
@@ -97,12 +97,12 @@ def create_serial_parse_function(serial_idx, key, v, tag_name="TagName"):
     return content_op, content_init, content_type + ">;", "", comments
 
 
-def create_serial_in_parallel_function(s_v, k, flg, tidx):
+def create_serial_in_parallel_function(s_v, k, flg, idx_str, tidx):
     name = camel_case(k)
     tag_name = f""" {name}::kTag """
     assert isinstance(s_v, list)
     contents_type = f"SerialParseFunctions<"
-    contents = f"""SerialParseFunctions(ParseFunctionInputs(
+    contents = f"""SerialParseFunctions("{name}_{idx_str}_serial_{tidx}", ParseFunctionInputs(
         false,calling_depth() + 1),"""
     for s_ in s_v:
         assert isinstance(s_, str)
@@ -136,30 +136,41 @@ def create_serial_in_parallel_function(s_v, k, flg, tidx):
     return contents, contents_type, flg
 
 
-def create_parallel_function_normal(v, k, idx, offset, out_name="output"):
+def create_parallel_function_normal(v, k, idx, offset, out_name="output", type_recursive=False):
     name = camel_case(k)
     tag_name = f""" {name}::kTag """
     contents_type = f"""ParallelParseFunctions<{len(v)}, """
+    sub_str = f"""parallel_{idx}"""
+    if type_recursive:
+        sub_str = f"""recursive_{idx}"""
+        contents_type = f"""RecursiveParseFunctions<"""
     content_func_def = f"""
-     parallel_funcs_{idx}(ParseFunctionInputs(false,calling_depth(), output.last_token_,output.cur_token_),
+     parallel_funcs_{idx}("{name}_parallel_{idx}",ParseFunctionInputs(false,calling_depth(), output.last_token_,output.cur_token_),
+     """
+    if type_recursive:
+        content_func_def = f"""
+     recursive_funcs_{idx}("{name}_recursive", ParseFunctionInputs(false,calling_depth(), output.last_token_,output.cur_token_),
      """
     flg = True
     for ii, s_v in enumerate(v):
         contents_, contents_type_, flg = create_serial_in_parallel_function(
-            s_v, k, flg, ii)
+            s_v, k, flg, sub_str, ii)
         if not flg:
             break
         content_func_def += contents_
         contents_type += contents_type_
     content_func_def = content_func_def[:-1] + ");"
     contents_type = contents_type[:-1] + "> "
+
     content_op = f"""
     static_assert(base::kNumberOfElements >= {len(v)});
     using parallel_funcs_bitset_map = decltype(parallel_funcs_{idx})::base::bitset_type;
-     parallel_funcs_{idx}.executed_mask( parallel_funcs_bitset_map(base::bitset_type::range<{offset}, {offset + len(v)}>(this->executed_mask_.value({offset})) ));
+     parallel_funcs_{idx}.executed_mask( parallel_funcs_bitset_map(this->executed_mask_.value({offset})) );
     {out_name} = parallel_funcs_{idx}();
-    this->executed_mask(base::bitset_type(parallel_funcs_{idx}.executed_mask().value() ,{offset}));
+    this->executed_mask_.set(parallel_funcs_{idx}.executed_mask().value() ,{offset});
     """
+    if type_recursive:
+        content_op = f"""{out_name} = recursive_funcs_{idx}();"""
     content_init = ""
     return content_op, content_init, contents_type,  content_func_def, flg
 
@@ -182,7 +193,8 @@ def create_parallel_function(serial_idx, key, v):
         if not flg_has_recursive:
             non_recursive_eles.append(s_v)
         else:
-            recursive_eles.append(s_v)
+            assert (len(s_v) > 1)
+            recursive_eles.append(s_v[1:])
 
     assert (len(non_recursive_eles) > 0)
 
@@ -205,7 +217,7 @@ def create_parallel_function(serial_idx, key, v):
             content_non_op = f"unreachable({tag_name});return output;"
             return content_non_op, "", "", "", flg
         content_r_op, content_r_init, content_r_type, content_r_func_def, flg = create_parallel_function_normal(
-            recursive_eles, k, 1, len(non_recursive_eles))
+            recursive_eles, k, 1, len(non_recursive_eles), "output", True)
         if not flg:
             content_r_op = f"unreachable({tag_name});return output;"
             return content_r_op, "", "", "", flg
@@ -382,11 +394,11 @@ class NAME : public ParseFunction<N> {                \\
     constexpr static ParseFunctionKind kind = ParseFunctionKind::TYPE;                     \\
     ~NAME() = default;                                                  \\
     template <typename... Params>                                       \\
-    explicit NAME(bool opt, Params... params) : base(opt, params...) {} \\
+    explicit NAME(bool opt, Params... params) : base(#NAME,opt, params...) {} \\
     template <typename... Params>                                       \\
-    explicit NAME(Params... params) : base(params...) {}                \\
+    explicit NAME(Params... params) : base(#NAME,params...) {}                \\
     explicit NAME(const ParseFunctionInputs& param)         \\
-        : base(param) {}                                               \\
+        : base(#NAME,param) {}                                               \\
     ParseFunctionOutputs operator()() override;             \\
   };
 
@@ -423,6 +435,7 @@ class NAME : public ParseFunction<N> {                \\
 #include "diag.h"
 #include "parse_function/function.h"
 #include "parse_function/parallel_function_impl.h"
+#include "parse_function/recursive_function_impl.h"
 #include "parse_function/serial_function_impl.h"
 #include "token.h"
 

include/basic/bitset.h

@@ -71,7 +71,9 @@ class Bitset {
     size_t m = std::min(N, N1);
     if (start_idx > 0) {
       lps_assert(kTag, N1 < N);
-      value_ = value_.to_ullong() || (other.to_ullong() << start_idx);
+      for (int i = 0; i < N1; i++) {
+        value_[start_idx + i] = other[i];
+      }
     } else {
 
       for (int i = 0; i < m; i++) {
@@ -108,7 +110,7 @@ class Bitset {
     return a;
   }
 
-  bool all() {
+  [[nodiscard]] bool all() const {
     size_t cnt = 0;
     for (int i = start_idx_; i < start_idx_ + len_; i++) {
       if (value_[i]) {

include/diag/kinds_lex.def

@@ -71,3 +71,9 @@ DIAG(expected_r_paren_in_the_define_with_parameters, "expected `)`.", "",
      Level::kError)
 DIAG(expected_vertws_after_slash_in_preprocessing, "expected `\\n` after `\\`.",
      "", Level::kError)
+DIAG(expected_directive_command_after_hash,
+     "expected directive command after `#`.", "", Level::kError)
+DIAG(expected_constant_expression_after_if_directive,
+     "expected `constant-expression` after `#if`.", "", Level::kError)
+DIAG(expected_new_line_after_if_constant_expression_directive,
+     "expected `\\n` after `#if constant-expression`.", "", Level::kError)

include/lex/base.h

@@ -151,10 +151,22 @@ class Base {
     return {c, sz};
   }
 
-  static inline void horzws_skipping(ptr_type& ptr) {
+  static inline bool horzws_skipping(ptr_type& ptr) {
     if (basic::str::ascii::is::HorzWs(*ptr)) {
       ptr.horzws_skipping();
     }
+    if (*ptr == '\\') {
+      if (!basic::str::ascii::is::VertWs(*(ptr + 1))) {
+        diag(ptr, ptr + 1,
+             diag::DiagKind::expected_vertws_after_slash_in_preprocessing);
+        throw basic::vfile::Eof();
+        return false;
+      }
+      ++ptr;
+      ++ptr;
+      horzws_skipping(ptr);
+    }
+    return true;
   }
   static inline CharSize advance(ptr_type& ptr) {
     uint32_t sz = 0;