Parser: Remove heavy elements from ParseFunctionOutputs.

Author: TuringKi

include/parse_function/parallel_function_impl.h

@@ -59,7 +59,7 @@ ParallelParseFunctions<NumElements, ParseFuncs...>::operator()() {
                   auto local_output = func();
                 }
                 if (local_output.work_) {  // only allow one of them work
-                  if (local_output.len() > max_len_match_output.len()) {
+                  if (local_output.len_ > max_len_match_output.len_) {
                     max_len_match_output = std::move(local_output);
                     max_len_match_idx = running_sub_func_idx;
                     if (!this->valid(
@@ -80,7 +80,6 @@ ParallelParseFunctions<NumElements, ParseFuncs...>::operator()() {
       parse_functions_);
   if (max_len_match_output.work_) {
     this->executed_mask_.set(max_len_match_idx);
-    output.node_ = std::move(max_len_match_output.node_);
     diag::infos() << basic::str::from(
         std::string(this->calling_depth(), '>'), " ", this->kName_,
         basic::tui::color::Shell::colorize(

include/parse_function/recursive_function_impl.h

@@ -113,7 +113,7 @@ void RecursiveParseFunctions<ParseFuncs...>::execute(
                   auto local_output = func();
                 }
                 if (local_output.work_) {  // only allow one of them work
-                  if (local_output.len() > max_len_match_output.len()) {
+                  if (local_output.len_ > max_len_match_output.len_) {
                     max_len_match_output = std::move(local_output);
                     max_len_match_idx = running_sub_func_idx;
                     if (!this->valid(
@@ -133,7 +133,6 @@ void RecursiveParseFunctions<ParseFuncs...>::execute(
       },
       parse_functions_);
   if (max_len_match_output.work_) {
-    output.node_ = std::move(max_len_match_output.node_);
     diag::infos() << basic::str::from(
         std::string(this->calling_depth(), '>'), " ", this->kName_,
         basic::tui::color::Shell::colorize(

include/parse_function/serial_function_impl.h

@@ -72,43 +72,36 @@ ParseFunctionOutputs SerialParseFunctions<ParseFuncs...>::operator()() {
                 if (running_sub_func_idx++ < running_sub_func_stack_top_idx) {
                   return;
                 }
-                auto make_try_again =
-                    [&](basic::Vector<4, diag::DiagInputs>&& diag_inputs) {
-                      if ((running_sub_func_idx - 1) == 0) {
-                        flg_continue = false;
-                        return;
-                      }
-                      flg_not_continue = true;
-                      running_sub_func_stack_top_idx =
-                          (running_sub_func_idx - 1) - 1;
-                      func.reset();
-                      auto tmp_output(path_stack.top());
-                      tmp_output.work_ = false;
-                      lps_assert("make_try_again", path_stack.size() > 0);
-
-                      path_stack.pop();
-
-                      path_stack.top().concat(std::move(tmp_output), false);
-
-                      if (!diag_inputs.empty()) {
-                        decltype(tmp_output) tmp_output0;
-                        tmp_output0.diag_inputs_ = std::move(diag_inputs);
-                        path_stack.top().concat(std::move(tmp_output0), false);
-                      }
-                      return;
-                    };
+                auto make_try_again = [&]() {
+                  if ((running_sub_func_idx - 1) == 0) {
+                    flg_continue = false;
+                    return;
+                  }
+                  flg_not_continue = true;
+                  running_sub_func_stack_top_idx =
+                      (running_sub_func_idx - 1) - 1;
+                  func.reset();
+                  auto tmp_output(path_stack.top());
+                  tmp_output.work_ = false;
+                  lps_assert("make_try_again", path_stack.size() > 0);
+
+                  path_stack.pop();
+
+                  path_stack.top().concat(std::move(tmp_output), false);
+                  return;
+                };
                 if (flg_continue) {
                   func.last_token(path_stack.top().last_token_);
                   func.cur_token(path_stack.top().cur_token_);
 
                   if (!func.ok_to_try() || (!func.valid() && !func.opt())) {
-                    make_try_again(basic::Vector<4, diag::DiagInputs>());
+                    make_try_again();
                     return;
                   }
 
                   auto local_output = func();
                   if (!local_output.work_ && !func.opt()) {
-                    make_try_again(std::move(local_output.diag_inputs_));
+                    make_try_again();
                     return;
                   }
 
@@ -129,7 +122,6 @@ ParseFunctionOutputs SerialParseFunctions<ParseFuncs...>::operator()() {
   }
   lps_assert("serial_function_impl", path_stack.size() > 0);
   if (path_stack.top().work_) {
-    path_stack.top().node_ = std::move(node);
     diag::infos() << basic::str::from(
         std::string(this->calling_depth(), '>'), " ", this->kName_,
         basic::tui::color::Shell::colorize(basic::str::from(" ok.\n"),

include/parser.h

@@ -44,9 +44,15 @@ class Context {
   using executed_func_type = std::function<void(Context*)>;
   void with(const executed_func_type& func) { func(this); }
   token::TokenLists& token_lists() { return token_lists_; }
+  const token::Token& start_token() const { return start_token_; }
+  void start_token(const token::Token& token) { start_token_ = token; }
+  size_t len_from_start(const token::Token& cur_token) {
+    return token_lists_.len(start_token_, cur_token);
+  }
 
  private:
   token::TokenLists token_lists_;
+  token::Token start_token_;
 };
 
 class ContextTrait {
@@ -115,26 +121,19 @@ class Tree {
 };
 
 struct ParseFunctionOutputs {
-  using diag_input_type = basic::Vector<4, diag::DiagInputs>;
-  using token_list_infos_type =
-      basic::Vector<16, const token::TokenLists::ele_type*>;
   explicit ParseFunctionOutputs() = default;
 
-#define SET_MOVE(A, B)                             \
-  (A)->work_ = (B).work_;                          \
-  (A)->last_token_ = (B).last_token_;              \
-  (A)->cur_token_ = (B).cur_token_;                \
-  (A)->diag_inputs_ = std::move((B).diag_inputs_); \
-  (A)->node_ = std::move((B).node_);               \
-  (A)->token_list_infos_ = std::move((B).token_list_infos_);
-
-#define SET(A, B)                       \
-  (A)->work_ = (B).work_;               \
-  (A)->last_token_ = (B).last_token_;   \
-  (A)->cur_token_ = (B).cur_token_;     \
-  (A)->diag_inputs_ = (B).diag_inputs_; \
-  (A)->node_ = (B).node_;               \
-  (A)->token_list_infos_ = (B).token_list_infos_;
+#define SET_MOVE(A, B)                \
+  (A)->work_ = (B).work_;             \
+  (A)->last_token_ = (B).last_token_; \
+  (A)->cur_token_ = (B).cur_token_;   \
+  (A)->len_ = (B).len_;
+
+#define SET(A, B)                     \
+  (A)->work_ = (B).work_;             \
+  (A)->last_token_ = (B).last_token_; \
+  (A)->cur_token_ = (B).cur_token_;   \
+  (A)->len_ = (B).len_;
 
   ParseFunctionOutputs(const ParseFunctionOutputs& other) {
     SET(this, other);
@@ -157,38 +156,22 @@ struct ParseFunctionOutputs {
 #undef SET_MOVE
 #undef SET
 
-  [[nodiscard]] size_t len() const {
-    return token_list_infos_.size();
-  }
-
-  void diag_record(diag::DiagInputs&& a) {
-    diag_inputs_.append(std::move(a));
-  }
-
   void concat(ParseFunctionOutputs&& other, bool opt = true) {
     if (!opt) {
-      for (auto& a : other.diag_inputs_) {
-        typename decltype(diag_inputs_)::ele_type b(std::move(a));
-        this->diag_inputs_.append(std::move(b));
-      }
       this->work_ = other.work_;
     }
     if (other.work_) {
       this->last_token_ = other.last_token_;
       this->cur_token_ = other.cur_token_;
       this->work_ = other.work_;
-      for (auto& a : other.token_list_infos_) {
-        this->token_list_infos_.append(a);
-      }
+      len_ += other.len_;
     }
   }
 
   bool work_{false};
   token::Token last_token_;
   token::Token cur_token_;
-  diag_input_type diag_inputs_;
-  token_list_infos_type token_list_infos_;
-  Tree::Node node_;
+  size_t len_{0};
 };
 
 struct ParseFunctionInputs {
@@ -361,8 +344,7 @@ class ParseFunction : public ContextTrait {
 
       if (output.cur_token_.kind() != token_kind) {
         if (!func->opt()) {
-          output.diag_record(
-              diag::record(output.cur_token_, diag_kind, {output.last_token_}));
+          //todo(@mxlol233): add diag.
         }
         output.work_ = false;
         return output;
@@ -372,10 +354,7 @@ class ParseFunction : public ContextTrait {
         output.last_token_ = output.cur_token_;
         output.work_ = true;
         output.cur_token_ = next_tok;
-        if (next_tok.kind() != token::details::TokenKind::unknown) {
-          output.token_list_infos_.append(
-              &(func->context()->token_lists().at(next_tok)));
-        }
+        ++output.len_;
       }
 
       return output;

include/src.h

@@ -233,7 +233,6 @@ class TokenLists {
 
   const ele_type& last(const Token& tok) const {
     lps_assert(kTag, has(tok));
-    auto info = Info::create(tok);
     const auto* ptr = at(tok).last();
     if (!ptr) {
       static const ele_type kEmptyTok;
@@ -244,7 +243,6 @@ class TokenLists {
 
   const ele_type& next(const Token& tok) const {
     lps_assert(kTag, has(tok));
-    auto info = Info::create(tok);
     const auto* ptr = at(tok).next();
     if (!ptr) {
       static const ele_type kEmptyTok;
@@ -258,6 +256,20 @@ class TokenLists {
     return has(info);
   }
 
+  size_t len(const Token& start, const Token& end) const {
+    lps_assert(kTag, has(start) && has(end));
+    const Token* t = &start;
+    int l = 0;
+
+    while (t != nullptr && *t != end) {
+      auto info = Info::create(*t);
+      t = at(info).next();
+      ++l;
+    }
+    lps_assert(kTag, t != nullptr);
+    return l;
+  }
+
   void append(const Token& tok, Info last_tok_info = {0, 0}) {
     auto info = Info::create(tok);
     if (lists_.contains(tok.file_id())) {

include/token.h

@@ -241,6 +241,15 @@ struct Token {
   } next_visitor_{0, 0};
 };
 
+inline bool operator==(const Token& a, const Token& b) {
+  return (a.file_id() == b.file_id()) && (a.offset() == b.offset()) &&
+         (a.kind() == b.kind());
+}
+
+inline bool operator!=(const Token& a, const Token& b) {
+  return !(a == b);
+}
+
 inline std::ostream& operator<<(std::ostream& s, const Token& tok) {
 
   s << "kind:" << tok.kind();

src/constant_expression.cc

@@ -41,11 +41,12 @@ bool Preprocessing::lex_conditional_expression(
       lexer.lex(next_tok);
       if (next_tok.kind() != token::details::TokenKind::unknown) {
         context->token_lists().append(next_tok);
+        context->start_token(next_tok);
+        params.cur_token_ = next_tok;
+        parser::details::ConditionalExpression func(context, params);
+        auto output = func();
+        int dummy = -1;
       }
-      params.cur_token_ = next_tok;
-      parser::details::ConditionalExpression func(context, params);
-      auto output = func();
-      int dummy = -1;
     });
   }
 

src/parser.cc

@@ -44,18 +44,10 @@ void Parser::parse(uint32_t file_id) {
       params.cur_token_ = next_tok;
       if (next_tok.kind() != token::details::TokenKind::unknown) {
         context->token_lists().append(next_tok);
-      }
-      details::TranslationUnit func(context, params);
-      auto output = func();
-      for (const auto& a : output.diag_inputs_) {
-        diag::doing(a.main_token_, a.kind_, a.context_tokens_);
-      }
-      if (output.work_) {
-        for (const auto& node : output.node_.sub_nodes_) {
-          if (node->kind_ != details::ParseFunctionKind::kUnknown) {
-            // todo(@mxlol233): handle the recursive tree nodes...
-          }
-        }
+        context->start_token(next_tok);
+        details::TranslationUnit func(context, params);
+        auto output = func();
+        if (output.work_) {}
       }
     });
   }