Replace list of tokens with appended tokens in AST construction

crates/radlr-ascript/build_db.rs

@@ -773,7 +773,7 @@ pub fn resolve_nonterm_types(db: &GrammarDatabase, adb: &mut AscriptDatabase) ->
             },
             None => AscriptType::Undefined,
           },
-          None => AscriptType::Aggregate(AscriptAggregateType::Vec { val_type: AscriptScalarType::Token }),
+          None => AscriptType::Scalar(AscriptScalarType::Token),
         }
       }
       _ => unreachable!(""),
@@ -891,9 +891,17 @@ fn resolve_nonterm_values(
               *ty = last.get_type().clone();
               *output_graph = Some(last);
             } else {
-              let last = GraphNode::Vec(GraphNodeVecInits(vec![last.clone()]), *expected_type);
-              *ty = last.get_type().clone();
-              *output_graph = Some(last);
+              match last.clone() {
+                GraphNode::TokSym(..) | GraphNode::Tok(..) | GraphNode::TokRule(..) => {
+                  *ty = last.get_type().clone();
+                  *output_graph = Some(last);
+                }
+                last => {
+                  let last = GraphNode::Vec(GraphNodeVecInits(vec![last.clone()]), *expected_type);
+                  *ty = last.get_type().clone();
+                  *output_graph = Some(last);
+                }
+              }
             }
           }
         };
@@ -910,7 +918,9 @@ fn resolve_nonterm_values(
           None => GraphNode::Undefined(AscriptType::Undefined),
         };
 
-        debug_assert!(matches!(first.get_type(), AscriptType::Aggregate(..)));
+        dbg!(&ty, &first, &last, item._debug_string_w_db_(db));
+
+        // debug_assert!(matches!(first.get_type(), AscriptType::Aggregate(..)));
         *ty = *expected_type;
 
         if ty.is_multi() {
@@ -1502,12 +1512,16 @@ fn create_graph_node<'a>(
     ASTNode::AST_TrimmedReference(val) => {
       let gn = create_graph_node(args.to_node(&val.reference), mut_args, nonterm)?;
 
-      Ok(GraphNode::Trim(
-        Rc::new(gn),
-        val.range.start_trim as isize,
-        val.range.end_trim as isize,
-        AscriptType::Scalar(AscriptScalarType::Token),
-      ))
+      if gn.get_type().is_unknown() {
+        Ok(gn)
+      } else {
+        Ok(GraphNode::Trim(
+          Rc::new(gn),
+          val.range.start_trim as isize,
+          val.range.end_trim as isize,
+          AscriptType::Scalar(AscriptScalarType::Token),
+        ))
+      }
     }
 
     node => todo!("handle graph resolve of node {node:#?}"),

crates/radlr-ascript/test.rs

@@ -322,6 +322,28 @@ fn test_temp() -> RadlrResult<()> {
   Ok(())
 }
 
+#[test]
+fn token_vector() -> RadlrResult<()> {
+  let source = r##"
+  IGNORE { c:sp }  
+
+  <> goal > tk:( "test" )+
+"##;
+
+  let db = RadlrGrammar::new().add_source_from_string(source, "", false)?.build_db("", Default::default())?;
+
+  let mut adb: AscriptDatabase = db.into();
+
+  dbg!(&adb);
+
+  let (_, strct) = adb.structs.pop_first().expect("should have that one struct here");
+  assert_eq!(adb.structs.len(), 1);
+
+  //assert_eq!(strct.properties.len(), 2);
+
+  Ok(())
+}
+
 #[test]
 fn builds_rum_lang() -> RadlrResult<()> {
   let source = r##"

crates/radlr-rust-runtime/parsers/ast.rs

@@ -3,7 +3,7 @@
 use crate::types::*;
 use std::fmt::Debug;
 
-pub trait Tk: Clone + Default + std::hash::Hash + Debug {
+pub trait Tk: Clone + Default + std::hash::Hash + Debug + std::ops::Add<Output = Self> {
   fn to_string(&self) -> String;
   fn trim(&self, start: usize, end: usize) -> Self;
   fn from_range(start: usize, end: usize, line_number: u32, line_offset: u32, id: u32, source: SharedSymbolBuffer) -> Self;
@@ -17,28 +17,6 @@ impl<Token: Tk, T: Default> Node<Token> for T {}
 
 pub type Reducer<Token, N> = fn(*mut [N], &[Token], Token) -> N;
 
-impl Tk for String {
-  fn from_range(start: usize, end: usize, _line_number: u32, _line_offset: u32, _id: u32, source: SharedSymbolBuffer) -> Self {
-    unsafe { Self::from_utf8_unchecked((&source[start..end]).to_vec()) }
-  }
-
-  fn from_slice(slice: &[Self]) -> Self {
-    slice.join("")
-  }
-
-  fn len(&self) -> usize {
-    String::len(&self)
-  }
-
-  fn to_string(&self) -> String {
-    self.clone()
-  }
-
-  fn trim(&self, _start: usize, _end: usize) -> Self {
-    self[_start.min(self.len())..self.len() - _end.min(self.len())].to_string()
-  }
-}
-
 impl Tk for Token {
   fn from_range(start: usize, end: usize, line_number: u32, line_offset: u32, _id: u32, source: SharedSymbolBuffer) -> Self {
     Token {

crates/radlr-rust-runtime/types/token.rs

@@ -104,6 +104,14 @@ impl Add for &Token {
   }
 }
 
+impl Add for Token {
+  type Output = Token;
+
+  fn add(self, rhs: Self) -> Self::Output {
+    Token { inner: self.inner + rhs.inner, input: self.input.clone() }
+  }
+}
+
 impl Add<Token> for u32 {
   type Output = Token;