Initial attempt to write a zero-copy parser

Add a simple binary for testing

Support menubar.ftl except of expressions

Support multiline patterns

Add EOF tests support

Support basic JSON fixtures

Support first set of comparison tests

Support most of the comments

Use serializer instead of deserializer

Uncomment some fixture tests with slight modifications

Support Junk

Use ps.take_if in more places

Support all reference fixtures

Fix benches

Support TermReference

Support variants and select expression

Update to rust 2018 preview 2

Use 2018 use extern macros

Clean up whitespace collecting

Remove try

Update bench to Rust 2018

Support parsing of 0.7 reference tests

Support CallExpressions

Author: Zibi Braniecki

fluent-syntax/Cargo.toml

@@ -4,6 +4,7 @@ description = """
 Parser/Serializer tools for Fluent Syntax. 
 """
 version = "0.1.1"
+edition = "2018"
 authors = [
     "Zibi Braniecki <[email protected]>",
     "Staś Małolepszy <[email protected]>"
@@ -16,7 +17,11 @@ keywords = ["localization", "l10n", "i18n", "intl", "internationalization"]
 categories = ["localization", "internationalization"]
 
 [dependencies]
+clap = "2.32"
 annotate-snippets = {version = "0.1", features = ["color"]}
 
 [dev-dependencies]
+serde = "1.0"
+serde_derive = "1.0"
+serde_json = "1.0"
 glob = "0.2"

fluent-syntax/benches/lib.rs

@@ -3,16 +3,16 @@
 extern crate fluent_syntax;
 extern crate test;
 
+use self::test::Bencher;
 use fluent_syntax::parser::parse;
 use std::fs::File;
 use std::io;
 use std::io::Read;
-use test::Bencher;
 
 fn read_file(path: &str) -> Result<String, io::Error> {
-    let mut f = try!(File::open(path));
+    let mut f = File::open(path)?;
     let mut s = String::new();
-    try!(f.read_to_string(&mut s));
+    f.read_to_string(&mut s)?;
     Ok(s)
 }
 

fluent-syntax/src/ast.rs

@@ -1,129 +1,136 @@
 #[derive(Debug, PartialEq)]
-pub struct Resource {
-    pub body: Vec<Entry>,
+pub struct Resource<'ast> {
+    pub body: Vec<ResourceEntry<'ast>>,
 }
 
 #[derive(Debug, PartialEq)]
-pub enum Entry {
-    Message(Message),
-    Term(Term),
-    Comment(Comment),
-    Junk { content: String },
+pub enum ResourceEntry<'ast> {
+    Entry(Entry<'ast>),
+    Junk(&'ast str),
 }
 
 #[derive(Debug, PartialEq)]
-pub struct Message {
-    pub id: Identifier,
-    pub value: Option<Pattern>,
-    pub attributes: Option<Vec<Attribute>>,
-    pub comment: Option<Comment>,
+pub enum Entry<'ast> {
+    Message(Message<'ast>),
+    Term(Term<'ast>),
+    Comment(Comment<'ast>),
 }
 
 #[derive(Debug, PartialEq)]
-pub struct Term {
-    pub id: Identifier,
-    pub value: Pattern,
-    pub attributes: Option<Vec<Attribute>>,
-    pub comment: Option<Comment>,
+pub struct Message<'ast> {
+    pub id: Identifier<'ast>,
+    pub value: Option<Pattern<'ast>>,
+    pub attributes: Vec<Attribute<'ast>>,
+    pub comment: Option<Comment<'ast>>,
 }
 
 #[derive(Debug, PartialEq)]
-pub struct Pattern {
-    pub elements: Vec<PatternElement>,
+pub struct Term<'ast> {
+    pub id: Identifier<'ast>,
+    pub value: Value<'ast>,
+    pub attributes: Vec<Attribute<'ast>>,
+    pub comment: Option<Comment<'ast>>,
 }
 
 #[derive(Debug, PartialEq)]
-pub enum PatternElement {
-    TextElement(String),
-    Placeable(Expression),
+pub enum Value<'ast> {
+    Pattern(Pattern<'ast>),
+    VariantList { variants: Vec<Variant<'ast>> },
 }
 
 #[derive(Debug, PartialEq)]
-pub enum Expression {
-    StringExpression {
-        value: String,
-    },
-    NumberExpression {
-        value: Number,
-    },
-    MessageReference {
-        id: Identifier,
-    },
-    ExternalArgument {
-        id: Identifier,
-    },
-    SelectExpression {
-        expression: Option<Box<Expression>>,
-        variants: Vec<Variant>,
-    },
-    AttributeExpression {
-        id: Identifier,
-        name: Identifier,
-    },
-    VariantExpression {
-        id: Identifier,
-        key: VarKey,
-    },
-    CallExpression {
-        callee: Function,
-        args: Vec<Argument>,
-    },
+pub struct Pattern<'ast> {
+    pub elements: Vec<PatternElement<'ast>>,
 }
 
 #[derive(Debug, PartialEq)]
-pub struct Attribute {
-    pub id: Identifier,
-    pub value: Pattern,
+pub enum PatternElement<'ast> {
+    TextElement(&'ast str),
+    Placeable(Expression<'ast>),
 }
 
 #[derive(Debug, PartialEq)]
-pub struct Variant {
-    pub key: VarKey,
-    pub value: Pattern,
-    pub default: bool,
+pub struct Attribute<'ast> {
+    pub id: Identifier<'ast>,
+    pub value: Pattern<'ast>,
 }
 
 #[derive(Debug, PartialEq)]
-pub enum VarKey {
-    VariantName(VariantName),
-    Number(Number),
+pub struct Identifier<'ast> {
+    pub name: &'ast str,
 }
 
 #[derive(Debug, PartialEq)]
-pub enum Argument {
-    Expression(Expression),
-    NamedArgument { name: Identifier, val: ArgValue },
+pub struct Function<'ast> {
+    pub name: &'ast str,
 }
 
 #[derive(Debug, PartialEq)]
-pub enum ArgValue {
-    Number(Number),
-    String(String),
+pub struct Variant<'ast> {
+    pub key: VariantKey<'ast>,
+    pub value: Value<'ast>,
+    pub default: bool,
 }
 
-#[derive(Debug, PartialEq, Eq, Hash)]
-pub struct Identifier {
-    pub name: String,
+#[derive(Debug, PartialEq)]
+pub enum VariantKey<'ast> {
+    Identifier { name: &'ast str },
+    NumberLiteral { value: &'ast str },
 }
 
 #[derive(Debug, PartialEq)]
-pub struct Number {
-    pub value: String,
+pub enum Comment<'ast> {
+    Comment { content: Vec<&'ast str> },
+    GroupComment { content: Vec<&'ast str> },
+    ResourceComment { content: Vec<&'ast str> },
 }
 
 #[derive(Debug, PartialEq)]
-pub struct VariantName {
-    pub name: String,
+pub enum InlineExpression<'ast> {
+    StringLiteral {
+        value: &'ast str,
+    },
+    NumberLiteral {
+        value: &'ast str,
+    },
+    VariableReference {
+        id: Identifier<'ast>,
+    },
+    CallExpression {
+        callee: Function<'ast>,
+        positional: Vec<InlineExpression<'ast>>,
+        named: Vec<NamedArgument<'ast>>,
+    },
+    AttributeExpression {
+        reference: Box<InlineExpression<'ast>>,
+        name: Identifier<'ast>,
+    },
+    VariantExpression {
+        reference: Box<InlineExpression<'ast>>,
+        key: VariantKey<'ast>,
+    },
+    MessageReference {
+        id: Identifier<'ast>,
+    },
+    TermReference {
+        id: Identifier<'ast>,
+    },
+    Placeable {
+        expression: Box<Expression<'ast>>,
+    },
 }
 
 #[derive(Debug, PartialEq)]
-pub enum Comment {
-    Comment { content: String },
-    GroupComment { content: String },
-    ResourceComment { content: String },
+pub struct NamedArgument<'ast> {
+    pub name: Identifier<'ast>,
+    pub value: InlineExpression<'ast>,
 }
 
 #[derive(Debug, PartialEq)]
-pub struct Function {
-    pub name: String,
+pub enum Expression<'ast> {
+    InlineExpression(InlineExpression<'ast>),
+    SelectExpression {
+        selector: InlineExpression<'ast>,
+        variants: Vec<Variant<'ast>>,
+    },
 }

fluent-syntax/src/bin/parser.rs

@@ -0,0 +1,58 @@
+extern crate clap;
+extern crate fluent_syntax;
+
+use std::fs::File;
+use std::io;
+use std::io::Read;
+
+use clap::App;
+
+use fluent_syntax::ast::Resource;
+use fluent_syntax::parser::parse;
+
+fn read_file(path: &str) -> Result<String, io::Error> {
+    let mut f = File::open(path)?;
+    let mut s = String::new();
+    f.read_to_string(&mut s)?;
+    Ok(s)
+}
+
+fn print_entries_resource(res: &Resource) {
+    println!("{:#?}", res);
+}
+
+fn main() {
+    let matches = App::new("Fluent Parser")
+        .version("1.0")
+        .about("Parses FTL file into an AST")
+        .args_from_usage(
+            "-s, --silence 'disable output'
+             <INPUT> 'Sets the input file to use'",
+        )
+        .get_matches();
+
+    let input = matches.value_of("INPUT").unwrap();
+
+    let source = read_file(&input).expect("Read file failed");
+
+    let res = parse(&source);
+
+    if matches.is_present("silence") {
+        return;
+    };
+
+    match res {
+        Ok(res) => print_entries_resource(&res),
+        Err((res, errors)) => {
+            print_entries_resource(&res);
+            println!("==============================\n");
+            if errors.len() == 1 {
+                println!("Parser encountered one error:");
+            } else {
+                println!("Parser encountered {} errors:", errors.len());
+            }
+            println!("-----------------------------");
+            println!("{:#?}", errors);
+        }
+    };
+}

fluent-syntax/src/lib.rs

@@ -1,2 +1,4 @@
+#![feature(box_syntax, box_patterns)]
+
 pub mod ast;
 pub mod parser;