add a lexer in rust

chyyuu-tsinghua-cs · chyyuu-tsinghua-cs · commit 9ba32ca361d4 · 2019-09-28T09:05:34.000+08:00
diff --git a/rust/lexer/lexer-in-rust/Cargo.toml b/rust/lexer/lexer-in-rust/Cargo.toml
@@ -0,0 +1,6 @@
+[package]
+name = "writing_an_interpreter_in_rust"
+version = "0.1.0"
+authors = ["Chris Aumann <me@chr4.org>"]
+
+[dependencies]
diff --git a/rust/lexer/lexer-in-rust/README.md b/rust/lexer/lexer-in-rust/README.md
@@ -0,0 +1,16 @@
+# Writing an interpreter in rust
+
+This is an implementation of the Monkey interpreter from Thorsten Ball's book [Writing an interpreter in Go](https://interpreterbook.com).
+
+[![Build Status](https://travis-ci.org/chr4/writing_an_interpreter_in_rust.svg?branch=master)](https://travis-ci.org/chr4/writing_an_interpreter_in_rust)
+
+## Usage
+
+```bash
+$ git clone https://github.com/chr4/writing_an_interpreter_in_rust
+$ cd writing_an_interpreter_in_rust
+$ cargo test
+$ cargo run
+```
+
+For more information, see this [blog post](https://chr4.org/blog/2016/12/09/writing-an-interpreter-in-rust/).
diff --git a/rust/lexer/lexer-in-rust/src/lexer.rs b/rust/lexer/lexer-in-rust/src/lexer.rs
@@ -0,0 +1,245 @@
+use token;
+use token::Token;
+
+use std::str::Chars;
+use std::iter::Peekable;
+
+pub struct Lexer<'a> {
+    input: Peekable<Chars<'a>>,
+}
+
+impl<'a> Lexer<'a> {
+    pub fn new(input: &str) -> Lexer {
+        Lexer { input: input.chars().peekable() }
+    }
+
+    fn read_char(&mut self) -> Option<char> {
+        self.input.next()
+    }
+
+    fn peek_char(&mut self) -> Option<&char> {
+        self.input.peek()
+    }
+
+    fn peek_char_eq(&mut self, ch: char) -> bool {
+        match self.peek_char() {
+            Some(&peek_ch) => peek_ch == ch,
+            None => false,
+        }
+    }
+
+    fn skip_whitespace(&mut self) {
+        while let Some(&c) = self.peek_char() {
+            if !c.is_whitespace() {
+                break;
+            }
+            self.read_char();
+        }
+    }
+
+    fn peek_is_letter(&mut self) -> bool {
+        match self.peek_char() {
+            Some(&ch) => is_letter(ch),
+            None => false,
+        }
+    }
+
+    fn read_identifier(&mut self, first: char) -> String {
+        let mut ident = String::new();
+        ident.push(first);
+
+        while self.peek_is_letter() {
+            ident.push(self.read_char().unwrap()); // TODO: unwrap()
+        }
+
+        ident
+    }
+
+    fn read_number(&mut self, first: char) -> String {
+        let mut number = String::new();
+        number.push(first);
+
+        while let Some(&c) = self.peek_char() {
+            if !c.is_numeric() {
+                break;
+            }
+            number.push(self.read_char().unwrap()); // TODO: unwrap()
+        }
+
+        number
+    }
+
+    pub fn next_token(&mut self) -> Token {
+        self.skip_whitespace();
+
+        match self.read_char() {
+            Some('=') => {
+                if self.peek_char_eq('=') {
+                    self.read_char();
+                    Token::Equal
+                } else {
+                    Token::Assign
+                }
+            }
+            Some('+') => Token::Plus,
+            Some('-') => Token::Minus,
+            Some('!') => {
+                if self.peek_char_eq('=') {
+                    self.read_char();
+                    Token::NotEqual
+                } else {
+                    Token::Bang
+                }
+            }
+            Some('/') => Token::Slash,
+            Some('*') => Token::Asterisk,
+            Some('<') => Token::LowerThan,
+            Some('>') => Token::GreaterThan,
+            Some(';') => Token::Semicolon,
+            Some(',') => Token::Comma,
+            Some('{') => Token::LeftBrace,
+            Some('}') => Token::RightBrace,
+            Some('(') => Token::LeftParenthesis,
+            Some(')') => Token::RightParenthesis,
+
+            Some(ch @ _) => {
+                if is_letter(ch) {
+                    let literal = self.read_identifier(ch);
+                    token::lookup_ident(&literal)
+                } else if ch.is_numeric() {
+                    Token::Integer(self.read_number(ch))
+                } else {
+                    Token::Illegal // TODO: Maybe we need ch here, to display a nice error message later?
+                }
+            }
+
+            // Handle EOF
+            None => Token::EndOfFile,
+        }
+    }
+}
+
+// is_letter checks whether a char is a valid alphabetic character or an underscore
+fn is_letter(ch: char) -> bool {
+    ch.is_alphabetic() || ch == '_'
+}
+
+#[test]
+fn is_letter_test() {
+    assert!(is_letter('_'));
+    assert!(is_letter('a'));
+    assert!(is_letter('Z'));
+
+    assert!(!is_letter('*'));
+    assert!(!is_letter('1'));
+}
+
+
+#[test]
+fn next_token_test() {
+
+    #[cfg_attr(rustfmt, rustfmt_skip)]
+    let input = "let five = 5;
+let ten = 10;
+
+let add = fn(x, y) {
+  x + y;
+};
+
+let result = add(five, ten);
+!-/*5;
+5 < 10 > 5;
+
+if (5 < 10) {
+    return true;
+} else {
+    return false;
+}
+
+10 == 10;
+10 != 9;
+";
+
+    let tests = vec![
+        Token::Let,
+        Token::Ident("five".to_string()),
+        Token::Assign,
+        Token::Integer("5".to_string()),
+        Token::Semicolon,
+        Token::Let,
+        Token::Ident("ten".to_string()),
+        Token::Assign,
+        Token::Integer("10".to_string()),
+        Token::Semicolon,
+        Token::Let,
+        Token::Ident("add".to_string()),
+        Token::Assign,
+        Token::Function,
+        Token::LeftParenthesis,
+        Token::Ident("x".to_string()),
+        Token::Comma,
+        Token::Ident("y".to_string()),
+        Token::RightParenthesis,
+        Token::LeftBrace,
+        Token::Ident("x".to_string()),
+        Token::Plus,
+        Token::Ident("y".to_string()),
+        Token::Semicolon,
+        Token::RightBrace,
+        Token::Semicolon,
+        Token::Let,
+        Token::Ident("result".to_string()),
+        Token::Assign,
+        Token::Ident("add".to_string()),
+        Token::LeftParenthesis,
+        Token::Ident("five".to_string()),
+        Token::Comma,
+        Token::Ident("ten".to_string()),
+        Token::RightParenthesis,
+        Token::Semicolon,
+        Token::Bang,
+        Token::Minus,
+        Token::Slash,
+        Token::Asterisk,
+        Token::Integer("5".to_string()),
+        Token::Semicolon,
+        Token::Integer("5".to_string()),
+        Token::LowerThan,
+        Token::Integer("10".to_string()),
+        Token::GreaterThan,
+        Token::Integer("5".to_string()),
+        Token::Semicolon,
+        Token::If,
+        Token::LeftParenthesis,
+        Token::Integer("5".to_string()),
+        Token::LowerThan,
+        Token::Integer("10".to_string()),
+        Token::RightParenthesis,
+        Token::LeftBrace,
+        Token::Return,
+        Token::True,
+        Token::Semicolon,
+        Token::RightBrace,
+        Token::Else,
+        Token::LeftBrace,
+        Token::Return,
+        Token::False,
+        Token::Semicolon,
+        Token::RightBrace,
+        Token::Integer("10".to_string()),
+        Token::Equal,
+        Token::Integer("10".to_string()),
+        Token::Semicolon,
+        Token::Integer("10".to_string()),
+        Token::NotEqual,
+        Token::Integer("9".to_string()),
+        Token::Semicolon,
+        Token::EndOfFile,
+    ];
+
+    let mut l = Lexer::new(input);
+    for t in tests {
+        let tok = l.next_token();
+        assert_eq!(tok, t);
+    }
+}
diff --git a/rust/lexer/lexer-in-rust/src/lib.rs b/rust/lexer/lexer-in-rust/src/lib.rs
@@ -0,0 +1,2 @@
+pub mod token;
+pub mod lexer;
diff --git a/rust/lexer/lexer-in-rust/src/main.rs b/rust/lexer/lexer-in-rust/src/main.rs
@@ -0,0 +1,29 @@
+pub mod lexer;
+pub mod token;
+
+use std::io::{self, BufRead, Write};
+use token::Token;
+use lexer::Lexer;
+
+// Start a custom repl
+fn main() {
+    let stdin = io::stdin();
+
+    loop {
+        // Stdout needs to be flushed, due to missing newline
+        print!(">> ");
+        io::stdout().flush().expect("Error flushing stdout");
+
+        let mut line = String::new();
+        stdin.lock().read_line(&mut line).expect("Error reading from stdin");
+        let mut lexer = Lexer::new(&mut line);
+
+        loop {
+            let tok = lexer.next_token();
+            println!("{:?}", tok);
+            if tok == Token::EndOfFile {
+                break;
+            }
+        }
+    }
+}
diff --git a/rust/lexer/lexer-in-rust/src/token.rs b/rust/lexer/lexer-in-rust/src/token.rs
@@ -0,0 +1,64 @@
+#[derive(Debug, PartialEq)]
+pub enum Token {
+    Illegal,
+    EndOfFile,
+
+    // Literals are stored as strings
+    Ident(String),
+    Integer(String),
+
+    // Operators
+    Assign,
+    Plus,
+    Minus,
+    Bang,
+    Asterisk,
+    Slash,
+    LowerThan,
+    GreaterThan,
+    Equal,
+    NotEqual,
+
+    // Delimiters
+    Comma,
+    Semicolon,
+    LeftParenthesis,
+    RightParenthesis,
+    LeftBrace,
+    RightBrace,
+
+    // Keywords
+    Function,
+    Let,
+    True,
+    False,
+    If,
+    Else,
+    Return,
+}
+
+impl Default for Token {
+    // Choose an Illegal identifier as default
+    // this should be overriden before being used
+    fn default() -> Token {
+        Token::Illegal
+    }
+}
+
+pub fn lookup_ident(ident: &str) -> Token {
+    match ident {
+        "fn" => Token::Function,
+        "let" => Token::Let,
+        "true" => Token::True,
+        "false" => Token::False,
+        "if" => Token::If,
+        "else" => Token::Else,
+        "return" => Token::Return,
+        _ => Token::Ident(ident.to_string()),
+    }
+}
+
+#[test]
+fn lookup_ident_test() {
+    assert_eq!(lookup_ident("fn"), Token::Function);
+}
