Skip to content

Commit 2e20b15

Browse files
andygroveandygrove
committed
Refactoring
1 parent 69a140a commit 2e20b15

File tree

5 files changed

+68
-58
lines changed

5 files changed

+68
-58
lines changed

‎examples/acme_parser.rs

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ impl SQLTokenizer<AcmeToken> for AcmeTokenizer {
3535
unimplemented!()
3636
}
3737

38-
fn next_token(&mut self) -> Result<Option<SQLToken<AcmeToken>>, TokenizerError> {
38+
fn next_token(&mut self, chars: &mut CharSeq) -> Result<Option<SQLToken<AcmeToken>>, TokenizerError> {
3939
// let mut arc = self.ansi_tokenizer.lock().unwrap();
4040
// match arc.peek_char() {
4141
// Some(&ch) => match ch {
@@ -67,14 +67,14 @@ struct AcmeParser {
6767

6868
impl SQLParser<AcmeToken, AcmeExpr> for AcmeParser {
6969

70-
fn parse_prefix(&mut self) -> Result<Box<SQLExpr<AcmeExpr>>, ParserError<AcmeToken>> {
70+
fn parse_prefix(&mut self, chars: &mut CharSeq) -> Result<Box<SQLExpr<AcmeExpr>>, ParserError<AcmeToken>> {
7171
//TODO: add custom overrides
72-
self.ansi_parser.lock().unwrap().parse_prefix()
72+
self.ansi_parser.lock().unwrap().parse_prefix(chars)
7373
}
7474

75-
fn parse_infix(&mut self, left: &SQLExpr<AcmeExpr>, precedence: usize) -> Result<Option<Box<SQLExpr<AcmeExpr>>>, ParserError<AcmeToken>> {
75+
fn parse_infix(&mut self, chars: &mut CharSeq, left: &SQLExpr<AcmeExpr>, precedence: usize) -> Result<Option<Box<SQLExpr<AcmeExpr>>>, ParserError<AcmeToken>> {
7676
//TODO: add custom overrides
77-
self.ansi_parser.lock().unwrap().parse_infix(left, precedence)
77+
self.ansi_parser.lock().unwrap().parse_infix(chars, left, precedence)
7878
}
7979
}
8080

@@ -83,7 +83,7 @@ fn main() {
8383
let sql = "1 + !! 5 * 2";
8484

8585
// ANSI SQL tokenizer
86-
let ansi_tokenizer = Arc::new(Mutex::new(ANSISQLTokenizer { chars: sql.chars().peekable() }));
86+
let ansi_tokenizer = Arc::new(Mutex::new(ANSISQLTokenizer { }));
8787

8888
// Custom ACME tokenizer
8989
let mut acme_tokenizer = Arc::new(Mutex::new(AcmeTokenizer {
@@ -95,9 +95,9 @@ fn main() {
9595
ansi_parser: Arc::new(Mutex::new(ANSISQLParser::new(acme_tokenizer)))
9696
}));
9797

98-
let expr = parse_expr(acme_parser).unwrap();
99-
100-
println!("Parsed: {:?}", expr);
98+
// let expr = parse_expr(acme_parser).unwrap();
99+
//
100+
// println!("Parsed: {:?}", expr);
101101

102102

103103
}

‎src/ansi/parser.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,9 @@ impl<TokenType> ANSISQLParser<TokenType> where TokenType: Debug + PartialEq {
2222
impl<TokenType, ExprType> SQLParser<TokenType, ExprType> for ANSISQLParser<TokenType>
2323
where TokenType: Debug + PartialEq, ExprType: Debug {
2424

25-
fn parse_prefix(&mut self) -> Result<Box<SQLExpr<ExprType>>, ParserError<TokenType>> {
25+
fn parse_prefix(&mut self, chars: &mut CharSeq) -> Result<Box<SQLExpr<ExprType>>, ParserError<TokenType>> {
2626

27-
match self.tokenizer.lock().unwrap().next_token()? {
27+
match self.tokenizer.lock().unwrap().next_token(chars)? {
2828
Some(SQLToken::Keyword(ref k)) => match k.to_uppercase().as_ref() {
2929
"INSERT" => unimplemented!(),
3030
"UPDATE" => unimplemented!(),
@@ -37,7 +37,7 @@ impl<TokenType, ExprType> SQLParser<TokenType, ExprType> for ANSISQLParser<Token
3737
}
3838
}
3939

40-
fn parse_infix(&mut self, _left: &SQLExpr<ExprType>, _precedence: usize) -> Result<Option<Box<SQLExpr<ExprType>>>, ParserError<TokenType>> {
40+
fn parse_infix(&mut self, _chars: &mut CharSeq, _left: &SQLExpr<ExprType>, _precedence: usize) -> Result<Option<Box<SQLExpr<ExprType>>>, ParserError<TokenType>> {
4141
unimplemented!()
4242
}
4343
}

‎src/ansi/tokenizer.rs

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,28 +5,27 @@ use std::str::Chars;
55

66
use super::super::tokenizer::*;
77

8-
pub struct ANSISQLTokenizer<'a> {
9-
pub chars: Peekable<Chars<'a>>
8+
pub struct ANSISQLTokenizer {
109
}
1110

12-
impl<'a, TokenType> SQLTokenizer<TokenType> for ANSISQLTokenizer<'a>
11+
impl<TokenType> SQLTokenizer<TokenType> for ANSISQLTokenizer
1312
where TokenType: Debug + PartialEq {
1413

1514
fn precedence(&self, _token: &SQLToken<TokenType>) -> usize {
1615
unimplemented!()
1716
}
1817

19-
fn next_token(&mut self) -> Result<Option<SQLToken<TokenType>>, TokenizerError> {
20-
match self.chars.next() {
18+
fn next_token(&mut self, chars: &mut CharSeq) -> Result<Option<SQLToken<TokenType>>, TokenizerError> {
19+
match chars.next() {
2120
Some(ch) => match ch {
2221
' ' | '\t' | '\n' => Ok(Some(SQLToken::Whitespace(ch))),
2322
'0' ... '9' => {
2423
let mut s = String::new();
2524
s.push(ch);
26-
while let Some(&ch) = self.chars.peek() {
25+
while let Some(&ch) = chars.peek() {
2726
match ch {
2827
'0' ... '9' => {
29-
self.chars.next(); // consume
28+
chars.next(); // consume
3029
s.push(ch);
3130
},
3231
_ => break

‎src/parser.rs

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -110,20 +110,20 @@ pub trait SQLParser<TokenType, ExprType>
110110
where TokenType: Debug + PartialEq, ExprType: Debug {
111111

112112
/// parse the prefix and stop once an infix operator is reached
113-
fn parse_prefix(&mut self) -> Result<Box<SQLExpr<ExprType>>, ParserError<TokenType>> ;
113+
fn parse_prefix(&mut self, chars: &mut CharSeq) -> Result<Box<SQLExpr<ExprType>>, ParserError<TokenType>> ;
114114
/// parse the next infix expression, returning None if the precedence has changed
115-
fn parse_infix(&mut self, left: &SQLExpr<ExprType>, precedence: usize) -> Result<Option<Box<SQLExpr<ExprType>>>, ParserError<TokenType>>;
115+
fn parse_infix(&mut self, chars: &mut CharSeq, left: &SQLExpr<ExprType>, precedence: usize) -> Result<Option<Box<SQLExpr<ExprType>>>, ParserError<TokenType>>;
116116
}
117117

118-
119-
pub fn parse_expr<'a, TokenType, ExprType>(parser: Arc<Mutex<SQLParser<TokenType, ExprType>>>)
120-
-> Result<Box<SQLExpr<ExprType>>, ParserError<TokenType>> where TokenType: Debug + PartialEq, ExprType: Debug {
121-
let mut guard = parser.lock().unwrap();
122-
123-
//Result<Box<SQLExpr<ExprType>>, ParserError<TokenType>>
124-
let x = guard.parse_prefix();
125-
x
126-
}
118+
//
119+
//pub fn parse_expr<'a, TokenType, ExprType>(parser: Arc<Mutex<SQLParser<TokenType, ExprType>>>)
120+
// -> Result<Box<SQLExpr<ExprType>>, ParserError<TokenType>> where TokenType: Debug + PartialEq, ExprType: Debug {
121+
// let mut guard = parser.lock().unwrap();
122+
//
123+
// //Result<Box<SQLExpr<ExprType>>, ParserError<TokenType>>
124+
// let x = guard.parse_prefix();
125+
// x
126+
//}
127127

128128

129129
//pub struct PrattParser<'a, TokenType, ExprType> {

‎src/tokenizer.rs

Lines changed: 39 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
use std::cmp::PartialEq;
22
use std::fmt::Debug;
3-
//use std::iter::Peekable;
4-
//use std::str::Chars;
5-
63

4+
/// Simple holder for a sequence of characters that supports iteration and mark/reset methods
75
pub struct CharSeq {
86
chars: Vec<char>,
97
i: usize,
@@ -12,6 +10,7 @@ pub struct CharSeq {
1210

1311
impl CharSeq {
1412

13+
/// Create a CharSeq from a string
1514
pub fn new(sql: &str) -> Self {
1615
CharSeq {
1716
chars: sql.chars().collect(),
@@ -20,14 +19,26 @@ impl CharSeq {
2019
}
2120
}
2221

22+
/// Mark the current index
2323
pub fn mark(&mut self) {
2424
self.m = self.i;
2525
}
2626

27+
/// Reset the index
2728
pub fn reset(&mut self) {
2829
self.i = self.m;
2930
}
3031

32+
/// Peek the next char
33+
pub fn peek(&mut self) -> Option<&char> {
34+
if self.i < self.chars.len() {
35+
Some(&self.chars[self.i])
36+
} else {
37+
None
38+
}
39+
}
40+
41+
/// Get the next char
3142
pub fn next(&mut self) -> Option<char> {
3243
if self.i < self.chars.len() {
3344
self.i += 1;
@@ -61,8 +72,8 @@ pub enum TokenizerError {
6172
#[derive(Debug,PartialEq)]
6273
pub enum SQLToken<T: Debug + PartialEq> {
6374
Whitespace(char),
64-
Keyword(String), //TODO: &str ?
65-
Identifier(String), //TODO: &str ?
75+
Keyword(String),
76+
Identifier(String),
6677
Literal(String), //TODO: need to model different types of literal
6778
Plus,
6879
Minus,
@@ -89,28 +100,28 @@ pub trait SQLTokenizer<TokenType>
89100
fn precedence(&self, token: &SQLToken<TokenType>) -> usize;
90101

91102
/// return a reference to the next token and advance the index
92-
fn next_token(&mut self) -> Result<Option<SQLToken<TokenType>>, TokenizerError>;
103+
fn next_token(&mut self, chars: &mut CharSeq) -> Result<Option<SQLToken<TokenType>>, TokenizerError>;
93104
}
94105

95-
//
96-
//pub fn tokenize<TokenType>(sql: &str, tokenizer: &mut SQLTokenizer<TokenType>) -> Result<Vec<SQLToken<TokenType>>, TokenizerError<TokenType>>
97-
// where TokenType: Debug + PartialEq
98-
// {
99-
//
100-
// let mut peekable = sql.chars().peekable();
101-
//
102-
// let mut tokens : Vec<SQLToken<TokenType>> = vec![];
103-
//
104-
// loop {
105-
// match tokenizer.next_token(&mut peekable)? {
106-
// Some(SQLToken::Whitespace(_)) => { /* ignore */ },
107-
// Some(token) => {
108-
// println!("Token: {:?}", token);
109-
// tokens.push(token)
110-
// },
111-
// None => break
112-
// }
113-
// }
114-
//
115-
// Ok(tokens)
116-
//}
106+
107+
pub fn tokenize<TokenType>(sql: &str, tokenizer: &mut SQLTokenizer<TokenType>) -> Result<Vec<SQLToken<TokenType>>, TokenizerError>
108+
where TokenType: Debug + PartialEq
109+
{
110+
111+
let mut chars = CharSeq::new(sql);
112+
113+
let mut tokens : Vec<SQLToken<TokenType>> = vec![];
114+
115+
loop {
116+
match tokenizer.next_token(&mut chars)? {
117+
Some(SQLToken::Whitespace(_)) => { /* ignore */ },
118+
Some(token) => {
119+
println!("Token: {:?}", token);
120+
tokens.push(token)
121+
},
122+
None => break
123+
}
124+
}
125+
126+
Ok(tokens)
127+
}

0 commit comments

Comments
 (0)