Skip to content

Shrink token::Lit's hash count from usize to u16. #38043

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Shrink token::Lit's hash count from usize to u16. #38043

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions src/grammar/verify.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ fn parse_token_list(file: &str) -> HashMap<String, token::Token> {
"ANDAND" => Token::AndAnd,
"AT" => Token::At,
"LBRACKET" => Token::OpenDelim(DelimToken::Bracket),
"LIT_STR_RAW" => Token::Literal(Lit::StrRaw(Name(0), 0), None),
"LIT_STR_RAW" => Token::Literal(0, Lit::StrRaw(Name(0)), None),
"RPAREN" => Token::CloseDelim(DelimToken::Paren),
"SLASH" => Token::BinOp(BinOpToken::Slash),
"COMMA" => Token::Comma,
Expand Down Expand Up @@ -112,7 +112,7 @@ fn parse_token_list(file: &str) -> HashMap<String, token::Token> {
"GT" => Token::Gt,
"LE" => Token::Le,
"LIT_BINARY" => Token::Literal(Lit::ByteStr(Name(0)), None),
"LIT_BINARY_RAW" => Token::Literal(Lit::ByteStrRaw(Name(0), 0), None),
"LIT_BINARY_RAW" => Token::Literal(0, Lit::ByteStrRaw(Name(0)), None),
"QUESTION" => Token::Question,
"SHEBANG" => Token::Shebang(Name(0)),
_ => continue,
Expand Down Expand Up @@ -204,16 +204,16 @@ fn parse_antlr_token(s: &str, tokens: &HashMap<String, token::Token>, surrogate_
Token::BinOp(..) => Token::BinOp(str_to_binop(content)),
Token::BinOpEq(..) => Token::BinOpEq(str_to_binop(&content[..content.len() - 1])),
Token::Literal(Lit::Str_(..), n) => Token::Literal(Lit::Str_(fix(content)), n),
Token::Literal(Lit::StrRaw(..), n) => Token::Literal(Lit::StrRaw(fix(content),
count(content)), n),
Token::Literal(Lit::StrRaw(..), n) => Token::Literal(Lit::StrRaw(count(content),
fix(content)), n),
Token::Literal(Lit::Char(..), n) => Token::Literal(Lit::Char(fixchar(content)), n),
Token::Literal(Lit::Byte(..), n) => Token::Literal(Lit::Byte(fixchar(content)), n),
Token::DocComment(..) => Token::DocComment(nm),
Token::Literal(Lit::Integer(..), n) => Token::Literal(Lit::Integer(nm), n),
Token::Literal(Lit::Float(..), n) => Token::Literal(Lit::Float(nm), n),
Token::Literal(Lit::ByteStr(..), n) => Token::Literal(Lit::ByteStr(nm), n),
Token::Literal(Lit::ByteStrRaw(..), n) => Token::Literal(Lit::ByteStrRaw(fix(content),
count(content)), n),
Token::Literal(Lit::ByteStrRaw(..), n) => Token::Literal(Lit::ByteStrRaw(count(content),
fix(content)), n),
Token::Ident(..) => Token::Ident(ast::Ident::with_empty_ctxt(nm)),
Token::Lifetime(..) => Token::Lifetime(ast::Ident::with_empty_ctxt(nm)),
ref t => t.clone()
Expand Down
4 changes: 2 additions & 2 deletions src/librustc_incremental/calculate_svh/svh_visitor.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1042,8 +1042,8 @@ impl<'a, 'hash, 'tcx> StrictVersionHashVisitor<'a, 'hash, 'tcx> {
token::Lit::Float(val) |
token::Lit::Str_(val) |
token::Lit::ByteStr(val) => val.as_str().hash(self.st),
token::Lit::StrRaw(val, n) |
token::Lit::ByteStrRaw(val, n) => {
token::Lit::StrRaw(n, val) |
token::Lit::ByteStrRaw(n, val) => {
val.as_str().hash(self.st);
n.hash(self.st);
}
Expand Down
2 changes: 1 addition & 1 deletion src/libsyntax/ast.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1079,7 +1079,7 @@ pub enum StrStyle {
/// A raw string, like `r##"foo"##`
///
/// The uint is the number of `#` symbols used
Raw(usize)
Raw(u16)
}

/// A literal
Expand Down
2 changes: 1 addition & 1 deletion src/libsyntax/diagnostics/plugin.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt,
},
(3, Some(&TokenTree::Token(_, token::Ident(ref code))),
Some(&TokenTree::Token(_, token::Comma)),
Some(&TokenTree::Token(_, token::Literal(token::StrRaw(description, _), None)))) => {
Some(&TokenTree::Token(_, token::Literal(token::StrRaw(_, description), None)))) => {
(code, Some(description))
}
_ => unreachable!()
Expand Down
4 changes: 4 additions & 0 deletions src/libsyntax/ext/build.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,7 @@ pub trait AstBuilder {
fn expr_usize(&self, span: Span, i: usize) -> P<ast::Expr>;
fn expr_isize(&self, sp: Span, i: isize) -> P<ast::Expr>;
fn expr_u8(&self, sp: Span, u: u8) -> P<ast::Expr>;
fn expr_u16(&self, sp: Span, u: u16) -> P<ast::Expr>;
fn expr_u32(&self, sp: Span, u: u32) -> P<ast::Expr>;
fn expr_bool(&self, sp: Span, value: bool) -> P<ast::Expr>;

Expand Down Expand Up @@ -738,6 +739,9 @@ impl<'a> AstBuilder for ExtCtxt<'a> {
fn expr_u32(&self, sp: Span, u: u32) -> P<ast::Expr> {
self.expr_lit(sp, ast::LitKind::Int(u as u64, ast::LitIntType::Unsigned(ast::UintTy::U32)))
}
fn expr_u16(&self, sp: Span, u: u16) -> P<ast::Expr> {
self.expr_lit(sp, ast::LitKind::Int(u as u64, ast::LitIntType::Unsigned(ast::UintTy::U16)))
}
fn expr_u8(&self, sp: Span, u: u8) -> P<ast::Expr> {
self.expr_lit(sp, ast::LitKind::Int(u as u64, ast::LitIntType::Unsigned(ast::UintTy::U8)))
}
Expand Down
6 changes: 3 additions & 3 deletions src/libsyntax/ext/quote.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -649,9 +649,9 @@ fn expr_mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
return mk_lit!("Str_", suf, mk_name(cx, sp, ast::Ident::with_empty_ctxt(ident)))
}

token::Literal(token::StrRaw(ident, n), suf) => {
return mk_lit!("StrRaw", suf, mk_name(cx, sp, ast::Ident::with_empty_ctxt(ident)),
cx.expr_usize(sp, n))
token::Literal(token::StrRaw(n, ident), suf) => {
return mk_lit!("StrRaw", suf, cx.expr_u16(sp, n),
mk_name(cx, sp, ast::Ident::with_empty_ctxt(ident)));
}

token::Ident(ident) => {
Expand Down
13 changes: 7 additions & 6 deletions src/libsyntax/parse/lexer/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1428,7 +1428,7 @@ impl<'a> StringReader<'a> {
Symbol::intern("??")
};
let suffix = self.scan_optional_raw_name();
return Ok(token::Literal(token::StrRaw(id, hash_count), suffix));
return Ok(token::Literal(token::StrRaw(hash_count, id), suffix));
}
'-' => {
if self.nextch_is('>') {
Expand Down Expand Up @@ -1597,6 +1597,7 @@ impl<'a> StringReader<'a> {
let mut hash_count = 0;
while self.ch_is('#') {
self.bump();
// njn: check for u16 overflow?
hash_count += 1;
}

Expand Down Expand Up @@ -1641,8 +1642,8 @@ impl<'a> StringReader<'a> {
self.bump();
}
self.bump();
return token::ByteStrRaw(self.name_from_to(content_start_bpos, content_end_bpos),
hash_count);
return token::ByteStrRaw(hash_count,
self.name_from_to(content_start_bpos, content_end_bpos));
}
}

Expand Down Expand Up @@ -1849,7 +1850,7 @@ mod tests {
assert_eq!(setup(&cm, &sh, "r###\"\"#a\\b\x00c\"\"###".to_string())
.next_token()
.tok,
token::Literal(token::StrRaw(Symbol::intern("\"#a\\b\x00c\""), 3), None));
token::Literal(token::StrRaw(3, Symbol::intern("\"#a\\b\x00c\"")), None));
}

#[test]
Expand Down Expand Up @@ -1882,10 +1883,10 @@ mod tests {
token::Literal(token::Integer(Symbol::intern("2")),
Some(Symbol::intern("us"))));
assert_eq!(setup(&cm, &sh, "r###\"raw\"###suffix".to_string()).next_token().tok,
token::Literal(token::StrRaw(Symbol::intern("raw"), 3),
token::Literal(token::StrRaw(3, Symbol::intern("raw")),
Some(Symbol::intern("suffix"))));
assert_eq!(setup(&cm, &sh, "br###\"raw\"###suffix".to_string()).next_token().tok,
token::Literal(token::ByteStrRaw(Symbol::intern("raw"), 3),
token::Literal(token::ByteStrRaw(3, Symbol::intern("raw")),
Some(Symbol::intern("suffix"))));
}

Expand Down
8 changes: 4 additions & 4 deletions src/libsyntax/parse/parser.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1528,14 +1528,14 @@ impl<'a> Parser<'a> {
let s = Symbol::intern(&parse::str_lit(&s.as_str()));
(true, LitKind::Str(s, ast::StrStyle::Cooked))
}
token::StrRaw(s, n) => {
token::StrRaw(n, s) => {
let s = Symbol::intern(&parse::raw_str_lit(&s.as_str()));
(true, LitKind::Str(s, ast::StrStyle::Raw(n)))
}
token::ByteStr(i) => {
(true, LitKind::ByteStr(parse::byte_str_lit(&i.as_str())))
}
token::ByteStrRaw(i, _) => {
token::ByteStrRaw(_, i) => {
(true, LitKind::ByteStr(Rc::new(i.to_string().into_bytes())))
}
};
Expand Down Expand Up @@ -5654,7 +5654,7 @@ impl<'a> Parser<'a> {
/// the `extern` keyword, if one is found.
fn parse_opt_abi(&mut self) -> PResult<'a, Option<abi::Abi>> {
match self.token {
token::Literal(token::Str_(s), suf) | token::Literal(token::StrRaw(s, _), suf) => {
token::Literal(token::Str_(s), suf) | token::Literal(token::StrRaw(_, s), suf) => {
let sp = self.span;
self.expect_no_suffix(sp, "ABI spec", suf);
self.bump();
Expand Down Expand Up @@ -6151,7 +6151,7 @@ impl<'a> Parser<'a> {
pub fn parse_optional_str(&mut self) -> Option<(Symbol, ast::StrStyle, Option<ast::Name>)> {
let ret = match self.token {
token::Literal(token::Str_(s), suf) => (s, ast::StrStyle::Cooked, suf),
token::Literal(token::StrRaw(s, n), suf) => (s, ast::StrStyle::Raw(n), suf),
token::Literal(token::StrRaw(n, s), suf) => (s, ast::StrStyle::Raw(n), suf),
_ => return None
};
self.bump();
Expand Down
4 changes: 2 additions & 2 deletions src/libsyntax/parse/token.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,9 @@ pub enum Lit {
Integer(ast::Name),
Float(ast::Name),
Str_(ast::Name),
StrRaw(ast::Name, usize), /* raw str delimited by n hash symbols */
StrRaw(u16, ast::Name), /* raw str delimited by n hash symbols */
ByteStr(ast::Name),
ByteStrRaw(ast::Name, usize), /* raw byte str delimited by n hash symbols */
ByteStrRaw(u16, ast::Name), /* raw byte str delimited by n hash symbols */
}

impl Lit {
Expand Down
11 changes: 6 additions & 5 deletions src/libsyntax/print/pprust.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -255,12 +255,13 @@ pub fn token_to_string(tok: &Token) -> String {
token::Float(c) => c.to_string(),
token::Integer(c) => c.to_string(),
token::Str_(s) => format!("\"{}\"", s),
token::StrRaw(s, n) => format!("r{delim}\"{string}\"{delim}",
delim=repeat("#", n),
token::StrRaw(n, s) => format!("r{delim}\"{string}\"{delim}",
delim=repeat("#", n as usize),
string=s),
token::ByteStr(v) => format!("b\"{}\"", v),
token::ByteStrRaw(s, n) => format!("br{delim}\"{string}\"{delim}",
delim=repeat("#", n),
token::ByteStrRaw(n, s) => format!("br{delim}\"{string}\"{delim}",
delim=repeat("#", n as
usize),
string=s),
};

Expand Down Expand Up @@ -687,7 +688,7 @@ pub trait PrintState<'a> {
}
ast::StrStyle::Raw(n) => {
(format!("r{delim}\"{string}\"{delim}",
delim=repeat("#", n),
delim=repeat("#", n as usize),
string=st))
}
};
Expand Down
5 changes: 3 additions & 2 deletions src/libsyntax/tokenstream.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,7 @@ impl TokenTree {

// Searches for the occurrences of `"#*` and returns the minimum number of `#`s
// required to wrap the text.
// njn: check for u16 overflow?
let num_of_hashes = stripped.chars()
.scan(0, |cnt, x| {
*cnt = if x == '"' {
Expand All @@ -177,7 +178,7 @@ impl TokenTree {
tts: vec![TokenTree::Token(sp, token::Ident(ast::Ident::from_str("doc"))),
TokenTree::Token(sp, token::Eq),
TokenTree::Token(sp, token::Literal(
token::StrRaw(Symbol::intern(&stripped), num_of_hashes), None))],
token::StrRaw(num_of_hashes, Symbol::intern(&stripped)), None))],
close_span: sp,
}))
}
Expand Down Expand Up @@ -303,7 +304,7 @@ impl TokenTree {
span: sp,
})
}
TokenTree::Token(sp, Token::Literal(Lit::StrRaw(s, n), _)) => {
TokenTree::Token(sp, Token::Literal(Lit::StrRaw(n, s), _)) => {
let l = LitKind::Str(Symbol::intern(&parse::raw_str_lit(&s.as_str())),
ast::StrStyle::Raw(n));
Some(Spanned {
Expand Down