Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit decb0c9

Browse files
Rollup merge of #137902 - nnethercote:ast-lexer-TokenKind, r=compiler-errors
Make `ast::TokenKind` more like `lexer::TokenKind` This is step 2 of rust-lang/compiler-team#831. r? `@spastorino`
2 parents 70b9968 + 53167c0 commit decb0c9

File tree

24 files changed

+396
-353
lines changed

24 files changed

+396
-353
lines changed

‎compiler/rustc_ast/src/token.rs

Lines changed: 170 additions & 126 deletions
Large diffs are not rendered by default.

‎compiler/rustc_ast/src/tokenstream.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -651,7 +651,7 @@ impl TokenStream {
651651
if attr_style == AttrStyle::Inner {
652652
vec![
653653
TokenTree::token_joint(token::Pound, span),
654-
TokenTree::token_joint_hidden(token::Not, span),
654+
TokenTree::token_joint_hidden(token::Bang, span),
655655
body,
656656
]
657657
} else {

‎compiler/rustc_ast/src/util/parser.rs

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
use rustc_span::kw;
22

33
use crate::ast::{self, BinOpKind, RangeLimits};
4-
use crate::token::{self, BinOpToken,Token};
4+
use crate::token::{self, Token};
55

66
/// Associative operator.
77
#[derive(Copy, Clone, PartialEq, Debug)]
@@ -34,26 +34,26 @@ impl AssocOp {
3434
use AssocOp::*;
3535
match t.kind {
3636
token::Eq => Some(Assign),
37-
token::BinOp(BinOpToken::Plus) => Some(Binary(BinOpKind::Add)),
38-
token::BinOp(BinOpToken::Minus) => Some(Binary(BinOpKind::Sub)),
39-
token::BinOp(BinOpToken::Star) => Some(Binary(BinOpKind::Mul)),
40-
token::BinOp(BinOpToken::Slash) => Some(Binary(BinOpKind::Div)),
41-
token::BinOp(BinOpToken::Percent) => Some(Binary(BinOpKind::Rem)),
42-
token::BinOp(BinOpToken::Caret) => Some(Binary(BinOpKind::BitXor)),
43-
token::BinOp(BinOpToken::And) => Some(Binary(BinOpKind::BitAnd)),
44-
token::BinOp(BinOpToken::Or) => Some(Binary(BinOpKind::BitOr)),
45-
token::BinOp(BinOpToken::Shl) => Some(Binary(BinOpKind::Shl)),
46-
token::BinOp(BinOpToken::Shr) => Some(Binary(BinOpKind::Shr)),
47-
token::BinOpEq(BinOpToken::Plus) => Some(AssignOp(BinOpKind::Add)),
48-
token::BinOpEq(BinOpToken::Minus) => Some(AssignOp(BinOpKind::Sub)),
49-
token::BinOpEq(BinOpToken::Star) => Some(AssignOp(BinOpKind::Mul)),
50-
token::BinOpEq(BinOpToken::Slash) => Some(AssignOp(BinOpKind::Div)),
51-
token::BinOpEq(BinOpToken::Percent) => Some(AssignOp(BinOpKind::Rem)),
52-
token::BinOpEq(BinOpToken::Caret) => Some(AssignOp(BinOpKind::BitXor)),
53-
token::BinOpEq(BinOpToken::And) => Some(AssignOp(BinOpKind::BitAnd)),
54-
token::BinOpEq(BinOpToken::Or) => Some(AssignOp(BinOpKind::BitOr)),
55-
token::BinOpEq(BinOpToken::Shl) => Some(AssignOp(BinOpKind::Shl)),
56-
token::BinOpEq(BinOpToken::Shr) => Some(AssignOp(BinOpKind::Shr)),
37+
token::Plus => Some(Binary(BinOpKind::Add)),
38+
token::Minus => Some(Binary(BinOpKind::Sub)),
39+
token::Star => Some(Binary(BinOpKind::Mul)),
40+
token::Slash => Some(Binary(BinOpKind::Div)),
41+
token::Percent => Some(Binary(BinOpKind::Rem)),
42+
token::Caret => Some(Binary(BinOpKind::BitXor)),
43+
token::And => Some(Binary(BinOpKind::BitAnd)),
44+
token::Or => Some(Binary(BinOpKind::BitOr)),
45+
token::Shl => Some(Binary(BinOpKind::Shl)),
46+
token::Shr => Some(Binary(BinOpKind::Shr)),
47+
token::PlusEq => Some(AssignOp(BinOpKind::Add)),
48+
token::MinusEq => Some(AssignOp(BinOpKind::Sub)),
49+
token::StarEq => Some(AssignOp(BinOpKind::Mul)),
50+
token::SlashEq => Some(AssignOp(BinOpKind::Div)),
51+
token::PercentEq => Some(AssignOp(BinOpKind::Rem)),
52+
token::CaretEq => Some(AssignOp(BinOpKind::BitXor)),
53+
token::AndEq => Some(AssignOp(BinOpKind::BitAnd)),
54+
token::OrEq => Some(AssignOp(BinOpKind::BitOr)),
55+
token::ShlEq => Some(AssignOp(BinOpKind::Shl)),
56+
token::ShrEq => Some(AssignOp(BinOpKind::Shr)),
5757
token::Lt => Some(Binary(BinOpKind::Lt)),
5858
token::Le => Some(Binary(BinOpKind::Le)),
5959
token::Ge => Some(Binary(BinOpKind::Ge)),

‎compiler/rustc_ast_pretty/src/pprust/state.rs

Lines changed: 23 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,7 @@ use std::sync::Arc;
1111

1212
use rustc_ast::attr::AttrIdGenerator;
1313
use rustc_ast::ptr::P;
14-
use rustc_ast::token::{
15-
self, BinOpToken, CommentKind, Delimiter, IdentIsRaw, Nonterminal, Token, TokenKind,
16-
};
14+
use rustc_ast::token::{self, CommentKind, Delimiter, IdentIsRaw, Nonterminal, Token, TokenKind};
1715
use rustc_ast::tokenstream::{Spacing, TokenStream, TokenTree};
1816
use rustc_ast::util::classify;
1917
use rustc_ast::util::comments::{Comment, CommentStyle};
@@ -319,7 +317,7 @@ fn space_between(tt1: &TokenTree, tt2: &TokenTree) -> bool {
319317
(tt1, Tok(Token { kind: Comma | Semi | Dot, .. }, _)) if !is_punct(tt1) => false,
320318

321319
// IDENT + `!`: `println!()`, but `if !x { ... }` needs a space after the `if`
322-
(Tok(Token { kind: Ident(sym, is_raw), span }, _), Tok(Token { kind: Not, .. }, _))
320+
(Tok(Token { kind: Ident(sym, is_raw), span }, _), Tok(Token { kind: Bang, .. }, _))
323321
if !Ident::new(*sym, *span).is_reserved() || matches!(is_raw, IdentIsRaw::Yes) =>
324322
{
325323
false
@@ -344,21 +342,6 @@ fn space_between(tt1: &TokenTree, tt2: &TokenTree) -> bool {
344342
}
345343
}
346344

347-
fn binop_to_string(op: BinOpToken) -> &'static str {
348-
match op {
349-
token::Plus => "+",
350-
token::Minus => "-",
351-
token::Star => "*",
352-
token::Slash => "/",
353-
token::Percent => "%",
354-
token::Caret => "^",
355-
token::And => "&",
356-
token::Or => "|",
357-
token::Shl => "<<",
358-
token::Shr => ">>",
359-
}
360-
}
361-
362345
pub fn doc_comment_to_string(
363346
comment_kind: CommentKind,
364347
attr_style: ast::AttrStyle,
@@ -913,12 +896,30 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
913896
token::Ne => "!=".into(),
914897
token::Ge => ">=".into(),
915898
token::Gt => ">".into(),
916-
token::Not => "!".into(),
899+
token::Bang => "!".into(),
917900
token::Tilde => "~".into(),
918901
token::OrOr => "||".into(),
919902
token::AndAnd => "&&".into(),
920-
token::BinOp(op) => binop_to_string(op).into(),
921-
token::BinOpEq(op) => format!("{}=", binop_to_string(op)).into(),
903+
token::Plus => "+".into(),
904+
token::Minus => "-".into(),
905+
token::Star => "*".into(),
906+
token::Slash => "/".into(),
907+
token::Percent => "%".into(),
908+
token::Caret => "^".into(),
909+
token::And => "&".into(),
910+
token::Or => "|".into(),
911+
token::Shl => "<<".into(),
912+
token::Shr => ">>".into(),
913+
token::PlusEq => "+=".into(),
914+
token::MinusEq => "-=".into(),
915+
token::StarEq => "*=".into(),
916+
token::SlashEq => "/=".into(),
917+
token::PercentEq => "%=".into(),
918+
token::CaretEq => "^=".into(),
919+
token::AndEq => "&=".into(),
920+
token::OrEq => "|=".into(),
921+
token::ShlEq => "<<=".into(),
922+
token::ShrEq => ">>=".into(),
922923

923924
/* Structural symbols */
924925
token::At => "@".into(),

‎compiler/rustc_expand/src/config.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -328,7 +328,7 @@ impl<'a> StripUnconfigured<'a> {
328328

329329
// For inner attributes, we do the same thing for the `!` in `#![attr]`.
330330
let mut trees = if cfg_attr.style == AttrStyle::Inner {
331-
let Some(TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }, _)) =
331+
let Some(TokenTree::Token(bang_token @ Token { kind: TokenKind::Bang, .. }, _)) =
332332
orig_trees.next()
333333
else {
334334
panic!("Bad tokens for attribute {cfg_attr:?}");

‎compiler/rustc_expand/src/mbe/macro_check.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -432,7 +432,7 @@ fn check_nested_occurrences(
432432
}
433433
(
434434
NestedMacroState::MacroRules,
435-
&TokenTree::Token(Token { kind: TokenKind::Not, .. }),
435+
&TokenTree::Token(Token { kind: TokenKind::Bang, .. }),
436436
) => {
437437
state = NestedMacroState::MacroRulesNot;
438438
}

‎compiler/rustc_expand/src/mbe/macro_rules.rs

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -690,7 +690,7 @@ fn has_compile_error_macro(rhs: &mbe::TokenTree) -> bool {
690690
&& let TokenKind::Ident(ident, _) = ident.kind
691691
&& ident == sym::compile_error
692692
&& let mbe::TokenTree::Token(bang) = bang
693-
&& let TokenKind::Not = bang.kind
693+
&& let TokenKind::Bang = bang.kind
694694
&& let mbe::TokenTree::Delimited(.., del) = args
695695
&& !del.delim.skip()
696696
{
@@ -1135,7 +1135,7 @@ fn check_matcher_core<'tt>(
11351135
&& matches!(kind, NonterminalKind::Pat(PatParam { inferred: true }))
11361136
&& matches!(
11371137
next_token,
1138-
TokenTree::Token(token) if *token == BinOp(token::BinOpToken::Or)
1138+
TokenTree::Token(token) if *token == token::Or
11391139
)
11401140
{
11411141
// It is suggestion to use pat_param, for example: $x:pat -> $x:pat_param.
@@ -1177,7 +1177,7 @@ fn check_matcher_core<'tt>(
11771177

11781178
if kind == NonterminalKind::Pat(PatWithOr)
11791179
&& sess.psess.edition.at_least_rust_2021()
1180-
&& next_token.is_token(&BinOp(token::BinOpToken::Or))
1180+
&& next_token.is_token(&token::Or)
11811181
{
11821182
let suggestion = quoted_tt_to_string(&TokenTree::MetaVarDecl(
11831183
span,
@@ -1296,7 +1296,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
12961296
const TOKENS: &[&str] = &["`=>`", "`,`", "`=`", "`|`", "`if`", "`in`"];
12971297
match tok {
12981298
TokenTree::Token(token) => match token.kind {
1299-
FatArrow | Comma | Eq | BinOp(token::Or) => IsInFollow::Yes,
1299+
FatArrow | Comma | Eq | Or => IsInFollow::Yes,
13001300
Ident(name, IdentIsRaw::No) if name == kw::If || name == kw::In => {
13011301
IsInFollow::Yes
13021302
}
@@ -1332,9 +1332,9 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
13321332
| Colon
13331333
| Eq
13341334
| Gt
1335-
| BinOp(token::Shr)
1335+
| Shr
13361336
| Semi
1337-
| BinOp(token::Or) => IsInFollow::Yes,
1337+
| Or => IsInFollow::Yes,
13381338
Ident(name, IdentIsRaw::No) if name == kw::As || name == kw::Where => {
13391339
IsInFollow::Yes
13401340
}

‎compiler/rustc_expand/src/mbe/quoted.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -302,8 +302,8 @@ fn parse_tree<'a>(
302302
/// `None`.
303303
fn kleene_op(token: &Token) -> Option<KleeneOp> {
304304
match token.kind {
305-
token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore),
306-
token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore),
305+
token::Star => Some(KleeneOp::ZeroOrMore),
306+
token::Plus => Some(KleeneOp::OneOrMore),
307307
token::Question => Some(KleeneOp::ZeroOrOne),
308308
_ => None,
309309
}

‎compiler/rustc_expand/src/proc_macro_server.rs

Lines changed: 33 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -180,28 +180,28 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
180180
Gt => op(">"),
181181
AndAnd => op("&&"),
182182
OrOr => op("||"),
183-
Not => op("!"),
183+
Bang => op("!"),
184184
Tilde => op("~"),
185-
BinOp(Plus) => op("+"),
186-
BinOp(Minus) => op("-"),
187-
BinOp(Star) => op("*"),
188-
BinOp(Slash) => op("/"),
189-
BinOp(Percent) => op("%"),
190-
BinOp(Caret) => op("^"),
191-
BinOp(And) => op("&"),
192-
BinOp(Or) => op("|"),
193-
BinOp(Shl) => op("<<"),
194-
BinOp(Shr) => op(">>"),
195-
BinOpEq(Plus) => op("+="),
196-
BinOpEq(Minus) => op("-="),
197-
BinOpEq(Star) => op("*="),
198-
BinOpEq(Slash) => op("/="),
199-
BinOpEq(Percent) => op("%="),
200-
BinOpEq(Caret) => op("^="),
201-
BinOpEq(And) => op("&="),
202-
BinOpEq(Or) => op("|="),
203-
BinOpEq(Shl) => op("<<="),
204-
BinOpEq(Shr) => op(">>="),
185+
Plus => op("+"),
186+
Minus => op("-"),
187+
Star => op("*"),
188+
Slash => op("/"),
189+
Percent => op("%"),
190+
Caret => op("^"),
191+
And => op("&"),
192+
Or => op("|"),
193+
Shl => op("<<"),
194+
Shr => op(">>"),
195+
PlusEq => op("+="),
196+
MinusEq => op("-="),
197+
StarEq => op("*="),
198+
SlashEq => op("/="),
199+
PercentEq => op("%="),
200+
CaretEq => op("^="),
201+
AndEq => op("&="),
202+
OrEq => op("|="),
203+
ShlEq => op("<<="),
204+
ShrEq => op(">>="),
205205
At => op("@"),
206206
Dot => op("."),
207207
DotDot => op(".."),
@@ -322,16 +322,16 @@ impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
322322
b'=' => Eq,
323323
b'<' => Lt,
324324
b'>' => Gt,
325-
b'!' => Not,
325+
b'!' => Bang,
326326
b'~' => Tilde,
327-
b'+' => BinOp(Plus),
328-
b'-' => BinOp(Minus),
329-
b'*' => BinOp(Star),
330-
b'/' => BinOp(Slash),
331-
b'%' => BinOp(Percent),
332-
b'^' => BinOp(Caret),
333-
b'&' => BinOp(And),
334-
b'|' => BinOp(Or),
327+
b'+' => Plus,
328+
b'-' => Minus,
329+
b'*' => Star,
330+
b'/' => Slash,
331+
b'%' => Percent,
332+
b'^' => Caret,
333+
b'&' => And,
334+
b'|' => Or,
335335
b'@' => At,
336336
b'.' => Dot,
337337
b',' => Comma,
@@ -372,10 +372,9 @@ impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
372372
suffix,
373373
span,
374374
}) if symbol.as_str().starts_with('-') => {
375-
let minus = BinOp(BinOpToken::Minus);
376375
let symbol = Symbol::intern(&symbol.as_str()[1..]);
377376
let integer = TokenKind::lit(token::Integer, symbol, suffix);
378-
let a = tokenstream::TokenTree::token_joint_hidden(minus, span);
377+
let a = tokenstream::TokenTree::token_joint_hidden(Minus, span);
379378
let b = tokenstream::TokenTree::token_alone(integer, span);
380379
smallvec![a, b]
381380
}
@@ -385,10 +384,9 @@ impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
385384
suffix,
386385
span,
387386
}) if symbol.as_str().starts_with('-') => {
388-
let minus = BinOp(BinOpToken::Minus);
389387
let symbol = Symbol::intern(&symbol.as_str()[1..]);
390388
let float = TokenKind::lit(token::Float, symbol, suffix);
391-
let a = tokenstream::TokenTree::token_joint_hidden(minus, span);
389+
let a = tokenstream::TokenTree::token_joint_hidden(Minus, span);
392390
let b = tokenstream::TokenTree::token_alone(float, span);
393391
smallvec![a, b]
394392
}
@@ -599,10 +597,7 @@ impl server::TokenStream for Rustc<'_, '_> {
599597
Ok(Self::TokenStream::from_iter([
600598
// FIXME: The span of the `-` token is lost when
601599
// parsing, so we cannot faithfully recover it here.
602-
tokenstream::TokenTree::token_joint_hidden(
603-
token::BinOp(token::Minus),
604-
e.span,
605-
),
600+
tokenstream::TokenTree::token_joint_hidden(token::Minus, e.span),
606601
tokenstream::TokenTree::token_alone(token::Literal(*token_lit), e.span),
607602
]))
608603
}

‎compiler/rustc_parse/src/lexer/mod.rs

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -384,17 +384,17 @@ impl<'psess, 'src> Lexer<'psess, 'src> {
384384
rustc_lexer::TokenKind::Colon => token::Colon,
385385
rustc_lexer::TokenKind::Dollar => token::Dollar,
386386
rustc_lexer::TokenKind::Eq => token::Eq,
387-
rustc_lexer::TokenKind::Bang => token::Not,
387+
rustc_lexer::TokenKind::Bang => token::Bang,
388388
rustc_lexer::TokenKind::Lt => token::Lt,
389389
rustc_lexer::TokenKind::Gt => token::Gt,
390-
rustc_lexer::TokenKind::Minus => token::BinOp(token::Minus),
391-
rustc_lexer::TokenKind::And => token::BinOp(token::And),
392-
rustc_lexer::TokenKind::Or => token::BinOp(token::Or),
393-
rustc_lexer::TokenKind::Plus => token::BinOp(token::Plus),
394-
rustc_lexer::TokenKind::Star => token::BinOp(token::Star),
395-
rustc_lexer::TokenKind::Slash => token::BinOp(token::Slash),
396-
rustc_lexer::TokenKind::Caret => token::BinOp(token::Caret),
397-
rustc_lexer::TokenKind::Percent => token::BinOp(token::Percent),
390+
rustc_lexer::TokenKind::Minus => token::Minus,
391+
rustc_lexer::TokenKind::And => token::And,
392+
rustc_lexer::TokenKind::Or => token::Or,
393+
rustc_lexer::TokenKind::Plus => token::Plus,
394+
rustc_lexer::TokenKind::Star => token::Star,
395+
rustc_lexer::TokenKind::Slash => token::Slash,
396+
rustc_lexer::TokenKind::Caret => token::Caret,
397+
rustc_lexer::TokenKind::Percent => token::Percent,
398398

399399
rustc_lexer::TokenKind::Unknown | rustc_lexer::TokenKind::InvalidIdent => {
400400
// Don't emit diagnostics for sequences of the same invalid token

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /