Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[PERF TEST] Make TokenKind Copy #96764

Closed
wants to merge 1 commit into from
Closed

[PERF TEST] Make TokenKind Copy #96764

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion compiler/rustc_ast/src/mut_visit.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -756,7 +756,10 @@ pub fn visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
return; // Avoid visiting the span for the second time.
}
token::Interpolated(nt) => {
visit_nonterminal(Lrc::make_mut(nt), vis);
let mut nt2 = nt.clone();
visit_nonterminal(&mut nt2, vis);
// njn: skip this step if nt == nt2?
*nt = Box::leak(Box::new(nt2));
}
_ => {}
}
Expand Down
147 changes: 143 additions & 4 deletions compiler/rustc_ast/src/token.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ use crate::ptr::P;
use crate::util::case::Case;

use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
use rustc_data_structures::sync::Lrc;
use rustc_macros::HashStable_Generic;
use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
use rustc_span::symbol::{kw, sym};
#[allow(hidden_glob_reexports)]
use rustc_span::symbol::{Ident, Symbol};
Expand Down Expand Up @@ -226,7 +226,23 @@ fn ident_can_begin_type(name: Symbol, span: Span, is_raw: bool) -> bool {
.contains(&name)
}

#[derive(Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
// njn: comment
// rustc_index::newtype_index! {
// pub struct NtIndex {
// ENCODABLE = custom
// DEBUG_FORMAT = "NtIndex({})"
// }
// }

// njn: comment
// njn: name?
//#[derive(Clone, Copy)]
//pub struct Nt(NtIndex);

//impl Nt {
//}

#[derive(Clone, Copy, PartialEq, Debug, HashStable_Generic)]
pub enum TokenKind {
/* Expression-operator symbols. */
Eq,
Expand Down Expand Up @@ -280,6 +296,7 @@ pub enum TokenKind {
/// treat regular and interpolated lifetime identifiers in the same way.
Lifetime(Symbol),

// njn: update comment
/// An embedded AST node, as produced by a macro. This only exists for
/// historical reasons. We'd like to get rid of it, for multiple reasons.
/// - It's conceptually very strange. Saying a token can contain an AST
Expand All @@ -289,7 +306,8 @@ pub enum TokenKind {
/// - It prevents `Token` from implementing `Copy`.
/// It adds complexity and likely slows things down. Please don't add new
/// occurrences of this token kind!
Interpolated(Lrc<Nonterminal>),
//Interpolated(Nt),
Interpolated(&'static Nonterminal),

/// A doc comment token.
/// `Symbol` is the doc comment's data excluding its "quotes" (`///`, `/**`, etc)
Expand All @@ -299,7 +317,128 @@ pub enum TokenKind {
Eof,
}

#[derive(Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
// njn: necessary?
impl<S: Encoder> Encodable<S> for TokenKind {
fn encode(&self, s: &mut S) {
match self {
Eq => s.emit_usize(0),
Lt => s.emit_usize(1),
Le => s.emit_usize(2),
EqEq => s.emit_usize(3),
Ne => s.emit_usize(4),
Ge => s.emit_usize(5),
Gt => s.emit_usize(6),
AndAnd => s.emit_usize(7),
OrOr => s.emit_usize(8),
Not => s.emit_usize(9),
Tilde => s.emit_usize(10),
BinOp(tok) => {
s.emit_usize(11);
tok.encode(s)
}
BinOpEq(tok) => {
s.emit_usize(12);
tok.encode(s)
}
At => s.emit_usize(13),
Dot => s.emit_usize(14),
DotDot => s.emit_usize(15),
DotDotDot => s.emit_usize(16),
DotDotEq => s.emit_usize(17),
Comma => s.emit_usize(18),
Semi => s.emit_usize(19),
Colon => s.emit_usize(20),
ModSep => s.emit_usize(21),
RArrow => s.emit_usize(22),
LArrow => s.emit_usize(23),
FatArrow => s.emit_usize(24),
Pound => s.emit_usize(25),
Dollar => s.emit_usize(26),
Question => s.emit_usize(27),
SingleQuote => s.emit_usize(28),
OpenDelim(delim) => {
s.emit_usize(29);
delim.encode(s)
}
CloseDelim(delim) => {
s.emit_usize(30);
delim.encode(s)
}
Literal(lit) => {
s.emit_usize(31);
lit.encode(s)
}
Ident(name, is_raw) => {
s.emit_usize(32);
name.encode(s);
is_raw.encode(s)
}
Lifetime(name) => {
s.emit_usize(33);
name.encode(s)
}
Interpolated(_nt) => {
s.emit_usize(34);
panic!("njn: impossible?");
}
DocComment(kind, style, sym) => {
s.emit_usize(35);
kind.encode(s);
style.encode(s);
sym.encode(s)
}
Eof => s.emit_usize(36),
}
}
}

// njn: necessary?
impl<D: Decoder> Decodable<D> for TokenKind {
fn decode(d: &mut D) -> TokenKind {
match d.read_usize() {
0 => Eq,
1 => Lt,
2 => Le,
3 => EqEq,
4 => Ne,
5 => Ge,
6 => Gt,
7 => AndAnd,
8 => OrOr,
9 => Not,
10 => Tilde,
11 => BinOp(Decodable::decode(d)),
12 => BinOpEq(Decodable::decode(d)),
13 => At,
14 => Dot,
15 => DotDot,
16 => DotDotDot,
17 => DotDotEq,
18 => Comma,
19 => Semi,
20 => Colon,
21 => ModSep,
22 => RArrow,
23 => LArrow,
24 => FatArrow,
25 => Pound,
26 => Dollar,
27 => Question,
28 => SingleQuote,
29 => OpenDelim(Decodable::decode(d)),
30 => CloseDelim(Decodable::decode(d)),
31 => Literal(Decodable::decode(d)),
32 => Ident(Decodable::decode(d), Decodable::decode(d)),
33 => Lifetime(Decodable::decode(d)),
34 => panic!("njn: unreachable?"),
35 => DocComment(Decodable::decode(d), Decodable::decode(d), Decodable::decode(d)),
36 => Eof,
_ => panic!("njn: bad"),
}
}
}

#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
pub struct Token {
pub kind: TokenKind,
pub span: Span,
Expand Down
6 changes: 5 additions & 1 deletion compiler/rustc_expand/src/mbe/transcribe.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,11 @@ pub(super) fn transcribe<'a>(
// `Delimiter::Invisible` to maintain parsing priorities.
// `Interpolated` is currently used for such groups in rustc parser.
marker.visit_span(&mut sp);
let token = TokenTree::token_alone(token::Interpolated(nt.clone()), sp);
let token = TokenTree::token_alone(
// njn: nt.clone()?
token::Interpolated(Box::leak(Box::new(nt.clone()))),
sp,
);
result.push(token);
}
MatchedSeq(..) => {
Expand Down
3 changes: 1 addition & 2 deletions compiler/rustc_expand/src/proc_macro.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ use rustc_ast as ast;
use rustc_ast::ptr::P;
use rustc_ast::token;
use rustc_ast::tokenstream::TokenStream;
use rustc_data_structures::sync::Lrc;
use rustc_errors::ErrorGuaranteed;
use rustc_parse::parser::ForceCollect;
use rustc_session::config::ProcMacroExecutionStrategy;
Expand Down Expand Up @@ -126,7 +125,7 @@ impl MultiItemModifier for DeriveProcMacro {
Annotatable::Stmt(stmt) => token::NtStmt(stmt),
_ => unreachable!(),
};
TokenStream::token_alone(token::Interpolated(Lrc::new(nt)), DUMMY_SP)
TokenStream::token_alone(token::Interpolated(Box::leak(Box::new(nt))), DUMMY_SP)
} else {
item.to_tokens()
};
Expand Down