diff --git a/src/librustc_expand/proc_macro_server.rs b/src/librustc_expand/proc_macro_server.rs index 790e1f0edc01b..b41490011af3f 100644 --- a/src/librustc_expand/proc_macro_server.rs +++ b/src/librustc_expand/proc_macro_server.rs @@ -1,5 +1,6 @@ use crate::base::ExtCtxt; +use rustc_parse::lexer::nfc_normalize; use rustc_parse::{nt_to_tokenstream, parse_stream_from_source_str}; use syntax::ast; use syntax::print::pprust; @@ -327,6 +328,7 @@ impl Ident { } } fn new(sym: Symbol, is_raw: bool, span: Span) -> Ident { + let sym = nfc_normalize(&sym.as_str()); let string = sym.as_str(); if !Self::is_valid(&string) { panic!("`{:?}` is not a valid identifier", string) diff --git a/src/librustc_parse/lexer/mod.rs b/src/librustc_parse/lexer/mod.rs index d69cd14d544db..30ec202e0ffe3 100644 --- a/src/librustc_parse/lexer/mod.rs +++ b/src/librustc_parse/lexer/mod.rs @@ -220,7 +220,7 @@ impl<'a> StringReader<'a> { if is_raw_ident { ident_start = ident_start + BytePos(2); } - let sym = self.nfc_symbol_from(ident_start); + let sym = nfc_normalize(self.str_from(ident_start)); if is_raw_ident { let span = self.mk_sp(start, self.pos); if !sym.can_be_raw() { @@ -469,20 +469,6 @@ impl<'a> StringReader<'a> { Symbol::intern(self.str_from_to(start, end)) } - /// As symbol_from, with the text normalized into Unicode NFC form. - fn nfc_symbol_from(&self, start: BytePos) -> Symbol { - use unicode_normalization::{is_nfc_quick, IsNormalized, UnicodeNormalization}; - debug!("taking an normalized ident from {:?} to {:?}", start, self.pos); - let sym = self.str_from(start); - match is_nfc_quick(sym.chars()) { - IsNormalized::Yes => Symbol::intern(sym), - _ => { - let sym_str: String = sym.chars().nfc().collect(); - Symbol::intern(&sym_str) - } - } - } - /// Slice of the source text spanning from `start` up to but excluding `end`. fn str_from_to(&self, start: BytePos, end: BytePos) -> &str { &self.src[self.src_index(start)..self.src_index(end)] @@ -651,3 +637,14 @@ impl<'a> StringReader<'a> { } } } + +pub fn nfc_normalize(string: &str) -> Symbol { + use unicode_normalization::{is_nfc_quick, IsNormalized, UnicodeNormalization}; + match is_nfc_quick(string.chars()) { + IsNormalized::Yes => Symbol::intern(string), + _ => { + let normalized_str: String = string.chars().nfc().collect(); + Symbol::intern(&normalized_str) + } + } +}