diff --git a/Cargo.toml b/Cargo.toml index 2fe0464b..7ceb7a8e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "proc-macro2" -version = "0.2.3" # remember to update html_root_url +version = "0.3.0" # remember to update html_root_url authors = ["Alex Crichton "] license = "MIT/Apache-2.0" readme = "README.md" diff --git a/src/lib.rs b/src/lib.rs index 1d9f0cfd..941fcd46 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -24,7 +24,7 @@ //! [ts]: https://doc.rust-lang.org/proc_macro/struct.TokenStream.html // Proc-macro2 types in rustdoc of other crates get linked to here. -#![doc(html_root_url = "https://docs.rs/proc-macro2/0.2.3")] +#![doc(html_root_url = "https://docs.rs/proc-macro2/0.3.0")] #![cfg_attr(feature = "nightly", feature(proc_macro))] @@ -35,8 +35,10 @@ extern crate proc_macro; extern crate unicode_xid; use std::fmt; -use std::str::FromStr; use std::iter::FromIterator; +use std::marker; +use std::rc::Rc; +use std::str::FromStr; #[macro_use] #[cfg(not(feature = "nightly"))] @@ -49,48 +51,62 @@ mod imp; #[cfg(feature = "nightly")] mod imp; -#[macro_use] -mod macros; - #[derive(Clone)] -pub struct TokenStream(imp::TokenStream); +pub struct TokenStream { + inner: imp::TokenStream, + _marker: marker::PhantomData>, +} + +pub struct LexError { + inner: imp::LexError, + _marker: marker::PhantomData>, +} + +impl TokenStream { + fn _new(inner: imp::TokenStream) -> TokenStream { + TokenStream { + inner, + _marker: marker::PhantomData, + } + } -pub struct LexError(imp::LexError); + pub fn empty() -> TokenStream { + TokenStream::_new(imp::TokenStream::empty()) + } + + pub fn is_empty(&self) -> bool { + self.inner.is_empty() + } +} impl FromStr for TokenStream { type Err = LexError; fn from_str(src: &str) -> Result { - match src.parse() { - Ok(e) => Ok(TokenStream(e)), - Err(e) => Err(LexError(e)), - } + let e = src.parse().map_err(|e| { + LexError { inner: e, _marker: marker::PhantomData } + })?; + Ok(TokenStream::_new(e)) } } #[cfg(feature = "proc-macro")] impl From for TokenStream { fn from(inner: proc_macro::TokenStream) -> TokenStream { - TokenStream(inner.into()) + TokenStream::_new(inner.into()) } } #[cfg(feature = "proc-macro")] impl From for proc_macro::TokenStream { fn from(inner: TokenStream) -> proc_macro::TokenStream { - inner.0.into() + inner.inner.into() } } -impl From for TokenStream { - fn from(tree: TokenTree) -> TokenStream { - TokenStream(tree.into()) - } -} - -impl> FromIterator for TokenStream { - fn from_iter>(streams: I) -> Self { - TokenStream(streams.into_iter().map(|t| t.into().0).collect()) +impl FromIterator for TokenStream { + fn from_iter>(streams: I) -> Self { + TokenStream::_new(streams.into_iter().collect()) } } @@ -99,17 +115,28 @@ impl IntoIterator for TokenStream { type IntoIter = TokenTreeIter; fn into_iter(self) -> TokenTreeIter { - TokenTreeIter(self.0.into_iter()) + TokenTreeIter { + inner: self.inner.into_iter(), + _marker: marker::PhantomData, + } } } -impl TokenStream { - pub fn empty() -> TokenStream { - TokenStream(imp::TokenStream::empty()) +impl fmt::Display for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.inner.fmt(f) } +} - pub fn is_empty(&self) -> bool { - self.0.is_empty() +impl fmt::Debug for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.inner.fmt(f) + } +} + +impl fmt::Debug for LexError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.inner.fmt(f) } } @@ -153,83 +180,145 @@ pub struct LineColumn { pub column: usize, } -#[derive(Copy, Clone, PartialEq, Eq)] -pub struct Span(imp::Span); +#[derive(Copy, Clone)] +#[cfg_attr(procmacro2_semver_exempt, derive(PartialEq, Eq))] +pub struct Span { + inner: imp::Span, + _marker: marker::PhantomData>, +} impl Span { + fn _new(inner: imp::Span) -> Span { + Span { + inner, + _marker: marker::PhantomData, + } + } + pub fn call_site() -> Span { - Span(imp::Span::call_site()) + Span::_new(imp::Span::call_site()) } + #[cfg(procmacro2_semver_exempt)] pub fn def_site() -> Span { - Span(imp::Span::def_site()) + Span::_new(imp::Span::def_site()) } /// Creates a new span with the same line/column information as `self` but /// that resolves symbols as though it were at `other`. pub fn resolved_at(&self, other: Span) -> Span { - Span(self.0.resolved_at(other.0)) + Span::_new(self.inner.resolved_at(other.inner)) } /// Creates a new span with the same name resolution behavior as `self` but /// with the line/column information of `other`. pub fn located_at(&self, other: Span) -> Span { - Span(self.0.located_at(other.0)) + Span::_new(self.inner.located_at(other.inner)) } /// This method is only available when the `"nightly"` feature is enabled. #[cfg(all(feature = "nightly", feature = "proc-macro"))] pub fn unstable(self) -> proc_macro::Span { - self.0.unstable() + self.inner.unstable() } #[cfg(procmacro2_semver_exempt)] pub fn source_file(&self) -> SourceFile { - SourceFile(self.0.source_file()) + SourceFile(self.inner.source_file()) } #[cfg(procmacro2_semver_exempt)] pub fn start(&self) -> LineColumn { - let imp::LineColumn{ line, column } = self.0.start(); + let imp::LineColumn{ line, column } = self.inner.start(); LineColumn { line: line, column: column } } #[cfg(procmacro2_semver_exempt)] pub fn end(&self) -> LineColumn { - let imp::LineColumn{ line, column } = self.0.end(); + let imp::LineColumn{ line, column } = self.inner.end(); LineColumn { line: line, column: column } } #[cfg(procmacro2_semver_exempt)] pub fn join(&self, other: Span) -> Option { - self.0.join(other.0).map(Span) + self.inner.join(other.inner).map(Span::_new) + } +} + +impl fmt::Debug for Span { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.inner.fmt(f) } } #[derive(Clone, Debug)] -pub struct TokenTree { - pub span: Span, - pub kind: TokenNode, +pub enum TokenTree { + Group(Group), + Term(Term), + Op(Op), + Literal(Literal), +} + +impl TokenTree { + pub fn span(&self) -> Span { + match *self { + TokenTree::Group(ref t) => t.span(), + TokenTree::Term(ref t) => t.span(), + TokenTree::Op(ref t) => t.span(), + TokenTree::Literal(ref t) => t.span(), + } + } + + pub fn set_span(&mut self, span: Span) { + match *self { + TokenTree::Group(ref mut t) => t.set_span(span), + TokenTree::Term(ref mut t) => t.set_span(span), + TokenTree::Op(ref mut t) => t.set_span(span), + TokenTree::Literal(ref mut t) => t.set_span(span), + } + } +} + +impl From for TokenTree { + fn from(g: Group) -> TokenTree { + TokenTree::Group(g) + } +} + +impl From for TokenTree { + fn from(g: Term) -> TokenTree { + TokenTree::Term(g) + } } -impl From for TokenTree { - fn from(kind: TokenNode) -> TokenTree { - TokenTree { span: Span::def_site(), kind: kind } +impl From for TokenTree { + fn from(g: Op) -> TokenTree { + TokenTree::Op(g) + } +} + +impl From for TokenTree { + fn from(g: Literal) -> TokenTree { + TokenTree::Literal(g) } } impl fmt::Display for TokenTree { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - TokenStream::from(self.clone()).fmt(f) + match *self { + TokenTree::Group(ref t) => t.fmt(f), + TokenTree::Term(ref t) => t.fmt(f), + TokenTree::Op(ref t) => t.fmt(f), + TokenTree::Literal(ref t) => t.fmt(f), + } } } #[derive(Clone, Debug)] -pub enum TokenNode { - Group(Delimiter, TokenStream), - Term(Term), - Op(char, Spacing), - Literal(Literal), +pub struct Group { + delimiter: Delimiter, + stream: TokenStream, + span: Span, } #[derive(Copy, Clone, Debug, Eq, PartialEq)] @@ -240,105 +329,255 @@ pub enum Delimiter { None, } -#[derive(Copy, Clone)] -pub struct Term(imp::Term); +impl Group { + pub fn new(delimiter: Delimiter, stream: TokenStream) -> Group { + Group { delimiter, stream, span: Span::call_site() } + } -impl Term { - pub fn intern(string: &str) -> Term { - Term(imp::Term::intern(string)) + pub fn delimiter(&self) -> Delimiter { + self.delimiter } - pub fn as_str(&self) -> &str { - self.0.as_str() + pub fn stream(&self) -> &TokenStream { + &self.stream + } + + pub fn span(&self) -> Span { + self.span + } + + pub fn set_span(&mut self, span: Span) { + self.span = span; } } +impl fmt::Display for Group { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.stream.fmt(f) + } +} + +#[derive(Copy, Clone, Debug)] +pub struct Op { + op: char, + spacing: Spacing, + span: Span, +} + #[derive(Copy, Clone, Debug, Eq, PartialEq)] pub enum Spacing { Alone, Joint, } +impl Op { + pub fn new(op: char, spacing: Spacing) -> Op { + Op { op, spacing, span: Span::call_site() } + } + + pub fn op(&self) -> char { + self.op + } + + pub fn spacing(&self) -> Spacing { + self.spacing + } + + pub fn span(&self) -> Span { + self.span + } + + pub fn set_span(&mut self, span: Span) { + self.span = span; + } +} + +impl fmt::Display for Op { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.op.fmt(f) + } +} + +#[derive(Copy, Clone)] +pub struct Term { + inner: imp::Term, + span: Span, + _marker: marker::PhantomData>, +} + +impl Term { + fn _new(inner: imp::Term, span: Span) -> Term { + Term { + inner, + span, + _marker: marker::PhantomData, + } + } + + pub fn new(string: &str, span: Span) -> Term { + Term::_new(imp::Term::intern(string), span) + } + + pub fn as_str(&self) -> &str { + self.inner.as_str() + } + + pub fn span(&self) -> Span { + self.span + } + + pub fn set_span(&mut self, span: Span) { + self.span = span; + } +} + +impl fmt::Display for Term { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.as_str().fmt(f) + } +} + +impl fmt::Debug for Term { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.inner.fmt(f) + } +} + #[derive(Clone)] -pub struct Literal(imp::Literal); +pub struct Literal { + inner: imp::Literal, + span: Span, + _marker: marker::PhantomData>, +} + +macro_rules! suffixed_int_literals { + ($($name:ident => $kind:ident,)*) => ($( + #[allow(unused_comparisons)] + pub fn $name(n: $kind) -> Literal { + assert!(n >= 0); + Literal::_new(n.into()) + } + )*) +} -macro_rules! int_literals { - ($($kind:ident,)*) => ($( - pub fn $kind(n: $kind) -> Literal { - Literal(n.into()) +macro_rules! unsuffixed_int_literals { + ($($name:ident => $kind:ident,)*) => ($( + #[allow(unused_comparisons)] + pub fn $name(n: $kind) -> Literal { + assert!(n >= 0); + Literal::_new(imp::Literal::integer(n as i64)) } )*) } impl Literal { - pub fn integer(s: i64) -> Literal { - Literal(imp::Literal::integer(s)) + fn _new(inner: imp::Literal) -> Literal { + Literal { + inner, + span: Span::call_site(), + _marker: marker::PhantomData, + } + } + + suffixed_int_literals! { + u8_suffixed => u8, + u16_suffixed => u16, + u32_suffixed => u32, + u64_suffixed => u64, + usize_suffixed => usize, + i8_suffixed => i8, + i16_suffixed => i16, + i32_suffixed => i32, + i64_suffixed => i64, + isize_suffixed => isize, } - int_literals! { - u8, u16, u32, u64, usize, - i8, i16, i32, i64, isize, + unsuffixed_int_literals! { + u8_unsuffixed => u8, + u16_unsuffixed => u16, + u32_unsuffixed => u32, + u64_unsuffixed => u64, + usize_unsuffixed => usize, + i8_unsuffixed => i8, + i16_unsuffixed => i16, + i32_unsuffixed => i32, + i64_unsuffixed => i64, + isize_unsuffixed => isize, } - pub fn float(f: f64) -> Literal { - Literal(imp::Literal::float(f)) + pub fn f64_unsuffixed(f: f64) -> Literal { + assert!(f.is_finite()); + assert!(f.is_sign_positive()); + Literal::_new(imp::Literal::float(f)) } - pub fn f64(f: f64) -> Literal { - Literal(f.into()) + pub fn f64_suffixed(f: f64) -> Literal { + assert!(f.is_finite()); + assert!(f.is_sign_positive()); + Literal::_new(f.into()) } - pub fn f32(f: f32) -> Literal { - Literal(f.into()) + pub fn f32_unsuffixed(f: f32) -> Literal { + assert!(f.is_finite()); + assert!(f.is_sign_positive()); + Literal::_new(imp::Literal::float(f as f64)) + } + + pub fn f32_suffixed(f: f32) -> Literal { + assert!(f.is_finite()); + assert!(f.is_sign_positive()); + Literal::_new(f.into()) } pub fn string(string: &str) -> Literal { - Literal(string.into()) + Literal::_new(string.into()) } pub fn character(ch: char) -> Literal { - Literal(ch.into()) + Literal::_new(ch.into()) } pub fn byte_string(s: &[u8]) -> Literal { - Literal(imp::Literal::byte_string(s)) + Literal::_new(imp::Literal::byte_string(s)) } - // ======================================================================= - // Not present upstream in proc_macro yet - - pub fn byte_char(b: u8) -> Literal { - Literal(imp::Literal::byte_char(b)) + pub fn span(&self) -> Span { + self.span } - pub fn doccomment(s: &str) -> Literal { - Literal(imp::Literal::doccomment(s)) + pub fn set_span(&mut self, span: Span) { + self.span = span; } +} - pub fn raw_string(s: &str, pounds: usize) -> Literal { - Literal(imp::Literal::raw_string(s, pounds)) +impl fmt::Debug for Literal { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.inner.fmt(f) } +} - pub fn raw_byte_string(s: &str, pounds: usize) -> Literal { - Literal(imp::Literal::raw_byte_string(s, pounds)) +impl fmt::Display for Literal { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.inner.fmt(f) } } -pub struct TokenTreeIter(imp::TokenTreeIter); +pub struct TokenTreeIter { + inner: imp::TokenTreeIter, + _marker: marker::PhantomData>, +} + impl Iterator for TokenTreeIter { type Item = TokenTree; fn next(&mut self) -> Option { - self.0.next() + self.inner.next() } } -forward_fmt!(Debug for LexError); -forward_fmt!(Debug for Literal); -forward_fmt!(Debug for Span); -forward_fmt!(Debug for Term); -forward_fmt!(Debug for TokenTreeIter); -forward_fmt!(Debug for TokenStream); -forward_fmt!(Display for Literal); -forward_fmt!(Display for TokenStream); +impl fmt::Debug for TokenTreeIter { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.inner.fmt(f) + } +} diff --git a/src/macros.rs b/src/macros.rs deleted file mode 100644 index 89eb554c..00000000 --- a/src/macros.rs +++ /dev/null @@ -1,9 +0,0 @@ -macro_rules! forward_fmt { - ($tr:ident for $ty:ident) => { - impl ::std::fmt::$tr for $ty { - fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { - ::std::fmt::$tr::fmt(&self.0, f) - } - } - } -} diff --git a/src/stable.rs b/src/stable.rs index cb376a95..1d2d9297 100644 --- a/src/stable.rs +++ b/src/stable.rs @@ -1,3 +1,5 @@ +#![allow(dead_code)] + use std::ascii; use std::borrow::Borrow; use std::cell::RefCell; @@ -14,7 +16,7 @@ use std::vec; use unicode_xid::UnicodeXID; use strnom::{Cursor, PResult, skip_whitespace, block_comment, whitespace, word_break}; -use {TokenTree, TokenNode, Delimiter, Spacing}; +use {TokenTree, Delimiter, Spacing, Group, Op}; #[derive(Clone, Debug)] pub struct TokenStream { @@ -67,7 +69,7 @@ impl FromStr for TokenStream { if skip_whitespace(input).len() != 0 { Err(LexError) } else { - Ok(output.0) + Ok(output.inner) } } Err(LexError) => Err(LexError), @@ -83,32 +85,32 @@ impl fmt::Display for TokenStream { write!(f, " ")?; } joint = false; - match tt.kind { - TokenNode::Group(delim, ref stream) => { - let (start, end) = match delim { + match *tt { + TokenTree::Group(ref g) => { + let (start, end) = match g.delimiter() { Delimiter::Parenthesis => ("(", ")"), Delimiter::Brace => ("{", "}"), Delimiter::Bracket => ("[", "]"), Delimiter::None => ("", ""), }; - if stream.0.inner.len() == 0 { + if g.stream().inner.inner.len() == 0 { write!(f, "{} {}", start, end)? } else { - write!(f, "{} {} {}", start, stream, end)? + write!(f, "{} {} {}", start, g.stream(), end)? } } - TokenNode::Term(ref sym) => write!(f, "{}", sym.as_str())?, - TokenNode::Op(ch, ref op) => { - write!(f, "{}", ch)?; - match *op { + TokenTree::Term(ref sym) => write!(f, "{}", sym.as_str())?, + TokenTree::Op(ref op) => { + write!(f, "{}", op.op())?; + match op.spacing() { Spacing::Alone => {} Spacing::Joint => joint = true, } } - TokenNode::Literal(ref literal) => { + TokenTree::Literal(ref literal) => { write!(f, "{}", literal)?; // handle comments - if (literal.0).0.starts_with("/") { + if literal.inner.0.starts_with("/") { write!(f, "\n")?; } } @@ -140,12 +142,12 @@ impl From for TokenStream { } } -impl iter::FromIterator for TokenStream { - fn from_iter>(streams: I) -> Self { +impl iter::FromIterator for TokenStream { + fn from_iter>(streams: I) -> Self { let mut v = Vec::new(); - for stream in streams.into_iter() { - v.extend(stream.inner); + for token in streams.into_iter() { + v.push(token); } TokenStream { inner: v } @@ -589,64 +591,58 @@ impl From for Literal { named!(token_stream -> ::TokenStream, map!( many0!(token_tree), - |trees| ::TokenStream(TokenStream { inner: trees }) + |trees| ::TokenStream::_new(TokenStream { inner: trees }) )); #[cfg(not(procmacro2_semver_exempt))] fn token_tree(input: Cursor) -> PResult { - let (input, kind) = token_kind(input)?; - Ok((input, TokenTree { - span: ::Span(Span {}), - kind: kind, - })) + token_kind(input) } #[cfg(procmacro2_semver_exempt)] fn token_tree(input: Cursor) -> PResult { let input = skip_whitespace(input); let lo = input.off; - let (input, kind) = token_kind(input)?; + let (input, mut token) = token_kind(input)?; let hi = input.off; - Ok((input, TokenTree { - span: ::Span(Span { - lo: lo, - hi: hi, - }), - kind: kind, - })) + token.set_span(::Span::_new(Span { + lo: lo, + hi: hi, + })); + Ok((input, token)) } -named!(token_kind -> TokenNode, alt!( - map!(delimited, |(d, s)| TokenNode::Group(d, s)) +named!(token_kind -> TokenTree, alt!( + map!(group, TokenTree::Group) | - map!(literal, TokenNode::Literal) // must be before symbol + map!(literal, TokenTree::Literal) // must be before symbol | symbol | - map!(op, |(op, kind)| TokenNode::Op(op, kind)) + map!(op, TokenTree::Op) )); -named!(delimited -> (Delimiter, ::TokenStream), alt!( +named!(group -> Group, alt!( delimited!( punct!("("), token_stream, punct!(")") - ) => { |ts| (Delimiter::Parenthesis, ts) } + ) => { |ts| Group::new(Delimiter::Parenthesis, ts) } | delimited!( punct!("["), token_stream, punct!("]") - ) => { |ts| (Delimiter::Bracket, ts) } + ) => { |ts| Group::new(Delimiter::Bracket, ts) } | delimited!( punct!("{"), token_stream, punct!("}") - ) => { |ts| (Delimiter::Brace, ts) } + ) => { |ts| Group::new(Delimiter::Brace, ts) } )); -fn symbol(mut input: Cursor) -> PResult { +fn symbol(mut input: Cursor) -> PResult { input = skip_whitespace(input); let mut chars = input.char_indices(); @@ -674,9 +670,9 @@ fn symbol(mut input: Cursor) -> PResult { } else { let a = &input.rest[..end]; if a == "_" { - Ok((input.advance(end), TokenNode::Op('_', Spacing::Alone))) + Ok((input.advance(end), Op::new('_', Spacing::Alone).into())) } else { - Ok((input.advance(end), TokenNode::Term(::Term::intern(a)))) + Ok((input.advance(end), ::Term::new(a, ::Span::call_site()).into())) } } } @@ -700,7 +696,7 @@ fn literal(input: Cursor) -> PResult<::Literal> { let start = input.len() - input_no_ws.len(); let len = input_no_ws.len() - a.len(); let end = start + len; - Ok((a, ::Literal(Literal(input.rest[start..end].to_string())))) + Ok((a, ::Literal::_new(Literal(input.rest[start..end].to_string())))) } Err(LexError) => Err(LexError), } @@ -1147,7 +1143,7 @@ fn digits(mut input: Cursor) -> PResult<()> { } } -fn op(input: Cursor) -> PResult<(char, Spacing)> { +fn op(input: Cursor) -> PResult { let input = skip_whitespace(input); match op_char(input) { Ok((rest, ch)) => { @@ -1155,7 +1151,7 @@ fn op(input: Cursor) -> PResult<(char, Spacing)> { Ok(_) => Spacing::Joint, Err(LexError) => Spacing::Alone, }; - Ok((rest, (ch, kind))) + Ok((rest, Op::new(ch, kind))) } Err(LexError) => Err(LexError), } diff --git a/src/unstable.rs b/src/unstable.rs index 6f971977..d0b3a002 100644 --- a/src/unstable.rs +++ b/src/unstable.rs @@ -1,3 +1,5 @@ +#![allow(dead_code)] + use std::ascii; use std::fmt; use std::iter; @@ -5,7 +7,7 @@ use std::str::FromStr; use proc_macro; -use {TokenTree, TokenNode, Delimiter, Spacing}; +use {TokenTree, Delimiter, Spacing, Group, Op}; #[derive(Clone)] pub struct TokenStream(proc_macro::TokenStream); @@ -49,40 +51,40 @@ impl From for proc_macro::TokenStream { } impl From for TokenStream { - fn from(tree: TokenTree) -> TokenStream { - TokenStream(proc_macro::TokenTree { - span: (tree.span.0).0, - kind: match tree.kind { - TokenNode::Group(delim, s) => { - let delim = match delim { - Delimiter::Parenthesis => proc_macro::Delimiter::Parenthesis, - Delimiter::Bracket => proc_macro::Delimiter::Bracket, - Delimiter::Brace => proc_macro::Delimiter::Brace, - Delimiter::None => proc_macro::Delimiter::None, - }; - proc_macro::TokenNode::Group(delim, (s.0).0) - } - TokenNode::Op(ch, kind) => { - let kind = match kind { - Spacing::Joint => proc_macro::Spacing::Joint, - Spacing::Alone => proc_macro::Spacing::Alone, - }; - proc_macro::TokenNode::Op(ch, kind) - } - TokenNode::Term(s) => { - proc_macro::TokenNode::Term((s.0).0) - } - TokenNode::Literal(l) => { - proc_macro::TokenNode::Literal((l.0).0) - } - }, - }.into()) - } -} - -impl iter::FromIterator for TokenStream { - fn from_iter>(streams: I) -> Self { - let streams = streams.into_iter().map(|s| s.0); + fn from(token: TokenTree) -> TokenStream { + let (span, kind) = match token { + TokenTree::Group(g) => { + let delim = match g.delimiter() { + Delimiter::Parenthesis => proc_macro::Delimiter::Parenthesis, + Delimiter::Bracket => proc_macro::Delimiter::Bracket, + Delimiter::Brace => proc_macro::Delimiter::Brace, + Delimiter::None => proc_macro::Delimiter::None, + }; + let span = g.span(); + let group = proc_macro::TokenNode::Group(delim, g.stream.inner.0); + (span, group) + } + TokenTree::Op(op) => { + let kind = match op.spacing() { + Spacing::Joint => proc_macro::Spacing::Joint, + Spacing::Alone => proc_macro::Spacing::Alone, + }; + (op.span(), proc_macro::TokenNode::Op(op.op(), kind)) + } + TokenTree::Term(s) => { + (s.span(), proc_macro::TokenNode::Term(s.inner.0)) + } + TokenTree::Literal(l) => { + (l.span(), proc_macro::TokenNode::Literal(l.inner.0)) + } + }; + TokenStream(proc_macro::TokenTree { span: span.inner.0, kind }.into()) + } +} + +impl iter::FromIterator for TokenStream { + fn from_iter>(streams: I) -> Self { + let streams = streams.into_iter().map(TokenStream::from); TokenStream(streams.collect::()) } } @@ -114,36 +116,38 @@ impl Iterator for TokenTreeIter { type Item = TokenTree; fn next(&mut self) -> Option { - let token = match self.0.next() { - Some(n) => n, - None => return None, - }; - Some(TokenTree { - span: ::Span(Span(token.span)), - kind: match token.kind { - proc_macro::TokenNode::Group(delim, s) => { - let delim = match delim { - proc_macro::Delimiter::Parenthesis => Delimiter::Parenthesis, - proc_macro::Delimiter::Bracket => Delimiter::Bracket, - proc_macro::Delimiter::Brace => Delimiter::Brace, - proc_macro::Delimiter::None => Delimiter::None, - }; - TokenNode::Group(delim, ::TokenStream(TokenStream(s))) - } - proc_macro::TokenNode::Op(ch, kind) => { - let kind = match kind { - proc_macro::Spacing::Joint => Spacing::Joint, - proc_macro::Spacing::Alone => Spacing::Alone, - }; - TokenNode::Op(ch, kind) - } - proc_macro::TokenNode::Term(s) => { - TokenNode::Term(::Term(Term(s))) - } - proc_macro::TokenNode::Literal(l) => { - TokenNode::Literal(::Literal(Literal(l))) - } - }, + let token = self.0.next()?; + let span = ::Span::_new(Span(token.span)); + Some(match token.kind { + proc_macro::TokenNode::Group(delim, s) => { + let delim = match delim { + proc_macro::Delimiter::Parenthesis => Delimiter::Parenthesis, + proc_macro::Delimiter::Bracket => Delimiter::Bracket, + proc_macro::Delimiter::Brace => Delimiter::Brace, + proc_macro::Delimiter::None => Delimiter::None, + }; + let stream = ::TokenStream::_new(TokenStream(s)); + let mut g = Group::new(delim, stream); + g.set_span(span); + g.into() + } + proc_macro::TokenNode::Op(ch, kind) => { + let kind = match kind { + proc_macro::Spacing::Joint => Spacing::Joint, + proc_macro::Spacing::Alone => Spacing::Alone, + }; + let mut o = Op::new(ch, kind); + o.span = span; + o.into() + } + proc_macro::TokenNode::Term(s) => { + ::Term::_new(Term(s), span).into() + } + proc_macro::TokenNode::Literal(l) => { + let mut l = ::Literal::_new(Literal(l)); + l.span = span; + l.into() + } }) } @@ -158,11 +162,9 @@ impl fmt::Debug for TokenTreeIter { } } -#[cfg(procmacro2_semver_exempt)] #[derive(Clone, PartialEq, Eq)] pub struct FileName(String); -#[cfg(procmacro2_semver_exempt)] impl fmt::Display for FileName { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { self.0.fmt(f) @@ -171,11 +173,9 @@ impl fmt::Display for FileName { // NOTE: We have to generate our own filename object here because we can't wrap // the one provided by proc_macro. -#[cfg(procmacro2_semver_exempt)] #[derive(Clone, PartialEq, Eq)] pub struct SourceFile(proc_macro::SourceFile, FileName); -#[cfg(procmacro2_semver_exempt)] impl SourceFile { fn new(sf: proc_macro::SourceFile) -> Self { let filename = FileName(sf.path().to_string()); @@ -192,21 +192,18 @@ impl SourceFile { } } -#[cfg(procmacro2_semver_exempt)] impl AsRef for SourceFile { fn as_ref(&self) -> &FileName { self.path() } } -#[cfg(procmacro2_semver_exempt)] impl fmt::Debug for SourceFile { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { self.0.fmt(f) } } -#[cfg(procmacro2_semver_exempt)] pub struct LineColumn { pub line: usize, pub column: usize, @@ -217,7 +214,7 @@ pub struct Span(proc_macro::Span); impl From for ::Span { fn from(proc_span: proc_macro::Span) -> ::Span { - ::Span(Span(proc_span)) + ::Span::_new(Span(proc_span)) } } @@ -242,24 +239,20 @@ impl Span { self.0 } - #[cfg(procmacro2_semver_exempt)] pub fn source_file(&self) -> SourceFile { SourceFile::new(self.0.source_file()) } - #[cfg(procmacro2_semver_exempt)] pub fn start(&self) -> LineColumn { let proc_macro::LineColumn{ line, column } = self.0.start(); LineColumn { line, column } } - #[cfg(procmacro2_semver_exempt)] pub fn end(&self) -> LineColumn { let proc_macro::LineColumn{ line, column } = self.0.end(); LineColumn { line, column } } - #[cfg(procmacro2_semver_exempt)] pub fn join(&self, other: Span) -> Option { self.0.join(other.0).map(Span) } diff --git a/tests/test.rs b/tests/test.rs index 2e481d16..7bd96f0c 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -2,26 +2,19 @@ extern crate proc_macro2; use std::str; -use proc_macro2::{Term, Literal, TokenStream}; - -#[cfg(procmacro2_semver_exempt)] -use proc_macro2::TokenNode; - -#[cfg(procmacro2_semver_exempt)] -#[cfg(not(feature = "nightly"))] -use proc_macro2::Span; +use proc_macro2::{Term, Literal, TokenStream, Span}; #[test] fn symbols() { - assert_eq!(Term::intern("foo").as_str(), "foo"); - assert_eq!(Term::intern("bar").as_str(), "bar"); + assert_eq!(Term::new("foo", Span::call_site()).as_str(), "foo"); + assert_eq!(Term::new("bar", Span::call_site()).as_str(), "bar"); } #[test] fn literals() { assert_eq!(Literal::string("foo").to_string(), "\"foo\""); assert_eq!(Literal::string("\"").to_string(), "\"\\\"\""); - assert_eq!(Literal::float(10.0).to_string(), "10.0"); + assert_eq!(Literal::f32_unsuffixed(10.0).to_string(), "10.0"); } #[test] @@ -78,6 +71,8 @@ fn fail() { #[cfg(procmacro2_semver_exempt)] #[test] fn span_test() { + use proc_macro2::TokenTree; + fn check_spans(p: &str, mut lines: &[(usize, usize, usize, usize)]) { let ts = p.parse::().unwrap(); check_spans_internal(ts, &mut lines); @@ -91,17 +86,18 @@ fn span_test() { if let Some((&(sline, scol, eline, ecol), rest)) = lines.split_first() { *lines = rest; - let start = i.span.start(); + let start = i.span().start(); assert_eq!(start.line, sline, "sline did not match for {}", i); assert_eq!(start.column, scol, "scol did not match for {}", i); - let end = i.span.end(); + let end = i.span().end(); assert_eq!(end.line, eline, "eline did not match for {}", i); assert_eq!(end.column, ecol, "ecol did not match for {}", i); - match i.kind { - TokenNode::Group(_, stream) => - check_spans_internal(stream, lines), + match i { + TokenTree::Group(ref g) => { + check_spans_internal(g.stream().clone(), lines); + } _ => {} } } @@ -146,11 +142,11 @@ fn span_join() { let source2 = "ccc\nddd".parse::().unwrap().into_iter().collect::>(); - assert!(source1[0].span.source_file() != source2[0].span.source_file()); - assert_eq!(source1[0].span.source_file(), source1[1].span.source_file()); + assert!(source1[0].span().source_file() != source2[0].span().source_file()); + assert_eq!(source1[0].span().source_file(), source1[1].span().source_file()); - let joined1 = source1[0].span.join(source1[1].span); - let joined2 = source1[0].span.join(source2[0].span); + let joined1 = source1[0].span().join(source1[1].span()); + let joined2 = source1[0].span().join(source2[0].span()); assert!(joined1.is_some()); assert!(joined2.is_none()); @@ -161,7 +157,7 @@ fn span_join() { assert_eq!(end.line, 2); assert_eq!(end.column, 3); - assert_eq!(joined1.unwrap().source_file(), source1[0].span.source_file()); + assert_eq!(joined1.unwrap().source_file(), source1[0].span().source_file()); } #[test] @@ -179,8 +175,8 @@ fn tricky_doc_commaent() { let stream = "/// doc".parse::().unwrap(); let tokens = stream.into_iter().collect::>(); assert!(tokens.len() == 1, "not length 1 -- {:?}", tokens); - match tokens[0].kind { - proc_macro2::TokenNode::Literal(_) => {} + match tokens[0] { + proc_macro2::TokenTree::Literal(_) => {} _ => panic!("wrong token {:?}", tokens[0]), } }