Skip to content
This repository has been archived by the owner on Oct 6, 2024. It is now read-only.

Commit

Permalink
Factor segment parsing to module
Browse files Browse the repository at this point in the history
  • Loading branch information
dtolnay committed Oct 21, 2020
1 parent 5fd1855 commit fe7898b
Show file tree
Hide file tree
Showing 2 changed files with 143 additions and 140 deletions.
145 changes: 5 additions & 140 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -142,13 +142,13 @@ extern crate proc_macro;

mod doc;
mod error;
mod segment;

use crate::doc::{do_paste_doc, is_pasted_doc};
use crate::error::{Error, Result};
use proc_macro::{
token_stream, Delimiter, Group, Ident, Punct, Spacing, Span, TokenStream, TokenTree,
};
use std::iter::{self, FromIterator, Peekable};
use crate::segment::Segment;
use proc_macro::{Delimiter, Group, Ident, Punct, Spacing, Span, TokenStream, TokenTree};
use std::iter::{self, FromIterator};
use std::panic;

#[proc_macro]
Expand Down Expand Up @@ -302,22 +302,6 @@ fn is_flat_group(input: &TokenStream) -> bool {
state == State::Ident || state == State::Literal || state == State::Lifetime
}

struct LitStr {
value: String,
span: Span,
}

struct Colon {
span: Span,
}

enum Segment {
String(String),
Apostrophe(Span),
Env(LitStr),
Modifier(Colon, Ident),
}

fn is_paste_operation(input: &TokenStream) -> bool {
let mut tokens = input.clone().into_iter();

Expand Down Expand Up @@ -347,7 +331,7 @@ fn parse_bracket_as_segments(input: TokenStream, scope: Span) -> Result<Vec<Segm
None => return Err(Error::new(scope, "expected `[< ... >]`")),
}

let segments = parse_segments(&mut tokens)?;
let segments = segment::parse(&mut tokens)?;

match &tokens.next() {
Some(TokenTree::Punct(punct)) if punct.as_char() == '>' => {}
Expand All @@ -364,125 +348,6 @@ fn parse_bracket_as_segments(input: TokenStream, scope: Span) -> Result<Vec<Segm
}
}

fn parse_segments(tokens: &mut Peekable<token_stream::IntoIter>) -> Result<Vec<Segment>> {
let mut segments = Vec::new();
while match tokens.peek() {
None => false,
Some(TokenTree::Punct(punct)) => punct.as_char() != '>',
Some(_) => true,
} {
match tokens.next().unwrap() {
TokenTree::Ident(ident) => {
let mut fragment = ident.to_string();
if fragment.starts_with("r#") {
fragment = fragment.split_off(2);
}
if fragment == "env"
&& match tokens.peek() {
Some(TokenTree::Punct(punct)) => punct.as_char() == '!',
_ => false,
}
{
let bang = tokens.next().unwrap(); // `!`
let expect_group = tokens.next();
let parenthesized = match &expect_group {
Some(TokenTree::Group(group))
if group.delimiter() == Delimiter::Parenthesis =>
{
group
}
Some(wrong) => return Err(Error::new(wrong.span(), "expected `(`")),
None => {
return Err(Error::new2(
ident.span(),
bang.span(),
"expected `(` after `env!`",
));
}
};
let mut inner = parenthesized.stream().into_iter();
let lit = match inner.next() {
Some(TokenTree::Literal(lit)) => lit,
Some(wrong) => {
return Err(Error::new(wrong.span(), "expected string literal"))
}
None => {
return Err(Error::new2(
ident.span(),
parenthesized.span(),
"expected string literal as argument to env! macro",
))
}
};
let lit_string = lit.to_string();
if lit_string.starts_with('"')
&& lit_string.ends_with('"')
&& lit_string.len() >= 2
{
// TODO: maybe handle escape sequences in the string if
// someone has a use case.
segments.push(Segment::Env(LitStr {
value: lit_string[1..lit_string.len() - 1].to_owned(),
span: lit.span(),
}));
} else {
return Err(Error::new(lit.span(), "expected string literal"));
}
if let Some(unexpected) = inner.next() {
return Err(Error::new(
unexpected.span(),
"unexpected token in env! macro",
));
}
} else {
segments.push(Segment::String(fragment));
}
}
TokenTree::Literal(lit) => {
let mut lit_string = lit.to_string();
if lit_string.contains(&['#', '\\', '.', '+'][..]) {
return Err(Error::new(lit.span(), "unsupported literal"));
}
lit_string = lit_string
.replace('"', "")
.replace('\'', "")
.replace('-', "_");
segments.push(Segment::String(lit_string));
}
TokenTree::Punct(punct) => match punct.as_char() {
'_' => segments.push(Segment::String("_".to_owned())),
'\'' => segments.push(Segment::Apostrophe(punct.span())),
':' => {
let colon_span = punct.span();
let colon = Colon { span: colon_span };
let ident = match tokens.next() {
Some(TokenTree::Ident(ident)) => ident,
wrong => {
let span = wrong.as_ref().map_or(colon_span, TokenTree::span);
return Err(Error::new(span, "expected identifier after `:`"));
}
};
segments.push(Segment::Modifier(colon, ident));
}
_ => return Err(Error::new(punct.span(), "unexpected punct")),
},
TokenTree::Group(group) => {
if group.delimiter() == Delimiter::None {
let mut inner = group.stream().into_iter().peekable();
let nested = parse_segments(&mut inner)?;
if let Some(unexpected) = inner.next() {
return Err(Error::new(unexpected.span(), "unexpected token"));
}
segments.extend(nested);
} else {
return Err(Error::new(group.span(), "unexpected token"));
}
}
}
}
Ok(segments)
}

fn paste_segments(span: Span, segments: &[Segment]) -> Result<TokenStream> {
let mut evaluated = Vec::new();
let mut is_lifetime = false;
Expand Down
138 changes: 138 additions & 0 deletions src/segment.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
use crate::error::{Error, Result};
use proc_macro::{token_stream, Delimiter, Ident, Span, TokenTree};
use std::iter::Peekable;

pub(crate) enum Segment {
String(String),
Apostrophe(Span),
Env(LitStr),
Modifier(Colon, Ident),
}

pub(crate) struct LitStr {
pub value: String,
pub span: Span,
}

pub(crate) struct Colon {
pub span: Span,
}

pub(crate) fn parse(tokens: &mut Peekable<token_stream::IntoIter>) -> Result<Vec<Segment>> {
let mut segments = Vec::new();
while match tokens.peek() {
None => false,
Some(TokenTree::Punct(punct)) => punct.as_char() != '>',
Some(_) => true,
} {
match tokens.next().unwrap() {
TokenTree::Ident(ident) => {
let mut fragment = ident.to_string();
if fragment.starts_with("r#") {
fragment = fragment.split_off(2);
}
if fragment == "env"
&& match tokens.peek() {
Some(TokenTree::Punct(punct)) => punct.as_char() == '!',
_ => false,
}
{
let bang = tokens.next().unwrap(); // `!`
let expect_group = tokens.next();
let parenthesized = match &expect_group {
Some(TokenTree::Group(group))
if group.delimiter() == Delimiter::Parenthesis =>
{
group
}
Some(wrong) => return Err(Error::new(wrong.span(), "expected `(`")),
None => {
return Err(Error::new2(
ident.span(),
bang.span(),
"expected `(` after `env!`",
));
}
};
let mut inner = parenthesized.stream().into_iter();
let lit = match inner.next() {
Some(TokenTree::Literal(lit)) => lit,
Some(wrong) => {
return Err(Error::new(wrong.span(), "expected string literal"))
}
None => {
return Err(Error::new2(
ident.span(),
parenthesized.span(),
"expected string literal as argument to env! macro",
))
}
};
let lit_string = lit.to_string();
if lit_string.starts_with('"')
&& lit_string.ends_with('"')
&& lit_string.len() >= 2
{
// TODO: maybe handle escape sequences in the string if
// someone has a use case.
segments.push(Segment::Env(LitStr {
value: lit_string[1..lit_string.len() - 1].to_owned(),
span: lit.span(),
}));
} else {
return Err(Error::new(lit.span(), "expected string literal"));
}
if let Some(unexpected) = inner.next() {
return Err(Error::new(
unexpected.span(),
"unexpected token in env! macro",
));
}
} else {
segments.push(Segment::String(fragment));
}
}
TokenTree::Literal(lit) => {
let mut lit_string = lit.to_string();
if lit_string.contains(&['#', '\\', '.', '+'][..]) {
return Err(Error::new(lit.span(), "unsupported literal"));
}
lit_string = lit_string
.replace('"', "")
.replace('\'', "")
.replace('-', "_");
segments.push(Segment::String(lit_string));
}
TokenTree::Punct(punct) => match punct.as_char() {
'_' => segments.push(Segment::String("_".to_owned())),
'\'' => segments.push(Segment::Apostrophe(punct.span())),
':' => {
let colon_span = punct.span();
let colon = Colon { span: colon_span };
let ident = match tokens.next() {
Some(TokenTree::Ident(ident)) => ident,
wrong => {
let span = wrong.as_ref().map_or(colon_span, TokenTree::span);
return Err(Error::new(span, "expected identifier after `:`"));
}
};
segments.push(Segment::Modifier(colon, ident));
}
_ => return Err(Error::new(punct.span(), "unexpected punct")),
},
TokenTree::Group(group) => {
if group.delimiter() == Delimiter::None {
let mut inner = group.stream().into_iter().peekable();
let nested = parse(&mut inner)?;
if let Some(unexpected) = inner.next() {
return Err(Error::new(unexpected.span(), "unexpected token"));
}
segments.extend(nested);
} else {
return Err(Error::new(group.span(), "unexpected token"));
}
}
}
}
Ok(segments)
}

0 comments on commit fe7898b

Please sign in to comment.