Track upstream proc_macro changes
* Rename `Term` to `Ident`
* Rename `Punct` to `Op`
* Remove `Term::as_str`
* Rename `Op::op` to `Punct::as_char`
* `Term::new` no longer accepts lifetimes or raw idents
* Lifetimes are lexed as a `Joint` `'` character followed by an `Ident`
* `Ident::new_raw` is a new `procmacro2_semver_exempt` API for creating raw
identifiers.
diff --git a/src/lib.rs b/src/lib.rs
index b7fbb14..286bf3a 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -43,14 +43,16 @@
//! [ts]: https://doc.rust-lang.org/proc_macro/struct.TokenStream.html
// Proc-macro2 types in rustdoc of other crates get linked to here.
-#![doc(html_root_url = "https://docs.rs/proc-macro2/0.3.8")]
+#![doc(html_root_url = "https://docs.rs/proc-macro2/0.4.0")]
#![cfg_attr(feature = "nightly", feature(proc_macro))]
#[cfg(feature = "proc-macro")]
extern crate proc_macro;
extern crate unicode_xid;
+use std::cmp::Ordering;
use std::fmt;
+use std::hash::{Hash, Hasher};
use std::iter::FromIterator;
use std::marker;
use std::rc::Rc;
@@ -127,6 +129,12 @@
}
}
+impl Extend<TokenTree> for TokenStream {
+ fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, streams: I) {
+ self.inner.extend(streams)
+ }
+}
+
impl FromIterator<TokenTree> for TokenStream {
fn from_iter<I: IntoIterator<Item = TokenTree>>(streams: I) -> Self {
TokenStream::_new(streams.into_iter().collect())
@@ -284,8 +292,8 @@
#[derive(Clone)]
pub enum TokenTree {
Group(Group),
- Term(Term),
- Op(Op),
+ Ident(Ident),
+ Punct(Punct),
Literal(Literal),
}
@@ -293,8 +301,8 @@
pub fn span(&self) -> Span {
match *self {
TokenTree::Group(ref t) => t.span(),
- TokenTree::Term(ref t) => t.span(),
- TokenTree::Op(ref t) => t.span(),
+ TokenTree::Ident(ref t) => t.span(),
+ TokenTree::Punct(ref t) => t.span(),
TokenTree::Literal(ref t) => t.span(),
}
}
@@ -302,8 +310,8 @@
pub fn set_span(&mut self, span: Span) {
match *self {
TokenTree::Group(ref mut t) => t.set_span(span),
- TokenTree::Term(ref mut t) => t.set_span(span),
- TokenTree::Op(ref mut t) => t.set_span(span),
+ TokenTree::Ident(ref mut t) => t.set_span(span),
+ TokenTree::Punct(ref mut t) => t.set_span(span),
TokenTree::Literal(ref mut t) => t.set_span(span),
}
}
@@ -315,15 +323,15 @@
}
}
-impl From<Term> for TokenTree {
- fn from(g: Term) -> TokenTree {
- TokenTree::Term(g)
+impl From<Ident> for TokenTree {
+ fn from(g: Ident) -> TokenTree {
+ TokenTree::Ident(g)
}
}
-impl From<Op> for TokenTree {
- fn from(g: Op) -> TokenTree {
- TokenTree::Op(g)
+impl From<Punct> for TokenTree {
+ fn from(g: Punct) -> TokenTree {
+ TokenTree::Punct(g)
}
}
@@ -337,8 +345,8 @@
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
TokenTree::Group(ref t) => t.fmt(f),
- TokenTree::Term(ref t) => t.fmt(f),
- TokenTree::Op(ref t) => t.fmt(f),
+ TokenTree::Ident(ref t) => t.fmt(f),
+ TokenTree::Punct(ref t) => t.fmt(f),
TokenTree::Literal(ref t) => t.fmt(f),
}
}
@@ -350,8 +358,8 @@
// so don't bother with an extra layer of indirection
match *self {
TokenTree::Group(ref t) => t.fmt(f),
- TokenTree::Term(ref t) => t.fmt(f),
- TokenTree::Op(ref t) => t.fmt(f),
+ TokenTree::Ident(ref t) => t.fmt(f),
+ TokenTree::Punct(ref t) => t.fmt(f),
TokenTree::Literal(ref t) => t.fmt(f),
}
}
@@ -415,8 +423,8 @@
}
}
-#[derive(Copy, Clone)]
-pub struct Op {
+#[derive(Clone)]
+pub struct Punct {
op: char,
spacing: Spacing,
span: Span,
@@ -428,16 +436,16 @@
Joint,
}
-impl Op {
- pub fn new(op: char, spacing: Spacing) -> Op {
- Op {
+impl Punct {
+ pub fn new(op: char, spacing: Spacing) -> Punct {
+ Punct {
op: op,
spacing: spacing,
span: Span::call_site(),
}
}
- pub fn op(&self) -> char {
+ pub fn as_char(&self) -> char {
self.op
}
@@ -454,15 +462,15 @@
}
}
-impl fmt::Display for Op {
+impl fmt::Display for Punct {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.op.fmt(f)
}
}
-impl fmt::Debug for Op {
+impl fmt::Debug for Punct {
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
- let mut debug = fmt.debug_struct("Op");
+ let mut debug = fmt.debug_struct("Punct");
debug.field("op", &self.op);
debug.field("spacing", &self.spacing);
#[cfg(procmacro2_semver_exempt)]
@@ -471,26 +479,31 @@
}
}
-#[derive(Copy, Clone)]
-pub struct Term {
- inner: imp::Term,
+#[derive(Clone)]
+pub struct Ident {
+ inner: imp::Ident,
_marker: marker::PhantomData<Rc<()>>,
}
-impl Term {
- fn _new(inner: imp::Term) -> Term {
- Term {
+impl Ident {
+ fn _new(inner: imp::Ident) -> Ident {
+ Ident {
inner: inner,
_marker: marker::PhantomData,
}
}
- pub fn new(string: &str, span: Span) -> Term {
- Term::_new(imp::Term::new(string, span.inner))
+ pub fn new(string: &str, span: Span) -> Ident {
+ Ident::_new(imp::Ident::new(string, span.inner))
}
- pub fn as_str(&self) -> &str {
- self.inner.as_str()
+ #[cfg(procmacro2_semver_exempt)]
+ pub fn new_raw(string: &str, span: Span) -> Ident {
+ Ident::_new_raw(string, span)
+ }
+
+ fn _new_raw(string: &str, span: Span) -> Ident {
+ Ident::_new(imp::Ident::new_raw(string, span.inner))
}
pub fn span(&self) -> Span {
@@ -502,13 +515,40 @@
}
}
-impl fmt::Display for Term {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- self.as_str().fmt(f)
+impl PartialEq for Ident {
+ fn eq(&self, other: &Ident) -> bool{
+ self.to_string() == other.to_string()
}
}
-impl fmt::Debug for Term {
+impl Eq for Ident {
+}
+
+impl PartialOrd for Ident {
+ fn partial_cmp(&self, other: &Ident) -> Option<Ordering> {
+ Some(self.cmp(other))
+ }
+}
+
+impl Ord for Ident {
+ fn cmp(&self, other: &Ident) -> Ordering {
+ self.to_string().cmp(&other.to_string())
+ }
+}
+
+impl Hash for Ident {
+ fn hash<H: Hasher>(&self, hasher: &mut H) {
+ self.to_string().hash(hasher)
+ }
+}
+
+impl fmt::Display for Ident {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ self.inner.fmt(f)
+ }
+}
+
+impl fmt::Debug for Ident {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.inner.fmt(f)
}
diff --git a/src/stable.rs b/src/stable.rs
index 9222793..a718db2 100644
--- a/src/stable.rs
+++ b/src/stable.rs
@@ -14,7 +14,7 @@
use strnom::{block_comment, skip_whitespace, whitespace, word_break, Cursor, PResult};
use unicode_xid::UnicodeXID;
-use {Delimiter, Group, Op, Spacing, TokenTree};
+use {Delimiter, Group, Punct, Spacing, TokenTree};
#[derive(Clone)]
pub struct TokenStream {
@@ -95,9 +95,9 @@
write!(f, "{} {} {}", start, tt.stream(), end)?
}
}
- TokenTree::Term(ref tt) => write!(f, "{}", tt.as_str())?,
- TokenTree::Op(ref tt) => {
- write!(f, "{}", tt.op())?;
+ TokenTree::Ident(ref tt) => write!(f, "{}", tt)?,
+ TokenTree::Punct(ref tt) => {
+ write!(f, "{}", tt.as_char())?;
match tt.spacing() {
Spacing::Alone => {}
Spacing::Joint => joint = true,
@@ -156,6 +156,12 @@
}
}
+impl Extend<TokenTree> for TokenStream {
+ fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, streams: I) {
+ self.inner.extend(streams);
+ }
+}
+
pub type TokenTreeIter = vec::IntoIter<TokenTree>;
impl IntoIterator for TokenStream {
@@ -170,6 +176,7 @@
#[derive(Clone, PartialEq, Eq, Debug)]
pub struct FileName(String);
+#[allow(dead_code)]
pub fn file_name(s: String) -> FileName {
FileName(s)
}
@@ -407,24 +414,34 @@
}
}
-#[derive(Copy, Clone)]
-pub struct Term {
+#[derive(Clone)]
+pub struct Ident {
intern: usize,
span: Span,
+ raw: bool,
}
thread_local!(static SYMBOLS: RefCell<Interner> = RefCell::new(Interner::new()));
-impl Term {
- pub fn new(string: &str, span: Span) -> Term {
+impl Ident {
+ fn _new(string: &str, raw: bool, span: Span) -> Ident {
validate_term(string);
- Term {
+ Ident {
intern: SYMBOLS.with(|s| s.borrow_mut().intern(string)),
span: span,
+ raw: raw,
}
}
+ pub fn new(string: &str, span: Span) -> Ident {
+ Ident::_new(string, false, span)
+ }
+
+ pub fn new_raw(string: &str, span: Span) -> Ident {
+ Ident::_new(string, true, span)
+ }
+
pub fn as_str(&self) -> &str {
SYMBOLS.with(|interner| {
let interner = interner.borrow();
@@ -443,20 +460,13 @@
}
fn validate_term(string: &str) {
- let validate = if string.starts_with('\'') {
- &string[1..]
- } else if string.starts_with("r#") {
- &string[2..]
- } else {
- string
- };
-
+ let validate = string;
if validate.is_empty() {
- panic!("Term is not allowed to be empty; use Option<Term>");
+ panic!("Ident is not allowed to be empty; use Option<Ident>");
}
if validate.bytes().all(|digit| digit >= b'0' && digit <= b'9') {
- panic!("Term cannot be a number; use Literal instead");
+ panic!("Ident cannot be a number; use Literal instead");
}
fn xid_ok(string: &str) -> bool {
@@ -474,16 +484,26 @@
}
if !xid_ok(validate) {
- panic!("{:?} is not a valid Term", string);
+ panic!("{:?} is not a valid Ident", string);
}
}
-impl fmt::Debug for Term {
+impl fmt::Display for Ident {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- let mut debug = f.debug_struct("Term");
+ if self.raw {
+ "r#".fmt(f)?;
+ }
+ self.as_str().fmt(f)
+ }
+}
+
+impl fmt::Debug for Ident {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ let mut debug = f.debug_struct("Ident");
debug.field("sym", &format_args!("{}", self.as_str()));
#[cfg(procmacro2_semver_exempt)]
debug.field("span", &self.span);
+ debug.field("raw", &self.raw);
debug.finish()
}
}
@@ -713,9 +733,9 @@
|
map!(literal, |l| TokenTree::Literal(::Literal::_new_stable(l))) // must be before symbol
|
- symbol
+ map!(op, TokenTree::Punct)
|
- map!(op, TokenTree::Op)
+ symbol_leading_ws
));
named!(group -> Group, alt!(
@@ -738,17 +758,14 @@
) => { |ts| Group::new(Delimiter::Brace, ::TokenStream::_new_stable(ts)) }
));
-fn symbol(mut input: Cursor) -> PResult<TokenTree> {
- input = skip_whitespace(input);
+fn symbol_leading_ws(input: Cursor) -> PResult<TokenTree> {
+ symbol(skip_whitespace(input))
+}
+fn symbol(input: Cursor) -> PResult<TokenTree> {
let mut chars = input.char_indices();
- let lifetime = input.starts_with("'");
- if lifetime {
- chars.next();
- }
-
- let raw = !lifetime && input.starts_with("r#");
+ let raw = input.starts_with("r#");
if raw {
chars.next();
chars.next();
@@ -768,27 +785,20 @@
}
let a = &input.rest[..end];
- if a == "r#_" || lifetime && a != "'static" && KEYWORDS.contains(&&a[1..]) {
+ if a == "r#_" {
Err(LexError)
} else if a == "_" {
- Ok((input.advance(end), Op::new('_', Spacing::Alone).into()))
+ Ok((input.advance(end), Punct::new('_', Spacing::Alone).into()))
} else {
- Ok((
- input.advance(end),
- ::Term::new(a, ::Span::call_site()).into(),
- ))
+ let ident = if raw {
+ ::Ident::_new_raw(&a[2..], ::Span::call_site())
+ } else {
+ ::Ident::new(a, ::Span::call_site())
+ };
+ Ok((input.advance(end), ident.into()))
}
}
-// From https://github.com/rust-lang/rust/blob/master/src/libsyntax_pos/symbol.rs
-static KEYWORDS: &'static [&'static str] = &[
- "abstract", "alignof", "as", "become", "box", "break", "const", "continue", "crate", "do",
- "else", "enum", "extern", "false", "final", "fn", "for", "if", "impl", "in", "let", "loop",
- "macro", "match", "mod", "move", "mut", "offsetof", "override", "priv", "proc", "pub", "pure",
- "ref", "return", "self", "Self", "sizeof", "static", "struct", "super", "trait", "true",
- "type", "typeof", "unsafe", "unsized", "use", "virtual", "where", "while", "yield",
-];
-
fn literal(input: Cursor) -> PResult<Literal> {
let input_no_ws = skip_whitespace(input);
@@ -1211,15 +1221,19 @@
}
}
-fn op(input: Cursor) -> PResult<Op> {
+fn op(input: Cursor) -> PResult<Punct> {
let input = skip_whitespace(input);
match op_char(input) {
+ Ok((rest, '\'')) => {
+ symbol(rest)?;
+ Ok((rest, Punct::new('\'', Spacing::Joint)))
+ }
Ok((rest, ch)) => {
let kind = match op_char(rest) {
Ok(_) => Spacing::Joint,
Err(LexError) => Spacing::Alone,
};
- Ok((rest, Op::new(ch, kind)))
+ Ok((rest, Punct::new(ch, kind)))
}
Err(LexError) => Err(LexError),
}
@@ -1238,7 +1252,7 @@
return Err(LexError);
}
};
- let recognized = "~!@#$%^&*-=+|;:,<.>/?";
+ let recognized = "~!@#$%^&*-=+|;:,<.>/?'";
if recognized.contains(first) {
Ok((input.advance(first.len_utf8()), first))
} else {
@@ -1249,13 +1263,13 @@
fn doc_comment(input: Cursor) -> PResult<Vec<TokenTree>> {
let mut trees = Vec::new();
let (rest, ((comment, inner), span)) = spanned(input, doc_comment_contents)?;
- trees.push(TokenTree::Op(Op::new('#', Spacing::Alone)));
+ trees.push(TokenTree::Punct(Punct::new('#', Spacing::Alone)));
if inner {
- trees.push(Op::new('!', Spacing::Alone).into());
+ trees.push(Punct::new('!', Spacing::Alone).into());
}
let mut stream = vec![
- TokenTree::Term(::Term::new("doc", span)),
- TokenTree::Op(Op::new('=', Spacing::Alone)),
+ TokenTree::Ident(::Ident::new("doc", span)),
+ TokenTree::Punct(Punct::new('=', Spacing::Alone)),
TokenTree::Literal(::Literal::string(comment)),
];
for tt in stream.iter_mut() {
diff --git a/src/unstable.rs b/src/unstable.rs
index d709f88..8026fda 100644
--- a/src/unstable.rs
+++ b/src/unstable.rs
@@ -8,7 +8,7 @@
use proc_macro;
use stable;
-use {Delimiter, Group, Op, Spacing, TokenTree};
+use {Delimiter, Group, Punct, Spacing, TokenTree};
#[derive(Clone)]
pub enum TokenStream {
@@ -123,16 +123,16 @@
group.set_span(span.inner.unwrap_nightly());
group.into()
}
- TokenTree::Op(tt) => {
+ TokenTree::Punct(tt) => {
let spacing = match tt.spacing() {
Spacing::Joint => proc_macro::Spacing::Joint,
Spacing::Alone => proc_macro::Spacing::Alone,
};
- let mut op = proc_macro::Op::new(tt.op(), spacing);
+ let mut op = proc_macro::Punct::new(tt.as_char(), spacing);
op.set_span(tt.span().inner.unwrap_nightly());
op.into()
}
- TokenTree::Term(tt) => tt.inner.unwrap_nightly().into(),
+ TokenTree::Ident(tt) => tt.inner.unwrap_nightly().into(),
TokenTree::Literal(tt) => tt.inner.unwrap_nightly().into(),
};
TokenStream::Nightly(tt.into())
@@ -157,6 +157,29 @@
}
}
+impl Extend<TokenTree> for TokenStream {
+ fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, streams: I) {
+ match self {
+ TokenStream::Nightly(tts) => {
+ *tts = tts.clone()
+ .into_iter()
+ .chain(
+ streams.into_iter()
+ .map(TokenStream::from)
+ .flat_map(|t| {
+ match t {
+ TokenStream::Nightly(tts) => tts.into_iter(),
+ _ => panic!()
+ }
+ })
+ )
+ .collect();
+ }
+ TokenStream::Stable(tts) => tts.extend(streams),
+ }
+ }
+}
+
impl fmt::Debug for TokenStream {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
@@ -225,16 +248,16 @@
g.set_span(::Span::_new(Span::Nightly(tt.span())));
g.into()
}
- proc_macro::TokenTree::Op(tt) => {
+ proc_macro::TokenTree::Punct(tt) => {
let spacing = match tt.spacing() {
proc_macro::Spacing::Joint => Spacing::Joint,
proc_macro::Spacing::Alone => Spacing::Alone,
};
- let mut o = Op::new(tt.op(), spacing);
+ let mut o = Punct::new(tt.as_char(), spacing);
o.set_span(::Span::_new(Span::Nightly(tt.span())));
o.into()
}
- proc_macro::TokenTree::Term(s) => ::Term::_new(Term::Nightly(s)).into(),
+ proc_macro::TokenTree::Ident(s) => ::Ident::_new(Ident::Nightly(s)).into(),
proc_macro::TokenTree::Literal(l) => ::Literal::_new(Literal::Nightly(l)).into(),
})
}
@@ -434,55 +457,64 @@
}
}
-#[derive(Copy, Clone)]
-pub enum Term {
- Nightly(proc_macro::Term),
- Stable(stable::Term),
+#[derive(Clone)]
+pub enum Ident {
+ Nightly(proc_macro::Ident),
+ Stable(stable::Ident),
}
-impl Term {
- pub fn new(string: &str, span: Span) -> Term {
+impl Ident {
+ pub fn new(string: &str, span: Span) -> Ident {
match span {
- Span::Nightly(s) => Term::Nightly(proc_macro::Term::new(string, s)),
- Span::Stable(s) => Term::Stable(stable::Term::new(string, s)),
+ Span::Nightly(s) => Ident::Nightly(proc_macro::Ident::new(string, s)),
+ Span::Stable(s) => Ident::Stable(stable::Ident::new(string, s)),
}
}
- pub fn as_str(&self) -> &str {
- match self {
- Term::Nightly(t) => t.as_str(),
- Term::Stable(t) => t.as_str(),
+ pub fn new_raw(string: &str, span: Span) -> Ident {
+ match span {
+ Span::Nightly(s) => Ident::Nightly(proc_macro::Ident::new_raw(string, s)),
+ Span::Stable(s) => Ident::Stable(stable::Ident::new_raw(string, s)),
}
}
pub fn span(&self) -> Span {
match self {
- Term::Nightly(t) => Span::Nightly(t.span()),
- Term::Stable(t) => Span::Stable(t.span()),
+ Ident::Nightly(t) => Span::Nightly(t.span()),
+ Ident::Stable(t) => Span::Stable(t.span()),
}
}
pub fn set_span(&mut self, span: Span) {
match (self, span) {
- (Term::Nightly(t), Span::Nightly(s)) => t.set_span(s),
- (Term::Stable(t), Span::Stable(s)) => t.set_span(s),
+ (Ident::Nightly(t), Span::Nightly(s)) => t.set_span(s),
+ (Ident::Stable(t), Span::Stable(s)) => t.set_span(s),
_ => mismatch(),
}
}
- fn unwrap_nightly(self) -> proc_macro::Term {
+ fn unwrap_nightly(self) -> proc_macro::Ident {
match self {
- Term::Nightly(s) => s,
- Term::Stable(_) => mismatch(),
+ Ident::Nightly(s) => s,
+ Ident::Stable(_) => mismatch(),
}
}
}
-impl fmt::Debug for Term {
+impl fmt::Display for Ident {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
- Term::Nightly(t) => t.fmt(f),
- Term::Stable(t) => t.fmt(f),
+ Ident::Nightly(t) => t.fmt(f),
+ Ident::Stable(t) => t.fmt(f),
+ }
+ }
+}
+
+impl fmt::Debug for Ident {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ match self {
+ Ident::Nightly(t) => t.fmt(f),
+ Ident::Stable(t) => t.fmt(f),
}
}
}