Update to latest syn-1.0.16
* fill back missing NOTICE,METADATA,*LICENSE* files
Bug: 150877376
Test: make
Test: atest --host -c --include-subdirs external/rust/crates
Change-Id: Ib5df6b8fb97764214e701a888e44fab3a4245800
diff --git a/src/attr.rs b/src/attr.rs
index 34009de..17b3793 100644
--- a/src/attr.rs
+++ b/src/attr.rs
@@ -111,6 +111,43 @@
///
/// [`parse_meta()`]: Attribute::parse_meta
/// [`parse_args()`]: Attribute::parse_args
+ ///
+ /// <p><br></p>
+ ///
+ /// # Doc comments
+ ///
+ /// The compiler transforms doc comments, such as `/// comment` and `/*!
+ /// comment */`, into attributes before macros are expanded. Each comment is
+ /// expanded into an attribute of the form `#[doc = r"comment"]`.
+ ///
+ /// As an example, the following `mod` items are expanded identically:
+ ///
+ /// ```
+ /// # use syn::{ItemMod, parse_quote};
+ /// let doc: ItemMod = parse_quote! {
+ /// /// Single line doc comments
+ /// /// We write so many!
+ /// /**
+ /// * Multi-line comments...
+ /// * May span many lines
+ /// */
+ /// mod example {
+ /// //! Of course, they can be inner too
+ /// /*! And fit in a single line */
+ /// }
+ /// };
+ /// let attr: ItemMod = parse_quote! {
+ /// #[doc = r" Single line doc comments"]
+ /// #[doc = r" We write so many!"]
+ /// #[doc = r" Multi-line comments...
+ /// May span many lines"]
+ /// mod example {
+ /// #![doc = r" Of course, they can be inner too"]
+ /// #![doc = r" And fit in a single line "]
+ /// }
+ /// };
+ /// assert_eq!(doc, attr);
+ /// ```
pub struct Attribute #manual_extra_traits {
pub pound_token: Token![#],
pub style: AttrStyle,
@@ -247,7 +284,7 @@
}
#[cfg(feature = "parsing")]
-fn error_expected_args(attr: &Attribute) -> Error {
+fn expected_parentheses(attr: &Attribute) -> String {
let style = match attr.style {
AttrStyle::Outer => "#",
AttrStyle::Inner(_) => "#!",
@@ -261,19 +298,23 @@
path += &segment.ident.to_string();
}
- let msg = format!("expected attribute arguments: {}[{}(...)]", style, path);
-
- #[cfg(feature = "printing")]
- return Error::new_spanned(attr, msg);
-
- #[cfg(not(feature = "printing"))]
- return Error::new(attr.bracket_token.span, msg);
+ format!("{}[{}(...)]", style, path)
}
#[cfg(feature = "parsing")]
fn enter_args<'a>(attr: &Attribute, input: ParseStream<'a>) -> Result<ParseBuffer<'a>> {
if input.is_empty() {
- return Err(error_expected_args(attr));
+ let expected = expected_parentheses(attr);
+ let msg = format!("expected attribute arguments in parentheses: {}", expected);
+ return Err(crate::error::new2(
+ attr.pound_token.span,
+ attr.bracket_token.span,
+ msg,
+ ));
+ } else if input.peek(Token![=]) {
+ let expected = expected_parentheses(attr);
+ let msg = format!("expected parentheses: {}", expected);
+ return Err(input.error(msg));
};
let content;
diff --git a/src/buffer.rs b/src/buffer.rs
index 551a5ac..5c2dd8a 100644
--- a/src/buffer.rs
+++ b/src/buffer.rs
@@ -221,7 +221,6 @@
/// Checks whether the cursor is currently pointing at the end of its valid
/// scope.
- #[inline]
pub fn eof(self) -> bool {
// We're at eof if we're at the end of our scope.
self.ptr == self.scope
@@ -342,6 +341,26 @@
Entry::End(..) => Span::call_site(),
}
}
+
+ /// Skip over the next token without cloning it. Returns `None` if this
+ /// cursor points to eof.
+ ///
+ /// This method treats `'lifetimes` as a single token.
+ pub(crate) fn skip(self) -> Option<Cursor<'a>> {
+ match self.entry() {
+ Entry::End(..) => None,
+
+ // Treat lifetimes as a single tt for the purposes of 'skip'.
+ Entry::Punct(op) if op.as_char() == '\'' && op.spacing() == Spacing::Joint => {
+ let next = unsafe { self.bump() };
+ match next.entry() {
+ Entry::Ident(_) => Some(unsafe { next.bump() }),
+ _ => Some(next),
+ }
+ }
+ _ => Some(unsafe { self.bump() }),
+ }
+ }
}
pub(crate) fn same_scope(a: Cursor, b: Cursor) -> bool {
diff --git a/src/custom_keyword.rs b/src/custom_keyword.rs
index 200e847..d46e277 100644
--- a/src/custom_keyword.rs
+++ b/src/custom_keyword.rs
@@ -95,7 +95,7 @@
}
#[doc(hidden)]
- #[allow(non_snake_case)]
+ #[allow(dead_code, non_snake_case)]
pub fn $ident<__S: $crate::export::IntoSpans<[$crate::export::Span; 1]>>(
span: __S,
) -> $ident {
diff --git a/src/custom_punctuation.rs b/src/custom_punctuation.rs
index 29fa448..8631b77 100644
--- a/src/custom_punctuation.rs
+++ b/src/custom_punctuation.rs
@@ -82,7 +82,7 @@
}
#[doc(hidden)]
- #[allow(non_snake_case)]
+ #[allow(dead_code, non_snake_case)]
pub fn $ident<__S: $crate::export::IntoSpans<custom_punctuation_repr!($($tt)+)>>(
spans: __S,
) -> $ident {
diff --git a/src/data.rs b/src/data.rs
index 2554506..0d6b1b5 100644
--- a/src/data.rs
+++ b/src/data.rs
@@ -238,12 +238,15 @@
use super::*;
use crate::ext::IdentExt;
+ use crate::parse::discouraged::Speculative;
use crate::parse::{Parse, ParseStream, Result};
impl Parse for Variant {
fn parse(input: ParseStream) -> Result<Self> {
+ let attrs = input.call(Attribute::parse_outer)?;
+ let _visibility: Visibility = input.parse()?;
Ok(Variant {
- attrs: input.call(Attribute::parse_outer)?,
+ attrs,
ident: input.parse()?,
fields: {
if input.peek(token::Brace) {
@@ -328,27 +331,39 @@
let pub_token = input.parse::<Token![pub]>()?;
if input.peek(token::Paren) {
- // TODO: optimize using advance_to
let ahead = input.fork();
- let mut content;
- parenthesized!(content in ahead);
+ let content;
+ let paren_token = parenthesized!(content in ahead);
if content.peek(Token![crate])
|| content.peek(Token![self])
|| content.peek(Token![super])
{
- return Ok(Visibility::Restricted(VisRestricted {
- pub_token,
- paren_token: parenthesized!(content in input),
- in_token: None,
- path: Box::new(Path::from(content.call(Ident::parse_any)?)),
- }));
+ let path = content.call(Ident::parse_any)?;
+
+ // Ensure there are no additional tokens within `content`.
+ // Without explicitly checking, we may misinterpret a tuple
+ // field as a restricted visibility, causing a parse error.
+ // e.g. `pub (crate::A, crate::B)` (Issue #720).
+ if content.is_empty() {
+ input.advance_to(&ahead);
+ return Ok(Visibility::Restricted(VisRestricted {
+ pub_token,
+ paren_token,
+ in_token: None,
+ path: Box::new(Path::from(path)),
+ }));
+ }
} else if content.peek(Token![in]) {
+ let in_token: Token![in] = content.parse()?;
+ let path = content.call(Path::parse_mod_style)?;
+
+ input.advance_to(&ahead);
return Ok(Visibility::Restricted(VisRestricted {
pub_token,
- paren_token: parenthesized!(content in input),
- in_token: Some(content.parse()?),
- path: Box::new(content.call(Path::parse_mod_style)?),
+ paren_token,
+ in_token: Some(in_token),
+ path: Box::new(path),
}));
}
}
diff --git a/src/discouraged.rs b/src/discouraged.rs
index 4d9ff93..29d1006 100644
--- a/src/discouraged.rs
+++ b/src/discouraged.rs
@@ -164,6 +164,30 @@
panic!("Fork was not derived from the advancing parse stream");
}
+ let (self_unexp, self_sp) = inner_unexpected(self);
+ let (fork_unexp, fork_sp) = inner_unexpected(fork);
+ if !Rc::ptr_eq(&self_unexp, &fork_unexp) {
+ match (fork_sp, self_sp) {
+ // Unexpected set on the fork, but not on `self`, copy it over.
+ (Some(span), None) => {
+ self_unexp.set(Unexpected::Some(span));
+ }
+ // Unexpected unset. Use chain to propagate errors from fork.
+ (None, None) => {
+ fork_unexp.set(Unexpected::Chain(self_unexp));
+
+ // Ensure toplevel 'unexpected' tokens from the fork don't
+ // bubble up the chain by replacing the root `unexpected`
+ // pointer, only 'unexpected' tokens from existing group
+ // parsers should bubble.
+ fork.unexpected
+ .set(Some(Rc::new(Cell::new(Unexpected::None))));
+ }
+ // Unexpected has been set on `self`. No changes needed.
+ (_, Some(_)) => {}
+ }
+ }
+
// See comment on `cell` in the struct definition.
self.cell
.set(unsafe { mem::transmute::<Cursor, Cursor<'static>>(fork.cursor()) })
diff --git a/src/error.rs b/src/error.rs
index 146d652..dbcbc07 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -1,4 +1,3 @@
-use std;
use std::fmt::{self, Debug, Display};
use std::iter::FromIterator;
use std::slice;
@@ -250,6 +249,17 @@
}
}
+#[cfg(all(feature = "parsing", any(feature = "full", feature = "derive")))]
+pub fn new2<T: Display>(start: Span, end: Span, message: T) -> Error {
+ Error {
+ messages: vec![ErrorMessage {
+ start_span: ThreadBound::new(start),
+ end_span: ThreadBound::new(end),
+ message: message.to_string(),
+ }],
+ }
+}
+
impl Debug for Error {
fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
if self.messages.len() == 1 {
diff --git a/src/expr.rs b/src/expr.rs
index 2874a46..b40f1ca 100644
--- a/src/expr.rs
+++ b/src/expr.rs
@@ -3,7 +3,10 @@
#[cfg(feature = "extra-traits")]
use crate::tt::TokenStreamHelper;
use proc_macro2::{Span, TokenStream};
-#[cfg(feature = "extra-traits")]
+#[cfg(feature = "printing")]
+use quote::IdentFragment;
+#[cfg(feature = "printing")]
+use std::fmt::{self, Display};
use std::hash::{Hash, Hasher};
#[cfg(all(feature = "parsing", feature = "full"))]
use std::mem;
@@ -998,7 +1001,8 @@
///
/// *This type is available if Syn is built with the `"derive"` or `"full"`
/// feature.*
- pub enum Member {
+ #[derive(Eq, PartialEq, Hash)]
+ pub enum Member #manual_extra_traits {
/// A named field like `self.x`.
Named(Ident),
/// An unnamed field like `self.0`.
@@ -1006,6 +1010,23 @@
}
}
+#[cfg(feature = "printing")]
+impl IdentFragment for Member {
+ fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+ match self {
+ Member::Named(m) => Display::fmt(m, formatter),
+ Member::Unnamed(m) => Display::fmt(&m.index, formatter),
+ }
+ }
+
+ fn span(&self) -> Option<Span> {
+ match self {
+ Member::Named(m) => Some(m.span()),
+ Member::Unnamed(m) => Some(m.span),
+ }
+ }
+}
+
ast_struct! {
/// The index of an unnamed tuple struct field.
///
@@ -1027,28 +1048,36 @@
}
}
-#[cfg(feature = "extra-traits")]
impl Eq for Index {}
-#[cfg(feature = "extra-traits")]
impl PartialEq for Index {
fn eq(&self, other: &Self) -> bool {
self.index == other.index
}
}
-#[cfg(feature = "extra-traits")]
impl Hash for Index {
fn hash<H: Hasher>(&self, state: &mut H) {
self.index.hash(state);
}
}
+#[cfg(feature = "printing")]
+impl IdentFragment for Index {
+ fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+ Display::fmt(&self.index, formatter)
+ }
+
+ fn span(&self) -> Option<Span> {
+ Some(self.span)
+ }
+}
+
#[cfg(feature = "full")]
ast_struct! {
#[derive(Default)]
pub struct Reserved {
- private: (),
+ _private: (),
}
}
@@ -1181,9 +1210,12 @@
pub(crate) mod parsing {
use super::*;
+ use crate::parse::discouraged::Speculative;
use crate::parse::{Parse, ParseStream, Result};
use crate::path;
+ crate::custom_keyword!(raw);
+
// When we're parsing expressions which occur before blocks, like in an if
// statement's condition, we cannot parse a struct literal.
//
@@ -1436,24 +1468,41 @@
// box <trailer>
#[cfg(feature = "full")]
fn unary_expr(input: ParseStream, allow_struct: AllowStruct) -> Result<Expr> {
- // TODO: optimize using advance_to
+ let begin = input.fork();
let ahead = input.fork();
- ahead.call(Attribute::parse_outer)?;
+ let attrs = ahead.call(Attribute::parse_outer)?;
if ahead.peek(Token![&])
|| ahead.peek(Token![box])
|| ahead.peek(Token![*])
|| ahead.peek(Token![!])
|| ahead.peek(Token![-])
{
- let attrs = input.call(Attribute::parse_outer)?;
+ input.advance_to(&ahead);
if input.peek(Token![&]) {
- Ok(Expr::Reference(ExprReference {
- attrs,
- and_token: input.parse()?,
- raw: Reserved::default(),
- mutability: input.parse()?,
- expr: Box::new(unary_expr(input, allow_struct)?),
- }))
+ let and_token: Token![&] = input.parse()?;
+ let raw: Option<raw> = if input.peek(raw)
+ && (input.peek2(Token![mut]) || input.peek2(Token![const]))
+ {
+ Some(input.parse()?)
+ } else {
+ None
+ };
+ let mutability: Option<Token![mut]> = input.parse()?;
+ if raw.is_some() && mutability.is_none() {
+ input.parse::<Token![const]>()?;
+ }
+ let expr = Box::new(unary_expr(input, allow_struct)?);
+ if raw.is_some() {
+ Ok(Expr::Verbatim(verbatim::between(begin, input)))
+ } else {
+ Ok(Expr::Reference(ExprReference {
+ attrs,
+ and_token,
+ raw: Reserved::default(),
+ mutability,
+ expr,
+ }))
+ }
} else if input.peek(Token![box]) {
Ok(Expr::Box(ExprBox {
attrs,
@@ -1474,12 +1523,12 @@
#[cfg(not(feature = "full"))]
fn unary_expr(input: ParseStream, allow_struct: AllowStruct) -> Result<Expr> {
- // TODO: optimize using advance_to
let ahead = input.fork();
- ahead.call(Attribute::parse_outer)?;
+ let attrs = ahead.call(Attribute::parse_outer)?;
if ahead.peek(Token![*]) || ahead.peek(Token![!]) || ahead.peek(Token![-]) {
+ input.advance_to(&ahead);
Ok(Expr::Unary(ExprUnary {
- attrs: input.call(Attribute::parse_outer)?,
+ attrs,
op: input.parse()?,
expr: Box::new(unary_expr(input, allow_struct)?),
}))
@@ -1905,7 +1954,7 @@
return parse_expr(input, expr, allow_struct, Precedence::Any);
};
- if input.peek(Token![.]) || input.peek(Token![?]) {
+ if input.peek(Token![.]) && !input.peek(Token![..]) || input.peek(Token![?]) {
expr = trailer_helper(input, expr)?;
attrs.extend(expr.replace_attrs(Vec::new()));
@@ -2399,6 +2448,7 @@
#[cfg(feature = "full")]
impl Parse for FieldValue {
fn parse(input: ParseStream) -> Result<Self> {
+ let attrs = input.call(Attribute::parse_outer)?;
let member: Member = input.parse()?;
let (colon_token, value) = if input.peek(Token![:]) || !member.is_named() {
let colon_token: Token![:] = input.parse()?;
@@ -2416,7 +2466,7 @@
};
Ok(FieldValue {
- attrs: Vec::new(),
+ attrs,
member,
colon_token,
expr: value,
@@ -2433,24 +2483,22 @@
let content;
let brace_token = braced!(content in input);
let inner_attrs = content.call(Attribute::parse_inner)?;
+ let attrs = private::attrs(outer_attrs, inner_attrs);
let mut fields = Punctuated::new();
- loop {
- let attrs = content.call(Attribute::parse_outer)?;
- // TODO: optimize using advance_to
- if content.fork().parse::<Member>().is_err() {
- if attrs.is_empty() {
- break;
- } else {
- return Err(content.error("expected struct field"));
- }
+ while !content.is_empty() {
+ if content.peek(Token![..]) {
+ return Ok(ExprStruct {
+ attrs,
+ brace_token,
+ path,
+ fields,
+ dot2_token: Some(content.parse()?),
+ rest: Some(Box::new(content.parse()?)),
+ });
}
- fields.push(FieldValue {
- attrs,
- ..content.parse()?
- });
-
+ fields.push(content.parse()?);
if !content.peek(Token![,]) {
break;
}
@@ -2458,21 +2506,13 @@
fields.push_punct(punct);
}
- let (dot2_token, rest) = if fields.empty_or_trailing() && content.peek(Token![..]) {
- let dot2_token: Token![..] = content.parse()?;
- let rest: Expr = content.parse()?;
- (Some(dot2_token), Some(Box::new(rest)))
- } else {
- (None, None)
- };
-
Ok(ExprStruct {
- attrs: private::attrs(outer_attrs, inner_attrs),
+ attrs,
brace_token,
path,
fields,
- dot2_token,
- rest,
+ dot2_token: None,
+ rest: None,
})
}
diff --git a/src/generics.rs b/src/generics.rs
index d357cbb..9feeab9 100644
--- a/src/generics.rs
+++ b/src/generics.rs
@@ -28,9 +28,6 @@
/// This type is a [syntax tree enum].
///
/// [syntax tree enum]: enum.Expr.html#syntax-tree-enums
- //
- // TODO: change syntax-tree-enum link to an intra rustdoc link, currently
- // blocked on https://github.com/rust-lang/rust/issues/62833
pub enum GenericParam {
/// A generic type parameter: `T: Into<String>`.
Type(TypeParam),
@@ -448,9 +445,6 @@
/// This type is a [syntax tree enum].
///
/// [syntax tree enum]: enum.Expr.html#syntax-tree-enums
- //
- // TODO: change syntax-tree-enum link to an intra rustdoc link, currently
- // blocked on https://github.com/rust-lang/rust/issues/62833
pub enum WherePredicate {
/// A type predicate in a `where` clause: `for<'c> Foo<'c>: Trait<'c>`.
Type(PredicateType),
diff --git a/src/item.rs b/src/item.rs
index a1ce7bf..83b82fd 100644
--- a/src/item.rs
+++ b/src/item.rs
@@ -7,6 +7,8 @@
use crate::tt::TokenStreamHelper;
#[cfg(feature = "extra-traits")]
use std::hash::{Hash, Hasher};
+#[cfg(feature = "parsing")]
+use std::mem;
ast_enum_of_structs! {
/// Things that can appear directly inside of a module or scope.
@@ -435,6 +437,32 @@
}
}
+impl Item {
+ #[cfg(feature = "parsing")]
+ pub(crate) fn replace_attrs(&mut self, new: Vec<Attribute>) -> Vec<Attribute> {
+ match self {
+ Item::ExternCrate(ItemExternCrate { attrs, .. })
+ | Item::Use(ItemUse { attrs, .. })
+ | Item::Static(ItemStatic { attrs, .. })
+ | Item::Const(ItemConst { attrs, .. })
+ | Item::Fn(ItemFn { attrs, .. })
+ | Item::Mod(ItemMod { attrs, .. })
+ | Item::ForeignMod(ItemForeignMod { attrs, .. })
+ | Item::Type(ItemType { attrs, .. })
+ | Item::Struct(ItemStruct { attrs, .. })
+ | Item::Enum(ItemEnum { attrs, .. })
+ | Item::Union(ItemUnion { attrs, .. })
+ | Item::Trait(ItemTrait { attrs, .. })
+ | Item::TraitAlias(ItemTraitAlias { attrs, .. })
+ | Item::Impl(ItemImpl { attrs, .. })
+ | Item::Macro(ItemMacro { attrs, .. })
+ | Item::Macro2(ItemMacro2 { attrs, .. }) => mem::replace(attrs, new),
+ Item::Verbatim(_) => Vec::new(),
+ Item::__Nonexhaustive => unreachable!(),
+ }
+ }
+}
+
#[cfg(feature = "extra-traits")]
impl Eq for ItemMacro2 {}
@@ -1128,6 +1156,7 @@
use crate::ext::IdentExt;
use crate::parse::discouraged::Speculative;
use crate::parse::{Parse, ParseStream, Result};
+ use crate::token::Brace;
use proc_macro2::{Delimiter, Group, Punct, Spacing, TokenTree};
use std::iter::{self, FromIterator};
@@ -1234,31 +1263,8 @@
Err(lookahead.error())
}?;
- {
- let item_attrs = match &mut item {
- Item::ExternCrate(item) => &mut item.attrs,
- Item::Use(item) => &mut item.attrs,
- Item::Static(item) => &mut item.attrs,
- Item::Const(item) => &mut item.attrs,
- Item::Fn(item) => &mut item.attrs,
- Item::Mod(item) => &mut item.attrs,
- Item::ForeignMod(item) => &mut item.attrs,
- Item::Type(item) => &mut item.attrs,
- Item::Struct(item) => &mut item.attrs,
- Item::Enum(item) => &mut item.attrs,
- Item::Union(item) => &mut item.attrs,
- Item::Trait(item) => &mut item.attrs,
- Item::TraitAlias(item) => &mut item.attrs,
- Item::Impl(item) => &mut item.attrs,
- Item::Macro(item) => &mut item.attrs,
- Item::Macro2(item) => &mut item.attrs,
- Item::Verbatim(_) => return Ok(item),
- Item::__Nonexhaustive => unreachable!(),
- };
- attrs.extend(item_attrs.drain(..));
- *item_attrs = attrs;
- }
-
+ attrs.extend(item.replace_attrs(Vec::new()));
+ item.replace_attrs(attrs);
Ok(item)
}
}
@@ -1463,6 +1469,54 @@
}
}
+ fn pop_variadic(args: &mut Punctuated<FnArg, Token![,]>) -> Option<Variadic> {
+ let trailing_punct = args.trailing_punct();
+
+ let last = match args.last_mut()? {
+ FnArg::Typed(last) => last,
+ _ => return None,
+ };
+
+ let ty = match last.ty.as_ref() {
+ Type::Verbatim(ty) => ty,
+ _ => return None,
+ };
+
+ let mut variadic = Variadic {
+ attrs: Vec::new(),
+ dots: parse2(ty.clone()).ok()?,
+ };
+
+ if let Pat::Verbatim(pat) = last.pat.as_ref() {
+ if pat.to_string() == "..." && !trailing_punct {
+ variadic.attrs = mem::replace(&mut last.attrs, Vec::new());
+ args.pop();
+ }
+ }
+
+ Some(variadic)
+ }
+
+ fn variadic_to_tokens(dots: &Token![...]) -> TokenStream {
+ TokenStream::from_iter(vec![
+ TokenTree::Punct({
+ let mut dot = Punct::new('.', Spacing::Joint);
+ dot.set_span(dots.spans[0]);
+ dot
+ }),
+ TokenTree::Punct({
+ let mut dot = Punct::new('.', Spacing::Joint);
+ dot.set_span(dots.spans[1]);
+ dot
+ }),
+ TokenTree::Punct({
+ let mut dot = Punct::new('.', Spacing::Alone);
+ dot.set_span(dots.spans[2]);
+ dot
+ }),
+ ])
+ }
+
impl Parse for ItemFn {
fn parse(input: ParseStream) -> Result<Self> {
let outer_attrs = input.call(Attribute::parse_outer)?;
@@ -1477,22 +1531,8 @@
let content;
let paren_token = parenthesized!(content in input);
- let inputs = content.parse_terminated(FnArg::parse)?;
- let variadic = inputs.last().as_ref().and_then(get_variadic);
-
- fn get_variadic(input: &&FnArg) -> Option<Variadic> {
- if let FnArg::Typed(PatType { ty, .. }) = input {
- if let Type::Verbatim(tokens) = &**ty {
- if let Ok(dots) = parse2(tokens.clone()) {
- return Some(Variadic {
- attrs: Vec::new(),
- dots,
- });
- }
- }
- }
- None
- }
+ let mut inputs = parse_fn_args(&content)?;
+ let variadic = pop_variadic(&mut inputs);
let output: ReturnType = input.parse()?;
let where_clause: Option<WhereClause> = input.parse()?;
@@ -1562,6 +1602,56 @@
}
}
+ fn parse_fn_args(input: ParseStream) -> Result<Punctuated<FnArg, Token![,]>> {
+ let mut args = Punctuated::new();
+ let mut has_receiver = false;
+
+ while !input.is_empty() {
+ let attrs = input.call(Attribute::parse_outer)?;
+
+ let arg = if let Some(dots) = input.parse::<Option<Token![...]>>()? {
+ FnArg::Typed(PatType {
+ attrs,
+ pat: Box::new(Pat::Verbatim(variadic_to_tokens(&dots))),
+ colon_token: Token,
+ ty: Box::new(Type::Verbatim(variadic_to_tokens(&dots))),
+ })
+ } else {
+ let mut arg: FnArg = input.parse()?;
+ match &mut arg {
+ FnArg::Receiver(receiver) if has_receiver => {
+ return Err(Error::new(
+ receiver.self_token.span,
+ "unexpected second method receiver",
+ ));
+ }
+ FnArg::Receiver(receiver) if !args.is_empty() => {
+ return Err(Error::new(
+ receiver.self_token.span,
+ "unexpected method receiver",
+ ));
+ }
+ FnArg::Receiver(receiver) => {
+ has_receiver = true;
+ receiver.attrs = attrs;
+ }
+ FnArg::Typed(arg) => arg.attrs = attrs,
+ }
+ arg
+ };
+ args.push_value(arg);
+
+ if input.is_empty() {
+ break;
+ }
+
+ let comma: Token![,] = input.parse()?;
+ args.push_punct(comma);
+ }
+
+ Ok(args)
+ }
+
fn fn_arg_typed(input: ParseStream) -> Result<PatType> {
// Hack to parse pre-2018 syntax in
// test/ui/rfc-2565-param-attrs/param-attrs-pretty.rs
@@ -1584,20 +1674,7 @@
pat: input.parse()?,
colon_token: input.parse()?,
ty: Box::new(match input.parse::<Option<Token![...]>>()? {
- Some(dot3) => {
- let args = vec![
- TokenTree::Punct(Punct::new('.', Spacing::Joint)),
- TokenTree::Punct(Punct::new('.', Spacing::Joint)),
- TokenTree::Punct(Punct::new('.', Spacing::Alone)),
- ];
- let tokens = TokenStream::from_iter(args.into_iter().zip(&dot3.spans).map(
- |(mut arg, span)| {
- arg.set_span(*span);
- arg
- },
- ));
- Type::Verbatim(tokens)
- }
+ Some(dot3) => Type::Verbatim(variadic_to_tokens(&dot3)),
None => input.parse()?,
}),
})
@@ -1718,25 +1795,8 @@
let content;
let paren_token = parenthesized!(content in input);
- let mut inputs = Punctuated::new();
- let mut variadic = None;
- while !content.is_empty() {
- let attrs = content.call(Attribute::parse_outer)?;
-
- if let Some(dots) = content.parse()? {
- variadic = Some(Variadic { attrs, dots });
- break;
- }
-
- let mut arg = content.call(fn_arg_typed)?;
- arg.attrs = attrs;
- inputs.push_value(FnArg::Typed(arg));
- if content.is_empty() {
- break;
- }
-
- inputs.push_punct(content.parse()?);
- }
+ let mut inputs = parse_fn_args(&content)?;
+ let variadic = pop_variadic(&mut inputs);
let output: ReturnType = input.parse()?;
let where_clause: Option<WhereClause> = input.parse()?;
@@ -2101,7 +2161,10 @@
impl Parse for TraitItem {
fn parse(input: ParseStream) -> Result<Self> {
+ let begin = input.fork();
let mut attrs = input.call(Attribute::parse_outer)?;
+ let vis: Visibility = input.parse()?;
+ let defaultness: Option<Token![default]> = input.parse()?;
let ahead = input.fork();
let lookahead = ahead.lookahead1();
@@ -2139,18 +2202,20 @@
Err(lookahead.error())
}?;
- {
- let item_attrs = match &mut item {
- TraitItem::Const(item) => &mut item.attrs,
- TraitItem::Method(item) => &mut item.attrs,
- TraitItem::Type(item) => &mut item.attrs,
- TraitItem::Macro(item) => &mut item.attrs,
- TraitItem::Verbatim(_) | TraitItem::__Nonexhaustive => unreachable!(),
- };
- attrs.extend(item_attrs.drain(..));
- *item_attrs = attrs;
+ match (vis, defaultness) {
+ (Visibility::Inherited, None) => {}
+ _ => return Ok(TraitItem::Verbatim(verbatim::between(begin, input))),
}
+ let item_attrs = match &mut item {
+ TraitItem::Const(item) => &mut item.attrs,
+ TraitItem::Method(item) => &mut item.attrs,
+ TraitItem::Type(item) => &mut item.attrs,
+ TraitItem::Macro(item) => &mut item.attrs,
+ TraitItem::Verbatim(_) | TraitItem::__Nonexhaustive => unreachable!(),
+ };
+ attrs.extend(item_attrs.drain(..));
+ *item_attrs = attrs;
Ok(item)
}
}
@@ -2190,7 +2255,8 @@
let content;
let paren_token = parenthesized!(content in input);
- let inputs = content.parse_terminated(FnArg::parse)?;
+ let mut inputs = parse_fn_args(&content)?;
+ let variadic = pop_variadic(&mut inputs);
let output: ReturnType = input.parse()?;
let where_clause: Option<WhereClause> = input.parse()?;
@@ -2221,7 +2287,7 @@
paren_token,
inputs,
output,
- variadic: None,
+ variadic,
generics: Generics {
where_clause,
..generics
@@ -2312,21 +2378,14 @@
Generics::default()
};
- let trait_ = {
- // TODO: optimize using advance_to
+ let trait_ = (|| -> Option<_> {
let ahead = input.fork();
- if ahead.parse::<Option<Token![!]>>().is_ok()
- && ahead.parse::<Path>().is_ok()
- && ahead.parse::<Token![for]>().is_ok()
- {
- let polarity: Option<Token![!]> = input.parse()?;
- let path: Path = input.parse()?;
- let for_token: Token![for] = input.parse()?;
- Some((polarity, path, for_token))
- } else {
- None
- }
- };
+ let polarity: Option<Token![!]> = ahead.parse().ok()?;
+ let path: Path = ahead.parse().ok()?;
+ let for_token: Token![for] = ahead.parse().ok()?;
+ input.advance_to(&ahead);
+ Some((polarity, path, for_token))
+ })();
let self_ty: Type = input.parse()?;
let where_clause: Option<WhereClause> = input.parse()?;
@@ -2358,6 +2417,7 @@
impl Parse for ImplItem {
fn parse(input: ParseStream) -> Result<Self> {
+ let begin = input.fork();
let mut attrs = input.call(Attribute::parse_outer)?;
let ahead = input.fork();
let vis: Visibility = ahead.parse()?;
@@ -2372,10 +2432,30 @@
};
let mut item = if lookahead.peek(Token![const]) {
- ahead.parse::<Token![const]>()?;
+ let const_token: Token![const] = ahead.parse()?;
let lookahead = ahead.lookahead1();
if lookahead.peek(Ident) {
- input.parse().map(ImplItem::Const)
+ input.advance_to(&ahead);
+ let ident: Ident = input.parse()?;
+ let colon_token: Token![:] = input.parse()?;
+ let ty: Type = input.parse()?;
+ if let Some(eq_token) = input.parse()? {
+ return Ok(ImplItem::Const(ImplItemConst {
+ attrs,
+ vis,
+ defaultness,
+ const_token,
+ ident,
+ colon_token,
+ ty,
+ eq_token,
+ expr: input.parse()?,
+ semi_token: input.parse()?,
+ }));
+ } else {
+ input.parse::<Token![;]>()?;
+ return Ok(ImplItem::Verbatim(verbatim::between(begin, input)));
+ }
} else if lookahead.peek(Token![unsafe])
|| lookahead.peek(Token![async])
|| lookahead.peek(Token![extern])
@@ -2392,7 +2472,41 @@
{
input.parse().map(ImplItem::Method)
} else if lookahead.peek(Token![type]) {
- input.parse().map(ImplItem::Type)
+ input.advance_to(&ahead);
+ let type_token: Token![type] = input.parse()?;
+ let ident: Ident = input.parse()?;
+ let mut generics: Generics = input.parse()?;
+ let colon_token: Option<Token![:]> = input.parse()?;
+ if colon_token.is_some() {
+ let mut first = true;
+ while !input.peek(Token![where])
+ && !input.peek(Token![=])
+ && !input.peek(Token![;])
+ {
+ if !first {
+ input.parse::<Token![+]>()?;
+ }
+ input.parse::<TypeParamBound>()?;
+ first = false;
+ }
+ }
+ generics.where_clause = input.parse()?;
+ if let Some(eq_token) = input.parse()? {
+ return Ok(ImplItem::Type(ImplItemType {
+ attrs,
+ vis,
+ defaultness,
+ type_token,
+ ident,
+ generics,
+ eq_token,
+ ty: input.parse()?,
+ semi_token: input.parse()?,
+ }));
+ } else {
+ input.parse::<Token![;]>()?;
+ return Ok(ImplItem::Verbatim(verbatim::between(begin, input)));
+ }
} else if vis.is_inherited() && defaultness.is_none() && lookahead.peek(existential) {
input.call(item_existential).map(ImplItem::Verbatim)
} else if vis.is_inherited()
@@ -2445,7 +2559,7 @@
impl Parse for ImplItemMethod {
fn parse(input: ParseStream) -> Result<Self> {
- let outer_attrs = input.call(Attribute::parse_outer)?;
+ let mut attrs = input.call(Attribute::parse_outer)?;
let vis: Visibility = input.parse()?;
let defaultness: Option<Token![default]> = input.parse()?;
let constness: Option<Token![const]> = input.parse()?;
@@ -2458,18 +2572,36 @@
let content;
let paren_token = parenthesized!(content in input);
- let inputs = content.parse_terminated(FnArg::parse)?;
+ let mut inputs = parse_fn_args(&content)?;
+ let variadic = pop_variadic(&mut inputs);
let output: ReturnType = input.parse()?;
let where_clause: Option<WhereClause> = input.parse()?;
- let content;
- let brace_token = braced!(content in input);
- let inner_attrs = content.call(Attribute::parse_inner)?;
- let stmts = content.call(Block::parse_within)?;
+ let block = if let Some(semi) = input.parse::<Option<Token![;]>>()? {
+ // Accept methods without a body in an impl block because
+ // rustc's *parser* does not reject them (the compilation error
+ // is emitted later than parsing) and it can be useful for macro
+ // DSLs.
+ let mut punct = Punct::new(';', Spacing::Alone);
+ punct.set_span(semi.span);
+ let tokens = TokenStream::from_iter(vec![TokenTree::Punct(punct)]);
+ Block {
+ brace_token: Brace::default(),
+ stmts: vec![Stmt::Item(Item::Verbatim(tokens))],
+ }
+ } else {
+ let content;
+ let brace_token = braced!(content in input);
+ attrs.extend(content.call(Attribute::parse_inner)?);
+ Block {
+ brace_token,
+ stmts: content.call(Block::parse_within)?,
+ }
+ };
Ok(ImplItemMethod {
- attrs: private::attrs(outer_attrs, inner_attrs),
+ attrs,
vis,
defaultness,
sig: Signature {
@@ -2482,13 +2614,13 @@
paren_token,
inputs,
output,
- variadic: None,
+ variadic,
generics: Generics {
where_clause,
..generics
},
},
- block: Block { brace_token, stmts },
+ block,
})
}
}
@@ -2558,6 +2690,7 @@
use crate::attr::FilterAttrs;
use crate::print::TokensOrDefault;
+ use crate::punctuated::Pair;
impl ToTokens for ItemExternCrate {
fn to_tokens(&self, tokens: &mut TokenStream) {
@@ -2922,6 +3055,14 @@
self.vis.to_tokens(tokens);
self.defaultness.to_tokens(tokens);
self.sig.to_tokens(tokens);
+ if self.block.stmts.len() == 1 {
+ if let Stmt::Item(Item::Verbatim(verbatim)) = &self.block.stmts[0] {
+ if verbatim.to_string() == ";" {
+ verbatim.to_tokens(tokens);
+ return;
+ }
+ }
+ }
self.block.brace_token.surround(tokens, |tokens| {
tokens.append_all(self.attrs.inner());
tokens.append_all(&self.block.stmts);
@@ -2992,23 +3133,31 @@
}
}
- fn has_variadic(inputs: &Punctuated<FnArg, Token![,]>) -> bool {
- let last = match inputs.last() {
- Some(last) => last,
- None => return false,
+ fn maybe_variadic_to_tokens(arg: &FnArg, tokens: &mut TokenStream) -> bool {
+ let arg = match arg {
+ FnArg::Typed(arg) => arg,
+ FnArg::Receiver(receiver) => {
+ receiver.to_tokens(tokens);
+ return false;
+ }
};
- let pat = match last {
- FnArg::Typed(pat) => pat,
- FnArg::Receiver(_) => return false,
- };
-
- let tokens = match pat.ty.as_ref() {
- Type::Verbatim(tokens) => tokens,
- _ => return false,
- };
-
- tokens.to_string() == "..."
+ match arg.ty.as_ref() {
+ Type::Verbatim(ty) if ty.to_string() == "..." => {
+ match arg.pat.as_ref() {
+ Pat::Verbatim(pat) if pat.to_string() == "..." => {
+ tokens.append_all(arg.attrs.outer());
+ pat.to_tokens(tokens);
+ }
+ _ => arg.to_tokens(tokens),
+ }
+ true
+ }
+ _ => {
+ arg.to_tokens(tokens);
+ false
+ }
+ }
}
impl ToTokens for Signature {
@@ -3021,8 +3170,19 @@
self.ident.to_tokens(tokens);
self.generics.to_tokens(tokens);
self.paren_token.surround(tokens, |tokens| {
- self.inputs.to_tokens(tokens);
- if self.variadic.is_some() && !has_variadic(&self.inputs) {
+ let mut last_is_variadic = false;
+ for input in self.inputs.pairs() {
+ match input {
+ Pair::Punctuated(input, comma) => {
+ maybe_variadic_to_tokens(input, tokens);
+ comma.to_tokens(tokens);
+ }
+ Pair::End(input) => {
+ last_is_variadic = maybe_variadic_to_tokens(input, tokens);
+ }
+ }
+ }
+ if self.variadic.is_some() && !last_is_variadic {
if !self.inputs.empty_or_trailing() {
<Token![,]>::default().to_tokens(tokens);
}
diff --git a/src/keyword.rs b/src/keyword.rs
deleted file mode 100644
index e69de29..0000000
--- a/src/keyword.rs
+++ /dev/null
diff --git a/src/lib.rs b/src/lib.rs
index 2a2ff61..c88a813 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -242,7 +242,7 @@
//! dynamic library libproc_macro from rustc toolchain.
// Syn types in rustdoc of other crates get linked to here.
-#![doc(html_root_url = "https://docs.rs/syn/1.0.7")]
+#![doc(html_root_url = "https://docs.rs/syn/1.0.16")]
#![deny(clippy::all, clippy::pedantic)]
// Ignored clippy lints.
#![allow(
@@ -265,6 +265,8 @@
clippy::empty_enum,
clippy::if_not_else,
clippy::items_after_statements,
+ clippy::match_same_arms,
+ clippy::missing_errors_doc,
clippy::module_name_repetitions,
clippy::must_use_candidate,
clippy::shadow_unrelated,
@@ -760,6 +762,8 @@
mod custom_keyword;
mod custom_punctuation;
mod sealed;
+mod span;
+mod thread;
#[cfg(feature = "parsing")]
mod lookahead;
@@ -767,13 +771,12 @@
#[cfg(feature = "parsing")]
pub mod parse;
-mod span;
+#[cfg(all(feature = "parsing", feature = "full"))]
+mod verbatim;
#[cfg(all(any(feature = "full", feature = "derive"), feature = "printing"))]
mod print;
-mod thread;
-
////////////////////////////////////////////////////////////////////////////////
#[allow(dead_code, non_camel_case_types)]
diff --git a/src/lit.rs b/src/lit.rs
index abc4ec2..0f271e4 100644
--- a/src/lit.rs
+++ b/src/lit.rs
@@ -403,18 +403,23 @@
impl LitInt {
pub fn new(repr: &str, span: Span) -> Self {
- if let Some((digits, suffix)) = value::parse_lit_int(repr) {
- let mut token = value::to_literal(repr);
- token.set_span(span);
- LitInt {
- repr: Box::new(LitIntRepr {
- token,
- digits,
- suffix,
- }),
- }
- } else {
- panic!("Not an integer literal: `{}`", repr);
+ let (digits, suffix) = match value::parse_lit_int(repr) {
+ Some(parse) => parse,
+ None => panic!("Not an integer literal: `{}`", repr),
+ };
+
+ let mut token = match value::to_literal(repr, &digits, &suffix) {
+ Some(token) => token,
+ None => panic!("Unsupported integer literal: `{}`", repr),
+ };
+
+ token.set_span(span);
+ LitInt {
+ repr: Box::new(LitIntRepr {
+ token,
+ digits,
+ suffix,
+ }),
}
}
@@ -492,18 +497,23 @@
impl LitFloat {
pub fn new(repr: &str, span: Span) -> Self {
- if let Some((digits, suffix)) = value::parse_lit_float(repr) {
- let mut token = value::to_literal(repr);
- token.set_span(span);
- LitFloat {
- repr: Box::new(LitFloatRepr {
- token,
- digits,
- suffix,
- }),
- }
- } else {
- panic!("Not a float literal: `{}`", repr);
+ let (digits, suffix) = match value::parse_lit_float(repr) {
+ Some(parse) => parse,
+ None => panic!("Not a float literal: `{}`", repr),
+ };
+
+ let mut token = match value::to_literal(repr, &digits, &suffix) {
+ Some(token) => token,
+ None => panic!("Unsupported float literal: `{}`", repr),
+ };
+
+ token.set_span(span);
+ LitFloat {
+ repr: Box::new(LitFloatRepr {
+ token,
+ digits,
+ suffix,
+ }),
}
}
@@ -691,7 +701,9 @@
#[cfg(feature = "parsing")]
pub mod parsing {
use super::*;
+ use crate::buffer::Cursor;
use crate::parse::{Parse, ParseStream, Result};
+ use proc_macro2::Punct;
impl Parse for Lit {
fn parse(input: ParseStream) -> Result<Self> {
@@ -699,25 +711,73 @@
if let Some((lit, rest)) = cursor.literal() {
return Ok((Lit::new(lit), rest));
}
- while let Some((ident, rest)) = cursor.ident() {
- let value = if ident == "true" {
- true
- } else if ident == "false" {
- false
- } else {
- break;
- };
- let lit_bool = LitBool {
- value,
- span: ident.span(),
- };
- return Ok((Lit::Bool(lit_bool), rest));
+
+ if let Some((ident, rest)) = cursor.ident() {
+ let value = ident == "true";
+ if value || ident == "false" {
+ let lit_bool = LitBool {
+ value,
+ span: ident.span(),
+ };
+ return Ok((Lit::Bool(lit_bool), rest));
+ }
}
+
+ if let Some((punct, rest)) = cursor.punct() {
+ if punct.as_char() == '-' {
+ if let Some((lit, rest)) = parse_negative_lit(punct, rest) {
+ return Ok((lit, rest));
+ }
+ }
+ }
+
Err(cursor.error("expected literal"))
})
}
}
+ fn parse_negative_lit(neg: Punct, cursor: Cursor) -> Option<(Lit, Cursor)> {
+ let (lit, rest) = cursor.literal()?;
+
+ let mut span = neg.span();
+ span = span.join(lit.span()).unwrap_or(span);
+
+ let mut repr = lit.to_string();
+ repr.insert(0, '-');
+
+ if !(repr.ends_with("f32") || repr.ends_with("f64")) {
+ if let Some((digits, suffix)) = value::parse_lit_int(&repr) {
+ if let Some(mut token) = value::to_literal(&repr, &digits, &suffix) {
+ token.set_span(span);
+ return Some((
+ Lit::Int(LitInt {
+ repr: Box::new(LitIntRepr {
+ token,
+ digits,
+ suffix,
+ }),
+ }),
+ rest,
+ ));
+ }
+ }
+ }
+
+ let (digits, suffix) = value::parse_lit_float(&repr)?;
+ let mut token = value::to_literal(&repr, &digits, &suffix)?;
+ token.set_span(span);
+ Some((
+ Lit::Float(LitFloat {
+ repr: Box::new(LitFloatRepr {
+ token,
+ digits,
+ suffix,
+ }),
+ }),
+ rest,
+ ))
+ }
+
impl Parse for LitStr {
fn parse(input: ParseStream) -> Result<Self> {
let head = input.fork();
@@ -1372,11 +1432,33 @@
}
}
- pub fn to_literal(s: &str) -> Literal {
- let stream = s.parse::<TokenStream>().unwrap();
- match stream.into_iter().next().unwrap() {
- TokenTree::Literal(l) => l,
- _ => unreachable!(),
+ pub fn to_literal(repr: &str, digits: &str, suffix: &str) -> Option<Literal> {
+ if repr.starts_with('-') {
+ if suffix == "f64" {
+ digits.parse().ok().map(Literal::f64_suffixed)
+ } else if suffix == "f32" {
+ digits.parse().ok().map(Literal::f32_suffixed)
+ } else if suffix == "i64" {
+ digits.parse().ok().map(Literal::i64_suffixed)
+ } else if suffix == "i32" {
+ digits.parse().ok().map(Literal::i32_suffixed)
+ } else if suffix == "i16" {
+ digits.parse().ok().map(Literal::i16_suffixed)
+ } else if suffix == "i8" {
+ digits.parse().ok().map(Literal::i8_suffixed)
+ } else if !suffix.is_empty() {
+ None
+ } else if digits.contains('.') {
+ digits.parse().ok().map(Literal::f64_unsuffixed)
+ } else {
+ digits.parse().ok().map(Literal::i64_unsuffixed)
+ }
+ } else {
+ let stream = repr.parse::<TokenStream>().unwrap();
+ match stream.into_iter().next().unwrap() {
+ TokenTree::Literal(l) => Some(l),
+ _ => unreachable!(),
+ }
}
}
}
diff --git a/src/parse.rs b/src/parse.rs
index 0dd698b..96eecce 100644
--- a/src/parse.rs
+++ b/src/parse.rs
@@ -261,13 +261,16 @@
// the cell.
cell: Cell<Cursor<'static>>,
marker: PhantomData<Cursor<'a>>,
- unexpected: Rc<Cell<Option<Span>>>,
+ unexpected: Cell<Option<Rc<Cell<Unexpected>>>>,
}
impl<'a> Drop for ParseBuffer<'a> {
fn drop(&mut self) {
- if !self.is_empty() && self.unexpected.get().is_none() {
- self.unexpected.set(Some(self.cursor().span()));
+ if !self.is_empty() {
+ let (inner, old_span) = inner_unexpected(self);
+ if old_span.is_none() {
+ inner.set(Unexpected::Some(self.cursor().span()));
+ }
}
}
}
@@ -371,36 +374,55 @@
unsafe { mem::transmute::<Cursor<'c>, Cursor<'a>>(to) }
}
-fn skip(input: ParseStream) -> bool {
- input
- .step(|cursor| {
- if let Some((_lifetime, rest)) = cursor.lifetime() {
- Ok((true, rest))
- } else if let Some((_token, rest)) = cursor.token_tree() {
- Ok((true, rest))
- } else {
- Ok((false, *cursor))
- }
- })
- .unwrap()
-}
-
pub(crate) fn new_parse_buffer(
scope: Span,
cursor: Cursor,
- unexpected: Rc<Cell<Option<Span>>>,
+ unexpected: Rc<Cell<Unexpected>>,
) -> ParseBuffer {
ParseBuffer {
scope,
// See comment on `cell` in the struct definition.
cell: Cell::new(unsafe { mem::transmute::<Cursor, Cursor<'static>>(cursor) }),
marker: PhantomData,
- unexpected,
+ unexpected: Cell::new(Some(unexpected)),
}
}
-pub(crate) fn get_unexpected(buffer: &ParseBuffer) -> Rc<Cell<Option<Span>>> {
- buffer.unexpected.clone()
+#[derive(Clone)]
+pub(crate) enum Unexpected {
+ None,
+ Some(Span),
+ Chain(Rc<Cell<Unexpected>>),
+}
+
+impl Default for Unexpected {
+ fn default() -> Self {
+ Unexpected::None
+ }
+}
+
+// We call this on Cell<Unexpected> and Cell<Option<T>> where temporarily
+// swapping in a None is cheap.
+fn cell_clone<T: Default + Clone>(cell: &Cell<T>) -> T {
+ let prev = cell.take();
+ let ret = prev.clone();
+ cell.set(prev);
+ ret
+}
+
+fn inner_unexpected(buffer: &ParseBuffer) -> (Rc<Cell<Unexpected>>, Option<Span>) {
+ let mut unexpected = get_unexpected(buffer);
+ loop {
+ match cell_clone(&unexpected) {
+ Unexpected::None => return (unexpected, None),
+ Unexpected::Some(span) => return (unexpected, Some(span)),
+ Unexpected::Chain(next) => unexpected = next,
+ }
+ }
+}
+
+pub(crate) fn get_unexpected(buffer: &ParseBuffer) -> Rc<Cell<Unexpected>> {
+ cell_clone(&buffer.unexpected).unwrap()
}
impl<'a> ParseBuffer<'a> {
@@ -562,14 +584,17 @@
/// }
/// ```
pub fn peek2<T: Peek>(&self, token: T) -> bool {
- let ahead = self.fork();
- skip(&ahead) && ahead.peek(token)
+ let _ = token;
+ self.cursor().skip().map_or(false, T::Token::peek)
}
/// Looks at the third-next token in the parse stream.
pub fn peek3<T: Peek>(&self, token: T) -> bool {
- let ahead = self.fork();
- skip(&ahead) && skip(&ahead) && ahead.peek(token)
+ let _ = token;
+ self.cursor()
+ .skip()
+ .and_then(Cursor::skip)
+ .map_or(false, T::Token::peek)
}
/// Parses zero or more occurrences of `T` separated by punctuation of type
@@ -841,8 +866,8 @@
cell: self.cell.clone(),
marker: PhantomData,
// Not the parent's unexpected. Nothing cares whether the clone
- // parses all the way.
- unexpected: Rc::new(Cell::new(None)),
+ // parses all the way unless we `advance_to`.
+ unexpected: Cell::new(Some(Rc::new(Cell::new(Unexpected::None)))),
}
}
@@ -953,6 +978,18 @@
Ok(node)
}
+ /// Returns the `Span` of the next token in the parse stream, or
+ /// `Span::call_site()` if this parse stream has completely exhausted its
+ /// input `TokenStream`.
+ pub fn span(&self) -> Span {
+ let cursor = self.cursor();
+ if cursor.eof() {
+ self.scope
+ } else {
+ crate::buffer::open_span_of_group(cursor)
+ }
+ }
+
/// Provides low-level access to the token representation underlying this
/// parse stream.
///
@@ -963,7 +1000,7 @@
}
fn check_unexpected(&self) -> Result<()> {
- match self.unexpected.get() {
+ match inner_unexpected(self).1 {
Some(span) => Err(Error::new(span, "unexpected token")),
None => Ok(()),
}
@@ -1095,7 +1132,7 @@
fn tokens_to_parse_buffer(tokens: &TokenBuffer) -> ParseBuffer {
let scope = Span::call_site();
let cursor = tokens.begin();
- let unexpected = Rc::new(Cell::new(None));
+ let unexpected = Rc::new(Cell::new(Unexpected::None));
new_parse_buffer(scope, cursor, unexpected)
}
@@ -1121,7 +1158,7 @@
fn __parse_scoped(self, scope: Span, tokens: TokenStream) -> Result<Self::Output> {
let buf = TokenBuffer::new2(tokens);
let cursor = buf.begin();
- let unexpected = Rc::new(Cell::new(None));
+ let unexpected = Rc::new(Cell::new(Unexpected::None));
let state = new_parse_buffer(scope, cursor, unexpected);
let node = self(&state)?;
state.check_unexpected()?;
diff --git a/src/parse_quote.rs b/src/parse_quote.rs
index 18a47b9..96e4765 100644
--- a/src/parse_quote.rs
+++ b/src/parse_quote.rs
@@ -56,8 +56,10 @@
/// or inner like `#![...]`
/// - [`Punctuated<T, P>`] — parses zero or more `T` separated by punctuation
/// `P` with optional trailing punctuation
+/// - [`Vec<Stmt>`] — parses the same as `Block::parse_within`
///
/// [`Punctuated<T, P>`]: punctuated::Punctuated
+/// [`Vec<Stmt>`]: Block::parse_within
///
/// # Panics
///
@@ -112,6 +114,8 @@
use crate::punctuated::Punctuated;
#[cfg(any(feature = "full", feature = "derive"))]
use crate::{attr, Attribute};
+#[cfg(feature = "full")]
+use crate::{Block, Stmt};
#[cfg(any(feature = "full", feature = "derive"))]
impl ParseQuote for Attribute {
@@ -129,3 +133,10 @@
Self::parse_terminated(input)
}
}
+
+#[cfg(feature = "full")]
+impl ParseQuote for Vec<Stmt> {
+ fn parse(input: ParseStream) -> Result<Self> {
+ Block::parse_within(input)
+ }
+}
diff --git a/src/pat.rs b/src/pat.rs
index 262129b..fac651e 100644
--- a/src/pat.rs
+++ b/src/pat.rs
@@ -386,11 +386,12 @@
use super::*;
use crate::ext::IdentExt;
- use crate::parse::{Parse, ParseStream, Result};
+ use crate::parse::{Parse, ParseBuffer, ParseStream, Result};
use crate::path;
impl Parse for Pat {
fn parse(input: ParseStream) -> Result<Self> {
+ let begin = input.fork();
let lookahead = input.lookahead1();
if lookahead.peek(Ident)
&& ({
@@ -434,7 +435,7 @@
} else if lookahead.peek(token::Bracket) {
input.call(pat_slice).map(Pat::Slice)
} else if lookahead.peek(Token![..]) && !input.peek(Token![...]) {
- input.call(pat_rest).map(Pat::Rest)
+ pat_range_half_open(input, begin)
} else {
Err(lookahead.error())
}
@@ -442,10 +443,11 @@
}
fn pat_path_or_macro_or_struct_or_range(input: ParseStream) -> Result<Pat> {
+ let begin = input.fork();
let (qself, path) = path::parsing::qpath(input, true)?;
if input.peek(Token![..]) {
- return pat_range(input, qself, path).map(Pat::Range);
+ return pat_range(input, begin, qself, path);
}
if qself.is_some() {
@@ -487,7 +489,7 @@
} else if input.peek(token::Paren) {
pat_tuple_struct(input, path).map(Pat::TupleStruct)
} else if input.peek(Token![..]) {
- pat_range(input, qself, path).map(Pat::Range)
+ pat_range(input, begin, qself, path)
} else {
Ok(Pat::Path(PatPath {
attrs: Vec::new(),
@@ -624,17 +626,44 @@
})
}
- fn pat_range(input: ParseStream, qself: Option<QSelf>, path: Path) -> Result<PatRange> {
- Ok(PatRange {
- attrs: Vec::new(),
- lo: Box::new(Expr::Path(ExprPath {
+ fn pat_range(
+ input: ParseStream,
+ begin: ParseBuffer,
+ qself: Option<QSelf>,
+ path: Path,
+ ) -> Result<Pat> {
+ let limits: RangeLimits = input.parse()?;
+ let hi = input.call(pat_lit_expr)?;
+ if let Some(hi) = hi {
+ Ok(Pat::Range(PatRange {
attrs: Vec::new(),
- qself,
- path,
- })),
- limits: input.parse()?,
- hi: input.call(pat_lit_expr)?,
- })
+ lo: Box::new(Expr::Path(ExprPath {
+ attrs: Vec::new(),
+ qself,
+ path,
+ })),
+ limits,
+ hi,
+ }))
+ } else {
+ Ok(Pat::Verbatim(verbatim::between(begin, input)))
+ }
+ }
+
+ fn pat_range_half_open(input: ParseStream, begin: ParseBuffer) -> Result<Pat> {
+ let limits: RangeLimits = input.parse()?;
+ let hi = input.call(pat_lit_expr)?;
+ if hi.is_some() {
+ Ok(Pat::Verbatim(verbatim::between(begin, input)))
+ } else {
+ match limits {
+ RangeLimits::HalfOpen(dot2_token) => Ok(Pat::Rest(PatRest {
+ attrs: Vec::new(),
+ dot2_token,
+ })),
+ RangeLimits::Closed(_) => Err(input.error("expected range upper bound")),
+ }
+ }
}
fn pat_tuple(input: ParseStream) -> Result<PatTuple> {
@@ -669,14 +698,21 @@
}
fn pat_lit_or_range(input: ParseStream) -> Result<Pat> {
- let lo = input.call(pat_lit_expr)?;
+ let begin = input.fork();
+ let lo = input.call(pat_lit_expr)?.unwrap();
if input.peek(Token![..]) {
- Ok(Pat::Range(PatRange {
- attrs: Vec::new(),
- lo,
- limits: input.parse()?,
- hi: input.call(pat_lit_expr)?,
- }))
+ let limits: RangeLimits = input.parse()?;
+ let hi = input.call(pat_lit_expr)?;
+ if let Some(hi) = hi {
+ Ok(Pat::Range(PatRange {
+ attrs: Vec::new(),
+ lo,
+ limits,
+ hi,
+ }))
+ } else {
+ Ok(Pat::Verbatim(verbatim::between(begin, input)))
+ }
} else {
Ok(Pat::Lit(PatLit {
attrs: Vec::new(),
@@ -685,7 +721,17 @@
}
}
- fn pat_lit_expr(input: ParseStream) -> Result<Box<Expr>> {
+ fn pat_lit_expr(input: ParseStream) -> Result<Option<Box<Expr>>> {
+ if input.is_empty()
+ || input.peek(Token![|])
+ || input.peek(Token![=>])
+ || input.peek(Token![:]) && !input.peek(Token![::])
+ || input.peek(Token![,])
+ || input.peek(Token![;])
+ {
+ return Ok(None);
+ }
+
let neg: Option<Token![-]> = input.parse()?;
let lookahead = input.lookahead1();
@@ -705,7 +751,7 @@
return Err(lookahead.error());
};
- Ok(Box::new(if let Some(neg) = neg {
+ Ok(Some(Box::new(if let Some(neg) = neg {
Expr::Unary(ExprUnary {
attrs: Vec::new(),
op: UnOp::Neg(neg),
@@ -713,7 +759,7 @@
})
} else {
expr
- }))
+ })))
}
fn pat_slice(input: ParseStream) -> Result<PatSlice> {
@@ -737,13 +783,6 @@
elems,
})
}
-
- fn pat_rest(input: ParseStream) -> Result<PatRest> {
- Ok(PatRest {
- attrs: Vec::new(),
- dot2_token: input.parse()?,
- })
- }
}
#[cfg(feature = "printing")]
diff --git a/src/punctuated.rs b/src/punctuated.rs
index 38c7bf4..e02ac44 100644
--- a/src/punctuated.rs
+++ b/src/punctuated.rs
@@ -76,22 +76,19 @@
self.iter().next()
}
+ /// Mutably borrows the first element in this sequence.
+ pub fn first_mut(&mut self) -> Option<&mut T> {
+ self.iter_mut().next()
+ }
+
/// Borrows the last element in this sequence.
pub fn last(&self) -> Option<&T> {
- if self.last.is_some() {
- self.last.as_ref().map(Box::as_ref)
- } else {
- self.inner.last().map(|pair| &pair.0)
- }
+ self.iter().next_back()
}
/// Mutably borrows the last element in this sequence.
pub fn last_mut(&mut self) -> Option<&mut T> {
- if self.last.is_some() {
- self.last.as_mut().map(Box::as_mut)
- } else {
- self.inner.last_mut().map(|pair| &mut pair.0)
- }
+ self.iter_mut().next_back()
}
/// Returns an iterator over borrowed syntax tree nodes of type `&T`.
diff --git a/src/stmt.rs b/src/stmt.rs
index e4277fd..acee5a3 100644
--- a/src/stmt.rs
+++ b/src/stmt.rs
@@ -47,8 +47,10 @@
pub mod parsing {
use super::*;
+ use crate::parse::discouraged::Speculative;
use crate::parse::{Parse, ParseStream, Result};
use crate::punctuated::Punctuated;
+ use proc_macro2::TokenStream;
impl Block {
/// Parse the body of a block as zero or more statements, possibly
@@ -106,8 +108,8 @@
pub fn parse_within(input: ParseStream) -> Result<Vec<Stmt>> {
let mut stmts = Vec::new();
loop {
- while input.peek(Token![;]) {
- input.parse::<Token![;]>()?;
+ while let Some(semi) = input.parse::<Option<Token![;]>>()? {
+ stmts.push(Stmt::Semi(Expr::Verbatim(TokenStream::new()), semi));
}
if input.is_empty() {
break;
@@ -146,55 +148,55 @@
}
fn parse_stmt(input: ParseStream, allow_nosemi: bool) -> Result<Stmt> {
- // TODO: optimize using advance_to
- let ahead = input.fork();
- ahead.call(Attribute::parse_outer)?;
+ let mut attrs = input.call(Attribute::parse_outer)?;
- if {
- let ahead = ahead.fork();
- // Only parse braces here; paren and bracket will get parsed as
- // expression statements
- ahead.call(Path::parse_mod_style).is_ok()
- && ahead.parse::<Token![!]>().is_ok()
- && (ahead.peek(token::Brace) || ahead.peek(Ident))
- } {
- stmt_mac(input)
- } else if ahead.peek(Token![let]) {
- stmt_local(input).map(Stmt::Local)
- } else if ahead.peek(Token![pub])
- || ahead.peek(Token![crate]) && !ahead.peek2(Token![::])
- || ahead.peek(Token![extern]) && !ahead.peek2(Token![::])
- || ahead.peek(Token![use])
- || ahead.peek(Token![static]) && (ahead.peek2(Token![mut]) || ahead.peek2(Ident))
- || ahead.peek(Token![const])
- || ahead.peek(Token![unsafe]) && !ahead.peek2(token::Brace)
- || ahead.peek(Token![async])
- && (ahead.peek2(Token![unsafe])
- || ahead.peek2(Token![extern])
- || ahead.peek2(Token![fn]))
- || ahead.peek(Token![fn])
- || ahead.peek(Token![mod])
- || ahead.peek(Token![type])
- || ahead.peek(item::parsing::existential) && ahead.peek2(Token![type])
- || ahead.peek(Token![struct])
- || ahead.peek(Token![enum])
- || ahead.peek(Token![union]) && ahead.peek2(Ident)
- || ahead.peek(Token![auto]) && ahead.peek2(Token![trait])
- || ahead.peek(Token![trait])
- || ahead.peek(Token![default])
- && (ahead.peek2(Token![unsafe]) || ahead.peek2(Token![impl]))
- || ahead.peek(Token![impl])
- || ahead.peek(Token![macro])
+ // brace-style macros; paren and bracket macros get parsed as
+ // expression statements.
+ let ahead = input.fork();
+ if let Ok(path) = ahead.call(Path::parse_mod_style) {
+ if ahead.peek(Token![!]) && (ahead.peek2(token::Brace) || ahead.peek2(Ident)) {
+ input.advance_to(&ahead);
+ return stmt_mac(input, attrs, path);
+ }
+ }
+
+ if input.peek(Token![let]) {
+ stmt_local(input, attrs).map(Stmt::Local)
+ } else if input.peek(Token![pub])
+ || input.peek(Token![crate]) && !input.peek2(Token![::])
+ || input.peek(Token![extern]) && !input.peek2(Token![::])
+ || input.peek(Token![use])
+ || input.peek(Token![static]) && (input.peek2(Token![mut]) || input.peek2(Ident))
+ || input.peek(Token![const])
+ || input.peek(Token![unsafe]) && !input.peek2(token::Brace)
+ || input.peek(Token![async])
+ && (input.peek2(Token![unsafe])
+ || input.peek2(Token![extern])
+ || input.peek2(Token![fn]))
+ || input.peek(Token![fn])
+ || input.peek(Token![mod])
+ || input.peek(Token![type])
+ || input.peek(item::parsing::existential) && input.peek2(Token![type])
+ || input.peek(Token![struct])
+ || input.peek(Token![enum])
+ || input.peek(Token![union]) && input.peek2(Ident)
+ || input.peek(Token![auto]) && input.peek2(Token![trait])
+ || input.peek(Token![trait])
+ || input.peek(Token![default])
+ && (input.peek2(Token![unsafe]) || input.peek2(Token![impl]))
+ || input.peek(Token![impl])
+ || input.peek(Token![macro])
{
- input.parse().map(Stmt::Item)
+ let mut item: Item = input.parse()?;
+ attrs.extend(item.replace_attrs(Vec::new()));
+ item.replace_attrs(attrs);
+ Ok(Stmt::Item(item))
} else {
- stmt_expr(input, allow_nosemi)
+ stmt_expr(input, allow_nosemi, attrs)
}
}
- fn stmt_mac(input: ParseStream) -> Result<Stmt> {
- let attrs = input.call(Attribute::parse_outer)?;
- let path = input.call(Path::parse_mod_style)?;
+ fn stmt_mac(input: ParseStream, attrs: Vec<Attribute>, path: Path) -> Result<Stmt> {
let bang_token: Token![!] = input.parse()?;
let ident: Option<Ident> = input.parse()?;
let (delimiter, tokens) = mac::parse_delimiter(input)?;
@@ -213,9 +215,9 @@
})))
}
- fn stmt_local(input: ParseStream) -> Result<Local> {
+ fn stmt_local(input: ParseStream, attrs: Vec<Attribute>) -> Result<Local> {
Ok(Local {
- attrs: input.call(Attribute::parse_outer)?,
+ attrs,
let_token: input.parse()?,
pat: {
let leading_vert: Option<Token![|]> = input.parse()?;
@@ -265,8 +267,11 @@
})
}
- fn stmt_expr(input: ParseStream, allow_nosemi: bool) -> Result<Stmt> {
- let mut attrs = input.call(Attribute::parse_outer)?;
+ fn stmt_expr(
+ input: ParseStream,
+ allow_nosemi: bool,
+ mut attrs: Vec<Attribute>,
+ ) -> Result<Stmt> {
let mut e = expr::parsing::expr_early(input)?;
attrs.extend(e.replace_attrs(Vec::new()));
diff --git a/src/token.rs b/src/token.rs
index 0b8c181..0b4ad88 100644
--- a/src/token.rs
+++ b/src/token.rs
@@ -88,7 +88,6 @@
//! [Printing]: https://docs.rs/quote/1.0/quote/trait.ToTokens.html
//! [`Span`]: https://docs.rs/proc-macro2/1.0/proc_macro2/struct.Span.html
-use std;
#[cfg(feature = "extra-traits")]
use std::cmp;
#[cfg(feature = "extra-traits")]
@@ -158,11 +157,12 @@
#[cfg(any(feature = "full", feature = "derive"))]
#[cfg(feature = "parsing")]
fn peek_impl(cursor: Cursor, peek: fn(ParseStream) -> bool) -> bool {
+ use crate::parse::Unexpected;
use std::cell::Cell;
use std::rc::Rc;
let scope = Span::call_site();
- let unexpected = Rc::new(Cell::new(None));
+ let unexpected = Rc::new(Cell::new(Unexpected::None));
let buffer = crate::parse::new_parse_buffer(scope, cursor, unexpected);
peek(&buffer)
}
@@ -855,7 +855,7 @@
}
pub fn punct<S: FromSpans>(input: ParseStream, token: &str) -> Result<S> {
- let mut spans = [input.cursor().span(); 3];
+ let mut spans = [input.span(); 3];
punct_helper(input, token, &mut spans)?;
Ok(S::from_spans(&spans))
}
diff --git a/src/ty.rs b/src/ty.rs
index 1e5b51f..b2e086b 100644
--- a/src/ty.rs
+++ b/src/ty.rs
@@ -784,9 +784,27 @@
impl Parse for TypeTuple {
fn parse(input: ParseStream) -> Result<Self> {
let content;
+ let paren_token = parenthesized!(content in input);
+
+ if content.is_empty() {
+ return Ok(TypeTuple {
+ paren_token,
+ elems: Punctuated::new(),
+ });
+ }
+
+ let first: Type = content.parse()?;
Ok(TypeTuple {
- paren_token: parenthesized!(content in input),
- elems: content.parse_terminated(Type::parse)?,
+ paren_token,
+ elems: {
+ let mut elems = Punctuated::new();
+ elems.push_value(first);
+ elems.push_punct(content.parse()?);
+ let rest: Punctuated<Type, Token![,]> =
+ content.parse_terminated(Parse::parse)?;
+ elems.extend(rest);
+ elems
+ },
})
}
}
diff --git a/src/verbatim.rs b/src/verbatim.rs
new file mode 100644
index 0000000..0686352
--- /dev/null
+++ b/src/verbatim.rs
@@ -0,0 +1,15 @@
+use crate::parse::{ParseBuffer, ParseStream};
+use proc_macro2::TokenStream;
+use std::iter;
+
+pub fn between<'a>(begin: ParseBuffer<'a>, end: ParseStream<'a>) -> TokenStream {
+ let end = end.cursor();
+ let mut cursor = begin.cursor();
+ let mut tokens = TokenStream::new();
+ while cursor != end {
+ let (tt, next) = cursor.token_tree().unwrap();
+ tokens.extend(iter::once(tt));
+ cursor = next;
+ }
+ tokens
+}