Add Speculative::advance_to for speculative parsing
diff --git a/src/parse/mod.rs b/src/parse/mod.rs
new file mode 100644
index 0000000..dc724ca
--- /dev/null
+++ b/src/parse/mod.rs
@@ -0,0 +1,1120 @@
+//! Parsing interface for parsing a token stream into a syntax tree node.
+//!
+//! Parsing in Syn is built on parser functions that take in a [`ParseStream`]
+//! and produce a [`Result<T>`] where `T` is some syntax tree node. Underlying
+//! these parser functions is a lower level mechanism built around the
+//! [`Cursor`] type. `Cursor` is a cheaply copyable cursor over a range of
+//! tokens in a token stream.
+//!
+//! [`ParseStream`]: type.ParseStream.html
+//! [`Result<T>`]: type.Result.html
+//! [`Cursor`]: ../buffer/index.html
+//!
+//! # Example
+//!
+//! Here is a snippet of parsing code to get a feel for the style of the
+//! library. We define data structures for a subset of Rust syntax including
+//! enums (not shown) and structs, then provide implementations of the [`Parse`]
+//! trait to parse these syntax tree data structures from a token stream.
+//!
+//! Once `Parse` impls have been defined, they can be called conveniently from a
+//! procedural macro through [`parse_macro_input!`] as shown at the bottom of
+//! the snippet. If the caller provides syntactically invalid input to the
+//! procedural macro, they will receive a helpful compiler error message
+//! pointing out the exact token that triggered the failure to parse.
+//!
+//! [`parse_macro_input!`]: ../macro.parse_macro_input.html
+//!
+//! ```edition2018
+//! extern crate proc_macro;
+//!
+//! use proc_macro::TokenStream;
+//! use syn::{braced, parse_macro_input, token, Field, Ident, Result, Token};
+//! use syn::parse::{Parse, ParseStream};
+//! use syn::punctuated::Punctuated;
+//!
+//! enum Item {
+//! Struct(ItemStruct),
+//! Enum(ItemEnum),
+//! }
+//!
+//! struct ItemStruct {
+//! struct_token: Token![struct],
+//! ident: Ident,
+//! brace_token: token::Brace,
+//! fields: Punctuated<Field, Token![,]>,
+//! }
+//! #
+//! # enum ItemEnum {}
+//!
+//! impl Parse for Item {
+//! fn parse(input: ParseStream) -> Result<Self> {
+//! let lookahead = input.lookahead1();
+//! if lookahead.peek(Token![struct]) {
+//! input.parse().map(Item::Struct)
+//! } else if lookahead.peek(Token![enum]) {
+//! input.parse().map(Item::Enum)
+//! } else {
+//! Err(lookahead.error())
+//! }
+//! }
+//! }
+//!
+//! impl Parse for ItemStruct {
+//! fn parse(input: ParseStream) -> Result<Self> {
+//! let content;
+//! Ok(ItemStruct {
+//! struct_token: input.parse()?,
+//! ident: input.parse()?,
+//! brace_token: braced!(content in input),
+//! fields: content.parse_terminated(Field::parse_named)?,
+//! })
+//! }
+//! }
+//! #
+//! # impl Parse for ItemEnum {
+//! # fn parse(input: ParseStream) -> Result<Self> {
+//! # unimplemented!()
+//! # }
+//! # }
+//!
+//! # const IGNORE: &str = stringify! {
+//! #[proc_macro]
+//! # };
+//! pub fn my_macro(tokens: TokenStream) -> TokenStream {
+//! let input = parse_macro_input!(tokens as Item);
+//!
+//! /* ... */
+//! # "".parse().unwrap()
+//! }
+//! ```
+//!
+//! # The `syn::parse*` functions
+//!
+//! The [`syn::parse`], [`syn::parse2`], and [`syn::parse_str`] functions serve
+//! as an entry point for parsing syntax tree nodes that can be parsed in an
+//! obvious default way. These functions can return any syntax tree node that
+//! implements the [`Parse`] trait, which includes most types in Syn.
+//!
+//! [`syn::parse`]: ../fn.parse.html
+//! [`syn::parse2`]: ../fn.parse2.html
+//! [`syn::parse_str`]: ../fn.parse_str.html
+//! [`Parse`]: trait.Parse.html
+//!
+//! ```edition2018
+//! use syn::Type;
+//!
+//! # fn run_parser() -> syn::Result<()> {
+//! let t: Type = syn::parse_str("std::collections::HashMap<String, Value>")?;
+//! # Ok(())
+//! # }
+//! #
+//! # fn main() {
+//! # run_parser().unwrap();
+//! # }
+//! ```
+//!
+//! The [`parse_quote!`] macro also uses this approach.
+//!
+//! [`parse_quote!`]: ../macro.parse_quote.html
+//!
+//! # The `Parser` trait
+//!
+//! Some types can be parsed in several ways depending on context. For example
+//! an [`Attribute`] can be either "outer" like `#[...]` or "inner" like
+//! `#![...]` and parsing the wrong one would be a bug. Similarly [`Punctuated`]
+//! may or may not allow trailing punctuation, and parsing it the wrong way
+//! would either reject valid input or accept invalid input.
+//!
+//! [`Attribute`]: ../struct.Attribute.html
+//! [`Punctuated`]: ../punctuated/index.html
+//!
+//! The `Parse` trait is not implemented in these cases because there is no good
+//! behavior to consider the default.
+//!
+//! ```edition2018,compile_fail
+//! # extern crate proc_macro;
+//! #
+//! # use syn::punctuated::Punctuated;
+//! # use syn::{PathSegment, Result, Token};
+//! #
+//! # fn f(tokens: proc_macro::TokenStream) -> Result<()> {
+//! #
+//! // Can't parse `Punctuated` without knowing whether trailing punctuation
+//! // should be allowed in this context.
+//! let path: Punctuated<PathSegment, Token![::]> = syn::parse(tokens)?;
+//! #
+//! # Ok(())
+//! # }
+//! ```
+//!
+//! In these cases the types provide a choice of parser functions rather than a
+//! single `Parse` implementation, and those parser functions can be invoked
+//! through the [`Parser`] trait.
+//!
+//! [`Parser`]: trait.Parser.html
+//!
+//! ```edition2018
+//! extern crate proc_macro;
+//!
+//! use proc_macro::TokenStream;
+//! use syn::parse::Parser;
+//! use syn::punctuated::Punctuated;
+//! use syn::{Attribute, Expr, PathSegment, Result, Token};
+//!
+//! fn call_some_parser_methods(input: TokenStream) -> Result<()> {
+//! // Parse a nonempty sequence of path segments separated by `::` punctuation
+//! // with no trailing punctuation.
+//! let tokens = input.clone();
+//! let parser = Punctuated::<PathSegment, Token![::]>::parse_separated_nonempty;
+//! let _path = parser.parse(tokens)?;
+//!
+//! // Parse a possibly empty sequence of expressions terminated by commas with
+//! // an optional trailing punctuation.
+//! let tokens = input.clone();
+//! let parser = Punctuated::<Expr, Token![,]>::parse_terminated;
+//! let _args = parser.parse(tokens)?;
+//!
+//! // Parse zero or more outer attributes but not inner attributes.
+//! let tokens = input.clone();
+//! let parser = Attribute::parse_outer;
+//! let _attrs = parser.parse(tokens)?;
+//!
+//! Ok(())
+//! }
+//! ```
+//!
+//! ---
+//!
+//! *This module is available if Syn is built with the `"parsing"` feature.*
+
+pub mod discouraged;
+
+use std::cell::Cell;
+use std::fmt::{self, Debug, Display};
+use std::marker::PhantomData;
+use std::mem;
+use std::ops::Deref;
+use std::rc::Rc;
+use std::str::FromStr;
+
+#[cfg(all(
+ not(all(target_arch = "wasm32", target_os = "unknown")),
+ feature = "proc-macro"
+))]
+use proc_macro;
+use proc_macro2::{self, Delimiter, Group, Literal, Punct, Span, TokenStream, TokenTree};
+
+use buffer::{Cursor, TokenBuffer};
+use error;
+use lookahead;
+use private;
+use punctuated::Punctuated;
+use token::Token;
+
+pub use error::{Error, Result};
+pub use lookahead::{Lookahead1, Peek};
+
+/// Parsing interface implemented by all types that can be parsed in a default
+/// way from a token stream.
+pub trait Parse: Sized {
+ fn parse(input: ParseStream) -> Result<Self>;
+}
+
+/// Input to a Syn parser function.
+///
+/// See the methods of this type under the documentation of [`ParseBuffer`]. For
+/// an overview of parsing in Syn, refer to the [module documentation].
+///
+/// [module documentation]: index.html
+pub type ParseStream<'a> = &'a ParseBuffer<'a>;
+
+/// Cursor position within a buffered token stream.
+///
+/// This type is more commonly used through the type alias [`ParseStream`] which
+/// is an alias for `&ParseBuffer`.
+///
+/// `ParseStream` is the input type for all parser functions in Syn. They have
+/// the signature `fn(ParseStream) -> Result<T>`.
+///
+/// ## Calling a parser function
+///
+/// There is no public way to construct a `ParseBuffer`. Instead, if you are
+/// looking to invoke a parser function that requires `ParseStream` as input,
+/// you will need to go through one of the public parsing entry points.
+///
+/// - The [`parse_macro_input!`] macro if parsing input of a procedural macro;
+/// - One of [the `syn::parse*` functions][syn-parse]; or
+/// - A method of the [`Parser`] trait.
+///
+/// [`parse_macro_input!`]: ../macro.parse_macro_input.html
+/// [syn-parse]: index.html#the-synparse-functions
+pub struct ParseBuffer<'a> {
+ // The identity of this Rc tracks the origin of forks.
+ // That is, any Rc which is Rc::ptr_eq are derived from the same cursor,
+ // and thus the cursor may be copied between them safely.
+ // Thus a new Rc must be created for a new buffer, and only be cloned on fork.
+ scope: Rc<Span>,
+ // Instead of Cell<Cursor<'a>> so that ParseBuffer<'a> is covariant in 'a.
+ // The rest of the code in this module needs to be careful that only a
+ // cursor derived from this `cell` is ever assigned to this `cell`.
+ //
+ // Cell<Cursor<'a>> cannot be covariant in 'a because then we could take a
+ // ParseBuffer<'a>, upcast to ParseBuffer<'short> for some lifetime shorter
+ // than 'a, and then assign a Cursor<'short> into the Cell.
+ //
+ // By extension, it would not be safe to expose an API that accepts a
+ // Cursor<'a> and trusts that it lives as long as the cursor currently in
+ // the cell.
+ cell: Cell<Cursor<'static>>,
+ marker: PhantomData<Cursor<'a>>,
+ unexpected: Rc<Cell<Option<Span>>>,
+}
+
+impl<'a> Drop for ParseBuffer<'a> {
+ fn drop(&mut self) {
+ if !self.is_empty() && self.unexpected.get().is_none() {
+ self.unexpected.set(Some(self.cursor().span()));
+ }
+ }
+}
+
+impl<'a> Display for ParseBuffer<'a> {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ Display::fmt(&self.cursor().token_stream(), f)
+ }
+}
+
+impl<'a> Debug for ParseBuffer<'a> {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ Debug::fmt(&self.cursor().token_stream(), f)
+ }
+}
+
+/// Cursor state associated with speculative parsing.
+///
+/// This type is the input of the closure provided to [`ParseStream::step`].
+///
+/// [`ParseStream::step`]: struct.ParseBuffer.html#method.step
+///
+/// # Example
+///
+/// ```edition2018
+/// use proc_macro2::TokenTree;
+/// use syn::Result;
+/// use syn::parse::ParseStream;
+///
+/// // This function advances the stream past the next occurrence of `@`. If
+/// // no `@` is present in the stream, the stream position is unchanged and
+/// // an error is returned.
+/// fn skip_past_next_at(input: ParseStream) -> Result<()> {
+/// input.step(|cursor| {
+/// let mut rest = *cursor;
+/// while let Some((tt, next)) = rest.token_tree() {
+/// match &tt {
+/// TokenTree::Punct(punct) if punct.as_char() == '@' => {
+/// return Ok(((), next));
+/// }
+/// _ => rest = next,
+/// }
+/// }
+/// Err(cursor.error("no `@` was found after this point"))
+/// })
+/// }
+/// #
+/// # fn remainder_after_skipping_past_next_at(
+/// # input: ParseStream,
+/// # ) -> Result<proc_macro2::TokenStream> {
+/// # skip_past_next_at(input)?;
+/// # input.parse()
+/// # }
+/// #
+/// # fn main() {
+/// # use syn::parse::Parser;
+/// # let remainder = remainder_after_skipping_past_next_at
+/// # .parse_str("a @ b c")
+/// # .unwrap();
+/// # assert_eq!(remainder.to_string(), "b c");
+/// # }
+/// ```
+#[derive(Copy, Clone)]
+pub struct StepCursor<'c, 'a> {
+ scope: Span,
+ // This field is covariant in 'c.
+ cursor: Cursor<'c>,
+ // This field is contravariant in 'c. Together these make StepCursor
+ // invariant in 'c. Also covariant in 'a. The user cannot cast 'c to a
+ // different lifetime but can upcast into a StepCursor with a shorter
+ // lifetime 'a.
+ //
+ // As long as we only ever construct a StepCursor for which 'c outlives 'a,
+ // this means if ever a StepCursor<'c, 'a> exists we are guaranteed that 'c
+ // outlives 'a.
+ marker: PhantomData<fn(Cursor<'c>) -> Cursor<'a>>,
+}
+
+impl<'c, 'a> Deref for StepCursor<'c, 'a> {
+ type Target = Cursor<'c>;
+
+ fn deref(&self) -> &Self::Target {
+ &self.cursor
+ }
+}
+
+impl<'c, 'a> StepCursor<'c, 'a> {
+ /// Triggers an error at the current position of the parse stream.
+ ///
+ /// The `ParseStream::step` invocation will return this same error without
+ /// advancing the stream state.
+ pub fn error<T: Display>(self, message: T) -> Error {
+ error::new_at(self.scope, self.cursor, message)
+ }
+}
+
+impl private {
+ pub fn advance_step_cursor<'c, 'a>(proof: StepCursor<'c, 'a>, to: Cursor<'c>) -> Cursor<'a> {
+ // Refer to the comments within the StepCursor definition. We use the
+ // fact that a StepCursor<'c, 'a> exists as proof that 'c outlives 'a.
+ // Cursor is covariant in its lifetime parameter so we can cast a
+ // Cursor<'c> to one with the shorter lifetime Cursor<'a>.
+ let _ = proof;
+ unsafe { mem::transmute::<Cursor<'c>, Cursor<'a>>(to) }
+ }
+}
+
+fn skip(input: ParseStream) -> bool {
+ input
+ .step(|cursor| {
+ if let Some((_lifetime, rest)) = cursor.lifetime() {
+ Ok((true, rest))
+ } else if let Some((_token, rest)) = cursor.token_tree() {
+ Ok((true, rest))
+ } else {
+ Ok((false, *cursor))
+ }
+ })
+ .unwrap()
+}
+
+impl private {
+ pub fn new_parse_buffer(
+ scope: Span,
+ cursor: Cursor,
+ unexpected: Rc<Cell<Option<Span>>>,
+ ) -> ParseBuffer {
+ ParseBuffer {
+ scope: Rc::new(scope),
+ // See comment on `cell` in the struct definition.
+ cell: Cell::new(unsafe { mem::transmute::<Cursor, Cursor<'static>>(cursor) }),
+ marker: PhantomData,
+ unexpected: unexpected,
+ }
+ }
+
+ pub fn get_unexpected(buffer: &ParseBuffer) -> Rc<Cell<Option<Span>>> {
+ buffer.unexpected.clone()
+ }
+}
+
+impl<'a> ParseBuffer<'a> {
+ /// Parses a syntax tree node of type `T`, advancing the position of our
+ /// parse stream past it.
+ pub fn parse<T: Parse>(&self) -> Result<T> {
+ T::parse(self)
+ }
+
+ /// Calls the given parser function to parse a syntax tree node of type `T`
+ /// from this stream.
+ ///
+ /// # Example
+ ///
+ /// The parser below invokes [`Attribute::parse_outer`] to parse a vector of
+ /// zero or more outer attributes.
+ ///
+ /// [`Attribute::parse_outer`]: ../struct.Attribute.html#method.parse_outer
+ ///
+ /// ```edition2018
+ /// use syn::{Attribute, Ident, Result, Token};
+ /// use syn::parse::{Parse, ParseStream};
+ ///
+ /// // Parses a unit struct with attributes.
+ /// //
+ /// // #[path = "s.tmpl"]
+ /// // struct S;
+ /// struct UnitStruct {
+ /// attrs: Vec<Attribute>,
+ /// struct_token: Token![struct],
+ /// name: Ident,
+ /// semi_token: Token![;],
+ /// }
+ ///
+ /// impl Parse for UnitStruct {
+ /// fn parse(input: ParseStream) -> Result<Self> {
+ /// Ok(UnitStruct {
+ /// attrs: input.call(Attribute::parse_outer)?,
+ /// struct_token: input.parse()?,
+ /// name: input.parse()?,
+ /// semi_token: input.parse()?,
+ /// })
+ /// }
+ /// }
+ /// ```
+ pub fn call<T>(&self, function: fn(ParseStream) -> Result<T>) -> Result<T> {
+ function(self)
+ }
+
+ /// Looks at the next token in the parse stream to determine whether it
+ /// matches the requested type of token.
+ ///
+ /// Does not advance the position of the parse stream.
+ ///
+ /// # Syntax
+ ///
+ /// Note that this method does not use turbofish syntax. Pass the peek type
+ /// inside of parentheses.
+ ///
+ /// - `input.peek(Token![struct])`
+ /// - `input.peek(Token![==])`
+ /// - `input.peek(Ident)` *(does not accept keywords)*
+ /// - `input.peek(Ident::peek_any)`
+ /// - `input.peek(Lifetime)`
+ /// - `input.peek(token::Brace)`
+ ///
+ /// # Example
+ ///
+ /// In this example we finish parsing the list of supertraits when the next
+ /// token in the input is either `where` or an opening curly brace.
+ ///
+ /// ```edition2018
+ /// use syn::{braced, token, Generics, Ident, Result, Token, TypeParamBound};
+ /// use syn::parse::{Parse, ParseStream};
+ /// use syn::punctuated::Punctuated;
+ ///
+ /// // Parses a trait definition containing no associated items.
+ /// //
+ /// // trait Marker<'de, T>: A + B<'de> where Box<T>: Clone {}
+ /// struct MarkerTrait {
+ /// trait_token: Token![trait],
+ /// ident: Ident,
+ /// generics: Generics,
+ /// colon_token: Option<Token![:]>,
+ /// supertraits: Punctuated<TypeParamBound, Token![+]>,
+ /// brace_token: token::Brace,
+ /// }
+ ///
+ /// impl Parse for MarkerTrait {
+ /// fn parse(input: ParseStream) -> Result<Self> {
+ /// let trait_token: Token![trait] = input.parse()?;
+ /// let ident: Ident = input.parse()?;
+ /// let mut generics: Generics = input.parse()?;
+ /// let colon_token: Option<Token![:]> = input.parse()?;
+ ///
+ /// let mut supertraits = Punctuated::new();
+ /// if colon_token.is_some() {
+ /// loop {
+ /// supertraits.push_value(input.parse()?);
+ /// if input.peek(Token![where]) || input.peek(token::Brace) {
+ /// break;
+ /// }
+ /// supertraits.push_punct(input.parse()?);
+ /// }
+ /// }
+ ///
+ /// generics.where_clause = input.parse()?;
+ /// let content;
+ /// let empty_brace_token = braced!(content in input);
+ ///
+ /// Ok(MarkerTrait {
+ /// trait_token: trait_token,
+ /// ident: ident,
+ /// generics: generics,
+ /// colon_token: colon_token,
+ /// supertraits: supertraits,
+ /// brace_token: empty_brace_token,
+ /// })
+ /// }
+ /// }
+ /// ```
+ pub fn peek<T: Peek>(&self, token: T) -> bool {
+ let _ = token;
+ T::Token::peek(self.cursor())
+ }
+
+ /// Looks at the second-next token in the parse stream.
+ ///
+ /// This is commonly useful as a way to implement contextual keywords.
+ ///
+ /// # Example
+ ///
+ /// This example needs to use `peek2` because the symbol `union` is not a
+ /// keyword in Rust. We can't use just `peek` and decide to parse a union if
+ /// the very next token is `union`, because someone is free to write a `mod
+ /// union` and a macro invocation that looks like `union::some_macro! { ...
+ /// }`. In other words `union` is a contextual keyword.
+ ///
+ /// ```edition2018
+ /// use syn::{Ident, ItemUnion, Macro, Result, Token};
+ /// use syn::parse::{Parse, ParseStream};
+ ///
+ /// // Parses either a union or a macro invocation.
+ /// enum UnionOrMacro {
+ /// // union MaybeUninit<T> { uninit: (), value: T }
+ /// Union(ItemUnion),
+ /// // lazy_static! { ... }
+ /// Macro(Macro),
+ /// }
+ ///
+ /// impl Parse for UnionOrMacro {
+ /// fn parse(input: ParseStream) -> Result<Self> {
+ /// if input.peek(Token![union]) && input.peek2(Ident) {
+ /// input.parse().map(UnionOrMacro::Union)
+ /// } else {
+ /// input.parse().map(UnionOrMacro::Macro)
+ /// }
+ /// }
+ /// }
+ /// ```
+ pub fn peek2<T: Peek>(&self, token: T) -> bool {
+ let ahead = self.fork();
+ skip(&ahead) && ahead.peek(token)
+ }
+
+ /// Looks at the third-next token in the parse stream.
+ pub fn peek3<T: Peek>(&self, token: T) -> bool {
+ let ahead = self.fork();
+ skip(&ahead) && skip(&ahead) && ahead.peek(token)
+ }
+
+ /// Parses zero or more occurrences of `T` separated by punctuation of type
+ /// `P`, with optional trailing punctuation.
+ ///
+ /// Parsing continues until the end of this parse stream. The entire content
+ /// of this parse stream must consist of `T` and `P`.
+ ///
+ /// # Example
+ ///
+ /// ```edition2018
+ /// # use quote::quote;
+ /// #
+ /// use syn::{parenthesized, token, Ident, Result, Token, Type};
+ /// use syn::parse::{Parse, ParseStream};
+ /// use syn::punctuated::Punctuated;
+ ///
+ /// // Parse a simplified tuple struct syntax like:
+ /// //
+ /// // struct S(A, B);
+ /// struct TupleStruct {
+ /// struct_token: Token![struct],
+ /// ident: Ident,
+ /// paren_token: token::Paren,
+ /// fields: Punctuated<Type, Token![,]>,
+ /// semi_token: Token![;],
+ /// }
+ ///
+ /// impl Parse for TupleStruct {
+ /// fn parse(input: ParseStream) -> Result<Self> {
+ /// let content;
+ /// Ok(TupleStruct {
+ /// struct_token: input.parse()?,
+ /// ident: input.parse()?,
+ /// paren_token: parenthesized!(content in input),
+ /// fields: content.parse_terminated(Type::parse)?,
+ /// semi_token: input.parse()?,
+ /// })
+ /// }
+ /// }
+ /// #
+ /// # fn main() {
+ /// # let input = quote! {
+ /// # struct S(A, B);
+ /// # };
+ /// # syn::parse2::<TupleStruct>(input).unwrap();
+ /// # }
+ /// ```
+ pub fn parse_terminated<T, P: Parse>(
+ &self,
+ parser: fn(ParseStream) -> Result<T>,
+ ) -> Result<Punctuated<T, P>> {
+ Punctuated::parse_terminated_with(self, parser)
+ }
+
+ /// Returns whether there are tokens remaining in this stream.
+ ///
+ /// This method returns true at the end of the content of a set of
+ /// delimiters, as well as at the very end of the complete macro input.
+ ///
+ /// # Example
+ ///
+ /// ```edition2018
+ /// use syn::{braced, token, Ident, Item, Result, Token};
+ /// use syn::parse::{Parse, ParseStream};
+ ///
+ /// // Parses a Rust `mod m { ... }` containing zero or more items.
+ /// struct Mod {
+ /// mod_token: Token![mod],
+ /// name: Ident,
+ /// brace_token: token::Brace,
+ /// items: Vec<Item>,
+ /// }
+ ///
+ /// impl Parse for Mod {
+ /// fn parse(input: ParseStream) -> Result<Self> {
+ /// let content;
+ /// Ok(Mod {
+ /// mod_token: input.parse()?,
+ /// name: input.parse()?,
+ /// brace_token: braced!(content in input),
+ /// items: {
+ /// let mut items = Vec::new();
+ /// while !content.is_empty() {
+ /// items.push(content.parse()?);
+ /// }
+ /// items
+ /// },
+ /// })
+ /// }
+ /// }
+ /// ```
+ pub fn is_empty(&self) -> bool {
+ self.cursor().eof()
+ }
+
+ /// Constructs a helper for peeking at the next token in this stream and
+ /// building an error message if it is not one of a set of expected tokens.
+ ///
+ /// # Example
+ ///
+ /// ```edition2018
+ /// use syn::{ConstParam, Ident, Lifetime, LifetimeDef, Result, Token, TypeParam};
+ /// use syn::parse::{Parse, ParseStream};
+ ///
+ /// // A generic parameter, a single one of the comma-separated elements inside
+ /// // angle brackets in:
+ /// //
+ /// // fn f<T: Clone, 'a, 'b: 'a, const N: usize>() { ... }
+ /// //
+ /// // On invalid input, lookahead gives us a reasonable error message.
+ /// //
+ /// // error: expected one of: identifier, lifetime, `const`
+ /// // |
+ /// // 5 | fn f<!Sized>() {}
+ /// // | ^
+ /// enum GenericParam {
+ /// Type(TypeParam),
+ /// Lifetime(LifetimeDef),
+ /// Const(ConstParam),
+ /// }
+ ///
+ /// impl Parse for GenericParam {
+ /// fn parse(input: ParseStream) -> Result<Self> {
+ /// let lookahead = input.lookahead1();
+ /// if lookahead.peek(Ident) {
+ /// input.parse().map(GenericParam::Type)
+ /// } else if lookahead.peek(Lifetime) {
+ /// input.parse().map(GenericParam::Lifetime)
+ /// } else if lookahead.peek(Token![const]) {
+ /// input.parse().map(GenericParam::Const)
+ /// } else {
+ /// Err(lookahead.error())
+ /// }
+ /// }
+ /// }
+ /// ```
+ pub fn lookahead1(&self) -> Lookahead1<'a> {
+ lookahead::new(*self.scope, self.cursor())
+ }
+
+ /// Forks a parse stream so that parsing tokens out of either the original
+ /// or the fork does not advance the position of the other.
+ ///
+ /// # Performance
+ ///
+ /// Forking a parse stream is a cheap fixed amount of work and does not
+ /// involve copying token buffers. Where you might hit performance problems
+ /// is if your macro ends up parsing a large amount of content more than
+ /// once.
+ ///
+ /// ```edition2018
+ /// # use syn::{Expr, Result};
+ /// # use syn::parse::ParseStream;
+ /// #
+ /// # fn bad(input: ParseStream) -> Result<Expr> {
+ /// // Do not do this.
+ /// if input.fork().parse::<Expr>().is_ok() {
+ /// return input.parse::<Expr>();
+ /// }
+ /// # unimplemented!()
+ /// # }
+ /// ```
+ ///
+ /// As a rule, avoid parsing an unbounded amount of tokens out of a forked
+ /// parse stream. Only use a fork when the amount of work performed against
+ /// the fork is small and bounded.
+ ///
+ /// For higher level speculative parsing, [`parse::discouraged::Speculative`]
+ /// is provided alongside matching tradeoffs to enable the pattern.
+ /// For a lower level but occasionally more performant way to perform
+ /// speculative parsing, consider using [`ParseStream::step`] instead.
+ ///
+ /// [`ParseStream::step`]: #method.step
+ /// [`parse::discouraged::Speculative`]: ./discouraged/trait.Speculative.html
+ ///
+ /// # Example
+ ///
+ /// The parse implementation shown here parses possibly restricted `pub`
+ /// visibilities.
+ ///
+ /// - `pub`
+ /// - `pub(crate)`
+ /// - `pub(self)`
+ /// - `pub(super)`
+ /// - `pub(in some::path)`
+ ///
+ /// To handle the case of visibilities inside of tuple structs, the parser
+ /// needs to distinguish parentheses that specify visibility restrictions
+ /// from parentheses that form part of a tuple type.
+ ///
+ /// ```edition2018
+ /// # struct A;
+ /// # struct B;
+ /// # struct C;
+ /// #
+ /// struct S(pub(crate) A, pub (B, C));
+ /// ```
+ ///
+ /// In this example input the first tuple struct element of `S` has
+ /// `pub(crate)` visibility while the second tuple struct element has `pub`
+ /// visibility; the parentheses around `(B, C)` are part of the type rather
+ /// than part of a visibility restriction.
+ ///
+ /// The parser uses a forked parse stream to check the first token inside of
+ /// parentheses after the `pub` keyword. This is a small bounded amount of
+ /// work performed against the forked parse stream.
+ ///
+ /// ```edition2018
+ /// use syn::{parenthesized, token, Ident, Path, Result, Token};
+ /// use syn::ext::IdentExt;
+ /// use syn::parse::{Parse, ParseStream};
+ ///
+ /// struct PubVisibility {
+ /// pub_token: Token![pub],
+ /// restricted: Option<Restricted>,
+ /// }
+ ///
+ /// struct Restricted {
+ /// paren_token: token::Paren,
+ /// in_token: Option<Token![in]>,
+ /// path: Path,
+ /// }
+ ///
+ /// impl Parse for PubVisibility {
+ /// fn parse(input: ParseStream) -> Result<Self> {
+ /// let pub_token: Token![pub] = input.parse()?;
+ ///
+ /// if input.peek(token::Paren) {
+ /// let ahead = input.fork();
+ /// let mut content;
+ /// parenthesized!(content in ahead);
+ ///
+ /// if content.peek(Token![crate])
+ /// || content.peek(Token![self])
+ /// || content.peek(Token![super])
+ /// {
+ /// return Ok(PubVisibility {
+ /// pub_token: pub_token,
+ /// restricted: Some(Restricted {
+ /// paren_token: parenthesized!(content in input),
+ /// in_token: None,
+ /// path: Path::from(content.call(Ident::parse_any)?),
+ /// }),
+ /// });
+ /// } else if content.peek(Token![in]) {
+ /// return Ok(PubVisibility {
+ /// pub_token: pub_token,
+ /// restricted: Some(Restricted {
+ /// paren_token: parenthesized!(content in input),
+ /// in_token: Some(content.parse()?),
+ /// path: content.call(Path::parse_mod_style)?,
+ /// }),
+ /// });
+ /// }
+ /// }
+ ///
+ /// Ok(PubVisibility {
+ /// pub_token: pub_token,
+ /// restricted: None,
+ /// })
+ /// }
+ /// }
+ /// ```
+ pub fn fork(&self) -> Self {
+ ParseBuffer {
+ scope: Rc::clone(&self.scope),
+ cell: self.cell.clone(),
+ marker: PhantomData,
+ // Not the parent's unexpected. Nothing cares whether the clone
+ // parses all the way.
+ unexpected: Rc::new(Cell::new(None)),
+ }
+ }
+
+ /// Triggers an error at the current position of the parse stream.
+ ///
+ /// # Example
+ ///
+ /// ```edition2018
+ /// use syn::{Expr, Result, Token};
+ /// use syn::parse::{Parse, ParseStream};
+ ///
+ /// // Some kind of loop: `while` or `for` or `loop`.
+ /// struct Loop {
+ /// expr: Expr,
+ /// }
+ ///
+ /// impl Parse for Loop {
+ /// fn parse(input: ParseStream) -> Result<Self> {
+ /// if input.peek(Token![while])
+ /// || input.peek(Token![for])
+ /// || input.peek(Token![loop])
+ /// {
+ /// Ok(Loop {
+ /// expr: input.parse()?,
+ /// })
+ /// } else {
+ /// Err(input.error("expected some kind of loop"))
+ /// }
+ /// }
+ /// }
+ /// ```
+ pub fn error<T: Display>(&self, message: T) -> Error {
+ error::new_at(*self.scope, self.cursor(), message)
+ }
+
+ /// Speculatively parses tokens from this parse stream, advancing the
+ /// position of this stream only if parsing succeeds.
+ ///
+ /// This is a powerful low-level API used for defining the `Parse` impls of
+ /// the basic built-in token types. It is not something that will be used
+ /// widely outside of the Syn codebase.
+ ///
+ /// # Example
+ ///
+ /// ```edition2018
+ /// use proc_macro2::TokenTree;
+ /// use syn::Result;
+ /// use syn::parse::ParseStream;
+ ///
+ /// // This function advances the stream past the next occurrence of `@`. If
+ /// // no `@` is present in the stream, the stream position is unchanged and
+ /// // an error is returned.
+ /// fn skip_past_next_at(input: ParseStream) -> Result<()> {
+ /// input.step(|cursor| {
+ /// let mut rest = *cursor;
+ /// while let Some((tt, next)) = rest.token_tree() {
+ /// match &tt {
+ /// TokenTree::Punct(punct) if punct.as_char() == '@' => {
+ /// return Ok(((), next));
+ /// }
+ /// _ => rest = next,
+ /// }
+ /// }
+ /// Err(cursor.error("no `@` was found after this point"))
+ /// })
+ /// }
+ /// #
+ /// # fn remainder_after_skipping_past_next_at(
+ /// # input: ParseStream,
+ /// # ) -> Result<proc_macro2::TokenStream> {
+ /// # skip_past_next_at(input)?;
+ /// # input.parse()
+ /// # }
+ /// #
+ /// # fn main() {
+ /// # use syn::parse::Parser;
+ /// # let remainder = remainder_after_skipping_past_next_at
+ /// # .parse_str("a @ b c")
+ /// # .unwrap();
+ /// # assert_eq!(remainder.to_string(), "b c");
+ /// # }
+ /// ```
+ pub fn step<F, R>(&self, function: F) -> Result<R>
+ where
+ F: for<'c> FnOnce(StepCursor<'c, 'a>) -> Result<(R, Cursor<'c>)>,
+ {
+ // Since the user's function is required to work for any 'c, we know
+ // that the Cursor<'c> they return is either derived from the input
+ // StepCursor<'c, 'a> or from a Cursor<'static>.
+ //
+ // It would not be legal to write this function without the invariant
+ // lifetime 'c in StepCursor<'c, 'a>. If this function were written only
+ // in terms of 'a, the user could take our ParseBuffer<'a>, upcast it to
+ // a ParseBuffer<'short> which some shorter lifetime than 'a, invoke
+ // `step` on their ParseBuffer<'short> with a closure that returns
+ // Cursor<'short>, and we would wrongly write that Cursor<'short> into
+ // the Cell intended to hold Cursor<'a>.
+ //
+ // In some cases it may be necessary for R to contain a Cursor<'a>.
+ // Within Syn we solve this using `private::advance_step_cursor` which
+ // uses the existence of a StepCursor<'c, 'a> as proof that it is safe
+ // to cast from Cursor<'c> to Cursor<'a>. If needed outside of Syn, it
+ // would be safe to expose that API as a method on StepCursor.
+ let (node, rest) = function(StepCursor {
+ scope: *self.scope,
+ cursor: self.cell.get(),
+ marker: PhantomData,
+ })?;
+ self.cell.set(rest);
+ Ok(node)
+ }
+
+ /// Provides low-level access to the token representation underlying this
+ /// parse stream.
+ ///
+ /// Cursors are immutable so no operations you perform against the cursor
+ /// will affect the state of this parse stream.
+ pub fn cursor(&self) -> Cursor<'a> {
+ self.cell.get()
+ }
+
+ fn check_unexpected(&self) -> Result<()> {
+ match self.unexpected.get() {
+ Some(span) => Err(Error::new(span, "unexpected token")),
+ None => Ok(()),
+ }
+ }
+}
+
+impl<T: Parse> Parse for Box<T> {
+ fn parse(input: ParseStream) -> Result<Self> {
+ input.parse().map(Box::new)
+ }
+}
+
+impl<T: Parse + Token> Parse for Option<T> {
+ fn parse(input: ParseStream) -> Result<Self> {
+ if T::peek(input.cursor()) {
+ Ok(Some(input.parse()?))
+ } else {
+ Ok(None)
+ }
+ }
+}
+
+impl Parse for TokenStream {
+ fn parse(input: ParseStream) -> Result<Self> {
+ input.step(|cursor| Ok((cursor.token_stream(), Cursor::empty())))
+ }
+}
+
+impl Parse for TokenTree {
+ fn parse(input: ParseStream) -> Result<Self> {
+ input.step(|cursor| match cursor.token_tree() {
+ Some((tt, rest)) => Ok((tt, rest)),
+ None => Err(cursor.error("expected token tree")),
+ })
+ }
+}
+
+impl Parse for Group {
+ fn parse(input: ParseStream) -> Result<Self> {
+ input.step(|cursor| {
+ for delim in &[Delimiter::Parenthesis, Delimiter::Brace, Delimiter::Bracket] {
+ if let Some((inside, span, rest)) = cursor.group(*delim) {
+ let mut group = Group::new(*delim, inside.token_stream());
+ group.set_span(span);
+ return Ok((group, rest));
+ }
+ }
+ Err(cursor.error("expected group token"))
+ })
+ }
+}
+
+impl Parse for Punct {
+ fn parse(input: ParseStream) -> Result<Self> {
+ input.step(|cursor| match cursor.punct() {
+ Some((punct, rest)) => Ok((punct, rest)),
+ None => Err(cursor.error("expected punctuation token")),
+ })
+ }
+}
+
+impl Parse for Literal {
+ fn parse(input: ParseStream) -> Result<Self> {
+ input.step(|cursor| match cursor.literal() {
+ Some((literal, rest)) => Ok((literal, rest)),
+ None => Err(cursor.error("expected literal token")),
+ })
+ }
+}
+
+/// Parser that can parse Rust tokens into a particular syntax tree node.
+///
+/// Refer to the [module documentation] for details about parsing in Syn.
+///
+/// [module documentation]: index.html
+///
+/// *This trait is available if Syn is built with the `"parsing"` feature.*
+pub trait Parser: Sized {
+ type Output;
+
+ /// Parse a proc-macro2 token stream into the chosen syntax tree node.
+ ///
+ /// This function will check that the input is fully parsed. If there are
+ /// any unparsed tokens at the end of the stream, an error is returned.
+ fn parse2(self, tokens: TokenStream) -> Result<Self::Output>;
+
+ /// Parse tokens of source code into the chosen syntax tree node.
+ ///
+ /// This function will check that the input is fully parsed. If there are
+ /// any unparsed tokens at the end of the stream, an error is returned.
+ ///
+ /// *This method is available if Syn is built with both the `"parsing"` and
+ /// `"proc-macro"` features.*
+ #[cfg(all(
+ not(all(target_arch = "wasm32", target_os = "unknown")),
+ feature = "proc-macro"
+ ))]
+ fn parse(self, tokens: proc_macro::TokenStream) -> Result<Self::Output> {
+ self.parse2(proc_macro2::TokenStream::from(tokens))
+ }
+
+ /// Parse a string of Rust code into the chosen syntax tree node.
+ ///
+ /// This function will check that the input is fully parsed. If there are
+ /// any unparsed tokens at the end of the string, an error is returned.
+ ///
+ /// # Hygiene
+ ///
+ /// Every span in the resulting syntax tree will be set to resolve at the
+ /// macro call site.
+ fn parse_str(self, s: &str) -> Result<Self::Output> {
+ self.parse2(proc_macro2::TokenStream::from_str(s)?)
+ }
+}
+
+fn tokens_to_parse_buffer(tokens: &TokenBuffer) -> ParseBuffer {
+ let scope = Span::call_site();
+ let cursor = tokens.begin();
+ let unexpected = Rc::new(Cell::new(None));
+ private::new_parse_buffer(scope, cursor, unexpected)
+}
+
+impl<F, T> Parser for F
+where
+ F: FnOnce(ParseStream) -> Result<T>,
+{
+ type Output = T;
+
+ fn parse2(self, tokens: TokenStream) -> Result<T> {
+ let buf = TokenBuffer::new2(tokens);
+ let state = tokens_to_parse_buffer(&buf);
+ let node = self(&state)?;
+ state.check_unexpected()?;
+ if state.is_empty() {
+ Ok(node)
+ } else {
+ Err(state.error("unexpected token"))
+ }
+ }
+}