Add Speculative::advance_to for speculative parsing
diff --git a/src/parse/discouraged.rs b/src/parse/discouraged.rs
new file mode 100644
index 0000000..96b03b9
--- /dev/null
+++ b/src/parse/discouraged.rs
@@ -0,0 +1,124 @@
+//! Extensions to the parsing API with niche applicability.
+
+use super::*;
+
+/// Extensions to the `ParseStream` API to support speculative parsing.
+pub trait Speculative {
+ /// Advance this parse stream to the position of a forked parse stream.
+ ///
+ /// This is the opposite operation to [`ParseStream::fork`].
+ /// You can fork a parse stream, perform some speculative parsing, then join
+ /// the original stream to the fork to "commit" the parsing from the fork to
+ /// the main stream.
+ ///
+ /// If you can avoid doing this, you should, as it limits the ability to
+ /// generate useful errors. That said, it is often the only way to parse
+ /// syntax of the form `A* B*` for arbitrary syntax `A` and `B`. The problem
+ /// is that when the fork fails to parse an `A`, it's impossible to tell
+ /// whether that was because of a syntax error and the user meant to provide
+ /// an `A`, or that the `A`s are finished and its time to start parsing `B`s.
+ /// Use with care.
+ ///
+ /// Also note that if `A` is a subset of `B`, `A* B*` can be parsed by parsing
+ /// `B*` and removing the leading members of `A` from the repetition, bypassing
+ /// the need to involve the downsides associated with speculative parsing.
+ ///
+ /// [`ParseStream::fork`]: ../struct.ParseBuffer.html#method.fork
+ ///
+ /// # Example
+ ///
+ /// There has been chatter about the possibility of making the colons in the
+ /// turbofish syntax like `path::to::<T>` no longer required by accepting
+ /// `path::to<T>` in expression position. Specifically, according to [RFC#2544],
+ /// [`PathSegment`] parsing should always try to consume a following `<` token
+ /// as the start of generic arguments, and reset to the `<` if that fails
+ /// (e.g. the token is acting as a less-than operator).
+ ///
+ /// This is the exact kind of parsing behavior which requires the "fork, try,
+ /// commit" behavior that [`ParseStream::fork`] discourages. With `advance_to`,
+ /// we can avoid having to parse the speculatively parsed content a second time.
+ ///
+ /// This change in behavior can be implemented in syn by replacing just the
+ /// `Parse` implementation for `PathSegment`:
+ ///
+ /// ```edition2018
+ /// # use syn::ext::IdentExt;
+ /// use syn::parse::discouraged::Speculative;
+ /// # use syn::parse::{Parse, ParseStream};
+ /// # use syn::{Ident, PathArguments, Result, Token};
+ ///
+ /// pub struct PathSegment {
+ /// pub ident: Ident,
+ /// pub arguments: PathArguments,
+ /// }
+ ///
+ /// # impl<T> From<T> for PathSegment
+ /// # where
+ /// # T: Into<Ident>,
+ /// # {
+ /// # fn from(ident: T) -> Self {
+ /// # PathSegment {
+ /// # ident: ident.into(),
+ /// # arguments: PathArguments::None,
+ /// # }
+ /// # }
+ /// # }
+ ///
+ ///
+ /// impl Parse for PathSegment {
+ /// fn parse(input: ParseStream) -> Result<Self> {
+ /// if input.peek(Token![super])
+ /// || input.peek(Token![self])
+ /// || input.peek(Token![Self])
+ /// || input.peek(Token![crate])
+ /// || input.peek(Token![extern])
+ /// {
+ /// let ident = input.call(Ident::parse_any)?;
+ /// return Ok(PathSegment::from(ident));
+ /// }
+ ///
+ /// let ident = input.parse()?;
+ /// if input.peek(Token![::]) && input.peek3(Token![<]) {
+ /// return Ok(PathSegment {
+ /// ident: ident,
+ /// arguments: PathArguments::AngleBracketed(input.parse()?),
+ /// });
+ /// }
+ /// if input.peek(Token![<]) && !input.peek(Token![<=]) {
+ /// let fork = input.fork();
+ /// if let Ok(arguments) = fork.parse() {
+ /// input.advance_to(&fork);
+ /// return Ok(PathSegment {
+ /// ident: ident,
+ /// arguments: PathArguments::AngleBracketed(arguments),
+ /// });
+ /// }
+ /// }
+ /// Ok(PathSegment::from(ident))
+ /// }
+ /// }
+ ///
+ /// # syn::parse_str::<PathSegment>("a<b,c>").unwrap();
+ /// ```
+ ///
+ /// [RFC#2544]: https://github.com/rust-lang/rfcs/pull/2544
+ /// [`PathSegment`]: ../../struct.PathSegment.html
+ ///
+ /// # Panics
+ ///
+ /// The forked stream that this joins with must be derived by forking this parse stream.
+ fn advance_to(&self, fork: &Self);
+}
+
+impl<'a> Speculative for ParseBuffer<'a> {
+ fn advance_to(&self, fork: &Self) {
+ // See comment on `scope` in the struct definition.
+ assert_eq!(
+ // Rc::ptr_eq for rustc < 1.17.0
+ &*self.scope as *const _, &*fork.scope as *const _,
+ "Fork was not derived from the advancing parse stream"
+ );
+ // See comment on `cell` in the struct definition.
+ self.cell.set(unsafe { mem::transmute::<Cursor, Cursor<'static>>(fork.cursor()) })
+ }
+}
diff --git a/src/parse.rs b/src/parse/mod.rs
similarity index 97%
rename from src/parse.rs
rename to src/parse/mod.rs
index c5786e7..dc724ca 100644
--- a/src/parse.rs
+++ b/src/parse/mod.rs
@@ -188,6 +188,8 @@
//!
//! *This module is available if Syn is built with the `"parsing"` feature.*
+pub mod discouraged;
+
use std::cell::Cell;
use std::fmt::{self, Debug, Display};
use std::marker::PhantomData;
@@ -248,7 +250,11 @@
/// [`parse_macro_input!`]: ../macro.parse_macro_input.html
/// [syn-parse]: index.html#the-synparse-functions
pub struct ParseBuffer<'a> {
- scope: Span,
+ // The identity of this Rc tracks the origin of forks.
+ // That is, any Rc which is Rc::ptr_eq are derived from the same cursor,
+ // and thus the cursor may be copied between them safely.
+ // Thus a new Rc must be created for a new buffer, and only be cloned on fork.
+ scope: Rc<Span>,
// Instead of Cell<Cursor<'a>> so that ParseBuffer<'a> is covariant in 'a.
// The rest of the code in this module needs to be careful that only a
// cursor derived from this `cell` is ever assigned to this `cell`.
@@ -397,7 +403,7 @@
unexpected: Rc<Cell<Option<Span>>>,
) -> ParseBuffer {
ParseBuffer {
- scope: scope,
+ scope: Rc::new(scope),
// See comment on `cell` in the struct definition.
cell: Cell::new(unsafe { mem::transmute::<Cursor, Cursor<'static>>(cursor) }),
marker: PhantomData,
@@ -715,7 +721,7 @@
/// }
/// ```
pub fn lookahead1(&self) -> Lookahead1<'a> {
- lookahead::new(self.scope, self.cursor())
+ lookahead::new(*self.scope, self.cursor())
}
/// Forks a parse stream so that parsing tokens out of either the original
@@ -745,10 +751,13 @@
/// parse stream. Only use a fork when the amount of work performed against
/// the fork is small and bounded.
///
+ /// For higher level speculative parsing, [`parse::discouraged::Speculative`]
+ /// is provided alongside matching tradeoffs to enable the pattern.
/// For a lower level but occasionally more performant way to perform
/// speculative parsing, consider using [`ParseStream::step`] instead.
///
/// [`ParseStream::step`]: #method.step
+ /// [`parse::discouraged::Speculative`]: ./discouraged/trait.Speculative.html
///
/// # Example
///
@@ -840,7 +849,7 @@
/// ```
pub fn fork(&self) -> Self {
ParseBuffer {
- scope: self.scope,
+ scope: Rc::clone(&self.scope),
cell: self.cell.clone(),
marker: PhantomData,
// Not the parent's unexpected. Nothing cares whether the clone
@@ -878,7 +887,7 @@
/// }
/// ```
pub fn error<T: Display>(&self, message: T) -> Error {
- error::new_at(self.scope, self.cursor(), message)
+ error::new_at(*self.scope, self.cursor(), message)
}
/// Speculatively parses tokens from this parse stream, advancing the
@@ -950,7 +959,7 @@
// to cast from Cursor<'c> to Cursor<'a>. If needed outside of Syn, it
// would be safe to expose that API as a method on StepCursor.
let (node, rest) = function(StepCursor {
- scope: self.scope,
+ scope: *self.scope,
cursor: self.cell.get(),
marker: PhantomData,
})?;
diff --git a/tests/test_parse_buffer.rs b/tests/test_parse_buffer.rs
new file mode 100644
index 0000000..88feaa9
--- /dev/null
+++ b/tests/test_parse_buffer.rs
@@ -0,0 +1,29 @@
+extern crate syn;
+
+use syn::{
+ parse::{Parse, ParseStream},
+ parse::discouraged::Speculative,
+ Result,
+};
+
+#[test]
+#[should_panic(expected = "Fork was not derived from the advancing parse stream")]
+fn smuggled_speculative_cursor() {
+ struct Smuggled(ParseStream<'static>);
+ impl Parse for Smuggled {
+ fn parse(input: ParseStream) -> Result<Self> {
+ Ok(Smuggled(unsafe { std::mem::transmute_copy(input) }))
+ }
+ }
+
+ struct BreakRules;
+ impl Parse for BreakRules {
+ fn parse(input: ParseStream) -> Result<Self> {
+ let Smuggled(fork) = syn::parse_str("").unwrap();
+ input.advance_to(fork);
+ Ok(Self)
+ }
+ }
+
+ syn::parse_str::<BreakRules>("").unwrap();
+}