Merge syn-error-experiment
diff --git a/src/lib.rs b/src/lib.rs
index 0b25ec2..f917e30 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -568,6 +568,8 @@
}
pub use gen::*;
+pub mod next;
+
////////////////////////////////////////////////////////////////////////////////
#[cfg(feature = "parsing")]
diff --git a/src/next/error.rs b/src/next/error.rs
new file mode 100644
index 0000000..05473f3
--- /dev/null
+++ b/src/next/error.rs
@@ -0,0 +1,86 @@
+use std;
+use std::fmt::{self, Display};
+use std::iter::FromIterator;
+
+use proc_macro2::{
+ Delimiter, Group, Ident, LexError, Literal, Punct, Spacing, Span, TokenStream, TokenTree,
+};
+
+use buffer::Cursor;
+
+/// The result of a Syn parser.
+pub type Result<T> = std::result::Result<T, Error>;
+
+/// Error returned when a Syn parser cannot parse the input tokens.
+#[derive(Debug)]
+pub struct Error {
+ span: Span,
+ message: String,
+}
+
+impl Error {
+ pub fn new<T: Display>(span: Span, message: T) -> Self {
+ Error {
+ span: span,
+ message: message.to_string(),
+ }
+ }
+
+ /// Render the error as an invocation of [`compile_error!`].
+ ///
+ /// The [`parse_macro_input!`] macro provides a convenient way to invoke
+ /// this method correctly in a procedural macro.
+ ///
+ /// [`compile_error!`]: https://doc.rust-lang.org/std/macro.compile_error.html
+ /// [`parse_macro_input!`]: ../macro.parse_macro_input.html
+ pub fn into_compile_error(self) -> TokenStream {
+ // compile_error!($message)
+ TokenStream::from_iter(vec![
+ TokenTree::Ident(Ident::new("compile_error", self.span)),
+ TokenTree::Punct({
+ let mut punct = Punct::new('!', Spacing::Alone);
+ punct.set_span(self.span);
+ punct
+ }),
+ TokenTree::Group({
+ let mut group = Group::new(Delimiter::Brace, {
+ TokenStream::from_iter(vec![TokenTree::Literal({
+ let mut string = Literal::string(&self.message);
+ string.set_span(self.span);
+ string
+ })])
+ });
+ group.set_span(self.span);
+ group
+ }),
+ ])
+ }
+}
+
+// Not public API.
+#[doc(hidden)]
+pub fn new_at<T: Display>(scope: Span, cursor: Cursor, message: T) -> Error {
+ if cursor.eof() {
+ Error::new(scope, format!("unexpected end of input, {}", message))
+ } else {
+ Error::new(cursor.span(), message)
+ }
+}
+
+impl Display for Error {
+ fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+ formatter.write_str(&self.message)
+ }
+}
+
+impl std::error::Error for Error {
+ fn description(&self) -> &str {
+ "parse error"
+ }
+}
+
+impl From<LexError> for Error {
+ fn from(err: LexError) -> Self {
+ Error::new(Span::call_site(), format!("{:?}", err))
+ }
+}
diff --git a/src/next/export.rs b/src/next/export.rs
new file mode 100644
index 0000000..cceed14
--- /dev/null
+++ b/src/next/export.rs
@@ -0,0 +1,4 @@
+pub use std::result::Result::{Err, Ok};
+
+#[cfg(feature = "proc-macro")]
+pub use proc_macro::TokenStream;
diff --git a/src/next/group.rs b/src/next/group.rs
new file mode 100644
index 0000000..40317bf
--- /dev/null
+++ b/src/next/group.rs
@@ -0,0 +1,90 @@
+use proc_macro2::Delimiter;
+
+use super::parse::{ParseBuffer, Result};
+use super::token;
+
+pub struct Braces<'a> {
+ pub token: token::Brace,
+ pub content: ParseBuffer<'a>,
+}
+
+impl<'a> ParseBuffer<'a> {
+ // Not public API.
+ #[doc(hidden)]
+ pub fn parse_braces(&self) -> Result<Braces<'a>> {
+ self.step_cursor(|cursor| {
+ if let Some((content, span, rest)) = cursor.group(Delimiter::Brace) {
+ let braces = Braces {
+ token: token::Brace(span),
+ content: ParseBuffer::new(span, cursor.advance(content)),
+ };
+ Ok((braces, rest))
+ } else {
+ Err(cursor.error("expected curly braces"))
+ }
+ })
+ }
+}
+
+/// Parse a set of curly braces and expose their content to subsequent parsers.
+///
+/// ```rust
+/// # extern crate syn;
+/// #
+/// use syn::{braced, Token};
+/// use syn::next::{token, Ident};
+/// use syn::next::parse::{Parse, ParseStream, Result};
+/// #
+/// # mod example {
+/// # use super::{syn, braced, token, Ident, Parse, ParseStream, Result};
+/// #
+/// # macro_rules! Token {
+/// # (struct) => {
+/// # syn::next::token::Struct
+/// # };
+/// # }
+/// #
+/// # type Field = Ident;
+///
+/// // Parse a simplified struct syntax like:
+/// //
+/// // struct S {
+/// // a: A,
+/// // b: B,
+/// // }
+/// struct Struct {
+/// pub struct_token: Token![struct],
+/// pub ident: Ident,
+/// pub brace_token: token::Brace,
+/// pub fields: Vec<Field>,
+/// }
+///
+/// impl Parse for Struct {
+/// fn parse(input: ParseStream) -> Result<Self> {
+/// let content;
+/// Ok(Struct {
+/// struct_token: input.parse()?,
+/// ident: input.parse()?,
+/// brace_token: braced!(content in input),
+/// fields: content.parse()?,
+/// })
+/// }
+/// }
+/// # }
+/// #
+/// # fn main() {}
+/// ```
+#[macro_export]
+macro_rules! braced {
+ ($content:ident in $cursor:expr) => {
+ match $crate::next::parse::ParseBuffer::parse_braces(&$cursor) {
+ $crate::next::export::Ok(braces) => {
+ $content = braces.content;
+ braces.token
+ }
+ $crate::next::export::Err(error) => {
+ return $crate::next::export::Err(error);
+ }
+ }
+ };
+}
diff --git a/src/next/lookahead.rs b/src/next/lookahead.rs
new file mode 100644
index 0000000..256ac05
--- /dev/null
+++ b/src/next/lookahead.rs
@@ -0,0 +1,91 @@
+use std::cell::RefCell;
+
+use buffer::Cursor;
+use proc_macro2::Span;
+
+use super::error;
+use super::parse::Error;
+use super::token::Token;
+
+/// Support for checking the next token in a stream to decide how to parse.
+///
+/// Use [`ParseStream::lookahead1`] to construct this object.
+///
+/// [`ParseStream::lookahead1`]: struct.ParseBuffer.html#method.lookahead1
+pub struct Lookahead1<'a> {
+ scope: Span,
+ cursor: Cursor<'a>,
+ comparisons: RefCell<Vec<String>>,
+}
+
+impl<'a> Lookahead1<'a> {
+ // Not public API.
+ #[doc(hidden)]
+ pub fn new(scope: Span, cursor: Cursor<'a>) -> Self {
+ Lookahead1 {
+ scope: scope,
+ cursor: cursor,
+ comparisons: RefCell::new(Vec::new()),
+ }
+ }
+
+ pub fn peek<T: Peek>(&self, token: T) -> bool {
+ let _ = token;
+ if T::Token::peek(self) {
+ return true;
+ }
+ self.comparisons.borrow_mut().push(T::Token::display());
+ false
+ }
+
+ pub fn error(self) -> Error {
+ let comparisons = self.comparisons.borrow();
+ match comparisons.len() {
+ 0 => if self.cursor.eof() {
+ Error::new(self.scope, "unexpected end of input")
+ } else {
+ Error::new(self.cursor.span(), "unexpected token")
+ },
+ 1 => {
+ let message = format!("expected {}", comparisons[0]);
+ error::new_at(self.scope, self.cursor, message)
+ }
+ _ => {
+ let join = comparisons.join(", ");
+ let message = format!("expected one of: {}", join);
+ error::new_at(self.scope, self.cursor, message)
+ }
+ }
+ }
+}
+
+/// Types that can be parsed by looking at just one token.
+///
+/// This trait is sealed and cannot be implemented for types outside of Syn.
+pub trait Peek: private::Sealed {
+ // Not public API.
+ #[doc(hidden)]
+ type Token: Token;
+}
+
+impl<F: FnOnce(TokenMarker) -> T, T: Token> Peek for F {
+ type Token = T;
+}
+
+pub enum TokenMarker {}
+
+// Not public API.
+#[doc(hidden)]
+pub fn is_token(lookahead: &Lookahead1, repr: &'static str) -> bool {
+ if let Some((token, _rest)) = lookahead.cursor.token_tree() {
+ token.to_string() == repr
+ } else {
+ false
+ }
+}
+
+mod private {
+ use super::{Token, TokenMarker};
+ pub trait Sealed {}
+ impl<F: FnOnce(TokenMarker) -> T, T: Token> Sealed for F {}
+}
diff --git a/src/next/mod.rs b/src/next/mod.rs
new file mode 100644
index 0000000..c9d4088
--- /dev/null
+++ b/src/next/mod.rs
@@ -0,0 +1,95 @@
+#[macro_use]
+pub mod token;
+
+#[macro_use]
+pub mod parse;
+
+#[macro_use]
+mod group;
+
+mod error;
+mod lookahead;
+mod span;
+
+pub use proc_macro2::Ident;
+
+// Not public API.
+#[doc(hidden)]
+pub mod export;
+
+use std::str::FromStr;
+
+use buffer::TokenBuffer;
+#[cfg(feature = "proc-macro")]
+use proc_macro;
+use proc_macro2::{self, Span};
+
+use self::parse::{Parse, ParseBuffer, Result};
+
+/// Parse tokens of source code into the chosen syntax tree node.
+#[cfg(feature = "proc-macro")]
+pub fn parse<T: Parse>(input: proc_macro::TokenStream) -> Result<T> {
+ parse2(proc_macro2::TokenStream::from(input))
+}
+
+/// Parse a proc-macro2 token stream into the chosen syntax tree node.
+pub fn parse2<T: Parse>(input: proc_macro2::TokenStream) -> Result<T> {
+ let buf = TokenBuffer::new2(input);
+ let state = ParseBuffer::new(Span::call_site(), buf.begin());
+ T::parse(&state)
+}
+
+/// Parse a string of Rust code into the chosen syntax tree node.
+pub fn parse_str<T: Parse>(input: &str) -> Result<T> {
+ let tokens = proc_macro2::TokenStream::from_str(input)?;
+ parse2(tokens)
+}
+
+/// Parse the input TokenStream of a macro, triggering a compile error if the
+/// tokens fail to parse.
+///
+/// # Intended usage
+///
+/// ```rust
+/// # extern crate proc_macro;
+/// # extern crate syn;
+/// #
+/// use proc_macro::TokenStream;
+/// use syn::parse_macro_input;
+/// use syn::next::parse::{Parse, ParseStream, Result};
+///
+/// struct MyMacroInput {
+/// /* ... */
+/// }
+///
+/// impl Parse for MyMacroInput {
+/// fn parse(input: ParseStream) -> Result<Self> {
+/// /* ... */
+/// # Ok(MyMacroInput {})
+/// }
+/// }
+///
+/// # const IGNORE: &str = stringify! {
+/// #[proc_macro]
+/// # };
+/// pub fn my_macro(tokens: TokenStream) -> TokenStream {
+/// let input = parse_macro_input!(tokens as MyMacroInput);
+///
+/// /* ... */
+/// # "".parse().unwrap()
+/// }
+/// #
+/// # fn main() {}
+/// ```
+#[cfg(feature = "proc-macro")]
+#[macro_export]
+macro_rules! parse_macro_input {
+ ($tokenstream:ident as $ty:ty) => {
+ match $crate::next::parse::<$ty>($tokenstream) {
+ $crate::next::export::Ok(data) => data,
+ $crate::next::export::Err(err) => {
+ return $crate::next::export::TokenStream::from(err.into_compile_error());
+ }
+ };
+ };
+}
diff --git a/src/next/parse.rs b/src/next/parse.rs
new file mode 100644
index 0000000..447b8e1
--- /dev/null
+++ b/src/next/parse.rs
@@ -0,0 +1,135 @@
+//! Parsing interface for parsing a token stream into a syntax tree node.
+
+use std::cell::Cell;
+use std::fmt::Display;
+use std::marker::PhantomData;
+use std::mem;
+use std::ops::Deref;
+
+use buffer::Cursor;
+use proc_macro2::{Ident, Span};
+
+use super::error;
+
+pub use super::error::{Error, Result};
+pub use super::lookahead::{Lookahead1, Peek};
+
+/// Parsing interface implemented by all types that can be parsed in a default
+/// way from a token stream.
+pub trait Parse: Sized {
+ fn parse(input: ParseStream) -> Result<Self>;
+}
+
+/// Input to a Syn parser function.
+pub type ParseStream<'a> = &'a ParseBuffer<'a>;
+
+/// Cursor position within a buffered token stream.
+#[derive(Clone)]
+pub struct ParseBuffer<'a> {
+ scope: Span,
+ cell: Cell<Cursor<'static>>,
+ marker: PhantomData<Cursor<'a>>,
+}
+
+// Not public API.
+#[doc(hidden)]
+#[derive(Copy, Clone)]
+pub struct StepCursor<'c, 'a> {
+ scope: Span,
+ cursor: Cursor<'c>,
+ marker: PhantomData<fn(Cursor<'c>) -> Cursor<'a>>,
+}
+
+impl<'c, 'a> Deref for StepCursor<'c, 'a> {
+ type Target = Cursor<'c>;
+
+ fn deref(&self) -> &Self::Target {
+ &self.cursor
+ }
+}
+
+impl<'c, 'a> StepCursor<'c, 'a> {
+ // Not public API.
+ #[doc(hidden)]
+ pub fn advance(self, other: Cursor<'c>) -> Cursor<'a> {
+ unsafe { mem::transmute::<Cursor<'c>, Cursor<'a>>(other) }
+ }
+
+ // Not public API.
+ #[doc(hidden)]
+ pub fn error<T: Display>(self, message: T) -> Error {
+ error::new_at(self.scope, self.cursor, message)
+ }
+}
+
+impl<'a> ParseBuffer<'a> {
+ // Not public API.
+ #[doc(hidden)]
+ pub fn new(scope: Span, cursor: Cursor<'a>) -> Self {
+ let extend = unsafe { mem::transmute::<Cursor<'a>, Cursor<'static>>(cursor) };
+ ParseBuffer {
+ scope: scope,
+ cell: Cell::new(extend),
+ marker: PhantomData,
+ }
+ }
+
+ pub fn cursor(&self) -> Cursor<'a> {
+ self.cell.get()
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.cursor().eof()
+ }
+
+ pub fn lookahead1(&self) -> Lookahead1<'a> {
+ Lookahead1::new(self.scope, self.cursor())
+ }
+
+ pub fn parse<T: Parse>(&self) -> Result<T> {
+ T::parse(self)
+ }
+
+ // Not public API.
+ #[doc(hidden)]
+ pub fn step_cursor<F, R>(&self, function: F) -> Result<R>
+ where
+ F: for<'c> FnOnce(StepCursor<'c, 'a>) -> Result<(R, Cursor<'c>)>,
+ {
+ match function(StepCursor {
+ scope: self.scope,
+ cursor: self.cell.get(),
+ marker: PhantomData,
+ }) {
+ Ok((ret, cursor)) => {
+ self.cell.set(cursor);
+ Ok(ret)
+ }
+ Err(err) => Err(err),
+ }
+ }
+}
+
+impl Parse for Ident {
+ fn parse(input: ParseStream) -> Result<Self> {
+ input.step_cursor(|cursor| {
+ if let Some((ident, rest)) = cursor.ident() {
+ Ok((ident, rest))
+ } else {
+ Err(cursor.error("expected identifier"))
+ }
+ })
+ }
+}
+
+// In reality the impl would be for Punctuated.
+impl<T: Parse> Parse for Vec<T> {
+ fn parse(input: ParseStream) -> Result<Self> {
+ let mut vec = Vec::new();
+ while !input.is_empty() {
+ let t = input.parse::<T>()?;
+ vec.push(t);
+ }
+ Ok(vec)
+ }
+}
diff --git a/src/next/span.rs b/src/next/span.rs
new file mode 100644
index 0000000..77d4448
--- /dev/null
+++ b/src/next/span.rs
@@ -0,0 +1,73 @@
+use proc_macro2::Span;
+
+use super::lookahead::TokenMarker;
+
+pub trait IntoSpans<S> {
+ // Not public API.
+ #[doc(hidden)]
+ fn into_spans(self) -> S;
+}
+
+impl<S> IntoSpans<S> for TokenMarker {
+ fn into_spans(self) -> S {
+ match self {}
+ }
+}
+
+impl IntoSpans<[Span; 1]> for Span {
+ fn into_spans(self) -> [Span; 1] {
+ [self]
+ }
+}
+
+impl IntoSpans<[Span; 2]> for Span {
+ fn into_spans(self) -> [Span; 2] {
+ [self, self]
+ }
+}
+
+impl IntoSpans<[Span; 3]> for Span {
+ fn into_spans(self) -> [Span; 3] {
+ [self, self, self]
+ }
+}
+
+impl IntoSpans<Self> for [Span; 1] {
+ fn into_spans(self) -> Self {
+ self
+ }
+}
+
+impl IntoSpans<Self> for [Span; 2] {
+ fn into_spans(self) -> Self {
+ self
+ }
+}
+
+impl IntoSpans<Self> for [Span; 3] {
+ fn into_spans(self) -> Self {
+ self
+ }
+}
+
+pub trait FromSpans: Sized {
+ fn from_spans(spans: &[Span]) -> Self;
+}
+
+impl FromSpans for [Span; 1] {
+ fn from_spans(spans: &[Span]) -> Self {
+ [spans[0]]
+ }
+}
+
+impl FromSpans for [Span; 2] {
+ fn from_spans(spans: &[Span]) -> Self {
+ [spans[0], spans[1]]
+ }
+}
+
+impl FromSpans for [Span; 3] {
+ fn from_spans(spans: &[Span]) -> Self {
+ [spans[0], spans[1], spans[2]]
+ }
+}
diff --git a/src/next/token.rs b/src/next/token.rs
new file mode 100644
index 0000000..5b80a14
--- /dev/null
+++ b/src/next/token.rs
@@ -0,0 +1,166 @@
+//! Tokens representing Rust punctuation, keywords, and delimiters.
+
+use std::ops::{Deref, DerefMut};
+
+use proc_macro2::{Spacing, Span};
+
+use super::error::Error;
+use super::lookahead;
+use super::parse::{Lookahead1, Parse, ParseStream, Result};
+use super::span::{FromSpans, IntoSpans};
+
+/// Marker trait for types that represent single tokens.
+///
+/// This trait is sealed and cannot be implemented for types outside of Syn.
+pub trait Token: private::Sealed {
+ // Not public API.
+ #[doc(hidden)]
+ fn peek(lookahead: &Lookahead1) -> bool;
+
+ // Not public API.
+ #[doc(hidden)]
+ fn display() -> String;
+}
+
+mod private {
+ pub trait Sealed {}
+}
+
+macro_rules! impl_token {
+ ($token:tt $name:ident #[$doc:meta]) => {
+ impl Token for $name {
+ fn peek(lookahead: &Lookahead1) -> bool {
+ lookahead::is_token(lookahead, $token)
+ }
+
+ fn display() -> String {
+ concat!("`", $token, "`").to_owned()
+ }
+ }
+
+ impl private::Sealed for $name {}
+ };
+}
+
+macro_rules! define_keywords {
+ ($($token:tt pub struct $name:ident #[$doc:meta])*) => {
+ $(
+ #[$doc]
+ #[derive(Debug)]
+ pub struct $name {
+ pub span: Span,
+ }
+
+ #[doc(hidden)]
+ #[allow(non_snake_case)]
+ pub fn $name<T: IntoSpans<[Span; 1]>>(span: T) -> $name {
+ $name {
+ span: span.into_spans()[0],
+ }
+ }
+
+ impl_token!($token $name #[$doc]);
+
+ impl Parse for $name {
+ fn parse(input: ParseStream) -> Result<Self> {
+ parse_keyword(input, $token).map($name)
+ }
+ }
+ )*
+ };
+}
+
+macro_rules! define_punctuation {
+ ($($token:tt pub struct $name:ident/$len:tt #[$doc:meta])*) => {
+ $(
+ #[$doc]
+ #[derive(Debug)]
+ pub struct $name {
+ pub spans: [Span; $len],
+ }
+
+ impl Deref for $name {
+ type Target = [Span; $len];
+
+ fn deref(&self) -> &Self::Target {
+ &self.spans
+ }
+ }
+
+ impl DerefMut for $name {
+ fn deref_mut(&mut self) -> &mut Self::Target {
+ &mut self.spans
+ }
+ }
+
+ #[doc(hidden)]
+ #[allow(non_snake_case)]
+ pub fn $name<T: IntoSpans<[Span; $len]>>(spans: T) -> $name {
+ $name {
+ spans: spans.into_spans(),
+ }
+ }
+
+ impl_token!($token $name #[$doc]);
+
+ impl Parse for $name {
+ fn parse(input: ParseStream) -> Result<Self> {
+ parse_punctuation(input, $token).map($name::<[Span; $len]>)
+ }
+ }
+ )*
+ };
+}
+
+define_keywords! {
+ "struct" pub struct Struct /// `struct`
+ "enum" pub struct Enum /// `enum`
+}
+
+define_punctuation! {
+ ":" pub struct Colon/1 /// `:`
+ "," pub struct Comma/1 /// `,`
+ ".." pub struct Dot2/2 /// `..`
+}
+
+/// `{...}`
+#[derive(Debug)]
+pub struct Brace(pub Span);
+
+fn parse_keyword(input: ParseStream, token: &str) -> Result<Span> {
+ input.step_cursor(|cursor| {
+ if let Some((ident, rest)) = cursor.ident() {
+ if ident == token {
+ return Ok((ident.span(), rest));
+ }
+ }
+ Err(cursor.error(format!("expected `{}`", token)))
+ })
+}
+
+fn parse_punctuation<S: FromSpans>(input: ParseStream, token: &str) -> Result<S> {
+ input.step_cursor(|cursor| {
+ let mut cursor = *cursor;
+ let mut spans = [cursor.span(); 3];
+ assert!(token.len() <= spans.len());
+
+ for (i, ch) in token.chars().enumerate() {
+ match cursor.punct() {
+ Some((punct, rest)) => {
+ spans[i] = punct.span();
+ if punct.as_char() != ch {
+ break;
+ } else if i == token.len() - 1 {
+ return Ok((S::from_spans(&spans), rest));
+ } else if punct.spacing() != Spacing::Joint {
+ break;
+ }
+ cursor = rest;
+ }
+ None => break,
+ }
+ }
+
+ Err(Error::new(spans[0], format!("expected `{}`", token)))
+ })
+}