Merge synom and parse modules
diff --git a/src/lib.rs b/src/lib.rs
index 6c8e915..ac61e79 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -441,8 +441,6 @@
#[cfg(feature = "parsing")]
pub mod ext;
pub mod punctuated;
-#[cfg(feature = "parsing")]
-pub mod synom;
#[cfg(all(
any(feature = "full", feature = "derive"),
feature = "extra-traits"
@@ -589,9 +587,6 @@
////////////////////////////////////////////////////////////////////////////////
#[cfg(feature = "parsing")]
-use synom::Parser;
-
-#[cfg(feature = "parsing")]
mod error;
#[cfg(feature = "parsing")]
use error::Error;
@@ -649,7 +644,7 @@
feature = "proc-macro"
))]
pub fn parse<T: parse::Parse>(tokens: proc_macro::TokenStream) -> Result<T, Error> {
- T::parse.parse(tokens)
+ parse::Parser::parse(T::parse, tokens)
}
/// Parse a proc-macro2 token stream into the chosen syntax tree node.
@@ -666,7 +661,7 @@
/// *This function is available if Syn is built with the `"parsing"` feature.*
#[cfg(feature = "parsing")]
pub fn parse2<T: parse::Parse>(tokens: proc_macro2::TokenStream) -> Result<T, Error> {
- T::parse.parse2(tokens)
+ parse::Parser::parse2(T::parse, tokens)
}
/// Parse a string of Rust code into the chosen syntax tree node.
@@ -699,7 +694,7 @@
/// ```
#[cfg(feature = "parsing")]
pub fn parse_str<T: parse::Parse>(s: &str) -> Result<T, Error> {
- T::parse.parse_str(s)
+ parse::Parser::parse_str(T::parse, s)
}
// FIXME the name parse_file makes it sound like you might pass in a path to a
diff --git a/src/parse.rs b/src/parse.rs
index 47b2a10..3419303 100644
--- a/src/parse.rs
+++ b/src/parse.rs
@@ -1,4 +1,154 @@
+// Copyright 2018 Syn Developers
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
//! Parsing interface for parsing a token stream into a syntax tree node.
+//!
+//! Parsing in Syn is built on parser functions that take in a [`Cursor`] and
+//! produce a [`PResult<T>`] where `T` is some syntax tree node. `Cursor` is a
+//! cheaply copyable cursor over a range of tokens in a token stream, and
+//! `PResult` is a result that packages together a parsed syntax tree node `T`
+//! with a stream of remaining unparsed tokens after `T` represented as another
+//! `Cursor`, or an [`Error`] if parsing failed.
+//!
+//! [`Cursor`]: ../buffer/index.html
+//! [`PResult<T>`]: type.PResult.html
+//! [`Error`]: struct.Error.html
+//!
+//! This `Cursor`- and `PResult`-based interface is convenient for parser
+//! combinators and parser implementations, but not necessarily when you just
+//! have some tokens that you want to parse. For that we expose the following
+//! two entry points.
+//!
+//! ## The `syn::parse*` functions
+//!
+//! The [`syn::parse`], [`syn::parse2`], and [`syn::parse_str`] functions serve
+//! as an entry point for parsing syntax tree nodes that can be parsed in an
+//! obvious default way. These functions can return any syntax tree node that
+//! implements the [`Synom`] trait, which includes most types in Syn.
+//!
+//! [`syn::parse`]: ../fn.parse.html
+//! [`syn::parse2`]: ../fn.parse2.html
+//! [`syn::parse_str`]: ../fn.parse_str.html
+//! [`Synom`]: trait.Synom.html
+//!
+//! ```
+//! use syn::Type;
+//!
+//! # fn run_parser() -> Result<(), syn::synom::Error> {
+//! let t: Type = syn::parse_str("std::collections::HashMap<String, Value>")?;
+//! # Ok(())
+//! # }
+//! #
+//! # fn main() {
+//! # run_parser().unwrap();
+//! # }
+//! ```
+//!
+//! The [`parse_quote!`] macro also uses this approach.
+//!
+//! [`parse_quote!`]: ../macro.parse_quote.html
+//!
+//! ## The `Parser` trait
+//!
+//! Some types can be parsed in several ways depending on context. For example
+//! an [`Attribute`] can be either "outer" like `#[...]` or "inner" like
+//! `#![...]` and parsing the wrong one would be a bug. Similarly [`Punctuated`]
+//! may or may not allow trailing punctuation, and parsing it the wrong way
+//! would either reject valid input or accept invalid input.
+//!
+//! [`Attribute`]: ../struct.Attribute.html
+//! [`Punctuated`]: ../punctuated/index.html
+//!
+//! The `Synom` trait is not implemented in these cases because there is no good
+//! behavior to consider the default.
+//!
+//! ```ignore
+//! // Can't parse `Punctuated` without knowing whether trailing punctuation
+//! // should be allowed in this context.
+//! let path: Punctuated<PathSegment, Token![::]> = syn::parse(tokens)?;
+//! ```
+//!
+//! In these cases the types provide a choice of parser functions rather than a
+//! single `Synom` implementation, and those parser functions can be invoked
+//! through the [`Parser`] trait.
+//!
+//! [`Parser`]: trait.Parser.html
+//!
+//! ```
+//! # #[macro_use]
+//! # extern crate syn;
+//! #
+//! # extern crate proc_macro2;
+//! # use proc_macro2::TokenStream;
+//! #
+//! use syn::synom::Parser;
+//! use syn::punctuated::Punctuated;
+//! use syn::{PathSegment, Expr, Attribute};
+//!
+//! # fn run_parsers() -> Result<(), syn::synom::Error> {
+//! # let tokens = TokenStream::new().into();
+//! // Parse a nonempty sequence of path segments separated by `::` punctuation
+//! // with no trailing punctuation.
+//! let parser = Punctuated::<PathSegment, Token![::]>::parse_separated_nonempty;
+//! let path = parser.parse(tokens)?;
+//!
+//! # let tokens = TokenStream::new().into();
+//! // Parse a possibly empty sequence of expressions terminated by commas with
+//! // an optional trailing punctuation.
+//! let parser = Punctuated::<Expr, Token![,]>::parse_terminated;
+//! let args = parser.parse(tokens)?;
+//!
+//! # let tokens = TokenStream::new().into();
+//! // Parse zero or more outer attributes but not inner attributes.
+//! named!(outer_attrs -> Vec<Attribute>, many0!(Attribute::old_parse_outer));
+//! let attrs = outer_attrs.parse(tokens)?;
+//! #
+//! # Ok(())
+//! # }
+//! #
+//! # fn main() {}
+//! ```
+//!
+//! # Implementing a parser function
+//!
+//! Parser functions are usually implemented using the [`nom`]-style parser
+//! combinator macros provided by Syn, but may also be implemented without
+//! macros be using the low-level [`Cursor`] API directly.
+//!
+//! [`nom`]: https://github.com/Geal/nom
+//!
+//! The following parser combinator macros are available and a `Synom` parsing
+//! example is provided for each one.
+//!
+//! - [`alt!`](../macro.alt.html)
+//! - [`braces!`](../macro.braces.html)
+//! - [`brackets!`](../macro.brackets.html)
+//! - [`call!`](../macro.call.html)
+//! - [`cond!`](../macro.cond.html)
+//! - [`cond_reduce!`](../macro.cond_reduce.html)
+//! - [`custom_keyword!`](../macro.custom_keyword.html)
+//! - [`do_parse!`](../macro.do_parse.html)
+//! - [`epsilon!`](../macro.epsilon.html)
+//! - [`input_end!`](../macro.input_end.html)
+//! - [`keyword!`](../macro.keyword.html)
+//! - [`many0!`](../macro.many0.html)
+//! - [`map!`](../macro.map.html)
+//! - [`not!`](../macro.not.html)
+//! - [`option!`](../macro.option.html)
+//! - [`parens!`](../macro.parens.html)
+//! - [`punct!`](../macro.punct.html)
+//! - [`reject!`](../macro.reject.html)
+//! - [`switch!`](../macro.switch.html)
+//! - [`syn!`](../macro.syn.html)
+//! - [`tuple!`](../macro.tuple.html)
+//! - [`value!`](../macro.value.html)
+//!
+//! *This module is available if Syn is built with the `"parsing"` feature.*
use std::cell::Cell;
use std::fmt::Display;
@@ -6,10 +156,16 @@
use std::mem;
use std::ops::Deref;
use std::rc::Rc;
+use std::str::FromStr;
-use proc_macro2::{Ident, Span};
+#[cfg(all(
+ not(all(target_arch = "wasm32", target_os = "unknown")),
+ feature = "proc-macro"
+))]
+use proc_macro;
+use proc_macro2::{self, Delimiter, Group, Ident, Literal, Punct, Span, TokenStream, TokenTree};
-use buffer::Cursor;
+use buffer::{Cursor, TokenBuffer};
use error;
use punctuated::Punctuated;
use token::Token;
@@ -228,3 +384,107 @@
}
}
}
+
+impl Parse for TokenStream {
+ fn parse(input: ParseStream) -> Result<Self> {
+ input.step(|cursor| Ok((cursor.token_stream(), Cursor::empty())))
+ }
+}
+
+impl Parse for TokenTree {
+ fn parse(input: ParseStream) -> Result<Self> {
+ input.step(|cursor| match cursor.token_tree() {
+ Some((tt, rest)) => Ok((tt, rest)),
+ None => Err(cursor.error("expected token tree")),
+ })
+ }
+}
+
+impl Parse for Group {
+ fn parse(input: ParseStream) -> Result<Self> {
+ input.step(|cursor| {
+ for delim in &[Delimiter::Parenthesis, Delimiter::Brace, Delimiter::Bracket] {
+ if let Some((inside, span, rest)) = cursor.group(*delim) {
+ let mut group = Group::new(*delim, inside.token_stream());
+ group.set_span(span);
+ return Ok((group, rest));
+ }
+ }
+ Err(cursor.error("expected group token"))
+ })
+ }
+}
+
+impl Parse for Punct {
+ fn parse(input: ParseStream) -> Result<Self> {
+ input.step(|cursor| match cursor.punct() {
+ Some((punct, rest)) => Ok((punct, rest)),
+ None => Err(cursor.error("expected punctuation token")),
+ })
+ }
+}
+
+impl Parse for Literal {
+ fn parse(input: ParseStream) -> Result<Self> {
+ input.step(|cursor| match cursor.literal() {
+ Some((literal, rest)) => Ok((literal, rest)),
+ None => Err(cursor.error("expected literal token")),
+ })
+ }
+}
+
+/// Parser that can parse Rust tokens into a particular syntax tree node.
+///
+/// Refer to the [module documentation] for details about parsing in Syn.
+///
+/// [module documentation]: index.html
+///
+/// *This trait is available if Syn is built with the `"parsing"` feature.*
+pub trait Parser: Sized {
+ type Output;
+
+ /// Parse a proc-macro2 token stream into the chosen syntax tree node.
+ fn parse2(self, tokens: TokenStream) -> Result<Self::Output>;
+
+ /// Parse tokens of source code into the chosen syntax tree node.
+ ///
+ /// *This method is available if Syn is built with both the `"parsing"` and
+ /// `"proc-macro"` features.*
+ #[cfg(all(
+ not(all(target_arch = "wasm32", target_os = "unknown")),
+ feature = "proc-macro"
+ ))]
+ fn parse(self, tokens: proc_macro::TokenStream) -> Result<Self::Output> {
+ self.parse2(proc_macro2::TokenStream::from(tokens))
+ }
+
+ /// Parse a string of Rust code into the chosen syntax tree node.
+ ///
+ /// # Hygiene
+ ///
+ /// Every span in the resulting syntax tree will be set to resolve at the
+ /// macro call site.
+ fn parse_str(self, s: &str) -> Result<Self::Output> {
+ self.parse2(proc_macro2::TokenStream::from_str(s)?)
+ }
+}
+
+impl<F, T> Parser for F
+where
+ F: FnOnce(ParseStream) -> Result<T>,
+{
+ type Output = T;
+
+ fn parse2(self, tokens: TokenStream) -> Result<T> {
+ let buf = TokenBuffer::new2(tokens);
+ let unexpected = Rc::new(Cell::new(None));
+ let state = ParseBuffer::new(Span::call_site(), buf.begin(), unexpected);
+ let node = self(&state)?;
+ state.check_unexpected()?;
+ if state.is_empty() {
+ Ok(node)
+ } else {
+ Err(state.error("unexpected token"))
+ }
+ }
+}
diff --git a/src/parse_quote.rs b/src/parse_quote.rs
index 9d65555..f4ad19c 100644
--- a/src/parse_quote.rs
+++ b/src/parse_quote.rs
@@ -88,9 +88,8 @@
////////////////////////////////////////////////////////////////////////////////
// Can parse any type that implements Synom.
-use parse::{Parse, ParseStream, Result};
+use parse::{Parse, Parser, ParseStream, Result};
use proc_macro2::TokenStream;
-use synom::Parser;
// Not public API.
#[doc(hidden)]
diff --git a/src/synom.rs b/src/synom.rs
index 4dabe38..e69de29 100644
--- a/src/synom.rs
+++ b/src/synom.rs
@@ -1,271 +0,0 @@
-// Copyright 2018 Syn Developers
-//
-// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
-// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
-// option. This file may not be copied, modified, or distributed
-// except according to those terms.
-
-//! Parsing interface for parsing a token stream into a syntax tree node.
-//!
-//! Parsing in Syn is built on parser functions that take in a [`Cursor`] and
-//! produce a [`PResult<T>`] where `T` is some syntax tree node. `Cursor` is a
-//! cheaply copyable cursor over a range of tokens in a token stream, and
-//! `PResult` is a result that packages together a parsed syntax tree node `T`
-//! with a stream of remaining unparsed tokens after `T` represented as another
-//! `Cursor`, or an [`Error`] if parsing failed.
-//!
-//! [`Cursor`]: ../buffer/index.html
-//! [`PResult<T>`]: type.PResult.html
-//! [`Error`]: struct.Error.html
-//!
-//! This `Cursor`- and `PResult`-based interface is convenient for parser
-//! combinators and parser implementations, but not necessarily when you just
-//! have some tokens that you want to parse. For that we expose the following
-//! two entry points.
-//!
-//! ## The `syn::parse*` functions
-//!
-//! The [`syn::parse`], [`syn::parse2`], and [`syn::parse_str`] functions serve
-//! as an entry point for parsing syntax tree nodes that can be parsed in an
-//! obvious default way. These functions can return any syntax tree node that
-//! implements the [`Synom`] trait, which includes most types in Syn.
-//!
-//! [`syn::parse`]: ../fn.parse.html
-//! [`syn::parse2`]: ../fn.parse2.html
-//! [`syn::parse_str`]: ../fn.parse_str.html
-//! [`Synom`]: trait.Synom.html
-//!
-//! ```
-//! use syn::Type;
-//!
-//! # fn run_parser() -> Result<(), syn::synom::Error> {
-//! let t: Type = syn::parse_str("std::collections::HashMap<String, Value>")?;
-//! # Ok(())
-//! # }
-//! #
-//! # fn main() {
-//! # run_parser().unwrap();
-//! # }
-//! ```
-//!
-//! The [`parse_quote!`] macro also uses this approach.
-//!
-//! [`parse_quote!`]: ../macro.parse_quote.html
-//!
-//! ## The `Parser` trait
-//!
-//! Some types can be parsed in several ways depending on context. For example
-//! an [`Attribute`] can be either "outer" like `#[...]` or "inner" like
-//! `#![...]` and parsing the wrong one would be a bug. Similarly [`Punctuated`]
-//! may or may not allow trailing punctuation, and parsing it the wrong way
-//! would either reject valid input or accept invalid input.
-//!
-//! [`Attribute`]: ../struct.Attribute.html
-//! [`Punctuated`]: ../punctuated/index.html
-//!
-//! The `Synom` trait is not implemented in these cases because there is no good
-//! behavior to consider the default.
-//!
-//! ```ignore
-//! // Can't parse `Punctuated` without knowing whether trailing punctuation
-//! // should be allowed in this context.
-//! let path: Punctuated<PathSegment, Token![::]> = syn::parse(tokens)?;
-//! ```
-//!
-//! In these cases the types provide a choice of parser functions rather than a
-//! single `Synom` implementation, and those parser functions can be invoked
-//! through the [`Parser`] trait.
-//!
-//! [`Parser`]: trait.Parser.html
-//!
-//! ```
-//! # #[macro_use]
-//! # extern crate syn;
-//! #
-//! # extern crate proc_macro2;
-//! # use proc_macro2::TokenStream;
-//! #
-//! use syn::synom::Parser;
-//! use syn::punctuated::Punctuated;
-//! use syn::{PathSegment, Expr, Attribute};
-//!
-//! # fn run_parsers() -> Result<(), syn::synom::Error> {
-//! # let tokens = TokenStream::new().into();
-//! // Parse a nonempty sequence of path segments separated by `::` punctuation
-//! // with no trailing punctuation.
-//! let parser = Punctuated::<PathSegment, Token![::]>::parse_separated_nonempty;
-//! let path = parser.parse(tokens)?;
-//!
-//! # let tokens = TokenStream::new().into();
-//! // Parse a possibly empty sequence of expressions terminated by commas with
-//! // an optional trailing punctuation.
-//! let parser = Punctuated::<Expr, Token![,]>::parse_terminated;
-//! let args = parser.parse(tokens)?;
-//!
-//! # let tokens = TokenStream::new().into();
-//! // Parse zero or more outer attributes but not inner attributes.
-//! named!(outer_attrs -> Vec<Attribute>, many0!(Attribute::old_parse_outer));
-//! let attrs = outer_attrs.parse(tokens)?;
-//! #
-//! # Ok(())
-//! # }
-//! #
-//! # fn main() {}
-//! ```
-//!
-//! # Implementing a parser function
-//!
-//! Parser functions are usually implemented using the [`nom`]-style parser
-//! combinator macros provided by Syn, but may also be implemented without
-//! macros be using the low-level [`Cursor`] API directly.
-//!
-//! [`nom`]: https://github.com/Geal/nom
-//!
-//! The following parser combinator macros are available and a `Synom` parsing
-//! example is provided for each one.
-//!
-//! - [`alt!`](../macro.alt.html)
-//! - [`braces!`](../macro.braces.html)
-//! - [`brackets!`](../macro.brackets.html)
-//! - [`call!`](../macro.call.html)
-//! - [`cond!`](../macro.cond.html)
-//! - [`cond_reduce!`](../macro.cond_reduce.html)
-//! - [`custom_keyword!`](../macro.custom_keyword.html)
-//! - [`do_parse!`](../macro.do_parse.html)
-//! - [`epsilon!`](../macro.epsilon.html)
-//! - [`input_end!`](../macro.input_end.html)
-//! - [`keyword!`](../macro.keyword.html)
-//! - [`many0!`](../macro.many0.html)
-//! - [`map!`](../macro.map.html)
-//! - [`not!`](../macro.not.html)
-//! - [`option!`](../macro.option.html)
-//! - [`parens!`](../macro.parens.html)
-//! - [`punct!`](../macro.punct.html)
-//! - [`reject!`](../macro.reject.html)
-//! - [`switch!`](../macro.switch.html)
-//! - [`syn!`](../macro.syn.html)
-//! - [`tuple!`](../macro.tuple.html)
-//! - [`value!`](../macro.value.html)
-//!
-//! *This module is available if Syn is built with the `"parsing"` feature.*
-
-use std::cell::Cell;
-use std::rc::Rc;
-use std::str::FromStr;
-
-#[cfg(all(
- not(all(target_arch = "wasm32", target_os = "unknown")),
- feature = "proc-macro"
-))]
-use proc_macro;
-use proc_macro2::{self, Delimiter, Group, Literal, Punct, Span, TokenStream, TokenTree};
-
-pub use error::Error;
-
-use buffer::{Cursor, TokenBuffer};
-use parse::{Parse, ParseBuffer, ParseStream, Result};
-
-impl Parse for TokenStream {
- fn parse(input: ParseStream) -> Result<Self> {
- input.step(|cursor| Ok((cursor.token_stream(), Cursor::empty())))
- }
-}
-
-impl Parse for TokenTree {
- fn parse(input: ParseStream) -> Result<Self> {
- input.step(|cursor| match cursor.token_tree() {
- Some((tt, rest)) => Ok((tt, rest)),
- None => Err(cursor.error("expected token tree")),
- })
- }
-}
-
-impl Parse for Group {
- fn parse(input: ParseStream) -> Result<Self> {
- input.step(|cursor| {
- for delim in &[Delimiter::Parenthesis, Delimiter::Brace, Delimiter::Bracket] {
- if let Some((inside, span, rest)) = cursor.group(*delim) {
- let mut group = Group::new(*delim, inside.token_stream());
- group.set_span(span);
- return Ok((group, rest));
- }
- }
- Err(cursor.error("expected group token"))
- })
- }
-}
-
-impl Parse for Punct {
- fn parse(input: ParseStream) -> Result<Self> {
- input.step(|cursor| match cursor.punct() {
- Some((punct, rest)) => Ok((punct, rest)),
- None => Err(cursor.error("expected punctuation token")),
- })
- }
-}
-
-impl Parse for Literal {
- fn parse(input: ParseStream) -> Result<Self> {
- input.step(|cursor| match cursor.literal() {
- Some((literal, rest)) => Ok((literal, rest)),
- None => Err(cursor.error("expected literal token")),
- })
- }
-}
-
-/// Parser that can parse Rust tokens into a particular syntax tree node.
-///
-/// Refer to the [module documentation] for details about parsing in Syn.
-///
-/// [module documentation]: index.html
-///
-/// *This trait is available if Syn is built with the `"parsing"` feature.*
-pub trait Parser: Sized {
- type Output;
-
- /// Parse a proc-macro2 token stream into the chosen syntax tree node.
- fn parse2(self, tokens: TokenStream) -> Result<Self::Output>;
-
- /// Parse tokens of source code into the chosen syntax tree node.
- ///
- /// *This method is available if Syn is built with both the `"parsing"` and
- /// `"proc-macro"` features.*
- #[cfg(all(
- not(all(target_arch = "wasm32", target_os = "unknown")),
- feature = "proc-macro"
- ))]
- fn parse(self, tokens: proc_macro::TokenStream) -> Result<Self::Output> {
- self.parse2(proc_macro2::TokenStream::from(tokens))
- }
-
- /// Parse a string of Rust code into the chosen syntax tree node.
- ///
- /// # Hygiene
- ///
- /// Every span in the resulting syntax tree will be set to resolve at the
- /// macro call site.
- fn parse_str(self, s: &str) -> Result<Self::Output> {
- self.parse2(proc_macro2::TokenStream::from_str(s)?)
- }
-}
-
-impl<F, T> Parser for F
-where
- F: FnOnce(ParseStream) -> Result<T>,
-{
- type Output = T;
-
- fn parse2(self, tokens: TokenStream) -> Result<T> {
- let buf = TokenBuffer::new2(tokens);
- let unexpected = Rc::new(Cell::new(None));
- let state = ParseBuffer::new(Span::call_site(), buf.begin(), unexpected);
- let node = self(&state)?;
- state.check_unexpected()?;
- if state.is_empty() {
- Ok(node)
- } else {
- Err(state.error("unexpected token"))
- }
- }
-}