Merge synom and parse modules
diff --git a/src/parse.rs b/src/parse.rs
index 47b2a10..3419303 100644
--- a/src/parse.rs
+++ b/src/parse.rs
@@ -1,4 +1,154 @@
+// Copyright 2018 Syn Developers
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
//! Parsing interface for parsing a token stream into a syntax tree node.
+//!
+//! Parsing in Syn is built on parser functions that take in a [`Cursor`] and
+//! produce a [`PResult<T>`] where `T` is some syntax tree node. `Cursor` is a
+//! cheaply copyable cursor over a range of tokens in a token stream, and
+//! `PResult` is a result that packages together a parsed syntax tree node `T`
+//! with a stream of remaining unparsed tokens after `T` represented as another
+//! `Cursor`, or an [`Error`] if parsing failed.
+//!
+//! [`Cursor`]: ../buffer/index.html
+//! [`PResult<T>`]: type.PResult.html
+//! [`Error`]: struct.Error.html
+//!
+//! This `Cursor`- and `PResult`-based interface is convenient for parser
+//! combinators and parser implementations, but not necessarily when you just
+//! have some tokens that you want to parse. For that we expose the following
+//! two entry points.
+//!
+//! ## The `syn::parse*` functions
+//!
+//! The [`syn::parse`], [`syn::parse2`], and [`syn::parse_str`] functions serve
+//! as an entry point for parsing syntax tree nodes that can be parsed in an
+//! obvious default way. These functions can return any syntax tree node that
+//! implements the [`Synom`] trait, which includes most types in Syn.
+//!
+//! [`syn::parse`]: ../fn.parse.html
+//! [`syn::parse2`]: ../fn.parse2.html
+//! [`syn::parse_str`]: ../fn.parse_str.html
+//! [`Synom`]: trait.Synom.html
+//!
+//! ```
+//! use syn::Type;
+//!
+//! # fn run_parser() -> Result<(), syn::synom::Error> {
+//! let t: Type = syn::parse_str("std::collections::HashMap<String, Value>")?;
+//! # Ok(())
+//! # }
+//! #
+//! # fn main() {
+//! # run_parser().unwrap();
+//! # }
+//! ```
+//!
+//! The [`parse_quote!`] macro also uses this approach.
+//!
+//! [`parse_quote!`]: ../macro.parse_quote.html
+//!
+//! ## The `Parser` trait
+//!
+//! Some types can be parsed in several ways depending on context. For example
+//! an [`Attribute`] can be either "outer" like `#[...]` or "inner" like
+//! `#![...]` and parsing the wrong one would be a bug. Similarly [`Punctuated`]
+//! may or may not allow trailing punctuation, and parsing it the wrong way
+//! would either reject valid input or accept invalid input.
+//!
+//! [`Attribute`]: ../struct.Attribute.html
+//! [`Punctuated`]: ../punctuated/index.html
+//!
+//! The `Synom` trait is not implemented in these cases because there is no good
+//! behavior to consider the default.
+//!
+//! ```ignore
+//! // Can't parse `Punctuated` without knowing whether trailing punctuation
+//! // should be allowed in this context.
+//! let path: Punctuated<PathSegment, Token![::]> = syn::parse(tokens)?;
+//! ```
+//!
+//! In these cases the types provide a choice of parser functions rather than a
+//! single `Synom` implementation, and those parser functions can be invoked
+//! through the [`Parser`] trait.
+//!
+//! [`Parser`]: trait.Parser.html
+//!
+//! ```
+//! # #[macro_use]
+//! # extern crate syn;
+//! #
+//! # extern crate proc_macro2;
+//! # use proc_macro2::TokenStream;
+//! #
+//! use syn::synom::Parser;
+//! use syn::punctuated::Punctuated;
+//! use syn::{PathSegment, Expr, Attribute};
+//!
+//! # fn run_parsers() -> Result<(), syn::synom::Error> {
+//! # let tokens = TokenStream::new().into();
+//! // Parse a nonempty sequence of path segments separated by `::` punctuation
+//! // with no trailing punctuation.
+//! let parser = Punctuated::<PathSegment, Token![::]>::parse_separated_nonempty;
+//! let path = parser.parse(tokens)?;
+//!
+//! # let tokens = TokenStream::new().into();
+//! // Parse a possibly empty sequence of expressions terminated by commas with
+//! // an optional trailing punctuation.
+//! let parser = Punctuated::<Expr, Token![,]>::parse_terminated;
+//! let args = parser.parse(tokens)?;
+//!
+//! # let tokens = TokenStream::new().into();
+//! // Parse zero or more outer attributes but not inner attributes.
+//! named!(outer_attrs -> Vec<Attribute>, many0!(Attribute::old_parse_outer));
+//! let attrs = outer_attrs.parse(tokens)?;
+//! #
+//! # Ok(())
+//! # }
+//! #
+//! # fn main() {}
+//! ```
+//!
+//! # Implementing a parser function
+//!
+//! Parser functions are usually implemented using the [`nom`]-style parser
+//! combinator macros provided by Syn, but may also be implemented without
+//! macros be using the low-level [`Cursor`] API directly.
+//!
+//! [`nom`]: https://github.com/Geal/nom
+//!
+//! The following parser combinator macros are available and a `Synom` parsing
+//! example is provided for each one.
+//!
+//! - [`alt!`](../macro.alt.html)
+//! - [`braces!`](../macro.braces.html)
+//! - [`brackets!`](../macro.brackets.html)
+//! - [`call!`](../macro.call.html)
+//! - [`cond!`](../macro.cond.html)
+//! - [`cond_reduce!`](../macro.cond_reduce.html)
+//! - [`custom_keyword!`](../macro.custom_keyword.html)
+//! - [`do_parse!`](../macro.do_parse.html)
+//! - [`epsilon!`](../macro.epsilon.html)
+//! - [`input_end!`](../macro.input_end.html)
+//! - [`keyword!`](../macro.keyword.html)
+//! - [`many0!`](../macro.many0.html)
+//! - [`map!`](../macro.map.html)
+//! - [`not!`](../macro.not.html)
+//! - [`option!`](../macro.option.html)
+//! - [`parens!`](../macro.parens.html)
+//! - [`punct!`](../macro.punct.html)
+//! - [`reject!`](../macro.reject.html)
+//! - [`switch!`](../macro.switch.html)
+//! - [`syn!`](../macro.syn.html)
+//! - [`tuple!`](../macro.tuple.html)
+//! - [`value!`](../macro.value.html)
+//!
+//! *This module is available if Syn is built with the `"parsing"` feature.*
use std::cell::Cell;
use std::fmt::Display;
@@ -6,10 +156,16 @@
use std::mem;
use std::ops::Deref;
use std::rc::Rc;
+use std::str::FromStr;
-use proc_macro2::{Ident, Span};
+#[cfg(all(
+ not(all(target_arch = "wasm32", target_os = "unknown")),
+ feature = "proc-macro"
+))]
+use proc_macro;
+use proc_macro2::{self, Delimiter, Group, Ident, Literal, Punct, Span, TokenStream, TokenTree};
-use buffer::Cursor;
+use buffer::{Cursor, TokenBuffer};
use error;
use punctuated::Punctuated;
use token::Token;
@@ -228,3 +384,107 @@
}
}
}
+
+impl Parse for TokenStream {
+ fn parse(input: ParseStream) -> Result<Self> {
+ input.step(|cursor| Ok((cursor.token_stream(), Cursor::empty())))
+ }
+}
+
+impl Parse for TokenTree {
+ fn parse(input: ParseStream) -> Result<Self> {
+ input.step(|cursor| match cursor.token_tree() {
+ Some((tt, rest)) => Ok((tt, rest)),
+ None => Err(cursor.error("expected token tree")),
+ })
+ }
+}
+
+impl Parse for Group {
+ fn parse(input: ParseStream) -> Result<Self> {
+ input.step(|cursor| {
+ for delim in &[Delimiter::Parenthesis, Delimiter::Brace, Delimiter::Bracket] {
+ if let Some((inside, span, rest)) = cursor.group(*delim) {
+ let mut group = Group::new(*delim, inside.token_stream());
+ group.set_span(span);
+ return Ok((group, rest));
+ }
+ }
+ Err(cursor.error("expected group token"))
+ })
+ }
+}
+
+impl Parse for Punct {
+ fn parse(input: ParseStream) -> Result<Self> {
+ input.step(|cursor| match cursor.punct() {
+ Some((punct, rest)) => Ok((punct, rest)),
+ None => Err(cursor.error("expected punctuation token")),
+ })
+ }
+}
+
+impl Parse for Literal {
+ fn parse(input: ParseStream) -> Result<Self> {
+ input.step(|cursor| match cursor.literal() {
+ Some((literal, rest)) => Ok((literal, rest)),
+ None => Err(cursor.error("expected literal token")),
+ })
+ }
+}
+
+/// Parser that can parse Rust tokens into a particular syntax tree node.
+///
+/// Refer to the [module documentation] for details about parsing in Syn.
+///
+/// [module documentation]: index.html
+///
+/// *This trait is available if Syn is built with the `"parsing"` feature.*
+pub trait Parser: Sized {
+ type Output;
+
+ /// Parse a proc-macro2 token stream into the chosen syntax tree node.
+ fn parse2(self, tokens: TokenStream) -> Result<Self::Output>;
+
+ /// Parse tokens of source code into the chosen syntax tree node.
+ ///
+ /// *This method is available if Syn is built with both the `"parsing"` and
+ /// `"proc-macro"` features.*
+ #[cfg(all(
+ not(all(target_arch = "wasm32", target_os = "unknown")),
+ feature = "proc-macro"
+ ))]
+ fn parse(self, tokens: proc_macro::TokenStream) -> Result<Self::Output> {
+ self.parse2(proc_macro2::TokenStream::from(tokens))
+ }
+
+ /// Parse a string of Rust code into the chosen syntax tree node.
+ ///
+ /// # Hygiene
+ ///
+ /// Every span in the resulting syntax tree will be set to resolve at the
+ /// macro call site.
+ fn parse_str(self, s: &str) -> Result<Self::Output> {
+ self.parse2(proc_macro2::TokenStream::from_str(s)?)
+ }
+}
+
+impl<F, T> Parser for F
+where
+ F: FnOnce(ParseStream) -> Result<T>,
+{
+ type Output = T;
+
+ fn parse2(self, tokens: TokenStream) -> Result<T> {
+ let buf = TokenBuffer::new2(tokens);
+ let unexpected = Rc::new(Cell::new(None));
+ let state = ParseBuffer::new(Span::call_site(), buf.begin(), unexpected);
+ let node = self(&state)?;
+ state.check_unexpected()?;
+ if state.is_empty() {
+ Ok(node)
+ } else {
+ Err(state.error("unexpected token"))
+ }
+ }
+}