Rewrite tokenization with `proc-macro2` tokens
This ended up being a bit larger of a commit than I intended! I imagine that
this'll be one of the larger of the commits working towards #142. The purpose of
this commit is to use an updated version of the `quote` crate which doesn't work
with strings but rather works with tokens form the `proc-macro2` crate. The
`proc-macro2` crate itself is based on the proposed API for `proc_macro` itself,
and will continue to mirror it. The hope is that we'll flip an easy switch
eventually to use compiler tokens, whereas for now we'll stick to string parsing
at the lowest layer.
The largest change here is the addition of span information to the AST. Building
on the previous PRs to refactor the AST this makes it relatively easy from a
user perspective to digest and use the AST still, it's just a few extra fields
on the side. The fallout from this was then quite large throughout the
`printing` feature of the crate. The `parsing`, `fold`, and `visit` features
then followed suit to get updated as well.
This commit also changes the the semantics of the AST somewhat as well.
Previously it was inferred what tokens should be printed, for example if you
have a closure argument `syn` would automatically not print the colon in `a: b`
if the type listed was "infer this type". Now the colon is a separate field and
must be in sync with the type listed as the colon/type will be printed
unconditionally (emitting no output if both are `None`).
diff --git a/synom/src/delimited.rs b/synom/src/delimited.rs
new file mode 100644
index 0000000..8e838a8
--- /dev/null
+++ b/synom/src/delimited.rs
@@ -0,0 +1,253 @@
+use std::iter::FromIterator;
+use std::slice;
+use std::vec;
+
+#[derive(Eq, PartialEq, Hash, Debug, Clone)]
+pub struct Delimited<T, D> {
+ inner: Vec<(T, Option<D>)>
+}
+
+impl<T, D> Delimited<T, D> {
+ pub fn new() -> Delimited<T, D> {
+ Delimited {
+ inner: Vec::new(),
+ }
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.inner.len() == 0
+ }
+
+ pub fn len(&self) -> usize {
+ self.inner.len()
+ }
+
+ pub fn get(&self, idx: usize) -> Element<&T, &D> {
+ let (ref t, ref d) = self.inner[idx];
+ match *d {
+ Some(ref d) => Element::Delimited(t, d),
+ None => Element::End(t),
+ }
+ }
+
+ pub fn get_mut(&mut self, idx: usize) -> Element<&mut T, &mut D> {
+ let (ref mut t, ref mut d) = self.inner[idx];
+ match *d {
+ Some(ref mut d) => Element::Delimited(t, d),
+ None => Element::End(t),
+ }
+ }
+
+ pub fn iter(&self) -> Iter<T, D> {
+ Iter { inner: self.inner.iter() }
+ }
+
+ pub fn into_iter(self) -> IntoIter<T, D> {
+ IntoIter { inner: self.inner.into_iter() }
+ }
+
+ pub fn items(&self) -> Items<T, D> {
+ Items { inner: self.inner.iter() }
+ }
+
+ pub fn push(&mut self, token: Element<T, D>) {
+ assert!(self.len() == 0 || self.trailing_delim());
+ match token {
+ Element::Delimited(t, d) => self.inner.push((t, Some(d))),
+ Element::End(t) => self.inner.push((t, None)),
+ }
+ }
+
+ pub fn push_first(&mut self, token: T) {
+ assert!(self.is_empty());
+ self.inner.push((token, None));
+ }
+
+ pub fn push_next(&mut self, token: T, delimiter: D) {
+ self.push_trailing(delimiter);
+ self.inner.push((token, None));
+ }
+
+ pub fn push_trailing(&mut self, delimiter: D) {
+ let len = self.len();
+ assert!(self.inner[len - 1].1.is_none());
+ self.inner[len - 1].1 = Some(delimiter);
+ }
+
+ pub fn push_default(&mut self, token: T) where D: Default {
+ if self.len() == 0 {
+ self.inner.push((token, None));
+ } else {
+ self.push_next(token, D::default());
+ }
+ }
+
+ pub fn pop(&mut self) -> Option<Element<T, D>> {
+ self.inner.pop().map(|e| {
+ match e {
+ (t, Some(d)) => Element::Delimited(t, d),
+ (t, None) => Element::End(t),
+ }
+ })
+ }
+
+ pub fn into_vec(self) -> Vec<T> {
+ self.inner.into_iter().map(|t| t.0).collect()
+ }
+
+ pub fn trailing_delim(&self) -> bool {
+ self.inner[self.inner.len() - 1].1.is_some()
+ }
+}
+
+impl<T, D> From<Vec<(T, Option<D>)>> for Delimited<T, D> {
+ fn from(v: Vec<(T, Option<D>)>) -> Self {
+ Delimited {
+ inner: v,
+ }
+ }
+}
+
+impl<T, D> From<Vec<T>> for Delimited<T, D>
+ where D: Default,
+{
+ fn from(v: Vec<T>) -> Self {
+ let last = v.len() - 1;
+ Delimited {
+ inner: v.into_iter().enumerate().map(|(i, item)| {
+ (item, if i == last {None} else {Some(D::default())})
+ }).collect(),
+ }
+ }
+}
+
+impl<T, D> FromIterator<Element<T, D>> for Delimited<T, D> {
+ fn from_iter<I: IntoIterator<Item = Element<T, D>>>(i: I) -> Self {
+ let mut ret = Delimited::new();
+ for element in i {
+ match element {
+ Element::Delimited(a, b) => ret.inner.push((a, Some(b))),
+ Element::End(a) => ret.inner.push((a, None)),
+ }
+ }
+ return ret
+ }
+}
+
+impl<T, D> Default for Delimited<T, D> {
+ fn default() -> Self {
+ Delimited::new()
+ }
+}
+
+pub struct Iter<'a, T: 'a, D: 'a> {
+ inner: slice::Iter<'a, (T, Option<D>)>,
+}
+
+impl<'a, T, D> Iterator for Iter<'a, T, D> {
+ type Item = Element<&'a T, &'a D>;
+
+ fn next(&mut self) -> Option<Element<&'a T, &'a D>> {
+ self.inner.next().map(|pair| {
+ match pair.1 {
+ Some(ref delimited) => Element::Delimited(&pair.0, delimited),
+ None => Element::End(&pair.0),
+ }
+ })
+ }
+}
+
+pub struct Items<'a, T: 'a, D: 'a> {
+ inner: slice::Iter<'a, (T, Option<D>)>,
+}
+
+impl<'a, T, D> Iterator for Items<'a, T, D> {
+ type Item = &'a T;
+
+ fn next(&mut self) -> Option<&'a T> {
+ self.inner.next().map(|pair| &pair.0)
+ }
+}
+
+pub struct IntoIter<T, D> {
+ inner: vec::IntoIter<(T, Option<D>)>,
+}
+
+impl<T, D> Iterator for IntoIter<T, D> {
+ type Item = Element<T, D>;
+
+ fn next(&mut self) -> Option<Element<T, D>> {
+ self.inner.next().map(|pair| {
+ match pair.1 {
+ Some(v) => Element::Delimited(pair.0, v),
+ None => Element::End(pair.0)
+ }
+ })
+ }
+}
+
+pub enum Element<T, D> {
+ Delimited(T, D),
+ End(T),
+}
+
+impl<T, D> Element<T, D> {
+ pub fn into_item(self) -> T {
+ match self {
+ Element::Delimited(t, _) |
+ Element::End(t) => t,
+ }
+ }
+
+ pub fn item(&self) -> &T {
+ match *self {
+ Element::Delimited(ref t, _) |
+ Element::End(ref t) => t,
+ }
+ }
+
+ pub fn item_mut(&mut self) -> &mut T {
+ match *self {
+ Element::Delimited(ref mut t, _) |
+ Element::End(ref mut t) => t,
+ }
+ }
+
+ pub fn delimiter(&self) -> Option<&D> {
+ match *self {
+ Element::Delimited(_, ref d) => Some(d),
+ Element::End(_) => None,
+ }
+ }
+}
+
+#[cfg(feature = "printing")]
+mod printing {
+ use super::*;
+ use quote::{Tokens, ToTokens};
+
+
+ impl<T, D> ToTokens for Delimited<T, D>
+ where T: ToTokens,
+ D: ToTokens,
+ {
+ fn to_tokens(&self, tokens: &mut Tokens) {
+ tokens.append_all(self.iter())
+ }
+ }
+
+ impl<T, D> ToTokens for Element<T, D>
+ where T: ToTokens,
+ D: ToTokens,
+ {
+ fn to_tokens(&self, tokens: &mut Tokens) {
+ match *self {
+ Element::Delimited(ref a, ref b) => {
+ a.to_tokens(tokens);
+ b.to_tokens(tokens);
+ }
+ Element::End(ref a) => a.to_tokens(tokens),
+ }
+ }
+ }
+}
diff --git a/synom/src/helper.rs b/synom/src/helper.rs
index 5bae69b..fc89c7b 100644
--- a/synom/src/helper.rs
+++ b/synom/src/helper.rs
@@ -1,5 +1,6 @@
use IResult;
use space::{skip_whitespace, word_break};
+use delimited::Delimited;
/// Parse a piece of punctuation like "+" or "+=".
///
@@ -152,7 +153,7 @@
/// punct!("<") >>
/// lifetimes: terminated_list!(punct!(","), lifetime) >>
/// punct!(">") >>
-/// (lifetimes)
+/// (lifetimes.into_vec())
/// )),
/// ty
/// ));
@@ -193,7 +194,7 @@
/// use syn::Mutability;
///
/// named!(mutability -> Mutability, alt!(
-/// keyword!("mut") => { |_| Mutability::Mutable }
+/// keyword!("mut") => { |_| Mutability::Mutable(Default::default()) }
/// |
/// epsilon!() => { |_| Mutability::Immutable }
/// ));
@@ -201,7 +202,7 @@
/// fn main() {
/// let input = "mut";
/// let parsed = mutability(input).expect("mutability");
-/// assert_eq!(parsed, Mutability::Mutable);
+/// assert_eq!(parsed, Mutability::Mutable(Default::default()));
///
/// let input = "";
/// let parsed = mutability(input).expect("mutability");
@@ -236,7 +237,8 @@
/// e = Expr {
/// node: ExprCall {
/// func: Box::new(e),
-/// args: vec![arg.1],
+/// args: vec![arg.1].into(),
+/// paren_token: Default::default(),
/// }.into(),
/// attrs: Vec::new(),
/// };
@@ -289,9 +291,11 @@
///
/// use syn::Expr;
/// use syn::parse::expr;
+/// use synom::delimited::Delimited;
///
/// named!(expr_list -> Vec<Expr>,
-/// separated_list!(punct!(","), expr)
+/// map!(separated_list!(punct!(","), expr),
+/// |v: Delimited<_, _>| v.into_vec())
/// );
///
/// fn main() {
@@ -308,10 +312,14 @@
///
/// use syn::Ident;
/// use syn::parse::ident;
+/// use synom::delimited::Delimited;
///
/// named!(run_on -> Vec<Ident>,
/// terminated!(
-/// separated_list!(keyword!("and"), preceded!(punct!("$"), ident)),
+/// map!(
+/// separated_list!(keyword!("and"), preceded!(punct!("$"), ident)),
+/// |v: Delimited<_, _>| v.into_vec()
+/// ),
/// punct!("...")
/// )
/// );
@@ -328,46 +336,11 @@
/// ```
#[macro_export]
macro_rules! separated_list {
- // Try to use this branch if possible - makes a difference in compile time.
- ($i:expr, punct!($sep:expr), $f:ident) => {
- $crate::helper::separated_list($i, $sep, $f, false)
- };
-
($i:expr, $sepmac:ident!( $($separgs:tt)* ), $fmac:ident!( $($fargs:tt)* )) => {{
- let mut res = ::std::vec::Vec::new();
- let mut input = $i;
-
- // get the first element
- match $fmac!(input, $($fargs)*) {
- $crate::IResult::Error => $crate::IResult::Done(input, res),
- $crate::IResult::Done(i, o) => {
- if i.len() == input.len() {
- $crate::IResult::Error
- } else {
- res.push(o);
- input = i;
-
- // get the separator first
- while let $crate::IResult::Done(i2, _) = $sepmac!(input, $($separgs)*) {
- if i2.len() == input.len() {
- break;
- }
-
- // get the element next
- if let $crate::IResult::Done(i3, o3) = $fmac!(i2, $($fargs)*) {
- if i3.len() == i2.len() {
- break;
- }
- res.push(o3);
- input = i3;
- } else {
- break;
- }
- }
- $crate::IResult::Done(input, res)
- }
- }
- }
+ $crate::helper::separated_list($i,
+ |d| $sepmac!(d, $($separgs)*),
+ |d| $fmac!(d, $($fargs)*),
+ false)
}};
($i:expr, $sepmac:ident!( $($separgs:tt)* ), $f:expr) => {
@@ -401,8 +374,9 @@
///
/// use syn::Expr;
/// use syn::parse::expr;
+/// use synom::delimited::Delimited;
///
-/// named!(expr_list -> Vec<Expr>,
+/// named!(expr_list -> Delimited<Expr, &str>,
/// terminated_list!(punct!(","), expr)
/// );
///
@@ -420,8 +394,9 @@
///
/// use syn::Ident;
/// use syn::parse::ident;
+/// use synom::delimited::Delimited;
///
-/// named!(run_on -> Vec<Ident>,
+/// named!(run_on -> Delimited<Ident, &str>,
/// terminated!(
/// terminated_list!(keyword!("and"), preceded!(punct!("$"), ident)),
/// punct!("...")
@@ -431,7 +406,7 @@
/// fn main() {
/// let input = "$expr and $ident and $pat and ...";
///
-/// let parsed = run_on(input).expect("run-on sentence");
+/// let parsed = run_on(input).expect("run-on sentence").into_vec();
/// assert_eq!(parsed.len(), 3);
/// assert_eq!(parsed[0], "expr");
/// assert_eq!(parsed[1], "ident");
@@ -440,49 +415,11 @@
/// ```
#[macro_export]
macro_rules! terminated_list {
- // Try to use this branch if possible - makes a difference in compile time.
- ($i:expr, punct!($sep:expr), $f:ident) => {
- $crate::helper::separated_list($i, $sep, $f, true)
- };
-
($i:expr, $sepmac:ident!( $($separgs:tt)* ), $fmac:ident!( $($fargs:tt)* )) => {{
- let mut res = ::std::vec::Vec::new();
- let mut input = $i;
-
- // get the first element
- match $fmac!(input, $($fargs)*) {
- $crate::IResult::Error => $crate::IResult::Done(input, res),
- $crate::IResult::Done(i, o) => {
- if i.len() == input.len() {
- $crate::IResult::Error
- } else {
- res.push(o);
- input = i;
-
- // get the separator first
- while let $crate::IResult::Done(i2, _) = $sepmac!(input, $($separgs)*) {
- if i2.len() == input.len() {
- break;
- }
-
- // get the element next
- if let $crate::IResult::Done(i3, o3) = $fmac!(i2, $($fargs)*) {
- if i3.len() == i2.len() {
- break;
- }
- res.push(o3);
- input = i3;
- } else {
- break;
- }
- }
- if let $crate::IResult::Done(after, _) = $sepmac!(input, $($separgs)*) {
- input = after;
- }
- $crate::IResult::Done(input, res)
- }
- }
- }
+ $crate::helper::separated_list($i,
+ |d| $sepmac!(d, $($separgs)*),
+ |d| $fmac!(d, $($fargs)*),
+ true)
}};
($i:expr, $sepmac:ident!( $($separgs:tt)* ), $f:expr) => {
@@ -500,47 +437,50 @@
// Not public API.
#[doc(hidden)]
-pub fn separated_list<'a, T>(mut input: &'a str,
- sep: &'static str,
- f: fn(&'a str) -> IResult<&'a str, T>,
- terminated: bool)
- -> IResult<&'a str, Vec<T>> {
- let mut res = Vec::new();
+pub fn separated_list<'a, F1, D, F2, T>(mut input: &'a str,
+ mut sep: F1,
+ mut parse: F2,
+ terminated: bool)
+ -> IResult<&'a str, Delimited<T, D>>
+ where F1: FnMut(&'a str) -> IResult<&'a str, D>,
+ F2: FnMut(&'a str) -> IResult<&'a str, T>,
+{
+ let mut res = Delimited::new();
// get the first element
- match f(input) {
+ match parse(input) {
IResult::Error => IResult::Done(input, res),
IResult::Done(i, o) => {
if i.len() == input.len() {
- IResult::Error
- } else {
- res.push(o);
- input = i;
-
- // get the separator first
- while let IResult::Done(i2, _) = punct(input, sep) {
- if i2.len() == input.len() {
- break;
- }
-
- // get the element next
- if let IResult::Done(i3, o3) = f(i2) {
- if i3.len() == i2.len() {
- break;
- }
- res.push(o3);
- input = i3;
- } else {
- break;
- }
- }
- if terminated {
- if let IResult::Done(after, _) = punct(input, sep) {
- input = after;
- }
- }
- IResult::Done(input, res)
+ return IResult::Error
}
+ input = i;
+ res.push_first(o);
+
+ // get the separator first
+ while let IResult::Done(i2, s) = sep(input) {
+ if i2.len() == input.len() {
+ break;
+ }
+
+ // get the element next
+ if let IResult::Done(i3, o3) = parse(i2) {
+ if i3.len() == i2.len() {
+ break;
+ }
+ res.push_next(o3, s);
+ input = i3;
+ } else {
+ break;
+ }
+ }
+ if terminated {
+ if let IResult::Done(after, sep) = sep(input) {
+ res.push_trailing(sep);
+ input = after;
+ }
+ }
+ IResult::Done(input, res)
}
}
}
diff --git a/synom/src/lib.rs b/synom/src/lib.rs
index 97c7833..0c90ece 100644
--- a/synom/src/lib.rs
+++ b/synom/src/lib.rs
@@ -23,12 +23,19 @@
extern crate unicode_xid;
+#[cfg(feature = "printing")]
+extern crate quote;
+
+#[cfg(feature = "parsing")]
#[doc(hidden)]
pub mod space;
+#[cfg(feature = "parsing")]
#[doc(hidden)]
pub mod helper;
+pub mod delimited;
+
/// The result of a parser.
#[derive(Debug, PartialEq, Eq, Clone)]
pub enum IResult<I, O> {
@@ -49,9 +56,10 @@
///
/// use syn::Ty;
/// use syn::parse::ty;
+ /// use synom::delimited::Delimited;
///
/// // One or more Rust types separated by commas.
- /// named!(comma_separated_types -> Vec<Ty>,
+ /// named!(comma_separated_types -> Delimited<Ty, &str>,
/// separated_nonempty_list!(punct!(","), ty)
/// );
///
@@ -88,8 +96,9 @@
/// # #[macro_use] extern crate synom;
/// # use syn::Ty;
/// # use syn::parse::ty;
+/// # use synom::delimited::Delimited;
/// // One or more Rust types separated by commas.
-/// named!(pub comma_separated_types -> Vec<Ty>,
+/// named!(pub comma_separated_types -> Delimited<Ty, &str>,
/// separated_nonempty_list!(punct!(","), ty)
/// );
/// # fn main() {}
@@ -255,7 +264,11 @@
/// extern crate syn;
/// #[macro_use] extern crate synom;
///
-/// use syn::parse::boolean;
+/// named!(boolean -> bool, alt!(
+/// keyword!("true") => { |_| true }
+/// |
+/// keyword!("false") => { |_| false }
+/// ));
///
/// // Parses a tuple of booleans like `(true, false, false)`, possibly with a
/// // dotdot indicating omitted values like `(true, true, .., true)`. Returns
@@ -278,7 +291,7 @@
/// // Allow trailing comma if there is no dotdot but there are elements.
/// cond!(!before.is_empty() && after.is_none(), option!(punct!(","))) >>
/// punct!(")") >>
-/// (before, after)
+/// (before.into_vec(), after)
/// ));
///
/// fn main() {
@@ -332,14 +345,18 @@
/// extern crate syn;
/// #[macro_use] extern crate synom;
///
-/// use syn::parse::boolean;
-///
/// #[derive(Debug, PartialEq)]
/// struct VariadicBools {
/// data: Vec<bool>,
/// variadic: bool,
/// }
///
+/// named!(boolean -> bool, alt!(
+/// keyword!("true") => { |_| true }
+/// |
+/// keyword!("false") => { |_| false }
+/// ));
+///
/// // Parse one or more comma-separated booleans, possibly ending in "..." to
/// // indicate there may be more.
/// named!(variadic_bools -> VariadicBools, do_parse!(
@@ -358,7 +375,7 @@
/// // Gives `Some("...")` for variadic and `None` otherwise. Perfect!
/// variadic: option!(cond_reduce!(trailing_comma.is_some(), punct!("..."))) >>
/// (VariadicBools {
-/// data: data,
+/// data: data.into_vec(),
/// variadic: variadic.is_some(),
/// })
/// ));
@@ -706,7 +723,7 @@
/// extern crate syn;
/// #[macro_use] extern crate synom;
///
-/// use syn::StrLit;
+/// use syn::Lit;
/// use syn::parse::string;
/// use synom::IResult;
///
@@ -714,7 +731,7 @@
/// named!(owned_string -> String,
/// map!(
/// terminated!(string, tag!("s")),
-/// |lit: StrLit| lit.value
+/// |lit: Lit| lit.to_string()
/// )
/// );
///
@@ -931,9 +948,10 @@
///
/// use syn::Ty;
/// use syn::parse::ty;
+/// use synom::delimited::Delimited;
///
/// // One or more Rust types separated by commas.
-/// named!(comma_separated_types -> Vec<Ty>,
+/// named!(comma_separated_types -> Delimited<Ty, &str>,
/// separated_nonempty_list!(punct!(","), ty)
/// );
///
@@ -949,7 +967,7 @@
#[macro_export]
macro_rules! separated_nonempty_list {
($i:expr, $sep:ident!( $($args:tt)* ), $submac:ident!( $($args2:tt)* )) => {{
- let mut res = ::std::vec::Vec::new();
+ let mut res = $crate::delimited::Delimited::new();
let mut input = $i;
// get the first element
@@ -959,19 +977,19 @@
if i.len() == input.len() {
$crate::IResult::Error
} else {
- res.push(o);
input = i;
+ res.push_first(o);
- while let $crate::IResult::Done(i2, _) = $sep!(input, $($args)*) {
+ while let $crate::IResult::Done(i2, s) = $sep!(input, $($args)*) {
if i2.len() == input.len() {
break;
}
if let $crate::IResult::Done(i3, o3) = $submac!(i2, $($args2)*) {
+ res.push_next(o3, s);
if i3.len() == i2.len() {
break;
}
- res.push(o3);
input = i3;
} else {
break;