Rewrite tokenization with `proc-macro2` tokens
This ended up being a bit larger of a commit than I intended! I imagine that
this'll be one of the larger of the commits working towards #142. The purpose of
this commit is to use an updated version of the `quote` crate which doesn't work
with strings but rather works with tokens form the `proc-macro2` crate. The
`proc-macro2` crate itself is based on the proposed API for `proc_macro` itself,
and will continue to mirror it. The hope is that we'll flip an easy switch
eventually to use compiler tokens, whereas for now we'll stick to string parsing
at the lowest layer.
The largest change here is the addition of span information to the AST. Building
on the previous PRs to refactor the AST this makes it relatively easy from a
user perspective to digest and use the AST still, it's just a few extra fields
on the side. The fallout from this was then quite large throughout the
`printing` feature of the crate. The `parsing`, `fold`, and `visit` features
then followed suit to get updated as well.
This commit also changes the the semantics of the AST somewhat as well.
Previously it was inferred what tokens should be printed, for example if you
have a closure argument `syn` would automatically not print the colon in `a: b`
if the type listed was "infer this type". Now the colon is a separate field and
must be in sync with the type listed as the colon/type will be printed
unconditionally (emitting no output if both are `None`).
diff --git a/src/aster/generics.rs b/src/aster/generics.rs
index caee562..8cc84d0 100644
--- a/src/aster/generics.rs
+++ b/src/aster/generics.rs
@@ -5,6 +5,8 @@
use aster::ty_param::TyParamBuilder;
use aster::where_predicate::WherePredicateBuilder;
+use delimited::Delimited;
+
pub struct GenericsBuilder<F = Identity> {
callback: F,
lifetimes: Vec<LifetimeDef>,
@@ -37,16 +39,16 @@
pub fn from_generics_with_callback(generics: Generics, callback: F) -> Self {
GenericsBuilder {
callback: callback,
- lifetimes: generics.lifetimes,
- ty_params: generics.ty_params,
- predicates: generics.where_clause.predicates,
+ lifetimes: generics.lifetimes.into_vec(),
+ ty_params: generics.ty_params.into_vec(),
+ predicates: generics.where_clause.predicates.into_vec(),
}
}
pub fn with(self, generics: Generics) -> Self {
- self.with_lifetimes(generics.lifetimes.into_iter())
- .with_ty_params(generics.ty_params.into_iter())
- .with_predicates(generics.where_clause.predicates.into_iter())
+ self.with_lifetimes(generics.lifetimes.into_vec().into_iter())
+ .with_ty_params(generics.ty_params.into_vec().into_iter())
+ .with_predicates(generics.where_clause.predicates.into_vec().into_iter())
}
pub fn with_lifetimes<I, L>(mut self, iter: I) -> Self
@@ -141,7 +143,7 @@
let lifetime = lifetime.into_lifetime();
for lifetime_def in &mut self.lifetimes {
- lifetime_def.bounds.push(lifetime.clone());
+ lifetime_def.bounds.push_default(lifetime.clone());
}
for ty_param in &mut self.ty_params {
@@ -174,14 +176,14 @@
pub fn strip_lifetimes(mut self) -> Self {
for lifetime in &mut self.lifetimes {
- lifetime.bounds = vec![];
+ lifetime.bounds = Delimited::new();
}
self
}
pub fn strip_ty_params(mut self) -> Self {
for ty_param in &mut self.ty_params {
- ty_param.bounds = vec![];
+ ty_param.bounds = Delimited::new();
}
self
}
@@ -193,10 +195,15 @@
pub fn build(self) -> F::Result {
self.callback.invoke(Generics {
- lifetimes: self.lifetimes,
- ty_params: self.ty_params,
- where_clause: WhereClause { predicates: self.predicates },
- })
+ lifetimes: self.lifetimes.into(),
+ ty_params: self.ty_params.into(),
+ where_clause: WhereClause {
+ predicates: self.predicates.into(),
+ where_token: Default::default(),
+ },
+ lt_token: Default::default(),
+ gt_token: Default::default(),
+ })
}
}
diff --git a/src/aster/lifetime.rs b/src/aster/lifetime.rs
index 3bdf372..ead21fe 100644
--- a/src/aster/lifetime.rs
+++ b/src/aster/lifetime.rs
@@ -1,5 +1,6 @@
use {Ident, Lifetime, LifetimeDef};
use aster::invoke::{Invoke, Identity};
+use delimited::Delimited;
// ////////////////////////////////////////////////////////////////////////////
@@ -36,7 +37,8 @@
LifetimeDef {
attrs: vec![],
lifetime: self,
- bounds: vec![],
+ bounds: Delimited::new(),
+ colon_token: Default::default(),
}
}
}
@@ -95,9 +97,10 @@
pub fn build(self) -> F::Result {
self.callback.invoke(LifetimeDef {
- attrs: vec![],
- lifetime: self.lifetime,
- bounds: self.bounds,
- })
+ attrs: vec![],
+ lifetime: self.lifetime,
+ bounds: self.bounds.into(),
+ colon_token: Default::default(),
+ })
}
}
diff --git a/src/aster/path.rs b/src/aster/path.rs
index e53b142..23d644e 100644
--- a/src/aster/path.rs
+++ b/src/aster/path.rs
@@ -1,5 +1,5 @@
use {AngleBracketedParameterData, Generics, Ident, Lifetime, ParenthesizedParameterData, Path,
- PathParameters, PathSegment, Ty, TypeBinding};
+ PathParameters, PathSegment, Ty, TypeBinding, FunctionRetTy};
use aster::ident::ToIdent;
use aster::invoke::{Invoke, Identity};
use aster::lifetime::IntoLifetime;
@@ -139,9 +139,10 @@
pub fn build(self) -> F::Result {
self.callback.invoke(Path {
- global: self.global,
- segments: self.segments,
- })
+ global: self.global,
+ segments: self.segments.into(),
+ leading_colon: None,
+ })
}
}
@@ -181,10 +182,10 @@
pub fn with_generics(self, generics: Generics) -> Self {
// Strip off the bounds.
- let lifetimes = generics.lifetimes.iter().map(|lifetime_def| lifetime_def.lifetime.clone());
+ let lifetimes = generics.lifetimes.iter().map(|lifetime_def| lifetime_def.item().lifetime.clone());
let tys =
- generics.ty_params.iter().map(|ty_param| TyBuilder::new().id(ty_param.ident.clone()));
+ generics.ty_params.iter().map(|ty_param| TyBuilder::new().id(ty_param.item().ident.clone()));
self.with_lifetimes(lifetimes).with_tys(tys)
}
@@ -252,23 +253,29 @@
pub fn build_return(self, output: Option<Ty>) -> F::Result {
let data = ParenthesizedParameterData {
- inputs: self.tys,
- output: output,
+ inputs: self.tys.into(),
+ output: match output {
+ Some(ty) => FunctionRetTy::Ty(ty, Default::default()),
+ None => FunctionRetTy::Default,
+ },
+ paren_token: Default::default(),
};
let parameters = PathParameters::Parenthesized(data);
self.callback.invoke(PathSegment {
- ident: self.id,
- parameters: parameters,
- })
+ ident: self.id,
+ parameters: parameters,
+ })
}
pub fn build(self) -> F::Result {
let data = AngleBracketedParameterData {
- lifetimes: self.lifetimes,
- types: self.tys,
- bindings: self.bindings,
+ lifetimes: self.lifetimes.into(),
+ types: self.tys.into(),
+ bindings: self.bindings.into(),
+ gt_token: Default::default(),
+ lt_token: Default::default(),
};
let parameters = PathParameters::AngleBracketed(data);
@@ -306,9 +313,10 @@
let id = self.id;
self.builder.with_binding(TypeBinding {
- ident: id,
- ty: ty,
- })
+ ident: id,
+ ty: ty,
+ eq_token: Default::default(),
+ })
}
}
diff --git a/src/aster/qpath.rs b/src/aster/qpath.rs
index 71f68b1..ea36837 100644
--- a/src/aster/qpath.rs
+++ b/src/aster/qpath.rs
@@ -3,6 +3,7 @@
use aster::invoke::{Invoke, Identity};
use aster::path::{PathBuilder, PathSegmentBuilder};
use aster::ty::TyBuilder;
+use delimited::Delimited;
// ////////////////////////////////////////////////////////////////////////////
@@ -74,7 +75,8 @@
{
let path = Path {
global: false,
- segments: vec![],
+ segments: Delimited::new(),
+ leading_colon: None,
};
self.as_().build(path).id(id)
}
@@ -84,7 +86,8 @@
{
let path = Path {
global: false,
- segments: vec![],
+ segments: Delimited::new(),
+ leading_colon: None,
};
self.as_().build(path).segment(id)
}
@@ -101,6 +104,9 @@
qself: QSelf {
ty: Box::new(self.ty),
position: path.segments.len(),
+ as_token: Default::default(),
+ gt_token: Default::default(),
+ lt_token: Default::default(),
},
path: path,
}
@@ -137,7 +143,7 @@
type Result = F::Result;
fn invoke(mut self, segment: PathSegment) -> F::Result {
- self.path.segments.push(segment);
+ self.path.segments.push_default(segment);
self.builder.build(self.qself, self.path)
}
}
diff --git a/src/aster/ty.rs b/src/aster/ty.rs
index f43e4a5..1d8962c 100644
--- a/src/aster/ty.rs
+++ b/src/aster/ty.rs
@@ -116,7 +116,10 @@
}
pub fn build_slice(self, ty: Ty) -> F::Result {
- self.build(Ty::Slice(TySlice { ty: Box::new(ty) }))
+ self.build(Ty::Slice(TySlice {
+ ty: Box::new(ty),
+ bracket_token: Default::default(),
+ }))
}
pub fn slice(self) -> TyBuilder<TySliceBuilder<F>> {
@@ -132,11 +135,15 @@
}
pub fn never(self) -> F::Result {
- self.build(Ty::Never(TyNever {}))
+ self.build(Ty::Never(TyNever {
+ bang_token: Default::default(),
+ }))
}
pub fn infer(self) -> F::Result {
- self.build(Ty::Infer(TyInfer {}))
+ self.build(Ty::Infer(TyInfer {
+ underscore_token: Default::default()
+ }))
}
pub fn option(self) -> TyBuilder<TyOptionBuilder<F>> {
@@ -221,7 +228,7 @@
where F: Invoke<Ty>
{
pub fn mut_(mut self) -> Self {
- self.mutability = Mutability::Mutable;
+ self.mutability = Mutability::Mutable(Default::default());
self
}
@@ -240,6 +247,7 @@
self.builder.build(Ty::Rptr(TyRptr {
lifetime: self.lifetime,
ty: Box::new(ty),
+ and_token: Default::default(),
}))
}
@@ -414,7 +422,9 @@
}
pub fn with_generics(self, generics: Generics) -> Self {
- self.with_lifetimes(generics.lifetimes.into_iter().map(|def| def.lifetime))
+ self.with_lifetimes(generics.lifetimes.iter().map(|def| {
+ Lifetime { ident: def.item().lifetime.ident.clone() }
+ }))
}
pub fn with_lifetimes<I, L>(mut self, lifetimes: I) -> Self
@@ -437,7 +447,8 @@
pub fn build(self) -> F::Result {
let bounds = self.bounds;
self.builder.build(Ty::ImplTrait(TyImplTrait {
- bounds: bounds,
+ bounds: bounds.into(),
+ impl_token: Default::default(),
}))
}
}
@@ -479,7 +490,11 @@
}
pub fn build(self) -> F::Result {
- self.builder.build(Ty::Tup(TyTup { tys: self.tys }))
+ self.builder.build(Ty::Tup(TyTup {
+ tys: self.tys.into(),
+ paren_token: Default::default(),
+ lone_comma: None,
+ }))
}
}
diff --git a/src/aster/ty_param.rs b/src/aster/ty_param.rs
index d8d13d1..7f82f6b 100644
--- a/src/aster/ty_param.rs
+++ b/src/aster/ty_param.rs
@@ -1,4 +1,5 @@
use {Ident, LifetimeDef, Path, PolyTraitRef, TraitBoundModifier, Ty, TyParam, TyParamBound};
+use BoundLifetimes;
use aster::invoke::{Invoke, Identity};
use aster::lifetime::{IntoLifetime, IntoLifetimeDef, LifetimeDefBuilder};
use aster::path::{IntoPath, PathBuilder};
@@ -43,7 +44,7 @@
TyParamBuilder {
callback: callback,
id: ty_param.ident,
- bounds: ty_param.bounds,
+ bounds: ty_param.bounds.into_vec(),
default: ty_param.default,
}
}
@@ -87,11 +88,13 @@
pub fn build(self) -> F::Result {
self.callback.invoke(TyParam {
- attrs: vec![],
- ident: self.id,
- bounds: self.bounds,
- default: self.default,
- })
+ attrs: vec![],
+ ident: self.id,
+ bounds: self.bounds.into(),
+ default: self.default,
+ colon_token: Default::default(),
+ eq_token: Default::default(),
+ })
}
}
@@ -165,7 +168,7 @@
{
let builder = TraitTyParamBoundBuilder {
builder: self,
- modifier: TraitBoundModifier::Maybe,
+ modifier: TraitBoundModifier::Maybe(Default::default()),
};
PolyTraitRefBuilder::with_callback(path, builder)
@@ -245,9 +248,12 @@
pub fn build(self) -> F::Result {
self.callback.invoke(PolyTraitRef {
- bound_lifetimes: self.lifetimes,
- trait_ref: self.trait_ref,
- })
+ bound_lifetimes: Some(BoundLifetimes {
+ lifetimes: self.lifetimes.into(),
+ ..Default::default()
+ }),
+ trait_ref: self.trait_ref,
+ })
}
}
diff --git a/src/aster/where_predicate.rs b/src/aster/where_predicate.rs
index 611d058..16cdfd8 100644
--- a/src/aster/where_predicate.rs
+++ b/src/aster/where_predicate.rs
@@ -1,5 +1,5 @@
use {Ident, Lifetime, LifetimeDef, Ty, TyParamBound, WhereBoundPredicate, WherePredicate,
- WhereRegionPredicate};
+ WhereRegionPredicate, BoundLifetimes};
use aster::invoke::{Invoke, Identity};
use aster::lifetime::{IntoLifetime, IntoLifetimeDef, LifetimeDefBuilder};
use aster::path::IntoPath;
@@ -201,9 +201,13 @@
pub fn build(self) -> F::Result {
let predicate = WhereBoundPredicate {
- bound_lifetimes: self.bound_lifetimes,
+ bound_lifetimes: Some(BoundLifetimes {
+ lifetimes: self.bound_lifetimes.into(),
+ ..Default::default()
+ }),
bounded_ty: self.ty,
- bounds: self.bounds,
+ bounds: self.bounds.into(),
+ colon_token: Default::default(),
};
self.callback.invoke(WherePredicate::BoundPredicate(predicate))
@@ -251,7 +255,8 @@
pub fn build(self) -> F::Result {
let predicate = WhereRegionPredicate {
lifetime: self.lifetime,
- bounds: self.bounds,
+ bounds: self.bounds.into(),
+ colon_token: Default::default(),
};
self.callback.invoke(WherePredicate::RegionPredicate(predicate))