Support multichar punctuation
diff --git a/src/lib.rs b/src/lib.rs
index b8807d5..57f5d4a 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -18,6 +18,7 @@
mod ast;
mod error;
mod lookahead;
+mod span;
pub use ast::*;
pub use proc_macro2::Ident;
diff --git a/src/lookahead.rs b/src/lookahead.rs
index b73d8b2..cbaa68e 100644
--- a/src/lookahead.rs
+++ b/src/lookahead.rs
@@ -68,10 +68,12 @@
type Token: Token;
}
-impl<F: FnOnce(Span) -> T, T: Token> Peek for F {
+impl<F: FnOnce(TokenMarker) -> T, T: Token> Peek for F {
type Token = T;
}
+pub enum TokenMarker {}
+
// Not public API.
#[doc(hidden)]
pub fn is_token(lookahead: &Lookahead1, repr: &'static str) -> bool {
@@ -83,7 +85,7 @@
}
mod private {
- use super::{Span, Token};
+ use super::{Token, TokenMarker};
pub trait Sealed {}
- impl<F, T: Token> Sealed for F where F: FnOnce(Span) -> T {}
+ impl<F: FnOnce(TokenMarker) -> T, T: Token> Sealed for F {}
}
diff --git a/src/span.rs b/src/span.rs
new file mode 100644
index 0000000..d734dee
--- /dev/null
+++ b/src/span.rs
@@ -0,0 +1,73 @@
+use proc_macro2::Span;
+
+use lookahead::TokenMarker;
+
+pub trait IntoSpans<S> {
+ // Not public API.
+ #[doc(hidden)]
+ fn into_spans(self) -> S;
+}
+
+impl<S> IntoSpans<S> for TokenMarker {
+ fn into_spans(self) -> S {
+ match self {}
+ }
+}
+
+impl IntoSpans<[Span; 1]> for Span {
+ fn into_spans(self) -> [Span; 1] {
+ [self]
+ }
+}
+
+impl IntoSpans<[Span; 2]> for Span {
+ fn into_spans(self) -> [Span; 2] {
+ [self, self]
+ }
+}
+
+impl IntoSpans<[Span; 3]> for Span {
+ fn into_spans(self) -> [Span; 3] {
+ [self, self, self]
+ }
+}
+
+impl IntoSpans<Self> for [Span; 1] {
+ fn into_spans(self) -> Self {
+ self
+ }
+}
+
+impl IntoSpans<Self> for [Span; 2] {
+ fn into_spans(self) -> Self {
+ self
+ }
+}
+
+impl IntoSpans<Self> for [Span; 3] {
+ fn into_spans(self) -> Self {
+ self
+ }
+}
+
+pub trait FromSpans: Sized {
+ fn from_spans(spans: &[Span]) -> Self;
+}
+
+impl FromSpans for [Span; 1] {
+ fn from_spans(spans: &[Span]) -> Self {
+ [spans[0]]
+ }
+}
+
+impl FromSpans for [Span; 2] {
+ fn from_spans(spans: &[Span]) -> Self {
+ [spans[0], spans[1]]
+ }
+}
+
+impl FromSpans for [Span; 3] {
+ fn from_spans(spans: &[Span]) -> Self {
+ [spans[0], spans[1], spans[2]]
+ }
+}
diff --git a/src/token.rs b/src/token.rs
index 31002d5..7305408 100644
--- a/src/token.rs
+++ b/src/token.rs
@@ -1,8 +1,13 @@
//! Tokens representing Rust punctuation, keywords, and delimiters.
-use proc_macro2::Span;
+use std::ops::{Deref, DerefMut};
+use proc_macro2::{Spacing, Span};
+
+use error::Error;
+use lookahead;
use parse::{Lookahead1, Parse, ParseStream, Result};
+use span::{FromSpans, IntoSpans};
/// Marker trait for types that represent single tokens.
///
@@ -30,17 +35,14 @@
(enum) => { $crate::token::Enum };
(:) => { $crate::token::Colon };
(,) => { $crate::token::Comma };
+ (..) => { $crate::token::Dot2 };
}
-macro_rules! define_token {
+macro_rules! impl_token {
($token:tt $name:ident #[$doc:meta]) => {
- #[$doc]
- #[derive(Debug)]
- pub struct $name(pub Span);
-
impl Token for $name {
fn peek(lookahead: &Lookahead1) -> bool {
- ::lookahead::is_token(lookahead, $token)
+ lookahead::is_token(lookahead, $token)
}
fn display() -> String {
@@ -53,9 +55,23 @@
}
macro_rules! define_keywords {
- ($($token:tt $name:ident #[$doc:meta])*) => {
+ ($($token:tt pub struct $name:ident #[$doc:meta])*) => {
$(
- define_token!($token $name #[$doc]);
+ #[$doc]
+ #[derive(Debug)]
+ pub struct $name {
+ pub span: Span,
+ }
+
+ #[doc(hidden)]
+ #[allow(non_snake_case)]
+ pub fn $name<T: IntoSpans<[Span; 1]>>(span: T) -> $name {
+ $name {
+ span: span.into_spans()[0],
+ }
+ }
+
+ impl_token!($token $name #[$doc]);
impl Parse for $name {
fn parse(input: ParseStream) -> Result<Self> {
@@ -67,13 +83,41 @@
}
macro_rules! define_punctuation {
- ($($token:tt $name:ident #[$doc:meta])*) => {
+ ($($token:tt pub struct $name:ident/$len:tt #[$doc:meta])*) => {
$(
- define_token!($token $name #[$doc]);
+ #[$doc]
+ #[derive(Debug)]
+ pub struct $name {
+ pub spans: [Span; $len],
+ }
+
+ impl Deref for $name {
+ type Target = [Span; $len];
+
+ fn deref(&self) -> &Self::Target {
+ &self.spans
+ }
+ }
+
+ impl DerefMut for $name {
+ fn deref_mut(&mut self) -> &mut Self::Target {
+ &mut self.spans
+ }
+ }
+
+ #[doc(hidden)]
+ #[allow(non_snake_case)]
+ pub fn $name<T: IntoSpans<[Span; $len]>>(spans: T) -> $name {
+ $name {
+ spans: spans.into_spans(),
+ }
+ }
+
+ impl_token!($token $name #[$doc]);
impl Parse for $name {
fn parse(input: ParseStream) -> Result<Self> {
- parse_punctuation(input, $token).map($name)
+ parse_punctuation(input, $token).map($name::<[Span; $len]>)
}
}
)*
@@ -81,13 +125,14 @@
}
define_keywords! {
- "struct" Struct /// `struct`
- "enum" Enum /// `enum`
+ "struct" pub struct Struct /// `struct`
+ "enum" pub struct Enum /// `enum`
}
define_punctuation! {
- ":" Colon /// `:`
- "," Comma /// `,`
+ ":" pub struct Colon/1 /// `:`
+ "," pub struct Comma/1 /// `,`
+ ".." pub struct Dot2/2 /// `..`
}
/// `{...}`
@@ -105,14 +150,29 @@
})
}
-fn parse_punctuation(input: ParseStream, token: &str) -> Result<Span> {
+fn parse_punctuation<S: FromSpans>(input: ParseStream, token: &str) -> Result<S> {
input.step_cursor(|cursor| {
- // TODO: support multi-character punctuation
- if let Some((punct, rest)) = cursor.punct() {
- if punct.as_char() == token.chars().next().unwrap() {
- return Ok((punct.span(), rest));
+ let mut cursor = *cursor;
+ let mut spans = [cursor.span(); 3];
+ assert!(token.len() <= spans.len());
+
+ for (i, ch) in token.chars().enumerate() {
+ match cursor.punct() {
+ Some((punct, rest)) => {
+ spans[i] = punct.span();
+ if punct.as_char() != ch {
+ break;
+ } else if i == token.len() - 1 {
+ return Ok((S::from_spans(&spans), rest));
+ } else if punct.spacing() != Spacing::Joint {
+ break;
+ }
+ cursor = rest;
+ }
+ None => break,
}
}
- Err(cursor.error(format!("expected `{}`", token)))
+
+ Err(Error::new(spans[0], format!("expected `{}`", token)))
})
}