Comment parsing
diff --git a/src/attr.rs b/src/attr.rs
index 0819f1c..30d8d96 100644
--- a/src/attr.rs
+++ b/src/attr.rs
@@ -79,19 +79,52 @@
use ident::parsing::ident;
use lit::{Lit, StrStyle};
use lit::parsing::lit;
+ use space::{block_comment, whitespace};
#[cfg(feature = "full")]
- named!(pub inner_attr -> Attribute, do_parse!(
- punct!("#") >>
- punct!("!") >>
- punct!("[") >>
- meta_item: meta_item >>
- punct!("]") >>
- (Attribute {
- style: AttrStyle::Inner,
- value: meta_item,
- is_sugared_doc: false,
- })
+ named!(pub inner_attr -> Attribute, alt!(
+ do_parse!(
+ punct!("#") >>
+ punct!("!") >>
+ punct!("[") >>
+ meta_item: meta_item >>
+ punct!("]") >>
+ (Attribute {
+ style: AttrStyle::Inner,
+ value: meta_item,
+ is_sugared_doc: false,
+ })
+ )
+ |
+ do_parse!(
+ punct!("//!") >>
+ content: take_until!("\n") >>
+ (Attribute {
+ style: AttrStyle::Inner,
+ value: MetaItem::NameValue(
+ "doc".into(),
+ Lit::Str(
+ format!("//!{}", content),
+ StrStyle::Cooked,
+ ),
+ ),
+ is_sugared_doc: true,
+ })
+ )
+ |
+ do_parse!(
+ option!(whitespace) >>
+ peek!(tag!("/*!")) >>
+ com: block_comment >>
+ (Attribute {
+ style: AttrStyle::Inner,
+ value: MetaItem::NameValue(
+ "doc".into(),
+ Lit::Str(com.to_owned(), StrStyle::Cooked),
+ ),
+ is_sugared_doc: true,
+ })
+ )
));
named!(pub outer_attr -> Attribute, alt!(
@@ -123,6 +156,20 @@
is_sugared_doc: true,
})
)
+ |
+ do_parse!(
+ option!(whitespace) >>
+ peek!(tag!("/**")) >>
+ com: block_comment >>
+ (Attribute {
+ style: AttrStyle::Outer,
+ value: MetaItem::NameValue(
+ "doc".into(),
+ Lit::Str(com.to_owned(), StrStyle::Cooked),
+ ),
+ is_sugared_doc: true,
+ })
+ )
));
named!(meta_item -> MetaItem, alt!(
@@ -153,27 +200,44 @@
impl ToTokens for Attribute {
fn to_tokens(&self, tokens: &mut Tokens) {
- match *self {
- Attribute {
- style: AttrStyle::Outer,
- value: MetaItem::NameValue(
- ref name,
- Lit::Str(ref value, StrStyle::Cooked),
- ),
- is_sugared_doc: true,
- } if name == "doc" && value.starts_with("///") => {
- tokens.append(&format!("{}\n", value));
- }
- _ => {
- tokens.append("#");
- if let AttrStyle::Inner = self.style {
- tokens.append("!");
+ if let Attribute {
+ style,
+ value: MetaItem::NameValue(
+ ref name,
+ Lit::Str(ref value, StrStyle::Cooked),
+ ),
+ is_sugared_doc: true,
+ } = *self {
+ if name == "doc" {
+ match style {
+ AttrStyle::Inner if value.starts_with("//!") => {
+ tokens.append(&format!("{}\n", value));
+ return;
+ }
+ AttrStyle::Inner if value.starts_with("/*!") => {
+ tokens.append(value);
+ return;
+ }
+ AttrStyle::Outer if value.starts_with("///") => {
+ tokens.append(&format!("{}\n", value));
+ return;
+ }
+ AttrStyle::Outer if value.starts_with("/**") => {
+ tokens.append(value);
+ return;
+ }
+ _ => {}
}
- tokens.append("[");
- self.value.to_tokens(tokens);
- tokens.append("]");
}
}
+
+ tokens.append("#");
+ if let AttrStyle::Inner = self.style {
+ tokens.append("!");
+ }
+ tokens.append("[");
+ self.value.to_tokens(tokens);
+ tokens.append("]");
}
}
diff --git a/src/helper.rs b/src/helper.rs
index 5086c3d..c1694ec 100644
--- a/src/helper.rs
+++ b/src/helper.rs
@@ -1,7 +1,7 @@
#![cfg(feature = "parsing")]
-use nom::{IResult, multispace};
-use unicode_xid::UnicodeXID;
+use nom::IResult;
+use space::{whitespace, word_break};
macro_rules! punct {
($i:expr, $punct:expr) => {
@@ -10,16 +10,15 @@
}
pub fn punct<'a>(input: &'a str, token: &'static str) -> IResult<&'a str, &'a str> {
- for (i, ch) in input.char_indices() {
- if !ch.is_whitespace() {
- return if input[i..].starts_with(token) {
- IResult::Done(&input[i + token.len()..], token)
- } else {
- IResult::Error
- };
- }
+ let input = match whitespace(input) {
+ IResult::Done(rest, _) => rest,
+ IResult::Error => input,
+ };
+ if input.starts_with(token) {
+ IResult::Done(&input[token.len()..], token)
+ } else {
+ IResult::Error
}
- IResult::Error
}
macro_rules! keyword {
@@ -40,17 +39,6 @@
}
}
-pub fn word_break(input: &str) -> IResult<&str, ()> {
- match input.chars().next() {
- Some(ch) if UnicodeXID::is_xid_continue(ch) => {
- IResult::Error
- }
- Some(_) | None => {
- IResult::Done(input, ())
- }
- }
-}
-
macro_rules! option {
($i:expr, $submac:ident!( $($args:tt)* )) => {
match $submac!($i, $($args)*) {
@@ -95,10 +83,3 @@
tap!($i, $name: call!($f) => $e);
};
}
-
-pub fn eat_spaces(input: &str) -> &str {
- match multispace(input) {
- IResult::Done(rest, _) => rest,
- IResult::Error => input,
- }
-}
diff --git a/src/ident.rs b/src/ident.rs
index beeeb00..adb4428 100644
--- a/src/ident.rs
+++ b/src/ident.rs
@@ -48,14 +48,14 @@
#[cfg(feature = "parsing")]
pub mod parsing {
use super::*;
- use nom::multispace;
+ use space::whitespace;
fn ident_ch(ch: char) -> bool {
ch.is_alphanumeric() || ch == '_'
}
named!(pub ident -> Ident, preceded!(
- option!(multispace),
+ option!(whitespace),
map!(take_while1!(ident_ch), Into::into)
));
}
diff --git a/src/krate.rs b/src/krate.rs
index c5b2126..0d12b59 100644
--- a/src/krate.rs
+++ b/src/krate.rs
@@ -10,13 +10,13 @@
pub mod parsing {
use super::*;
use attr::parsing::inner_attr;
+ use space::whitespace;
use item::parsing::item;
- use nom::multispace;
named!(pub krate -> Crate, do_parse!(
attrs: many0!(inner_attr) >>
items: many0!(item) >>
- option!(multispace) >>
+ option!(whitespace) >>
(Crate {
attrs: attrs,
items: items,
diff --git a/src/lib.rs b/src/lib.rs
index 50a754c..7c7bd13 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -126,6 +126,8 @@
MacroInput,
};
+mod space;
+
mod ty;
pub use ty::{
AngleBracketedParameterData,
@@ -158,7 +160,7 @@
mod parsing {
use super::*;
use {generics, macro_input, ty};
- use nom;
+ use nom::IResult;
#[cfg(feature = "full")]
use {expr, item, krate};
@@ -194,16 +196,16 @@
unwrap("where clause", generics::parsing::where_clause, input)
}
- fn unwrap<T>(name: &'static str, f: fn(&str) -> nom::IResult<&str, T>, input: &str) -> Result<T, String> {
+ fn unwrap<T>(name: &'static str, f: fn(&str) -> IResult<&str, T>, input: &str) -> Result<T, String> {
match f(input) {
- nom::IResult::Done(rest, t) => {
+ IResult::Done(rest, t) => {
if rest.is_empty() {
Ok(t)
} else {
Err(format!("remaining tokens after {}: {:?}", name, rest))
}
}
- nom::IResult::Error => Err(format!("failed to parse {}: {:?}", name, input)),
+ IResult::Error => Err(format!("failed to parse {}: {:?}", name, input)),
}
}
}
diff --git a/src/lit.rs b/src/lit.rs
index be41cd5..6f1b903 100644
--- a/src/lit.rs
+++ b/src/lit.rs
@@ -55,7 +55,7 @@
pub mod parsing {
use super::*;
use escape::{cooked_string, raw_string};
- use helper::eat_spaces;
+ use space::whitespace;
use nom::IResult;
named!(pub lit -> Lit, alt!(
@@ -83,7 +83,10 @@
));
named!(pub int -> (u64, IntTy), tuple!(
- digits,
+ preceded!(
+ option!(whitespace),
+ digits
+ ),
alt!(
tag!("isize") => { |_| IntTy::Isize }
|
@@ -110,7 +113,6 @@
));
pub fn digits(input: &str) -> IResult<&str, u64> {
- let input = eat_spaces(input);
let mut value = 0u64;
let mut len = 0;
let mut bytes = input.bytes().peekable();
diff --git a/src/macro_input.rs b/src/macro_input.rs
index c2b5900..976b7ce 100644
--- a/src/macro_input.rs
+++ b/src/macro_input.rs
@@ -21,8 +21,8 @@
use attr::parsing::outer_attr;
use data::parsing::{visibility, struct_body, enum_body};
use generics::parsing::generics;
+ use space::whitespace;
use ident::parsing::ident;
- use nom::multispace;
named!(pub macro_input -> MacroInput, do_parse!(
attrs: many0!(outer_attr) >>
@@ -47,7 +47,7 @@
body: Body::Enum(body),
})
) >>
- option!(multispace) >>
+ option!(whitespace) >>
(item)
));
}
diff --git a/src/nom.rs b/src/nom.rs
index 8df705c..ea066c7 100644
--- a/src/nom.rs
+++ b/src/nom.rs
@@ -10,25 +10,6 @@
Error,
}
-/// Recognizes spaces, tabs, carriage returns and line feeds
-pub fn multispace(input: &str) -> IResult<&str, &str> {
- if input.is_empty() {
- return IResult::Error;
- }
-
- for (idx, item) in input.char_indices() {
- let chr = item;
- if !(chr == ' ' || chr == '\t' || chr == '\r' || chr == '\n') {
- if idx == 0 {
- return IResult::Error;
- } else {
- return IResult::Done(&input[idx..], &input[0..idx]);
- }
- }
- }
- IResult::Done("", input)
-}
-
macro_rules! named {
($name:ident -> $o:ty, $submac:ident!( $($args:tt)* )) => {
fn $name(i: &str) -> $crate::nom::IResult<&str, $o> {
diff --git a/src/space.rs b/src/space.rs
new file mode 100644
index 0000000..ae0036d
--- /dev/null
+++ b/src/space.rs
@@ -0,0 +1,83 @@
+#![cfg(feature = "parsing")]
+
+use nom::IResult;
+use unicode_xid::UnicodeXID;
+
+pub fn whitespace(input: &str) -> IResult<&str, ()> {
+ if input.is_empty() {
+ return IResult::Error;
+ }
+
+ let mut start = 0;
+ let mut chars = input.char_indices();
+ while let Some((i, ch)) = chars.next() {
+ let s = &input[start + i..];
+ if ch == '/' {
+ if s.starts_with("//")
+ && (!s.starts_with("///") || s.starts_with("////"))
+ && !s.starts_with("//!") {
+ if let Some(len) = s.find('\n') {
+ start += i + len + 1;
+ chars = input[start..].char_indices();
+ continue;
+ }
+ break;
+ } else if s.starts_with("/*")
+ && !s.starts_with("/**")
+ && !s.starts_with("/*!") {
+ match block_comment(s) {
+ IResult::Done(_, com) => {
+ start += i + com.len();
+ chars = input[start..].char_indices();
+ continue;
+ }
+ IResult::Error => {
+ return IResult::Error;
+ }
+ }
+ }
+ }
+ if !ch.is_whitespace() {
+ return if start + i > 0 {
+ IResult::Done(s, ())
+ } else {
+ IResult::Error
+ };
+ }
+ }
+ IResult::Done("", ())
+}
+
+pub fn block_comment(input: &str) -> IResult<&str, &str> {
+ if !input.starts_with("/*") {
+ return IResult::Error;
+ }
+
+ let mut depth = 0;
+ let mut chars = input.char_indices();
+ while let Some((i, _)) = chars.next() {
+ let s = &input[i..];
+ if s.starts_with("/*") {
+ depth += 1;
+ chars.next(); // eat '*'
+ } else if s.starts_with("*/") {
+ depth -= 1;
+ if depth == 0 {
+ return IResult::Done(&input[i + 2..], &input[..i + 2]);
+ }
+ chars.next(); // eat '/'
+ }
+ }
+ IResult::Error
+}
+
+pub fn word_break(input: &str) -> IResult<&str, ()> {
+ match input.chars().next() {
+ Some(ch) if UnicodeXID::is_xid_continue(ch) => {
+ IResult::Error
+ }
+ Some(_) | None => {
+ IResult::Done(input, ())
+ }
+ }
+}
diff --git a/tests/cases/comment.rs b/tests/cases/comment.rs
new file mode 100644
index 0000000..d7ab22b
--- /dev/null
+++ b/tests/cases/comment.rs
@@ -0,0 +1,24 @@
+//! inner line doc comment
+
+/*! inner block doc comment
+ *
+ * /* nested block comment */
+ */
+
+// line comment
+
+/* block comment
+ *
+ * /* nested block comment */
+ */
+
+//// banner comment
+
+/// line doc comment
+
+/** block doc comment
+ *
+ * /* nested block comment */
+ */
+
+struct Success;