Fix a panic in `cooked_byte` on utf-8 chars
Don't want to slice on the wrong boundary!
Closes #54
diff --git a/src/stable.rs b/src/stable.rs
index 6312ace..ffa077a 100644
--- a/src/stable.rs
+++ b/src/stable.rs
@@ -904,7 +904,13 @@
};
if ok {
match bytes.next() {
- Some((offset, _)) => Ok((input.advance(offset), ())),
+ Some((offset, _)) => {
+ if input.chars().as_str().is_char_boundary(offset) {
+ Ok((input.advance(offset), ()))
+ } else {
+ Err(LexError)
+ }
+ }
None => Ok((input.advance(input.len()), ())),
}
} else {
diff --git a/tests/test.rs b/tests/test.rs
index 19f7b27..7c7275d 100644
--- a/tests/test.rs
+++ b/tests/test.rs
@@ -1,5 +1,7 @@
extern crate proc_macro2;
+use std::str;
+
use proc_macro2::{Term, Literal, TokenStream};
#[cfg(procmacro2_semver_exempt)]
@@ -161,3 +163,10 @@
assert_eq!(joined1.unwrap().source_file(), source1[0].span.source_file());
}
+
+#[test]
+fn no_panic() {
+ let s = str::from_utf8(b"b\'\xc2\x86 \x00\x00\x00^\"").unwrap();
+ assert!(s.parse::<proc_macro2::TokenStream>().is_err());
+}
+