blob: 047e63007748eb3c87fec0552412b15553e2069f [file] [log] [blame]
Alex Crichton76a5cc82017-05-23 07:01:44 -07001use std::ascii;
Alex Crichton44bffbc2017-05-19 17:51:59 -07002use std::borrow::Borrow;
3use std::cell::RefCell;
4use std::collections::HashMap;
5use std::fmt;
6use std::iter;
7use std::ops;
8use std::rc::Rc;
9use std::str::FromStr;
10use std::vec;
11
12use proc_macro;
David Tolnayb1032662017-05-31 15:52:28 -070013use unicode_xid::UnicodeXID;
David Tolnay744a6b82017-06-01 11:34:29 -070014use strnom::{PResult, skip_whitespace, block_comment, whitespace, word_break};
Alex Crichton44bffbc2017-05-19 17:51:59 -070015
16use {TokenTree, TokenKind, Delimiter, OpKind};
17
David Tolnay977f8282017-05-31 17:41:33 -070018#[derive(Clone, Debug)]
Alex Crichton44bffbc2017-05-19 17:51:59 -070019pub struct TokenStream {
20 inner: Vec<TokenTree>,
21}
22
23#[derive(Debug)]
24pub struct LexError;
25
26impl TokenStream {
27 pub fn empty() -> TokenStream {
28 TokenStream { inner: Vec::new() }
29 }
30
31 pub fn is_empty(&self) -> bool {
32 self.inner.len() == 0
33 }
34}
35
36impl FromStr for TokenStream {
37 type Err = LexError;
38
39 fn from_str(src: &str) -> Result<TokenStream, LexError> {
40 match token_stream(src) {
David Tolnay1218e122017-06-01 11:13:45 -070041 Ok((input, output)) => {
Alex Crichton44bffbc2017-05-19 17:51:59 -070042 if skip_whitespace(input).len() != 0 {
43 Err(LexError)
44 } else {
45 Ok(output)
46 }
47 }
David Tolnay1218e122017-06-01 11:13:45 -070048 Err(LexError) => Err(LexError),
Alex Crichton44bffbc2017-05-19 17:51:59 -070049 }
50 }
51}
52
53impl fmt::Display for TokenStream {
54 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
55 let mut joint = false;
56 for (i, tt) in self.inner.iter().enumerate() {
57 if i != 0 && !joint {
58 write!(f, " ")?;
59 }
60 joint = false;
61 match tt.kind {
62 TokenKind::Sequence(delim, ref stream) => {
63 let (start, end) = match delim {
64 Delimiter::Parenthesis => ("(", ")"),
65 Delimiter::Brace => ("{", "}"),
66 Delimiter::Bracket => ("[", "]"),
67 Delimiter::None => ("", ""),
68 };
Alex Crichton852d53d2017-05-19 19:25:08 -070069 if stream.0.inner.len() == 0 {
70 write!(f, "{} {}", start, end)?
71 } else {
72 write!(f, "{} {} {}", start, stream, end)?
73 }
Alex Crichton44bffbc2017-05-19 17:51:59 -070074 }
Alex Crichton9c2fb0a2017-05-26 08:49:31 -070075 TokenKind::Word(ref sym) => write!(f, "{}", sym.as_str())?,
Alex Crichton44bffbc2017-05-19 17:51:59 -070076 TokenKind::Op(ch, ref op) => {
77 write!(f, "{}", ch)?;
78 match *op {
79 OpKind::Alone => {}
80 OpKind::Joint => joint = true,
81 }
82 }
83 TokenKind::Literal(ref literal) => {
84 write!(f, "{}", literal)?;
85 // handle comments
86 if (literal.0).0.starts_with("/") {
87 write!(f, "\n")?;
88 }
89 }
90 }
91 }
92
93 Ok(())
94 }
95}
96
97impl From<proc_macro::TokenStream> for TokenStream {
98 fn from(inner: proc_macro::TokenStream) -> TokenStream {
99 inner.to_string().parse().expect("compiler token stream parse failed")
100 }
101}
102
103impl From<TokenStream> for proc_macro::TokenStream {
104 fn from(inner: TokenStream) -> proc_macro::TokenStream {
105 inner.to_string().parse().expect("failed to parse to compiler tokens")
106 }
107}
108
109
110impl From<TokenTree> for TokenStream {
111 fn from(tree: TokenTree) -> TokenStream {
112 TokenStream { inner: vec![tree] }
113 }
114}
115
116impl iter::FromIterator<TokenStream> for TokenStream {
117 fn from_iter<I: IntoIterator<Item=TokenStream>>(streams: I) -> Self {
118 let mut v = Vec::new();
119
120 for stream in streams.into_iter() {
121 v.extend(stream.inner);
122 }
123
124 TokenStream { inner: v }
125 }
126}
127
128pub type TokenIter = vec::IntoIter<TokenTree>;
129
130impl IntoIterator for TokenStream {
131 type Item = TokenTree;
132 type IntoIter = TokenIter;
133
134 fn into_iter(self) -> TokenIter {
135 self.inner.into_iter()
136 }
137}
138
David Tolnay977f8282017-05-31 17:41:33 -0700139#[derive(Clone, Copy, Default, Debug)]
Alex Crichton44bffbc2017-05-19 17:51:59 -0700140pub struct Span;
141
142impl Span {
143 pub fn call_site() -> Span {
144 Span
145 }
146}
147
David Tolnay977f8282017-05-31 17:41:33 -0700148#[derive(Copy, Clone, Debug)]
Alex Crichton44bffbc2017-05-19 17:51:59 -0700149pub struct Symbol(usize);
150
151thread_local!(static SYMBOLS: RefCell<Interner> = RefCell::new(Interner::new()));
152
153impl<'a> From<&'a str> for Symbol {
154 fn from(string: &'a str) -> Symbol {
155 Symbol(SYMBOLS.with(|s| s.borrow_mut().intern(string)))
156 }
157}
158
159impl ops::Deref for Symbol {
160 type Target = str;
161
162 fn deref(&self) -> &str {
163 SYMBOLS.with(|interner| {
164 let interner = interner.borrow();
165 let s = interner.get(self.0);
166 unsafe {
167 &*(s as *const str)
168 }
169 })
170 }
171}
172
173struct Interner {
174 string_to_index: HashMap<MyRc, usize>,
175 index_to_string: Vec<Rc<String>>,
176}
177
178#[derive(Hash, Eq, PartialEq)]
179struct MyRc(Rc<String>);
180
181impl Borrow<str> for MyRc {
182 fn borrow(&self) -> &str {
183 &self.0
184 }
185}
186
187impl Interner {
188 fn new() -> Interner {
189 Interner {
190 string_to_index: HashMap::new(),
191 index_to_string: Vec::new(),
192 }
193 }
194
195 fn intern(&mut self, s: &str) -> usize {
196 if let Some(&idx) = self.string_to_index.get(s) {
197 return idx
198 }
199 let s = Rc::new(s.to_string());
200 self.index_to_string.push(s.clone());
201 self.string_to_index.insert(MyRc(s), self.index_to_string.len() - 1);
202 self.index_to_string.len() - 1
203 }
204
205 fn get(&self, idx: usize) -> &str {
206 &self.index_to_string[idx]
207 }
208}
209
David Tolnay977f8282017-05-31 17:41:33 -0700210#[derive(Clone, Debug)]
Alex Crichton44bffbc2017-05-19 17:51:59 -0700211pub struct Literal(String);
212
Alex Crichton852d53d2017-05-19 19:25:08 -0700213impl Literal {
Alex Crichton9c2fb0a2017-05-26 08:49:31 -0700214 pub fn byte_char(byte: u8) -> Literal {
Alex Crichton76a5cc82017-05-23 07:01:44 -0700215 match byte {
216 0 => Literal(format!("b'\\0'")),
217 b'\"' => Literal(format!("b'\"'")),
218 n => {
219 let mut escaped = "b'".to_string();
220 escaped.extend(ascii::escape_default(n).map(|c| c as char));
221 escaped.push('\'');
222 Literal(escaped)
223 }
224 }
225 }
226
Alex Crichton9c2fb0a2017-05-26 08:49:31 -0700227 pub fn byte_string(bytes: &[u8]) -> Literal {
Alex Crichton852d53d2017-05-19 19:25:08 -0700228 let mut escaped = "b\"".to_string();
229 for b in bytes {
230 match *b {
231 b'\0' => escaped.push_str(r"\0"),
232 b'\t' => escaped.push_str(r"\t"),
233 b'\n' => escaped.push_str(r"\n"),
234 b'\r' => escaped.push_str(r"\r"),
235 b'"' => escaped.push_str("\\\""),
236 b'\\' => escaped.push_str("\\\\"),
237 b'\x20' ... b'\x7E' => escaped.push(*b as char),
238 _ => escaped.push_str(&format!("\\x{:02X}", b)),
239 }
240 }
241 escaped.push('"');
242 Literal(escaped)
243 }
Alex Crichton76a5cc82017-05-23 07:01:44 -0700244
245 pub fn doccomment(s: &str) -> Literal {
246 Literal(s.to_string())
247 }
Alex Crichton9c2fb0a2017-05-26 08:49:31 -0700248
249 pub fn float(s: &str) -> Literal {
250 Literal(s.to_string())
251 }
252
253 pub fn integer(s: &str) -> Literal {
254 Literal(s.to_string())
255 }
Alex Crichton31316622017-05-26 12:54:47 -0700256
257 pub fn raw_string(s: &str, pounds: usize) -> Literal {
258 let mut ret = format!("r");
259 ret.extend((0..pounds).map(|_| "#"));
260 ret.push('"');
261 ret.push_str(s);
262 ret.push('"');
263 ret.extend((0..pounds).map(|_| "#"));
264 Literal(ret)
265 }
266
267 pub fn raw_byte_string(s: &str, pounds: usize) -> Literal {
Alex Crichton7ed6d282017-05-26 13:42:50 -0700268 let mut ret = format!("br");
Alex Crichton31316622017-05-26 12:54:47 -0700269 ret.extend((0..pounds).map(|_| "#"));
270 ret.push('"');
271 ret.push_str(s);
272 ret.push('"');
273 ret.extend((0..pounds).map(|_| "#"));
274 Literal(ret)
275 }
Alex Crichton852d53d2017-05-19 19:25:08 -0700276}
277
Alex Crichton44bffbc2017-05-19 17:51:59 -0700278impl fmt::Display for Literal {
279 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
280 self.0.fmt(f)
281 }
282}
283
Alex Crichton9c2fb0a2017-05-26 08:49:31 -0700284macro_rules! ints {
Alex Crichton44bffbc2017-05-19 17:51:59 -0700285 ($($t:ty,)*) => {$(
286 impl From<$t> for Literal {
287 fn from(t: $t) -> Literal {
Alex Crichton852d53d2017-05-19 19:25:08 -0700288 Literal(format!(concat!("{}", stringify!($t)), t))
Alex Crichton44bffbc2017-05-19 17:51:59 -0700289 }
290 }
291 )*}
292}
293
Alex Crichton9c2fb0a2017-05-26 08:49:31 -0700294ints! {
Alex Crichton852d53d2017-05-19 19:25:08 -0700295 u8, u16, u32, u64, usize,
296 i8, i16, i32, i64, isize,
Alex Crichton9c2fb0a2017-05-26 08:49:31 -0700297}
298
299macro_rules! floats {
300 ($($t:ty,)*) => {$(
301 impl From<$t> for Literal {
302 fn from(t: $t) -> Literal {
303 assert!(!t.is_nan());
304 assert!(!t.is_infinite());
305 Literal(format!(concat!("{}", stringify!($t)), t))
306 }
307 }
308 )*}
309}
310
311floats! {
Alex Crichton852d53d2017-05-19 19:25:08 -0700312 f32, f64,
313}
314
Alex Crichton44bffbc2017-05-19 17:51:59 -0700315impl<'a> From<&'a str> for Literal {
316 fn from(t: &'a str) -> Literal {
317 let mut s = t.chars().flat_map(|c| c.escape_default()).collect::<String>();
318 s.push('"');
319 s.insert(0, '"');
320 Literal(s)
321 }
322}
323
324impl From<char> for Literal {
325 fn from(t: char) -> Literal {
Alex Crichton2d0cf0b2017-05-26 14:00:16 -0700326 Literal(format!("'{}'", t.escape_default().collect::<String>()))
Alex Crichton44bffbc2017-05-19 17:51:59 -0700327 }
328}
329
330named!(token_stream -> TokenStream,
David Tolnaye227d5b2017-06-01 11:57:10 -0700331 map!(many0!(token_tree), |trees| TokenStream { inner: trees }));
Alex Crichton44bffbc2017-05-19 17:51:59 -0700332
333named!(token_tree -> TokenTree,
334 map!(token_kind, |s: TokenKind| {
335 TokenTree {
336 span: ::Span(Span),
337 kind: s,
338 }
339 }));
340
341named!(token_kind -> TokenKind, alt!(
David Tolnay643177b2017-06-01 12:02:44 -0700342 map!(delimited, |(d, s)| TokenKind::Sequence(d, ::TokenStream(s)))
Alex Crichton44bffbc2017-05-19 17:51:59 -0700343 |
Michael Layzellbad58012017-05-24 20:42:52 -0400344 map!(literal, |l| TokenKind::Literal(::Literal(l))) // must be before symbol
Alex Crichton44bffbc2017-05-19 17:51:59 -0700345 |
Michael Layzellbad58012017-05-24 20:42:52 -0400346 map!(symbol, |w| TokenKind::Word(::Symbol(w)))
Alex Crichton44bffbc2017-05-19 17:51:59 -0700347 |
David Tolnay643177b2017-06-01 12:02:44 -0700348 map!(op, |(op, kind)| TokenKind::Op(op, kind))
Alex Crichton44bffbc2017-05-19 17:51:59 -0700349));
350
351named!(delimited -> (Delimiter, TokenStream), alt!(
352 delimited!(
353 punct!("("),
354 token_stream,
355 punct!(")")
356 ) => { |ts| (Delimiter::Parenthesis, ts) }
357 |
358 delimited!(
359 punct!("["),
360 token_stream,
361 punct!("]")
362 ) => { |ts| (Delimiter::Bracket, ts) }
363 |
364 delimited!(
365 punct!("{"),
366 token_stream,
367 punct!("}")
368 ) => { |ts| (Delimiter::Brace, ts) }
369));
370
371named!(symbol -> Symbol, alt!(
372 lifetime
373 |
374 map!(word, Symbol::from)
375));
376
377named!(lifetime -> Symbol, preceded!(
378 punct!("'"),
379 alt!(
380 // TODO: can we get rid of this allocation?
381 map!(word, |id| Symbol::from(&format!("'{}", id)[..]))
382 |
383 map!(keyword!("static"), |_| Symbol::from("'static"))
384 )
385));
386
David Tolnay1218e122017-06-01 11:13:45 -0700387fn word(mut input: &str) -> PResult<&str> {
Alex Crichton44bffbc2017-05-19 17:51:59 -0700388 input = skip_whitespace(input);
389
390 let mut chars = input.char_indices();
391 match chars.next() {
392 Some((_, ch)) if UnicodeXID::is_xid_start(ch) || ch == '_' => {}
David Tolnay1218e122017-06-01 11:13:45 -0700393 _ => return Err(LexError),
Alex Crichton44bffbc2017-05-19 17:51:59 -0700394 }
395
396 for (i, ch) in chars {
397 if !UnicodeXID::is_xid_continue(ch) {
David Tolnay1218e122017-06-01 11:13:45 -0700398 return Ok((&input[i..], &input[..i]))
Alex Crichton44bffbc2017-05-19 17:51:59 -0700399 }
400 }
401
David Tolnay1218e122017-06-01 11:13:45 -0700402 Ok(("", input))
Alex Crichton44bffbc2017-05-19 17:51:59 -0700403}
404
David Tolnay1218e122017-06-01 11:13:45 -0700405fn literal(input: &str) -> PResult<Literal> {
Alex Crichton44bffbc2017-05-19 17:51:59 -0700406 let input_no_ws = skip_whitespace(input);
407
408 match literal_nocapture(input_no_ws) {
David Tolnay1218e122017-06-01 11:13:45 -0700409 Ok((a, ())) => {
Alex Crichton44bffbc2017-05-19 17:51:59 -0700410 let start = input.len() - input_no_ws.len();
411 let len = input_no_ws.len() - a.len();
412 let end = start + len;
David Tolnay1218e122017-06-01 11:13:45 -0700413 Ok((a, Literal(input[start..end].to_string())))
Alex Crichton44bffbc2017-05-19 17:51:59 -0700414 }
David Tolnay1218e122017-06-01 11:13:45 -0700415 Err(LexError) => Err(LexError),
Alex Crichton44bffbc2017-05-19 17:51:59 -0700416 }
417}
418
419named!(literal_nocapture -> (), alt!(
420 string
421 |
422 byte_string
423 |
424 byte
425 |
426 character
427 |
428 float
429 |
430 int
431 |
432 boolean
433 |
434 doc_comment
435));
436
437named!(string -> (), alt!(
438 quoted_string
439 |
440 preceded!(
441 punct!("r"),
442 raw_string
443 ) => { |_| () }
444));
445
446named!(quoted_string -> (), delimited!(
447 punct!("\""),
448 cooked_string,
449 tag!("\"")
450));
451
David Tolnay1218e122017-06-01 11:13:45 -0700452fn cooked_string(input: &str) -> PResult<()> {
Alex Crichton44bffbc2017-05-19 17:51:59 -0700453 let mut chars = input.char_indices().peekable();
454 while let Some((byte_offset, ch)) = chars.next() {
455 match ch {
456 '"' => {
David Tolnay1218e122017-06-01 11:13:45 -0700457 return Ok((&input[byte_offset..], ()));
Alex Crichton44bffbc2017-05-19 17:51:59 -0700458 }
459 '\r' => {
460 if let Some((_, '\n')) = chars.next() {
461 // ...
462 } else {
463 break;
464 }
465 }
466 '\\' => {
467 match chars.next() {
468 Some((_, 'x')) => {
469 if !backslash_x_char(&mut chars) {
470 break
471 }
472 }
473 Some((_, 'n')) |
474 Some((_, 'r')) |
475 Some((_, 't')) |
476 Some((_, '\\')) |
477 Some((_, '\'')) |
478 Some((_, '"')) |
479 Some((_, '0')) => {}
480 Some((_, 'u')) => {
481 if !backslash_u(&mut chars) {
482 break
483 }
484 }
485 Some((_, '\n')) | Some((_, '\r')) => {
486 while let Some(&(_, ch)) = chars.peek() {
487 if ch.is_whitespace() {
488 chars.next();
489 } else {
490 break;
491 }
492 }
493 }
494 _ => break,
495 }
496 }
497 _ch => {}
498 }
499 }
David Tolnay1218e122017-06-01 11:13:45 -0700500 Err(LexError)
Alex Crichton44bffbc2017-05-19 17:51:59 -0700501}
502
503named!(byte_string -> (), alt!(
504 delimited!(
505 punct!("b\""),
506 cooked_byte_string,
507 tag!("\"")
508 ) => { |_| () }
509 |
510 preceded!(
511 punct!("br"),
512 raw_string
513 ) => { |_| () }
514));
515
David Tolnay1218e122017-06-01 11:13:45 -0700516fn cooked_byte_string(mut input: &str) -> PResult<()> {
Alex Crichton44bffbc2017-05-19 17:51:59 -0700517 let mut bytes = input.bytes().enumerate();
518 'outer: while let Some((offset, b)) = bytes.next() {
519 match b {
520 b'"' => {
David Tolnay1218e122017-06-01 11:13:45 -0700521 return Ok((&input[offset..], ()));
Alex Crichton44bffbc2017-05-19 17:51:59 -0700522 }
523 b'\r' => {
524 if let Some((_, b'\n')) = bytes.next() {
525 // ...
526 } else {
527 break;
528 }
529 }
530 b'\\' => {
531 match bytes.next() {
532 Some((_, b'x')) => {
533 if !backslash_x_byte(&mut bytes) {
534 break
535 }
536 }
537 Some((_, b'n')) |
538 Some((_, b'r')) |
539 Some((_, b't')) |
540 Some((_, b'\\')) |
541 Some((_, b'0')) |
542 Some((_, b'\'')) |
543 Some((_, b'"')) => {}
544 Some((newline, b'\n')) |
545 Some((newline, b'\r')) => {
546 let rest = &input[newline + 1..];
547 for (offset, ch) in rest.char_indices() {
548 if !ch.is_whitespace() {
549 input = &rest[offset..];
550 bytes = input.bytes().enumerate();
551 continue 'outer;
552 }
553 }
554 break;
555 }
556 _ => break,
557 }
558 }
559 b if b < 0x80 => {}
560 _ => break,
561 }
562 }
David Tolnay1218e122017-06-01 11:13:45 -0700563 Err(LexError)
Alex Crichton44bffbc2017-05-19 17:51:59 -0700564}
565
David Tolnay1218e122017-06-01 11:13:45 -0700566fn raw_string(input: &str) -> PResult<()> {
Alex Crichton44bffbc2017-05-19 17:51:59 -0700567 let mut chars = input.char_indices();
568 let mut n = 0;
569 while let Some((byte_offset, ch)) = chars.next() {
570 match ch {
571 '"' => {
572 n = byte_offset;
573 break;
574 }
575 '#' => {}
David Tolnay1218e122017-06-01 11:13:45 -0700576 _ => return Err(LexError),
Alex Crichton44bffbc2017-05-19 17:51:59 -0700577 }
578 }
579 for (byte_offset, ch) in chars {
580 match ch {
581 '"' if input[byte_offset + 1..].starts_with(&input[..n]) => {
582 let rest = &input[byte_offset + 1 + n..];
David Tolnay1218e122017-06-01 11:13:45 -0700583 return Ok((rest, ()))
Alex Crichton44bffbc2017-05-19 17:51:59 -0700584 }
585 '\r' => {}
586 _ => {}
587 }
588 }
David Tolnay1218e122017-06-01 11:13:45 -0700589 Err(LexError)
Alex Crichton44bffbc2017-05-19 17:51:59 -0700590}
591
592named!(byte -> (), do_parse!(
593 punct!("b") >>
594 tag!("'") >>
595 cooked_byte >>
596 tag!("'") >>
597 (())
598));
599
David Tolnay1218e122017-06-01 11:13:45 -0700600fn cooked_byte(input: &str) -> PResult<()> {
Alex Crichton44bffbc2017-05-19 17:51:59 -0700601 let mut bytes = input.bytes().enumerate();
602 let ok = match bytes.next().map(|(_, b)| b) {
603 Some(b'\\') => {
604 match bytes.next().map(|(_, b)| b) {
605 Some(b'x') => backslash_x_byte(&mut bytes),
606 Some(b'n') |
607 Some(b'r') |
608 Some(b't') |
609 Some(b'\\') |
610 Some(b'0') |
611 Some(b'\'') |
612 Some(b'"') => true,
613 _ => false,
614 }
615 }
616 b => b.is_some(),
617 };
618 if ok {
619 match bytes.next() {
David Tolnay1218e122017-06-01 11:13:45 -0700620 Some((offset, _)) => Ok((&input[offset..], ())),
621 None => Ok(("", ())),
Alex Crichton44bffbc2017-05-19 17:51:59 -0700622 }
623 } else {
David Tolnay1218e122017-06-01 11:13:45 -0700624 Err(LexError)
Alex Crichton44bffbc2017-05-19 17:51:59 -0700625 }
626}
627
628named!(character -> (), do_parse!(
629 punct!("'") >>
630 cooked_char >>
631 tag!("'") >>
632 (())
633));
634
David Tolnay1218e122017-06-01 11:13:45 -0700635fn cooked_char(input: &str) -> PResult<()> {
Alex Crichton44bffbc2017-05-19 17:51:59 -0700636 let mut chars = input.char_indices();
637 let ok = match chars.next().map(|(_, ch)| ch) {
638 Some('\\') => {
639 match chars.next().map(|(_, ch)| ch) {
640 Some('x') => backslash_x_char(&mut chars),
641 Some('u') => backslash_u(&mut chars),
642 Some('n') |
643 Some('r') |
644 Some('t') |
645 Some('\\') |
646 Some('0') |
647 Some('\'') |
648 Some('"') => true,
649 _ => false,
650 }
651 }
652 ch => ch.is_some(),
653 };
654 if ok {
David Tolnay1218e122017-06-01 11:13:45 -0700655 Ok((chars.as_str(), ()))
Alex Crichton44bffbc2017-05-19 17:51:59 -0700656 } else {
David Tolnay1218e122017-06-01 11:13:45 -0700657 Err(LexError)
Alex Crichton44bffbc2017-05-19 17:51:59 -0700658 }
659}
660
661macro_rules! next_ch {
662 ($chars:ident @ $pat:pat $(| $rest:pat)*) => {
663 match $chars.next() {
664 Some((_, ch)) => match ch {
665 $pat $(| $rest)* => ch,
666 _ => return false,
667 },
668 None => return false
669 }
670 };
671}
672
673fn backslash_x_char<I>(chars: &mut I) -> bool
674 where I: Iterator<Item = (usize, char)>
675{
676 next_ch!(chars @ '0'...'7');
677 next_ch!(chars @ '0'...'9' | 'a'...'f' | 'A'...'F');
678 true
679}
680
681fn backslash_x_byte<I>(chars: &mut I) -> bool
682 where I: Iterator<Item = (usize, u8)>
683{
684 next_ch!(chars @ b'0'...b'9' | b'a'...b'f' | b'A'...b'F');
685 next_ch!(chars @ b'0'...b'9' | b'a'...b'f' | b'A'...b'F');
686 true
687}
688
689fn backslash_u<I>(chars: &mut I) -> bool
690 where I: Iterator<Item = (usize, char)>
691{
692 next_ch!(chars @ '{');
693 next_ch!(chars @ '0'...'9' | 'a'...'f' | 'A'...'F');
694 let b = next_ch!(chars @ '0'...'9' | 'a'...'f' | 'A'...'F' | '}');
695 if b == '}' {
696 return true
697 }
698 let c = next_ch!(chars @ '0'...'9' | 'a'...'f' | 'A'...'F' | '}');
699 if c == '}' {
700 return true
701 }
702 let d = next_ch!(chars @ '0'...'9' | 'a'...'f' | 'A'...'F' | '}');
703 if d == '}' {
704 return true
705 }
706 let e = next_ch!(chars @ '0'...'9' | 'a'...'f' | 'A'...'F' | '}');
707 if e == '}' {
708 return true
709 }
710 let f = next_ch!(chars @ '0'...'9' | 'a'...'f' | 'A'...'F' | '}');
711 if f == '}' {
712 return true
713 }
714 next_ch!(chars @ '}');
715 true
716}
717
David Tolnay744a6b82017-06-01 11:34:29 -0700718fn float(input: &str) -> PResult<()> {
719 let (rest, ()) = float_digits(input)?;
720 for suffix in &["f32", "f64"] {
721 if rest.starts_with(suffix) {
722 return word_break(&rest[suffix.len()..]);
723 }
724 }
725 word_break(rest)
726}
Alex Crichton44bffbc2017-05-19 17:51:59 -0700727
David Tolnay744a6b82017-06-01 11:34:29 -0700728fn float_digits(input: &str) -> PResult<()> {
Alex Crichton44bffbc2017-05-19 17:51:59 -0700729 let mut chars = input.chars().peekable();
730 match chars.next() {
731 Some(ch) if ch >= '0' && ch <= '9' => {}
David Tolnay1218e122017-06-01 11:13:45 -0700732 _ => return Err(LexError),
Alex Crichton44bffbc2017-05-19 17:51:59 -0700733 }
734
735 let mut len = 1;
736 let mut has_dot = false;
737 let mut has_exp = false;
738 while let Some(&ch) = chars.peek() {
739 match ch {
740 '0'...'9' | '_' => {
741 chars.next();
742 len += 1;
743 }
744 '.' => {
745 if has_dot {
746 break;
747 }
748 chars.next();
749 if chars.peek()
750 .map(|&ch| ch == '.' || UnicodeXID::is_xid_start(ch))
751 .unwrap_or(false) {
David Tolnay1218e122017-06-01 11:13:45 -0700752 return Err(LexError);
Alex Crichton44bffbc2017-05-19 17:51:59 -0700753 }
754 len += 1;
755 has_dot = true;
756 }
757 'e' | 'E' => {
758 chars.next();
759 len += 1;
760 has_exp = true;
761 break;
762 }
763 _ => break,
764 }
765 }
766
767 let rest = &input[len..];
768 if !(has_dot || has_exp || rest.starts_with("f32") || rest.starts_with("f64")) {
David Tolnay1218e122017-06-01 11:13:45 -0700769 return Err(LexError);
Alex Crichton44bffbc2017-05-19 17:51:59 -0700770 }
771
772 if has_exp {
773 let mut has_exp_value = false;
774 while let Some(&ch) = chars.peek() {
775 match ch {
776 '+' | '-' => {
777 if has_exp_value {
778 break;
779 }
780 chars.next();
781 len += 1;
782 }
783 '0'...'9' => {
784 chars.next();
785 len += 1;
786 has_exp_value = true;
787 }
788 '_' => {
789 chars.next();
790 len += 1;
791 }
792 _ => break,
793 }
794 }
795 if !has_exp_value {
David Tolnay1218e122017-06-01 11:13:45 -0700796 return Err(LexError);
Alex Crichton44bffbc2017-05-19 17:51:59 -0700797 }
798 }
799
David Tolnay1218e122017-06-01 11:13:45 -0700800 Ok((&input[len..], ()))
Alex Crichton44bffbc2017-05-19 17:51:59 -0700801}
802
David Tolnay744a6b82017-06-01 11:34:29 -0700803fn int(input: &str) -> PResult<()> {
804 let (rest, ()) = digits(input)?;
805 for suffix in &[
806 "isize",
807 "i8",
808 "i16",
809 "i32",
810 "i64",
811 "i128",
812 "usize",
813 "u8",
814 "u16",
815 "u32",
816 "u64",
817 "u128",
818 ] {
819 if rest.starts_with(suffix) {
820 return word_break(&rest[suffix.len()..]);
821 }
822 }
823 word_break(rest)
824}
Alex Crichton44bffbc2017-05-19 17:51:59 -0700825
David Tolnay1218e122017-06-01 11:13:45 -0700826fn digits(mut input: &str) -> PResult<()> {
Alex Crichton44bffbc2017-05-19 17:51:59 -0700827 let base = if input.starts_with("0x") {
828 input = &input[2..];
829 16
830 } else if input.starts_with("0o") {
831 input = &input[2..];
832 8
833 } else if input.starts_with("0b") {
834 input = &input[2..];
835 2
836 } else {
837 10
838 };
839
Alex Crichton44bffbc2017-05-19 17:51:59 -0700840 let mut len = 0;
841 let mut empty = true;
842 for b in input.bytes() {
843 let digit = match b {
844 b'0'...b'9' => (b - b'0') as u64,
845 b'a'...b'f' => 10 + (b - b'a') as u64,
846 b'A'...b'F' => 10 + (b - b'A') as u64,
847 b'_' => {
848 if empty && base == 10 {
David Tolnay1218e122017-06-01 11:13:45 -0700849 return Err(LexError);
Alex Crichton44bffbc2017-05-19 17:51:59 -0700850 }
851 len += 1;
852 continue;
853 }
854 _ => break,
855 };
856 if digit >= base {
David Tolnay1218e122017-06-01 11:13:45 -0700857 return Err(LexError);
Alex Crichton44bffbc2017-05-19 17:51:59 -0700858 }
Alex Crichton44bffbc2017-05-19 17:51:59 -0700859 len += 1;
860 empty = false;
861 }
862 if empty {
David Tolnay1218e122017-06-01 11:13:45 -0700863 Err(LexError)
Alex Crichton44bffbc2017-05-19 17:51:59 -0700864 } else {
David Tolnay1218e122017-06-01 11:13:45 -0700865 Ok((&input[len..], ()))
Alex Crichton44bffbc2017-05-19 17:51:59 -0700866 }
867}
868
869named!(boolean -> (), alt!(
870 keyword!("true") => { |_| () }
871 |
872 keyword!("false") => { |_| () }
873));
874
David Tolnay1218e122017-06-01 11:13:45 -0700875fn op(input: &str) -> PResult<(char, OpKind)> {
David Tolnayea75c5f2017-05-31 23:40:33 -0700876 let input = skip_whitespace(input);
877 match op_char(input) {
David Tolnay1218e122017-06-01 11:13:45 -0700878 Ok((rest, ch)) => {
David Tolnayea75c5f2017-05-31 23:40:33 -0700879 let kind = match op_char(rest) {
David Tolnay1218e122017-06-01 11:13:45 -0700880 Ok(_) => OpKind::Joint,
881 Err(LexError) => OpKind::Alone,
David Tolnayea75c5f2017-05-31 23:40:33 -0700882 };
David Tolnay1218e122017-06-01 11:13:45 -0700883 Ok((rest, (ch, kind)))
David Tolnayea75c5f2017-05-31 23:40:33 -0700884 }
David Tolnay1218e122017-06-01 11:13:45 -0700885 Err(LexError) => Err(LexError),
Alex Crichton44bffbc2017-05-19 17:51:59 -0700886 }
887}
888
David Tolnay1218e122017-06-01 11:13:45 -0700889fn op_char(input: &str) -> PResult<char> {
David Tolnayea75c5f2017-05-31 23:40:33 -0700890 let mut chars = input.chars();
891 let first = match chars.next() {
892 Some(ch) => ch,
893 None => {
David Tolnay1218e122017-06-01 11:13:45 -0700894 return Err(LexError);
David Tolnayea75c5f2017-05-31 23:40:33 -0700895 }
896 };
897 let recognized = "~!@#$%^&*-=+|;:,<.>/?";
898 if recognized.contains(first) {
David Tolnay1218e122017-06-01 11:13:45 -0700899 Ok((chars.as_str(), first))
Alex Crichton44bffbc2017-05-19 17:51:59 -0700900 } else {
David Tolnay1218e122017-06-01 11:13:45 -0700901 Err(LexError)
Alex Crichton44bffbc2017-05-19 17:51:59 -0700902 }
903}
904
Alex Crichton44bffbc2017-05-19 17:51:59 -0700905named!(doc_comment -> (), alt!(
906 do_parse!(
907 punct!("//!") >>
908 take_until!("\n") >>
909 (())
910 )
911 |
912 do_parse!(
913 option!(whitespace) >>
914 peek!(tag!("/*!")) >>
915 block_comment >>
916 (())
917 )
918 |
919 do_parse!(
920 punct!("///") >>
921 not!(tag!("/")) >>
922 take_until!("\n") >>
923 (())
924 )
925 |
926 do_parse!(
927 option!(whitespace) >>
928 peek!(tuple!(tag!("/**"), not!(tag!("*")))) >>
929 block_comment >>
930 (())
931 )
932));
933
934#[cfg(test)]
935mod tests {
936 use super::*;
937
938 #[test]
939 fn symbols() {
940 assert_eq!(&*Symbol::from("foo"), "foo");
941 assert_eq!(&*Symbol::from("bar"), "bar");
942 }
943
944 #[test]
945 fn literals() {
946 assert_eq!(Literal::from("foo").to_string(), "\"foo\"");
947 assert_eq!(Literal::from("\"").to_string(), "\"\\\"\"");
948 }
949
950 #[test]
951 fn roundtrip() {
952 fn roundtrip(p: &str) {
953 println!("parse: {}", p);
954 let s = p.parse::<TokenStream>().unwrap().to_string();
955 println!("first: {}", s);
956 let s2 = s.to_string().parse::<TokenStream>().unwrap().to_string();
957 assert_eq!(s, s2);
958 }
959 roundtrip("a");
960 roundtrip("<<");
961 roundtrip("<<=");
962 roundtrip("
963 /// a
964 wut
965 ");
966 roundtrip("
967 1
968 1.0
969 1f32
970 2f64
971 1usize
972 4isize
973 4e10
974 1_000
975 1_0i32
976 8u8
977 9
978 0
Michael Layzell9ee75c92017-05-31 19:44:26 -0400979 0xffffffffffffffffffffffffffffffff
Alex Crichton44bffbc2017-05-19 17:51:59 -0700980 ");
981 }
David Tolnay744a6b82017-06-01 11:34:29 -0700982
983 #[test]
984 fn fail() {
985 fn fail(p: &str) {
986 if p.parse::<TokenStream>().is_ok() {
987 panic!("should have failed to parse: {}", p);
988 }
989 }
990 fail("1x");
991 fail("1u80");
992 fail("1f320");
993 }
Alex Crichton44bffbc2017-05-19 17:51:59 -0700994}