blob: 801a1df131ea56ef17f6aee3cc33887ff09db710 [file] [log] [blame]
Alex Crichtona914a612018-04-04 07:48:44 -07001#![cfg_attr(not(procmacro2_semver_exempt), allow(dead_code))]
Alex Crichtonaf5bad42018-03-27 14:45:10 -07002
Alex Crichton44bffbc2017-05-19 17:51:59 -07003use std::borrow::Borrow;
4use std::cell::RefCell;
David Tolnay1ebe3972018-01-02 20:14:20 -08005#[cfg(procmacro2_semver_exempt)]
Nika Layzellf8d5f212017-12-11 14:07:02 -05006use std::cmp;
Alex Crichton44bffbc2017-05-19 17:51:59 -07007use std::collections::HashMap;
8use std::fmt;
9use std::iter;
Alex Crichton44bffbc2017-05-19 17:51:59 -070010use std::rc::Rc;
11use std::str::FromStr;
12use std::vec;
13
David Tolnayb28f38a2018-03-31 22:02:29 +020014use strnom::{block_comment, skip_whitespace, whitespace, word_break, Cursor, PResult};
David Tolnayb1032662017-05-31 15:52:28 -070015use unicode_xid::UnicodeXID;
Alex Crichton44bffbc2017-05-19 17:51:59 -070016
Alex Crichtonf3888432018-05-16 09:11:05 -070017use {Delimiter, Group, Punct, Spacing, TokenTree};
Alex Crichton44bffbc2017-05-19 17:51:59 -070018
David Tolnay034205f2018-04-22 16:45:28 -070019#[derive(Clone)]
Alex Crichton44bffbc2017-05-19 17:51:59 -070020pub struct TokenStream {
21 inner: Vec<TokenTree>,
22}
23
24#[derive(Debug)]
25pub struct LexError;
26
27impl TokenStream {
28 pub fn empty() -> TokenStream {
29 TokenStream { inner: Vec::new() }
30 }
31
32 pub fn is_empty(&self) -> bool {
33 self.inner.len() == 0
34 }
35}
36
David Tolnay1ebe3972018-01-02 20:14:20 -080037#[cfg(procmacro2_semver_exempt)]
Nika Layzella9dbc182017-12-30 14:50:13 -050038fn get_cursor(src: &str) -> Cursor {
39 // Create a dummy file & add it to the codemap
40 CODEMAP.with(|cm| {
41 let mut cm = cm.borrow_mut();
42 let name = format!("<parsed string {}>", cm.files.len());
43 let span = cm.add_file(&name, src);
44 Cursor {
45 rest: src,
46 off: span.lo,
47 }
48 })
49}
50
David Tolnay1ebe3972018-01-02 20:14:20 -080051#[cfg(not(procmacro2_semver_exempt))]
Nika Layzella9dbc182017-12-30 14:50:13 -050052fn get_cursor(src: &str) -> Cursor {
David Tolnayb28f38a2018-03-31 22:02:29 +020053 Cursor { rest: src }
Nika Layzella9dbc182017-12-30 14:50:13 -050054}
55
Alex Crichton44bffbc2017-05-19 17:51:59 -070056impl FromStr for TokenStream {
57 type Err = LexError;
58
59 fn from_str(src: &str) -> Result<TokenStream, LexError> {
Nika Layzellf8d5f212017-12-11 14:07:02 -050060 // Create a dummy file & add it to the codemap
Nika Layzella9dbc182017-12-30 14:50:13 -050061 let cursor = get_cursor(src);
Nika Layzellf8d5f212017-12-11 14:07:02 -050062
63 match token_stream(cursor) {
David Tolnay1218e122017-06-01 11:13:45 -070064 Ok((input, output)) => {
Alex Crichton44bffbc2017-05-19 17:51:59 -070065 if skip_whitespace(input).len() != 0 {
66 Err(LexError)
67 } else {
Alex Crichton30a4e9e2018-04-27 17:02:19 -070068 Ok(output)
Alex Crichton44bffbc2017-05-19 17:51:59 -070069 }
70 }
David Tolnay1218e122017-06-01 11:13:45 -070071 Err(LexError) => Err(LexError),
Alex Crichton44bffbc2017-05-19 17:51:59 -070072 }
73 }
74}
75
76impl fmt::Display for TokenStream {
77 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
78 let mut joint = false;
79 for (i, tt) in self.inner.iter().enumerate() {
80 if i != 0 && !joint {
81 write!(f, " ")?;
82 }
83 joint = false;
Alex Crichtonaf5bad42018-03-27 14:45:10 -070084 match *tt {
85 TokenTree::Group(ref tt) => {
86 let (start, end) = match tt.delimiter() {
Alex Crichton44bffbc2017-05-19 17:51:59 -070087 Delimiter::Parenthesis => ("(", ")"),
88 Delimiter::Brace => ("{", "}"),
89 Delimiter::Bracket => ("[", "]"),
90 Delimiter::None => ("", ""),
91 };
Alex Crichton30a4e9e2018-04-27 17:02:19 -070092 if tt.stream().into_iter().next().is_none() {
Alex Crichton852d53d2017-05-19 19:25:08 -070093 write!(f, "{} {}", start, end)?
94 } else {
Alex Crichtonaf5bad42018-03-27 14:45:10 -070095 write!(f, "{} {} {}", start, tt.stream(), end)?
Alex Crichton852d53d2017-05-19 19:25:08 -070096 }
Alex Crichton44bffbc2017-05-19 17:51:59 -070097 }
Alex Crichtonf3888432018-05-16 09:11:05 -070098 TokenTree::Ident(ref tt) => write!(f, "{}", tt)?,
99 TokenTree::Punct(ref tt) => {
100 write!(f, "{}", tt.as_char())?;
Alex Crichtonaf5bad42018-03-27 14:45:10 -0700101 match tt.spacing() {
Alex Crichton1a7f7622017-07-05 17:47:15 -0700102 Spacing::Alone => {}
103 Spacing::Joint => joint = true,
Alex Crichton44bffbc2017-05-19 17:51:59 -0700104 }
105 }
Alex Crichtonb2c94622018-04-04 07:36:41 -0700106 TokenTree::Literal(ref tt) => write!(f, "{}", tt)?,
Alex Crichton44bffbc2017-05-19 17:51:59 -0700107 }
108 }
109
110 Ok(())
111 }
112}
113
David Tolnay034205f2018-04-22 16:45:28 -0700114impl fmt::Debug for TokenStream {
115 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
116 f.write_str("TokenStream ")?;
117 f.debug_list().entries(self.clone()).finish()
118 }
119}
120
Alex Crichton0e8e7f42018-02-22 06:15:13 -0800121#[cfg(feature = "proc-macro")]
122impl From<::proc_macro::TokenStream> for TokenStream {
123 fn from(inner: ::proc_macro::TokenStream) -> TokenStream {
David Tolnayb28f38a2018-03-31 22:02:29 +0200124 inner
125 .to_string()
126 .parse()
127 .expect("compiler token stream parse failed")
Alex Crichton44bffbc2017-05-19 17:51:59 -0700128 }
129}
130
Alex Crichton0e8e7f42018-02-22 06:15:13 -0800131#[cfg(feature = "proc-macro")]
132impl From<TokenStream> for ::proc_macro::TokenStream {
133 fn from(inner: TokenStream) -> ::proc_macro::TokenStream {
David Tolnayb28f38a2018-03-31 22:02:29 +0200134 inner
135 .to_string()
136 .parse()
137 .expect("failed to parse to compiler tokens")
Alex Crichton44bffbc2017-05-19 17:51:59 -0700138 }
139}
140
Alex Crichton44bffbc2017-05-19 17:51:59 -0700141impl From<TokenTree> for TokenStream {
142 fn from(tree: TokenTree) -> TokenStream {
143 TokenStream { inner: vec![tree] }
144 }
145}
146
Alex Crichtonaf5bad42018-03-27 14:45:10 -0700147impl iter::FromIterator<TokenTree> for TokenStream {
David Tolnayb28f38a2018-03-31 22:02:29 +0200148 fn from_iter<I: IntoIterator<Item = TokenTree>>(streams: I) -> Self {
Alex Crichton44bffbc2017-05-19 17:51:59 -0700149 let mut v = Vec::new();
150
Alex Crichtonaf5bad42018-03-27 14:45:10 -0700151 for token in streams.into_iter() {
152 v.push(token);
Alex Crichton44bffbc2017-05-19 17:51:59 -0700153 }
154
155 TokenStream { inner: v }
156 }
157}
158
Alex Crichtonf3888432018-05-16 09:11:05 -0700159impl Extend<TokenTree> for TokenStream {
160 fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, streams: I) {
161 self.inner.extend(streams);
162 }
163}
164
Alex Crichton1a7f7622017-07-05 17:47:15 -0700165pub type TokenTreeIter = vec::IntoIter<TokenTree>;
Alex Crichton44bffbc2017-05-19 17:51:59 -0700166
167impl IntoIterator for TokenStream {
168 type Item = TokenTree;
Alex Crichton1a7f7622017-07-05 17:47:15 -0700169 type IntoIter = TokenTreeIter;
Alex Crichton44bffbc2017-05-19 17:51:59 -0700170
Alex Crichton1a7f7622017-07-05 17:47:15 -0700171 fn into_iter(self) -> TokenTreeIter {
Alex Crichton44bffbc2017-05-19 17:51:59 -0700172 self.inner.into_iter()
173 }
174}
175
Nika Layzellb35a9a32017-12-30 14:34:35 -0500176#[derive(Clone, PartialEq, Eq, Debug)]
177pub struct FileName(String);
178
Alex Crichtonf3888432018-05-16 09:11:05 -0700179#[allow(dead_code)]
Alex Crichton30a4e9e2018-04-27 17:02:19 -0700180pub fn file_name(s: String) -> FileName {
181 FileName(s)
182}
183
Nika Layzellb35a9a32017-12-30 14:34:35 -0500184impl fmt::Display for FileName {
185 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
186 self.0.fmt(f)
187 }
188}
189
Nika Layzellf8d5f212017-12-11 14:07:02 -0500190#[derive(Clone, PartialEq, Eq)]
191pub struct SourceFile {
Nika Layzellb35a9a32017-12-30 14:34:35 -0500192 name: FileName,
Nika Layzellf8d5f212017-12-11 14:07:02 -0500193}
194
195impl SourceFile {
196 /// Get the path to this source file as a string.
Nika Layzellb35a9a32017-12-30 14:34:35 -0500197 pub fn path(&self) -> &FileName {
Nika Layzellf8d5f212017-12-11 14:07:02 -0500198 &self.name
199 }
200
201 pub fn is_real(&self) -> bool {
202 // XXX(nika): Support real files in the future?
203 false
204 }
205}
206
Nika Layzellb35a9a32017-12-30 14:34:35 -0500207impl AsRef<FileName> for SourceFile {
208 fn as_ref(&self) -> &FileName {
209 self.path()
Nika Layzellf8d5f212017-12-11 14:07:02 -0500210 }
211}
212
213impl fmt::Debug for SourceFile {
214 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
215 f.debug_struct("SourceFile")
Nika Layzellb35a9a32017-12-30 14:34:35 -0500216 .field("path", &self.path())
Nika Layzellf8d5f212017-12-11 14:07:02 -0500217 .field("is_real", &self.is_real())
218 .finish()
219 }
220}
221
222#[derive(Clone, Copy, Debug, PartialEq, Eq)]
223pub struct LineColumn {
224 pub line: usize,
225 pub column: usize,
226}
227
David Tolnay1ebe3972018-01-02 20:14:20 -0800228#[cfg(procmacro2_semver_exempt)]
Nika Layzellf8d5f212017-12-11 14:07:02 -0500229thread_local! {
230 static CODEMAP: RefCell<Codemap> = RefCell::new(Codemap {
231 // NOTE: We start with a single dummy file which all call_site() and
232 // def_site() spans reference.
233 files: vec![FileInfo {
234 name: "<unspecified>".to_owned(),
235 span: Span { lo: 0, hi: 0 },
236 lines: vec![0],
237 }],
238 });
239}
240
David Tolnay1ebe3972018-01-02 20:14:20 -0800241#[cfg(procmacro2_semver_exempt)]
Nika Layzellf8d5f212017-12-11 14:07:02 -0500242struct FileInfo {
243 name: String,
244 span: Span,
245 lines: Vec<usize>,
246}
247
David Tolnay1ebe3972018-01-02 20:14:20 -0800248#[cfg(procmacro2_semver_exempt)]
Nika Layzellf8d5f212017-12-11 14:07:02 -0500249impl FileInfo {
250 fn offset_line_column(&self, offset: usize) -> LineColumn {
David Tolnayb28f38a2018-03-31 22:02:29 +0200251 assert!(self.span_within(Span {
252 lo: offset as u32,
253 hi: offset as u32
254 }));
Nika Layzellf8d5f212017-12-11 14:07:02 -0500255 let offset = offset - self.span.lo as usize;
256 match self.lines.binary_search(&offset) {
257 Ok(found) => LineColumn {
258 line: found + 1,
David Tolnayb28f38a2018-03-31 22:02:29 +0200259 column: 0,
Nika Layzellf8d5f212017-12-11 14:07:02 -0500260 },
261 Err(idx) => LineColumn {
262 line: idx,
David Tolnayb28f38a2018-03-31 22:02:29 +0200263 column: offset - self.lines[idx - 1],
Nika Layzellf8d5f212017-12-11 14:07:02 -0500264 },
265 }
266 }
267
268 fn span_within(&self, span: Span) -> bool {
269 span.lo >= self.span.lo && span.hi <= self.span.hi
270 }
271}
272
Alex Crichtona914a612018-04-04 07:48:44 -0700273/// Computesthe offsets of each line in the given source string.
David Tolnay1ebe3972018-01-02 20:14:20 -0800274#[cfg(procmacro2_semver_exempt)]
Nika Layzella0a7c3d2017-12-30 14:52:39 -0500275fn lines_offsets(s: &str) -> Vec<usize> {
Nika Layzellf8d5f212017-12-11 14:07:02 -0500276 let mut lines = vec![0];
277 let mut prev = 0;
Nika Layzella0a7c3d2017-12-30 14:52:39 -0500278 while let Some(len) = s[prev..].find('\n') {
Nika Layzellf8d5f212017-12-11 14:07:02 -0500279 prev += len + 1;
280 lines.push(prev);
281 }
282 lines
283}
284
David Tolnay1ebe3972018-01-02 20:14:20 -0800285#[cfg(procmacro2_semver_exempt)]
Nika Layzellf8d5f212017-12-11 14:07:02 -0500286struct Codemap {
287 files: Vec<FileInfo>,
288}
289
David Tolnay1ebe3972018-01-02 20:14:20 -0800290#[cfg(procmacro2_semver_exempt)]
Nika Layzellf8d5f212017-12-11 14:07:02 -0500291impl Codemap {
292 fn next_start_pos(&self) -> u32 {
293 // Add 1 so there's always space between files.
294 //
295 // We'll always have at least 1 file, as we initialize our files list
296 // with a dummy file.
297 self.files.last().unwrap().span.hi + 1
298 }
299
300 fn add_file(&mut self, name: &str, src: &str) -> Span {
Nika Layzella0a7c3d2017-12-30 14:52:39 -0500301 let lines = lines_offsets(src);
Nika Layzellf8d5f212017-12-11 14:07:02 -0500302 let lo = self.next_start_pos();
303 // XXX(nika): Shouild we bother doing a checked cast or checked add here?
David Tolnayb28f38a2018-03-31 22:02:29 +0200304 let span = Span {
305 lo: lo,
306 hi: lo + (src.len() as u32),
307 };
Nika Layzellf8d5f212017-12-11 14:07:02 -0500308
309 self.files.push(FileInfo {
310 name: name.to_owned(),
311 span: span,
312 lines: lines,
313 });
314
315 span
316 }
317
318 fn fileinfo(&self, span: Span) -> &FileInfo {
319 for file in &self.files {
320 if file.span_within(span) {
321 return file;
322 }
323 }
324 panic!("Invalid span with no related FileInfo!");
325 }
326}
327
David Tolnay034205f2018-04-22 16:45:28 -0700328#[derive(Clone, Copy, PartialEq, Eq)]
David Tolnayddfca052017-12-31 10:41:24 -0500329pub struct Span {
David Tolnay1ebe3972018-01-02 20:14:20 -0800330 #[cfg(procmacro2_semver_exempt)]
David Tolnayddfca052017-12-31 10:41:24 -0500331 lo: u32,
David Tolnay1ebe3972018-01-02 20:14:20 -0800332 #[cfg(procmacro2_semver_exempt)]
David Tolnayddfca052017-12-31 10:41:24 -0500333 hi: u32,
334}
Alex Crichton44bffbc2017-05-19 17:51:59 -0700335
336impl Span {
David Tolnay1ebe3972018-01-02 20:14:20 -0800337 #[cfg(not(procmacro2_semver_exempt))]
Alex Crichton44bffbc2017-05-19 17:51:59 -0700338 pub fn call_site() -> Span {
David Tolnay79105e52017-12-31 11:03:04 -0500339 Span {}
340 }
341
David Tolnay1ebe3972018-01-02 20:14:20 -0800342 #[cfg(procmacro2_semver_exempt)]
David Tolnay79105e52017-12-31 11:03:04 -0500343 pub fn call_site() -> Span {
344 Span { lo: 0, hi: 0 }
Alex Crichton44bffbc2017-05-19 17:51:59 -0700345 }
Alex Crichtone6085b72017-11-21 07:24:25 -0800346
347 pub fn def_site() -> Span {
David Tolnay79105e52017-12-31 11:03:04 -0500348 Span::call_site()
Nika Layzellf8d5f212017-12-11 14:07:02 -0500349 }
350
David Tolnay4e8e3972018-01-05 18:10:22 -0800351 pub fn resolved_at(&self, _other: Span) -> Span {
352 // Stable spans consist only of line/column information, so
353 // `resolved_at` and `located_at` only select which span the
354 // caller wants line/column information from.
355 *self
356 }
357
358 pub fn located_at(&self, other: Span) -> Span {
359 other
360 }
361
David Tolnay1ebe3972018-01-02 20:14:20 -0800362 #[cfg(procmacro2_semver_exempt)]
Nika Layzellf8d5f212017-12-11 14:07:02 -0500363 pub fn source_file(&self) -> SourceFile {
364 CODEMAP.with(|cm| {
365 let cm = cm.borrow();
366 let fi = cm.fileinfo(*self);
367 SourceFile {
Nika Layzellb35a9a32017-12-30 14:34:35 -0500368 name: FileName(fi.name.clone()),
Nika Layzellf8d5f212017-12-11 14:07:02 -0500369 }
370 })
371 }
372
David Tolnay1ebe3972018-01-02 20:14:20 -0800373 #[cfg(procmacro2_semver_exempt)]
Nika Layzellf8d5f212017-12-11 14:07:02 -0500374 pub fn start(&self) -> LineColumn {
375 CODEMAP.with(|cm| {
376 let cm = cm.borrow();
377 let fi = cm.fileinfo(*self);
378 fi.offset_line_column(self.lo as usize)
379 })
380 }
381
David Tolnay1ebe3972018-01-02 20:14:20 -0800382 #[cfg(procmacro2_semver_exempt)]
Nika Layzellf8d5f212017-12-11 14:07:02 -0500383 pub fn end(&self) -> LineColumn {
384 CODEMAP.with(|cm| {
385 let cm = cm.borrow();
386 let fi = cm.fileinfo(*self);
387 fi.offset_line_column(self.hi as usize)
388 })
389 }
390
David Tolnay1ebe3972018-01-02 20:14:20 -0800391 #[cfg(procmacro2_semver_exempt)]
Nika Layzellf8d5f212017-12-11 14:07:02 -0500392 pub fn join(&self, other: Span) -> Option<Span> {
393 CODEMAP.with(|cm| {
394 let cm = cm.borrow();
395 // If `other` is not within the same FileInfo as us, return None.
396 if !cm.fileinfo(*self).span_within(other) {
397 return None;
398 }
399 Some(Span {
400 lo: cmp::min(self.lo, other.lo),
401 hi: cmp::max(self.hi, other.hi),
402 })
403 })
Alex Crichtone6085b72017-11-21 07:24:25 -0800404 }
Alex Crichton44bffbc2017-05-19 17:51:59 -0700405}
406
David Tolnay034205f2018-04-22 16:45:28 -0700407impl fmt::Debug for Span {
408 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
409 #[cfg(procmacro2_semver_exempt)]
410 return write!(f, "bytes({}..{})", self.lo, self.hi);
411
412 #[cfg(not(procmacro2_semver_exempt))]
413 write!(f, "Span")
414 }
415}
416
Alex Crichtonf3888432018-05-16 09:11:05 -0700417#[derive(Clone)]
418pub struct Ident {
David Tolnay041bcd42017-06-03 09:18:04 -0700419 intern: usize,
Alex Crichtonb2c94622018-04-04 07:36:41 -0700420 span: Span,
Alex Crichtonf3888432018-05-16 09:11:05 -0700421 raw: bool,
David Tolnay041bcd42017-06-03 09:18:04 -0700422}
Alex Crichton44bffbc2017-05-19 17:51:59 -0700423
424thread_local!(static SYMBOLS: RefCell<Interner> = RefCell::new(Interner::new()));
425
Alex Crichtonf3888432018-05-16 09:11:05 -0700426impl Ident {
427 fn _new(string: &str, raw: bool, span: Span) -> Ident {
David Tolnay489c6422018-04-07 08:37:28 -0700428 validate_term(string);
429
Alex Crichtonf3888432018-05-16 09:11:05 -0700430 Ident {
David Tolnay041bcd42017-06-03 09:18:04 -0700431 intern: SYMBOLS.with(|s| s.borrow_mut().intern(string)),
Alex Crichtona914a612018-04-04 07:48:44 -0700432 span: span,
Alex Crichtonf3888432018-05-16 09:11:05 -0700433 raw: raw,
David Tolnay041bcd42017-06-03 09:18:04 -0700434 }
Alex Crichton44bffbc2017-05-19 17:51:59 -0700435 }
Alex Crichton44bffbc2017-05-19 17:51:59 -0700436
Alex Crichtonf3888432018-05-16 09:11:05 -0700437 pub fn new(string: &str, span: Span) -> Ident {
438 Ident::_new(string, false, span)
439 }
440
441 pub fn new_raw(string: &str, span: Span) -> Ident {
442 Ident::_new(string, true, span)
443 }
444
David Tolnay10effeb2018-01-06 11:07:49 -0800445 pub fn as_str(&self) -> &str {
Alex Crichton44bffbc2017-05-19 17:51:59 -0700446 SYMBOLS.with(|interner| {
447 let interner = interner.borrow();
David Tolnay041bcd42017-06-03 09:18:04 -0700448 let s = interner.get(self.intern);
David Tolnayb28f38a2018-03-31 22:02:29 +0200449 unsafe { &*(s as *const str) }
Alex Crichton44bffbc2017-05-19 17:51:59 -0700450 })
451 }
Alex Crichtonb2c94622018-04-04 07:36:41 -0700452
453 pub fn span(&self) -> Span {
454 self.span
455 }
456
457 pub fn set_span(&mut self, span: Span) {
458 self.span = span;
459 }
Alex Crichton44bffbc2017-05-19 17:51:59 -0700460}
461
David Tolnay489c6422018-04-07 08:37:28 -0700462fn validate_term(string: &str) {
Alex Crichtonf3888432018-05-16 09:11:05 -0700463 let validate = string;
David Tolnay489c6422018-04-07 08:37:28 -0700464 if validate.is_empty() {
Alex Crichtonf3888432018-05-16 09:11:05 -0700465 panic!("Ident is not allowed to be empty; use Option<Ident>");
David Tolnay489c6422018-04-07 08:37:28 -0700466 }
467
468 if validate.bytes().all(|digit| digit >= b'0' && digit <= b'9') {
Alex Crichtonf3888432018-05-16 09:11:05 -0700469 panic!("Ident cannot be a number; use Literal instead");
David Tolnay489c6422018-04-07 08:37:28 -0700470 }
471
472 fn xid_ok(string: &str) -> bool {
473 let mut chars = string.chars();
474 let first = chars.next().unwrap();
475 if !(UnicodeXID::is_xid_start(first) || first == '_') {
476 return false;
477 }
478 for ch in chars {
479 if !UnicodeXID::is_xid_continue(ch) {
480 return false;
481 }
482 }
483 true
484 }
485
486 if !xid_ok(validate) {
Alex Crichtonf3888432018-05-16 09:11:05 -0700487 panic!("{:?} is not a valid Ident", string);
David Tolnay489c6422018-04-07 08:37:28 -0700488 }
489}
490
Alex Crichtonf3888432018-05-16 09:11:05 -0700491impl fmt::Display for Ident {
David Tolnay8ad3e3e2017-06-03 16:45:00 -0700492 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
Alex Crichtonf3888432018-05-16 09:11:05 -0700493 if self.raw {
494 "r#".fmt(f)?;
495 }
496 self.as_str().fmt(f)
497 }
498}
499
500impl fmt::Debug for Ident {
501 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
502 let mut debug = f.debug_struct("Ident");
David Tolnay034205f2018-04-22 16:45:28 -0700503 debug.field("sym", &format_args!("{}", self.as_str()));
504 #[cfg(procmacro2_semver_exempt)]
505 debug.field("span", &self.span);
Alex Crichtonf3888432018-05-16 09:11:05 -0700506 debug.field("raw", &self.raw);
David Tolnay034205f2018-04-22 16:45:28 -0700507 debug.finish()
David Tolnay8ad3e3e2017-06-03 16:45:00 -0700508 }
509}
510
Alex Crichton44bffbc2017-05-19 17:51:59 -0700511struct Interner {
512 string_to_index: HashMap<MyRc, usize>,
513 index_to_string: Vec<Rc<String>>,
514}
515
516#[derive(Hash, Eq, PartialEq)]
517struct MyRc(Rc<String>);
518
519impl Borrow<str> for MyRc {
520 fn borrow(&self) -> &str {
521 &self.0
522 }
523}
524
525impl Interner {
526 fn new() -> Interner {
527 Interner {
528 string_to_index: HashMap::new(),
529 index_to_string: Vec::new(),
530 }
531 }
532
David Tolnayb28f38a2018-03-31 22:02:29 +0200533 fn intern(&mut self, s: &str) -> usize {
Alex Crichton44bffbc2017-05-19 17:51:59 -0700534 if let Some(&idx) = self.string_to_index.get(s) {
David Tolnayb28f38a2018-03-31 22:02:29 +0200535 return idx;
Alex Crichton44bffbc2017-05-19 17:51:59 -0700536 }
537 let s = Rc::new(s.to_string());
538 self.index_to_string.push(s.clone());
David Tolnayb28f38a2018-03-31 22:02:29 +0200539 self.string_to_index
540 .insert(MyRc(s), self.index_to_string.len() - 1);
Alex Crichton44bffbc2017-05-19 17:51:59 -0700541 self.index_to_string.len() - 1
542 }
543
David Tolnayb28f38a2018-03-31 22:02:29 +0200544 fn get(&self, idx: usize) -> &str {
545 &self.index_to_string[idx]
546 }
Alex Crichton44bffbc2017-05-19 17:51:59 -0700547}
548
David Tolnay034205f2018-04-22 16:45:28 -0700549#[derive(Clone)]
Alex Crichtonb2c94622018-04-04 07:36:41 -0700550pub struct Literal {
551 text: String,
552 span: Span,
553}
Alex Crichton44bffbc2017-05-19 17:51:59 -0700554
Alex Crichtona914a612018-04-04 07:48:44 -0700555macro_rules! suffixed_numbers {
556 ($($name:ident => $kind:ident,)*) => ($(
557 pub fn $name(n: $kind) -> Literal {
558 Literal::_new(format!(concat!("{}", stringify!($kind)), n))
559 }
560 )*)
561}
562
563macro_rules! unsuffixed_numbers {
564 ($($name:ident => $kind:ident,)*) => ($(
565 pub fn $name(n: $kind) -> Literal {
566 Literal::_new(n.to_string())
567 }
568 )*)
569}
570
Alex Crichton852d53d2017-05-19 19:25:08 -0700571impl Literal {
Alex Crichtonb2c94622018-04-04 07:36:41 -0700572 fn _new(text: String) -> Literal {
573 Literal {
Alex Crichtona914a612018-04-04 07:48:44 -0700574 text: text,
Alex Crichtonb2c94622018-04-04 07:36:41 -0700575 span: Span::call_site(),
576 }
577 }
578
Alex Crichtona914a612018-04-04 07:48:44 -0700579 suffixed_numbers! {
580 u8_suffixed => u8,
581 u16_suffixed => u16,
582 u32_suffixed => u32,
583 u64_suffixed => u64,
584 usize_suffixed => usize,
585 i8_suffixed => i8,
586 i16_suffixed => i16,
587 i32_suffixed => i32,
588 i64_suffixed => i64,
589 isize_suffixed => isize,
590
591 f32_suffixed => f32,
592 f64_suffixed => f64,
593 }
594
595 unsuffixed_numbers! {
596 u8_unsuffixed => u8,
597 u16_unsuffixed => u16,
598 u32_unsuffixed => u32,
599 u64_unsuffixed => u64,
600 usize_unsuffixed => usize,
601 i8_unsuffixed => i8,
602 i16_unsuffixed => i16,
603 i32_unsuffixed => i32,
604 i64_unsuffixed => i64,
605 isize_unsuffixed => isize,
606 }
607
608 pub fn f32_unsuffixed(f: f32) -> Literal {
609 let mut s = f.to_string();
610 if !s.contains(".") {
611 s.push_str(".0");
Alex Crichton76a5cc82017-05-23 07:01:44 -0700612 }
Alex Crichtona914a612018-04-04 07:48:44 -0700613 Literal::_new(s)
614 }
615
616 pub fn f64_unsuffixed(f: f64) -> Literal {
617 let mut s = f.to_string();
618 if !s.contains(".") {
619 s.push_str(".0");
620 }
621 Literal::_new(s)
622 }
623
624 pub fn string(t: &str) -> Literal {
625 let mut s = t.chars()
626 .flat_map(|c| c.escape_default())
627 .collect::<String>();
628 s.push('"');
629 s.insert(0, '"');
630 Literal::_new(s)
631 }
632
633 pub fn character(t: char) -> Literal {
634 Literal::_new(format!("'{}'", t.escape_default().collect::<String>()))
Alex Crichton76a5cc82017-05-23 07:01:44 -0700635 }
636
Alex Crichton9c2fb0a2017-05-26 08:49:31 -0700637 pub fn byte_string(bytes: &[u8]) -> Literal {
Alex Crichton852d53d2017-05-19 19:25:08 -0700638 let mut escaped = "b\"".to_string();
639 for b in bytes {
640 match *b {
641 b'\0' => escaped.push_str(r"\0"),
642 b'\t' => escaped.push_str(r"\t"),
643 b'\n' => escaped.push_str(r"\n"),
644 b'\r' => escaped.push_str(r"\r"),
645 b'"' => escaped.push_str("\\\""),
646 b'\\' => escaped.push_str("\\\\"),
David Tolnayb28f38a2018-03-31 22:02:29 +0200647 b'\x20'...b'\x7E' => escaped.push(*b as char),
Alex Crichton852d53d2017-05-19 19:25:08 -0700648 _ => escaped.push_str(&format!("\\x{:02X}", b)),
649 }
650 }
651 escaped.push('"');
Alex Crichtonb2c94622018-04-04 07:36:41 -0700652 Literal::_new(escaped)
Alex Crichton76a5cc82017-05-23 07:01:44 -0700653 }
Alex Crichton9c2fb0a2017-05-26 08:49:31 -0700654
Alex Crichtonb2c94622018-04-04 07:36:41 -0700655 pub fn span(&self) -> Span {
656 self.span
Alex Crichton31316622017-05-26 12:54:47 -0700657 }
658
Alex Crichtonb2c94622018-04-04 07:36:41 -0700659 pub fn set_span(&mut self, span: Span) {
660 self.span = span;
Alex Crichton31316622017-05-26 12:54:47 -0700661 }
Alex Crichton852d53d2017-05-19 19:25:08 -0700662}
663
Alex Crichton44bffbc2017-05-19 17:51:59 -0700664impl fmt::Display for Literal {
665 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
Alex Crichtonb2c94622018-04-04 07:36:41 -0700666 self.text.fmt(f)
Alex Crichton44bffbc2017-05-19 17:51:59 -0700667 }
668}
669
David Tolnay034205f2018-04-22 16:45:28 -0700670impl fmt::Debug for Literal {
671 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
672 let mut debug = fmt.debug_struct("Literal");
673 debug.field("lit", &format_args!("{}", self.text));
674 #[cfg(procmacro2_semver_exempt)]
675 debug.field("span", &self.span);
676 debug.finish()
677 }
678}
679
Alex Crichton30a4e9e2018-04-27 17:02:19 -0700680fn token_stream(mut input: Cursor) -> PResult<TokenStream> {
Alex Crichton1eb96a02018-04-04 13:07:35 -0700681 let mut trees = Vec::new();
682 loop {
683 let input_no_ws = skip_whitespace(input);
684 if input_no_ws.rest.len() == 0 {
David Tolnay48ea5042018-04-23 19:17:35 -0700685 break;
Alex Crichton1eb96a02018-04-04 13:07:35 -0700686 }
687 if let Ok((a, tokens)) = doc_comment(input_no_ws) {
688 input = a;
689 trees.extend(tokens);
David Tolnay48ea5042018-04-23 19:17:35 -0700690 continue;
Alex Crichton1eb96a02018-04-04 13:07:35 -0700691 }
692
693 let (a, tt) = match token_tree(input_no_ws) {
694 Ok(p) => p,
695 Err(_) => break,
696 };
697 trees.push(tt);
698 input = a;
699 }
Alex Crichton30a4e9e2018-04-27 17:02:19 -0700700 Ok((input, TokenStream { inner: trees }))
Alex Crichton1eb96a02018-04-04 13:07:35 -0700701}
Alex Crichton44bffbc2017-05-19 17:51:59 -0700702
David Tolnay1ebe3972018-01-02 20:14:20 -0800703#[cfg(not(procmacro2_semver_exempt))]
Alex Crichton1eb96a02018-04-04 13:07:35 -0700704fn spanned<'a, T>(
705 input: Cursor<'a>,
706 f: fn(Cursor<'a>) -> PResult<'a, T>,
707) -> PResult<'a, (T, ::Span)> {
708 let (a, b) = f(skip_whitespace(input))?;
Alex Crichton30a4e9e2018-04-27 17:02:19 -0700709 Ok((a, ((b, ::Span::_new_stable(Span {})))))
David Tolnayddfca052017-12-31 10:41:24 -0500710}
711
David Tolnay1ebe3972018-01-02 20:14:20 -0800712#[cfg(procmacro2_semver_exempt)]
Alex Crichton1eb96a02018-04-04 13:07:35 -0700713fn spanned<'a, T>(
714 input: Cursor<'a>,
715 f: fn(Cursor<'a>) -> PResult<'a, T>,
716) -> PResult<'a, (T, ::Span)> {
Nika Layzellf8d5f212017-12-11 14:07:02 -0500717 let input = skip_whitespace(input);
718 let lo = input.off;
Alex Crichton1eb96a02018-04-04 13:07:35 -0700719 let (a, b) = f(input)?;
720 let hi = a.off;
Alex Crichton30a4e9e2018-04-27 17:02:19 -0700721 let span = ::Span::_new_stable(Span { lo: lo, hi: hi });
Alex Crichton1eb96a02018-04-04 13:07:35 -0700722 Ok((a, (b, span)))
723}
724
725fn token_tree(input: Cursor) -> PResult<TokenTree> {
726 let (rest, (mut tt, span)) = spanned(input, token_kind)?;
727 tt.set_span(span);
728 Ok((rest, tt))
Nika Layzellf8d5f212017-12-11 14:07:02 -0500729}
Alex Crichton44bffbc2017-05-19 17:51:59 -0700730
Alex Crichtonaf5bad42018-03-27 14:45:10 -0700731named!(token_kind -> TokenTree, alt!(
732 map!(group, TokenTree::Group)
Alex Crichton44bffbc2017-05-19 17:51:59 -0700733 |
Alex Crichton30a4e9e2018-04-27 17:02:19 -0700734 map!(literal, |l| TokenTree::Literal(::Literal::_new_stable(l))) // must be before symbol
Alex Crichton44bffbc2017-05-19 17:51:59 -0700735 |
Alex Crichtonf3888432018-05-16 09:11:05 -0700736 map!(op, TokenTree::Punct)
Alex Crichton44bffbc2017-05-19 17:51:59 -0700737 |
Alex Crichtonf3888432018-05-16 09:11:05 -0700738 symbol_leading_ws
Alex Crichton44bffbc2017-05-19 17:51:59 -0700739));
740
Alex Crichtonaf5bad42018-03-27 14:45:10 -0700741named!(group -> Group, alt!(
Alex Crichton44bffbc2017-05-19 17:51:59 -0700742 delimited!(
743 punct!("("),
744 token_stream,
745 punct!(")")
Alex Crichton30a4e9e2018-04-27 17:02:19 -0700746 ) => { |ts| Group::new(Delimiter::Parenthesis, ::TokenStream::_new_stable(ts)) }
Alex Crichton44bffbc2017-05-19 17:51:59 -0700747 |
748 delimited!(
749 punct!("["),
750 token_stream,
751 punct!("]")
Alex Crichton30a4e9e2018-04-27 17:02:19 -0700752 ) => { |ts| Group::new(Delimiter::Bracket, ::TokenStream::_new_stable(ts)) }
Alex Crichton44bffbc2017-05-19 17:51:59 -0700753 |
754 delimited!(
755 punct!("{"),
756 token_stream,
757 punct!("}")
Alex Crichton30a4e9e2018-04-27 17:02:19 -0700758 ) => { |ts| Group::new(Delimiter::Brace, ::TokenStream::_new_stable(ts)) }
Alex Crichton44bffbc2017-05-19 17:51:59 -0700759));
760
Alex Crichtonf3888432018-05-16 09:11:05 -0700761fn symbol_leading_ws(input: Cursor) -> PResult<TokenTree> {
762 symbol(skip_whitespace(input))
763}
Alex Crichton44bffbc2017-05-19 17:51:59 -0700764
Alex Crichtonf3888432018-05-16 09:11:05 -0700765fn symbol(input: Cursor) -> PResult<TokenTree> {
Alex Crichton44bffbc2017-05-19 17:51:59 -0700766 let mut chars = input.char_indices();
David Tolnaya202d502017-06-01 12:26:55 -0700767
Alex Crichtonf3888432018-05-16 09:11:05 -0700768 let raw = input.starts_with("r#");
David Tolnaya13d1422018-03-31 21:27:48 +0200769 if raw {
770 chars.next();
771 chars.next();
772 }
773
Alex Crichton44bffbc2017-05-19 17:51:59 -0700774 match chars.next() {
775 Some((_, ch)) if UnicodeXID::is_xid_start(ch) || ch == '_' => {}
David Tolnay1218e122017-06-01 11:13:45 -0700776 _ => return Err(LexError),
Alex Crichton44bffbc2017-05-19 17:51:59 -0700777 }
778
David Tolnay214c94c2017-06-01 12:42:56 -0700779 let mut end = input.len();
Alex Crichton44bffbc2017-05-19 17:51:59 -0700780 for (i, ch) in chars {
781 if !UnicodeXID::is_xid_continue(ch) {
David Tolnay214c94c2017-06-01 12:42:56 -0700782 end = i;
783 break;
Alex Crichton44bffbc2017-05-19 17:51:59 -0700784 }
785 }
786
David Tolnaya13d1422018-03-31 21:27:48 +0200787 let a = &input.rest[..end];
Alex Crichtonf3888432018-05-16 09:11:05 -0700788 if a == "r#_" {
David Tolnay214c94c2017-06-01 12:42:56 -0700789 Err(LexError)
790 } else {
Alex Crichtonf3888432018-05-16 09:11:05 -0700791 let ident = if raw {
792 ::Ident::_new_raw(&a[2..], ::Span::call_site())
793 } else {
794 ::Ident::new(a, ::Span::call_site())
795 };
796 Ok((input.advance(end), ident.into()))
David Tolnay214c94c2017-06-01 12:42:56 -0700797 }
Alex Crichton44bffbc2017-05-19 17:51:59 -0700798}
799
Alex Crichton30a4e9e2018-04-27 17:02:19 -0700800fn literal(input: Cursor) -> PResult<Literal> {
Alex Crichton44bffbc2017-05-19 17:51:59 -0700801 let input_no_ws = skip_whitespace(input);
802
803 match literal_nocapture(input_no_ws) {
David Tolnay1218e122017-06-01 11:13:45 -0700804 Ok((a, ())) => {
Alex Crichton44bffbc2017-05-19 17:51:59 -0700805 let start = input.len() - input_no_ws.len();
806 let len = input_no_ws.len() - a.len();
807 let end = start + len;
David Tolnayb28f38a2018-03-31 22:02:29 +0200808 Ok((
809 a,
Alex Crichton30a4e9e2018-04-27 17:02:19 -0700810 Literal::_new(input.rest[start..end].to_string()),
David Tolnayb28f38a2018-03-31 22:02:29 +0200811 ))
Alex Crichton44bffbc2017-05-19 17:51:59 -0700812 }
David Tolnay1218e122017-06-01 11:13:45 -0700813 Err(LexError) => Err(LexError),
Alex Crichton44bffbc2017-05-19 17:51:59 -0700814 }
815}
816
817named!(literal_nocapture -> (), alt!(
818 string
819 |
820 byte_string
821 |
822 byte
823 |
824 character
825 |
826 float
827 |
828 int
Alex Crichton44bffbc2017-05-19 17:51:59 -0700829));
830
831named!(string -> (), alt!(
832 quoted_string
833 |
834 preceded!(
835 punct!("r"),
836 raw_string
837 ) => { |_| () }
838));
839
840named!(quoted_string -> (), delimited!(
841 punct!("\""),
842 cooked_string,
843 tag!("\"")
844));
845
Nika Layzellf8d5f212017-12-11 14:07:02 -0500846fn cooked_string(input: Cursor) -> PResult<()> {
Alex Crichton44bffbc2017-05-19 17:51:59 -0700847 let mut chars = input.char_indices().peekable();
848 while let Some((byte_offset, ch)) = chars.next() {
849 match ch {
850 '"' => {
Nika Layzellf8d5f212017-12-11 14:07:02 -0500851 return Ok((input.advance(byte_offset), ()));
Alex Crichton44bffbc2017-05-19 17:51:59 -0700852 }
853 '\r' => {
854 if let Some((_, '\n')) = chars.next() {
855 // ...
856 } else {
857 break;
858 }
859 }
David Tolnayb28f38a2018-03-31 22:02:29 +0200860 '\\' => match chars.next() {
861 Some((_, 'x')) => {
862 if !backslash_x_char(&mut chars) {
863 break;
Alex Crichton44bffbc2017-05-19 17:51:59 -0700864 }
Alex Crichton44bffbc2017-05-19 17:51:59 -0700865 }
David Tolnayb28f38a2018-03-31 22:02:29 +0200866 Some((_, 'n')) | Some((_, 'r')) | Some((_, 't')) | Some((_, '\\'))
867 | Some((_, '\'')) | Some((_, '"')) | Some((_, '0')) => {}
868 Some((_, 'u')) => {
869 if !backslash_u(&mut chars) {
870 break;
871 }
872 }
873 Some((_, '\n')) | Some((_, '\r')) => {
874 while let Some(&(_, ch)) = chars.peek() {
875 if ch.is_whitespace() {
876 chars.next();
877 } else {
878 break;
879 }
880 }
881 }
882 _ => break,
883 },
Alex Crichton44bffbc2017-05-19 17:51:59 -0700884 _ch => {}
885 }
886 }
David Tolnay1218e122017-06-01 11:13:45 -0700887 Err(LexError)
Alex Crichton44bffbc2017-05-19 17:51:59 -0700888}
889
890named!(byte_string -> (), alt!(
891 delimited!(
892 punct!("b\""),
893 cooked_byte_string,
894 tag!("\"")
895 ) => { |_| () }
896 |
897 preceded!(
898 punct!("br"),
899 raw_string
900 ) => { |_| () }
901));
902
Nika Layzellf8d5f212017-12-11 14:07:02 -0500903fn cooked_byte_string(mut input: Cursor) -> PResult<()> {
Alex Crichton44bffbc2017-05-19 17:51:59 -0700904 let mut bytes = input.bytes().enumerate();
905 'outer: while let Some((offset, b)) = bytes.next() {
906 match b {
907 b'"' => {
Nika Layzellf8d5f212017-12-11 14:07:02 -0500908 return Ok((input.advance(offset), ()));
Alex Crichton44bffbc2017-05-19 17:51:59 -0700909 }
910 b'\r' => {
911 if let Some((_, b'\n')) = bytes.next() {
912 // ...
913 } else {
914 break;
915 }
916 }
David Tolnayb28f38a2018-03-31 22:02:29 +0200917 b'\\' => match bytes.next() {
918 Some((_, b'x')) => {
919 if !backslash_x_byte(&mut bytes) {
Alex Crichton44bffbc2017-05-19 17:51:59 -0700920 break;
921 }
Alex Crichton44bffbc2017-05-19 17:51:59 -0700922 }
David Tolnayb28f38a2018-03-31 22:02:29 +0200923 Some((_, b'n')) | Some((_, b'r')) | Some((_, b't')) | Some((_, b'\\'))
924 | Some((_, b'0')) | Some((_, b'\'')) | Some((_, b'"')) => {}
925 Some((newline, b'\n')) | Some((newline, b'\r')) => {
926 let rest = input.advance(newline + 1);
927 for (offset, ch) in rest.char_indices() {
928 if !ch.is_whitespace() {
929 input = rest.advance(offset);
930 bytes = input.bytes().enumerate();
931 continue 'outer;
932 }
933 }
934 break;
935 }
936 _ => break,
937 },
Alex Crichton44bffbc2017-05-19 17:51:59 -0700938 b if b < 0x80 => {}
939 _ => break,
940 }
941 }
David Tolnay1218e122017-06-01 11:13:45 -0700942 Err(LexError)
Alex Crichton44bffbc2017-05-19 17:51:59 -0700943}
944
Nika Layzellf8d5f212017-12-11 14:07:02 -0500945fn raw_string(input: Cursor) -> PResult<()> {
Alex Crichton44bffbc2017-05-19 17:51:59 -0700946 let mut chars = input.char_indices();
947 let mut n = 0;
948 while let Some((byte_offset, ch)) = chars.next() {
949 match ch {
950 '"' => {
951 n = byte_offset;
952 break;
953 }
954 '#' => {}
David Tolnay1218e122017-06-01 11:13:45 -0700955 _ => return Err(LexError),
Alex Crichton44bffbc2017-05-19 17:51:59 -0700956 }
957 }
958 for (byte_offset, ch) in chars {
959 match ch {
Nika Layzellf8d5f212017-12-11 14:07:02 -0500960 '"' if input.advance(byte_offset + 1).starts_with(&input.rest[..n]) => {
961 let rest = input.advance(byte_offset + 1 + n);
David Tolnayb28f38a2018-03-31 22:02:29 +0200962 return Ok((rest, ()));
Alex Crichton44bffbc2017-05-19 17:51:59 -0700963 }
964 '\r' => {}
965 _ => {}
966 }
967 }
David Tolnay1218e122017-06-01 11:13:45 -0700968 Err(LexError)
Alex Crichton44bffbc2017-05-19 17:51:59 -0700969}
970
971named!(byte -> (), do_parse!(
972 punct!("b") >>
973 tag!("'") >>
974 cooked_byte >>
975 tag!("'") >>
976 (())
977));
978
Nika Layzellf8d5f212017-12-11 14:07:02 -0500979fn cooked_byte(input: Cursor) -> PResult<()> {
Alex Crichton44bffbc2017-05-19 17:51:59 -0700980 let mut bytes = input.bytes().enumerate();
981 let ok = match bytes.next().map(|(_, b)| b) {
David Tolnayb28f38a2018-03-31 22:02:29 +0200982 Some(b'\\') => match bytes.next().map(|(_, b)| b) {
983 Some(b'x') => backslash_x_byte(&mut bytes),
984 Some(b'n') | Some(b'r') | Some(b't') | Some(b'\\') | Some(b'0') | Some(b'\'')
985 | Some(b'"') => true,
986 _ => false,
987 },
Alex Crichton44bffbc2017-05-19 17:51:59 -0700988 b => b.is_some(),
989 };
990 if ok {
991 match bytes.next() {
Alex Crichton8c030332018-01-16 08:07:36 -0800992 Some((offset, _)) => {
993 if input.chars().as_str().is_char_boundary(offset) {
994 Ok((input.advance(offset), ()))
995 } else {
996 Err(LexError)
997 }
998 }
Nika Layzellf8d5f212017-12-11 14:07:02 -0500999 None => Ok((input.advance(input.len()), ())),
Alex Crichton44bffbc2017-05-19 17:51:59 -07001000 }
1001 } else {
David Tolnay1218e122017-06-01 11:13:45 -07001002 Err(LexError)
Alex Crichton44bffbc2017-05-19 17:51:59 -07001003 }
1004}
1005
1006named!(character -> (), do_parse!(
1007 punct!("'") >>
1008 cooked_char >>
1009 tag!("'") >>
1010 (())
1011));
1012
Nika Layzellf8d5f212017-12-11 14:07:02 -05001013fn cooked_char(input: Cursor) -> PResult<()> {
Alex Crichton44bffbc2017-05-19 17:51:59 -07001014 let mut chars = input.char_indices();
1015 let ok = match chars.next().map(|(_, ch)| ch) {
David Tolnayb28f38a2018-03-31 22:02:29 +02001016 Some('\\') => match chars.next().map(|(_, ch)| ch) {
1017 Some('x') => backslash_x_char(&mut chars),
1018 Some('u') => backslash_u(&mut chars),
1019 Some('n') | Some('r') | Some('t') | Some('\\') | Some('0') | Some('\'') | Some('"') => {
1020 true
Alex Crichton44bffbc2017-05-19 17:51:59 -07001021 }
David Tolnayb28f38a2018-03-31 22:02:29 +02001022 _ => false,
1023 },
Alex Crichton44bffbc2017-05-19 17:51:59 -07001024 ch => ch.is_some(),
1025 };
1026 if ok {
Nika Layzellf8d5f212017-12-11 14:07:02 -05001027 match chars.next() {
1028 Some((idx, _)) => Ok((input.advance(idx), ())),
1029 None => Ok((input.advance(input.len()), ())),
1030 }
Alex Crichton44bffbc2017-05-19 17:51:59 -07001031 } else {
David Tolnay1218e122017-06-01 11:13:45 -07001032 Err(LexError)
Alex Crichton44bffbc2017-05-19 17:51:59 -07001033 }
1034}
1035
1036macro_rules! next_ch {
1037 ($chars:ident @ $pat:pat $(| $rest:pat)*) => {
1038 match $chars.next() {
1039 Some((_, ch)) => match ch {
1040 $pat $(| $rest)* => ch,
1041 _ => return false,
1042 },
1043 None => return false
1044 }
1045 };
1046}
1047
1048fn backslash_x_char<I>(chars: &mut I) -> bool
David Tolnayb28f38a2018-03-31 22:02:29 +02001049where
1050 I: Iterator<Item = (usize, char)>,
Alex Crichton44bffbc2017-05-19 17:51:59 -07001051{
1052 next_ch!(chars @ '0'...'7');
1053 next_ch!(chars @ '0'...'9' | 'a'...'f' | 'A'...'F');
1054 true
1055}
1056
1057fn backslash_x_byte<I>(chars: &mut I) -> bool
David Tolnayb28f38a2018-03-31 22:02:29 +02001058where
1059 I: Iterator<Item = (usize, u8)>,
Alex Crichton44bffbc2017-05-19 17:51:59 -07001060{
1061 next_ch!(chars @ b'0'...b'9' | b'a'...b'f' | b'A'...b'F');
1062 next_ch!(chars @ b'0'...b'9' | b'a'...b'f' | b'A'...b'F');
1063 true
1064}
1065
1066fn backslash_u<I>(chars: &mut I) -> bool
David Tolnayb28f38a2018-03-31 22:02:29 +02001067where
1068 I: Iterator<Item = (usize, char)>,
Alex Crichton44bffbc2017-05-19 17:51:59 -07001069{
1070 next_ch!(chars @ '{');
1071 next_ch!(chars @ '0'...'9' | 'a'...'f' | 'A'...'F');
David Tolnay8d109342017-12-25 18:24:45 -05001072 loop {
1073 let c = next_ch!(chars @ '0'...'9' | 'a'...'f' | 'A'...'F' | '_' | '}');
1074 if c == '}' {
1075 return true;
1076 }
Alex Crichton44bffbc2017-05-19 17:51:59 -07001077 }
Alex Crichton44bffbc2017-05-19 17:51:59 -07001078}
1079
Nika Layzellf8d5f212017-12-11 14:07:02 -05001080fn float(input: Cursor) -> PResult<()> {
David Tolnay744a6b82017-06-01 11:34:29 -07001081 let (rest, ()) = float_digits(input)?;
1082 for suffix in &["f32", "f64"] {
1083 if rest.starts_with(suffix) {
Nika Layzellf8d5f212017-12-11 14:07:02 -05001084 return word_break(rest.advance(suffix.len()));
David Tolnay744a6b82017-06-01 11:34:29 -07001085 }
1086 }
1087 word_break(rest)
1088}
Alex Crichton44bffbc2017-05-19 17:51:59 -07001089
Nika Layzellf8d5f212017-12-11 14:07:02 -05001090fn float_digits(input: Cursor) -> PResult<()> {
Alex Crichton44bffbc2017-05-19 17:51:59 -07001091 let mut chars = input.chars().peekable();
1092 match chars.next() {
1093 Some(ch) if ch >= '0' && ch <= '9' => {}
David Tolnay1218e122017-06-01 11:13:45 -07001094 _ => return Err(LexError),
Alex Crichton44bffbc2017-05-19 17:51:59 -07001095 }
1096
1097 let mut len = 1;
1098 let mut has_dot = false;
1099 let mut has_exp = false;
1100 while let Some(&ch) = chars.peek() {
1101 match ch {
1102 '0'...'9' | '_' => {
1103 chars.next();
1104 len += 1;
1105 }
1106 '.' => {
1107 if has_dot {
1108 break;
1109 }
1110 chars.next();
David Tolnayb28f38a2018-03-31 22:02:29 +02001111 if chars
1112 .peek()
1113 .map(|&ch| ch == '.' || UnicodeXID::is_xid_start(ch))
1114 .unwrap_or(false)
1115 {
David Tolnay1218e122017-06-01 11:13:45 -07001116 return Err(LexError);
Alex Crichton44bffbc2017-05-19 17:51:59 -07001117 }
1118 len += 1;
1119 has_dot = true;
1120 }
1121 'e' | 'E' => {
1122 chars.next();
1123 len += 1;
1124 has_exp = true;
1125 break;
1126 }
1127 _ => break,
1128 }
1129 }
1130
Nika Layzellf8d5f212017-12-11 14:07:02 -05001131 let rest = input.advance(len);
Alex Crichton44bffbc2017-05-19 17:51:59 -07001132 if !(has_dot || has_exp || rest.starts_with("f32") || rest.starts_with("f64")) {
David Tolnay1218e122017-06-01 11:13:45 -07001133 return Err(LexError);
Alex Crichton44bffbc2017-05-19 17:51:59 -07001134 }
1135
1136 if has_exp {
1137 let mut has_exp_value = false;
1138 while let Some(&ch) = chars.peek() {
1139 match ch {
1140 '+' | '-' => {
1141 if has_exp_value {
1142 break;
1143 }
1144 chars.next();
1145 len += 1;
1146 }
1147 '0'...'9' => {
1148 chars.next();
1149 len += 1;
1150 has_exp_value = true;
1151 }
1152 '_' => {
1153 chars.next();
1154 len += 1;
1155 }
1156 _ => break,
1157 }
1158 }
1159 if !has_exp_value {
David Tolnay1218e122017-06-01 11:13:45 -07001160 return Err(LexError);
Alex Crichton44bffbc2017-05-19 17:51:59 -07001161 }
1162 }
1163
Nika Layzellf8d5f212017-12-11 14:07:02 -05001164 Ok((input.advance(len), ()))
Alex Crichton44bffbc2017-05-19 17:51:59 -07001165}
1166
Nika Layzellf8d5f212017-12-11 14:07:02 -05001167fn int(input: Cursor) -> PResult<()> {
David Tolnay744a6b82017-06-01 11:34:29 -07001168 let (rest, ()) = digits(input)?;
1169 for suffix in &[
David Tolnay48ea5042018-04-23 19:17:35 -07001170 "isize", "i8", "i16", "i32", "i64", "i128", "usize", "u8", "u16", "u32", "u64", "u128",
David Tolnay744a6b82017-06-01 11:34:29 -07001171 ] {
1172 if rest.starts_with(suffix) {
Nika Layzellf8d5f212017-12-11 14:07:02 -05001173 return word_break(rest.advance(suffix.len()));
David Tolnay744a6b82017-06-01 11:34:29 -07001174 }
1175 }
1176 word_break(rest)
1177}
Alex Crichton44bffbc2017-05-19 17:51:59 -07001178
Nika Layzellf8d5f212017-12-11 14:07:02 -05001179fn digits(mut input: Cursor) -> PResult<()> {
Alex Crichton44bffbc2017-05-19 17:51:59 -07001180 let base = if input.starts_with("0x") {
Nika Layzellf8d5f212017-12-11 14:07:02 -05001181 input = input.advance(2);
Alex Crichton44bffbc2017-05-19 17:51:59 -07001182 16
1183 } else if input.starts_with("0o") {
Nika Layzellf8d5f212017-12-11 14:07:02 -05001184 input = input.advance(2);
Alex Crichton44bffbc2017-05-19 17:51:59 -07001185 8
1186 } else if input.starts_with("0b") {
Nika Layzellf8d5f212017-12-11 14:07:02 -05001187 input = input.advance(2);
Alex Crichton44bffbc2017-05-19 17:51:59 -07001188 2
1189 } else {
1190 10
1191 };
1192
Alex Crichton44bffbc2017-05-19 17:51:59 -07001193 let mut len = 0;
1194 let mut empty = true;
1195 for b in input.bytes() {
1196 let digit = match b {
1197 b'0'...b'9' => (b - b'0') as u64,
1198 b'a'...b'f' => 10 + (b - b'a') as u64,
1199 b'A'...b'F' => 10 + (b - b'A') as u64,
1200 b'_' => {
1201 if empty && base == 10 {
David Tolnay1218e122017-06-01 11:13:45 -07001202 return Err(LexError);
Alex Crichton44bffbc2017-05-19 17:51:59 -07001203 }
1204 len += 1;
1205 continue;
1206 }
1207 _ => break,
1208 };
1209 if digit >= base {
David Tolnay1218e122017-06-01 11:13:45 -07001210 return Err(LexError);
Alex Crichton44bffbc2017-05-19 17:51:59 -07001211 }
Alex Crichton44bffbc2017-05-19 17:51:59 -07001212 len += 1;
1213 empty = false;
1214 }
1215 if empty {
David Tolnay1218e122017-06-01 11:13:45 -07001216 Err(LexError)
Alex Crichton44bffbc2017-05-19 17:51:59 -07001217 } else {
Nika Layzellf8d5f212017-12-11 14:07:02 -05001218 Ok((input.advance(len), ()))
Alex Crichton44bffbc2017-05-19 17:51:59 -07001219 }
1220}
1221
Alex Crichtonf3888432018-05-16 09:11:05 -07001222fn op(input: Cursor) -> PResult<Punct> {
David Tolnayea75c5f2017-05-31 23:40:33 -07001223 let input = skip_whitespace(input);
1224 match op_char(input) {
Alex Crichtonf3888432018-05-16 09:11:05 -07001225 Ok((rest, '\'')) => {
1226 symbol(rest)?;
1227 Ok((rest, Punct::new('\'', Spacing::Joint)))
1228 }
David Tolnay1218e122017-06-01 11:13:45 -07001229 Ok((rest, ch)) => {
David Tolnayea75c5f2017-05-31 23:40:33 -07001230 let kind = match op_char(rest) {
Alex Crichton1a7f7622017-07-05 17:47:15 -07001231 Ok(_) => Spacing::Joint,
1232 Err(LexError) => Spacing::Alone,
David Tolnayea75c5f2017-05-31 23:40:33 -07001233 };
Alex Crichtonf3888432018-05-16 09:11:05 -07001234 Ok((rest, Punct::new(ch, kind)))
David Tolnayea75c5f2017-05-31 23:40:33 -07001235 }
David Tolnay1218e122017-06-01 11:13:45 -07001236 Err(LexError) => Err(LexError),
Alex Crichton44bffbc2017-05-19 17:51:59 -07001237 }
1238}
1239
Nika Layzellf8d5f212017-12-11 14:07:02 -05001240fn op_char(input: Cursor) -> PResult<char> {
David Tolnay3a592ad2018-04-22 21:20:24 -07001241 if input.starts_with("//") || input.starts_with("/*") {
1242 // Do not accept `/` of a comment as an op.
1243 return Err(LexError);
1244 }
1245
David Tolnayea75c5f2017-05-31 23:40:33 -07001246 let mut chars = input.chars();
1247 let first = match chars.next() {
1248 Some(ch) => ch,
1249 None => {
David Tolnay1218e122017-06-01 11:13:45 -07001250 return Err(LexError);
David Tolnayea75c5f2017-05-31 23:40:33 -07001251 }
1252 };
Alex Crichtonf3888432018-05-16 09:11:05 -07001253 let recognized = "~!@#$%^&*-=+|;:,<.>/?'";
David Tolnayea75c5f2017-05-31 23:40:33 -07001254 if recognized.contains(first) {
Nika Layzellf8d5f212017-12-11 14:07:02 -05001255 Ok((input.advance(first.len_utf8()), first))
Alex Crichton44bffbc2017-05-19 17:51:59 -07001256 } else {
David Tolnay1218e122017-06-01 11:13:45 -07001257 Err(LexError)
Alex Crichton44bffbc2017-05-19 17:51:59 -07001258 }
1259}
1260
Alex Crichton1eb96a02018-04-04 13:07:35 -07001261fn doc_comment(input: Cursor) -> PResult<Vec<TokenTree>> {
1262 let mut trees = Vec::new();
1263 let (rest, ((comment, inner), span)) = spanned(input, doc_comment_contents)?;
Alex Crichtonf3888432018-05-16 09:11:05 -07001264 trees.push(TokenTree::Punct(Punct::new('#', Spacing::Alone)));
Alex Crichton1eb96a02018-04-04 13:07:35 -07001265 if inner {
Alex Crichtonf3888432018-05-16 09:11:05 -07001266 trees.push(Punct::new('!', Spacing::Alone).into());
Alex Crichton1eb96a02018-04-04 13:07:35 -07001267 }
1268 let mut stream = vec![
Alex Crichtonf3888432018-05-16 09:11:05 -07001269 TokenTree::Ident(::Ident::new("doc", span)),
1270 TokenTree::Punct(Punct::new('=', Spacing::Alone)),
Alex Crichton1eb96a02018-04-04 13:07:35 -07001271 TokenTree::Literal(::Literal::string(comment)),
1272 ];
1273 for tt in stream.iter_mut() {
1274 tt.set_span(span);
1275 }
1276 trees.push(Group::new(Delimiter::Bracket, stream.into_iter().collect()).into());
1277 for tt in trees.iter_mut() {
1278 tt.set_span(span);
1279 }
1280 Ok((rest, trees))
1281}
1282
1283named!(doc_comment_contents -> (&str, bool), alt!(
Alex Crichton44bffbc2017-05-19 17:51:59 -07001284 do_parse!(
1285 punct!("//!") >>
Alex Crichton1eb96a02018-04-04 13:07:35 -07001286 s: take_until_newline_or_eof!() >>
1287 ((s, true))
Alex Crichton44bffbc2017-05-19 17:51:59 -07001288 )
1289 |
1290 do_parse!(
1291 option!(whitespace) >>
1292 peek!(tag!("/*!")) >>
Alex Crichton1eb96a02018-04-04 13:07:35 -07001293 s: block_comment >>
1294 ((s, true))
Alex Crichton44bffbc2017-05-19 17:51:59 -07001295 )
1296 |
1297 do_parse!(
1298 punct!("///") >>
1299 not!(tag!("/")) >>
Alex Crichton1eb96a02018-04-04 13:07:35 -07001300 s: take_until_newline_or_eof!() >>
1301 ((s, false))
Alex Crichton44bffbc2017-05-19 17:51:59 -07001302 )
1303 |
1304 do_parse!(
1305 option!(whitespace) >>
1306 peek!(tuple!(tag!("/**"), not!(tag!("*")))) >>
Alex Crichton1eb96a02018-04-04 13:07:35 -07001307 s: block_comment >>
1308 ((s, false))
Alex Crichton44bffbc2017-05-19 17:51:59 -07001309 )
1310));