blob: cdfc7e08a8085680bd15afc072322cae72d545d9 [file] [log] [blame]
Carl Lerche058ff472019-02-13 16:23:52 -08001use crate::types;
2
David Tolnay14d463e2019-02-15 14:23:51 -08003use indexmap::IndexMap;
Carl Lerche058ff472019-02-13 16:23:52 -08004use syn::{Data, DataStruct, DeriveInput, Ident, Item};
5
6use std::collections::BTreeMap;
7use std::fs::File;
8use std::io::Read;
9use std::path::Path;
10
11const SYN_CRATE_ROOT: &str = "../src/lib.rs";
12const TOKEN_SRC: &str = "../src/token.rs";
13const IGNORED_MODS: &[&str] = &["fold", "visit", "visit_mut"];
14const EXTRA_TYPES: &[&str] = &["Lifetime"];
Carl Lerche058ff472019-02-13 16:23:52 -080015
16// NOTE: BTreeMap is used here instead of HashMap to have deterministic output.
17type ItemLookup = BTreeMap<Ident, AstItem>;
18type TokenLookup = BTreeMap<String, String>;
19
20/// Parse the contents of `src` and return a list of AST types.
David Tolnayf9bb8ff2019-02-15 13:10:14 -080021pub fn parse() -> types::Definitions {
Carl Lerche058ff472019-02-13 16:23:52 -080022 let mut item_lookup = BTreeMap::new();
23 load_file(SYN_CRATE_ROOT, &[], &mut item_lookup).unwrap();
24
25 let token_lookup = load_token_file(TOKEN_SRC).unwrap();
26
David Tolnayf9bb8ff2019-02-15 13:10:14 -080027 let types = item_lookup
Carl Lerche058ff472019-02-13 16:23:52 -080028 .values()
29 .map(|item| introspect_item(item, &item_lookup, &token_lookup))
David Tolnayf9bb8ff2019-02-15 13:10:14 -080030 .collect();
31
David Tolnay47fe7402019-02-15 14:35:25 -080032 let tokens = token_lookup
33 .into_iter()
34 .map(|(name, ty)| (ty, name))
35 .collect();
David Tolnayf9bb8ff2019-02-15 13:10:14 -080036
37 types::Definitions { types, tokens }
Carl Lerche058ff472019-02-13 16:23:52 -080038}
39
40/// Data extracted from syn source
41#[derive(Clone)]
42pub struct AstItem {
43 ast: DeriveInput,
44 features: Vec<syn::Attribute>,
45}
46
David Tolnayf9bb8ff2019-02-15 13:10:14 -080047fn introspect_item(item: &AstItem, items: &ItemLookup, tokens: &TokenLookup) -> types::Node {
Carl Lerche058ff472019-02-13 16:23:52 -080048 let features = introspect_features(&item.features);
49
50 match &item.ast.data {
David Tolnayc2be7b22019-02-15 18:48:31 -080051 Data::Enum(ref data) => types::Node {
52 ident: item.ast.ident.to_string(),
Carl Lerche058ff472019-02-13 16:23:52 -080053 features,
David Tolnayc2be7b22019-02-15 18:48:31 -080054 data: types::Data::Enum(introspect_enum(data, items, tokens)),
55 },
56 Data::Struct(ref data) => types::Node {
57 ident: item.ast.ident.to_string(),
Carl Lerche058ff472019-02-13 16:23:52 -080058 features,
David Tolnayc2be7b22019-02-15 18:48:31 -080059 data: {
60 if data.fields.iter().all(|f| is_pub(&f.vis)) {
61 types::Data::Struct(introspect_struct(data, items, tokens))
62 } else {
63 types::Data::Private
64 }
65 },
66 },
Carl Lerche058ff472019-02-13 16:23:52 -080067 Data::Union(..) => panic!("Union not supported"),
68 }
69}
70
71fn introspect_enum(
Carl Lerche058ff472019-02-13 16:23:52 -080072 item: &syn::DataEnum,
73 items: &ItemLookup,
74 tokens: &TokenLookup,
David Tolnayc2be7b22019-02-15 18:48:31 -080075) -> Vec<types::Variant> {
76 item.variants
Carl Lerche058ff472019-02-13 16:23:52 -080077 .iter()
78 .map(|variant| {
79 let fields = match &variant.fields {
80 syn::Fields::Unnamed(fields) => fields
81 .unnamed
82 .iter()
83 .map(|field| introspect_type(&field.ty, items, tokens))
84 .collect(),
85 syn::Fields::Unit => vec![],
86 _ => panic!("Enum representation not supported"),
87 };
88
David Tolnayfa67ab02019-02-15 20:17:30 -080089 types::Variant {
90 ident: variant.ident.to_string(),
91 fields,
92 }
Carl Lerche058ff472019-02-13 16:23:52 -080093 })
David Tolnayc2be7b22019-02-15 18:48:31 -080094 .collect()
Carl Lerche058ff472019-02-13 16:23:52 -080095}
96
97fn introspect_struct(
Carl Lerche058ff472019-02-13 16:23:52 -080098 item: &syn::DataStruct,
99 items: &ItemLookup,
100 tokens: &TokenLookup,
David Tolnayc2be7b22019-02-15 18:48:31 -0800101) -> IndexMap<String, types::Type> {
102 match &item.fields {
Carl Lerche058ff472019-02-13 16:23:52 -0800103 syn::Fields::Named(fields) => fields
104 .named
105 .iter()
106 .map(|field| {
David Tolnay14d463e2019-02-15 14:23:51 -0800107 (
Carl Lerche058ff472019-02-13 16:23:52 -0800108 field.ident.as_ref().unwrap().to_string(),
109 introspect_type(&field.ty, items, tokens),
110 )
111 })
112 .collect(),
David Tolnay14d463e2019-02-15 14:23:51 -0800113 syn::Fields::Unit => IndexMap::new(),
Carl Lerche058ff472019-02-13 16:23:52 -0800114 _ => panic!("Struct representation not supported"),
David Tolnayc2be7b22019-02-15 18:48:31 -0800115 }
Carl Lerche058ff472019-02-13 16:23:52 -0800116}
117
118fn introspect_type(item: &syn::Type, items: &ItemLookup, tokens: &TokenLookup) -> types::Type {
119 match item {
120 syn::Type::Path(syn::TypePath {
121 qself: None,
122 ref path,
123 }) => {
124 let last = path.segments.last().unwrap().into_value();
125
126 match &last.ident.to_string()[..] {
127 "Option" => {
128 let nested = introspect_type(first_arg(&last.arguments), items, tokens);
129 types::Type::Option(Box::new(nested))
130 }
131 "Punctuated" => {
132 let nested = introspect_type(first_arg(&last.arguments), items, tokens);
133 let punct = match introspect_type(last_arg(&last.arguments), items, tokens) {
134 types::Type::Token(s) => s,
135 _ => panic!(),
136 };
137
David Tolnayfa67ab02019-02-15 20:17:30 -0800138 types::Type::Punctuated(types::Punctuated {
139 element: Box::new(nested),
140 punct,
141 })
Carl Lerche058ff472019-02-13 16:23:52 -0800142 }
143 "Vec" => {
144 let nested = introspect_type(first_arg(&last.arguments), items, tokens);
145 types::Type::Vec(Box::new(nested))
146 }
147 "Box" => {
148 let nested = introspect_type(first_arg(&last.arguments), items, tokens);
149 types::Type::Box(Box::new(nested))
150 }
151 "Brace" | "Bracket" | "Paren" | "Group" => {
David Tolnay295141b2019-02-15 12:45:33 -0800152 types::Type::Group(last.ident.to_string())
Carl Lerche058ff472019-02-13 16:23:52 -0800153 }
David Tolnay47fe7402019-02-15 14:35:25 -0800154 "TokenStream" | "Literal" | "Ident" | "Span" => {
155 types::Type::Ext(last.ident.to_string())
156 }
Carl Lerche058ff472019-02-13 16:23:52 -0800157 "String" | "u32" | "usize" | "bool" => types::Type::Std(last.ident.to_string()),
158 _ => {
159 if items.get(&last.ident).is_some() {
David Tolnayd3076572019-02-15 13:32:44 -0800160 types::Type::Syn(last.ident.to_string())
Carl Lerche058ff472019-02-13 16:23:52 -0800161 } else {
162 unimplemented!("{}", last.ident.to_string());
163 }
164 }
165 }
166 }
167 syn::Type::Tuple(syn::TypeTuple { ref elems, .. }) => {
168 let tys = elems
169 .iter()
170 .map(|ty| introspect_type(&ty, items, tokens))
171 .collect();
172 types::Type::Tuple(tys)
173 }
174 syn::Type::Macro(syn::TypeMacro { ref mac })
175 if mac.path.segments.last().unwrap().into_value().ident == "Token" =>
176 {
177 let content = mac.tts.to_string();
178 let ty = tokens.get(&content).unwrap().to_string();
179
David Tolnay157c7eb2019-02-15 13:21:48 -0800180 types::Type::Token(ty)
Carl Lerche058ff472019-02-13 16:23:52 -0800181 }
182 _ => panic!("{}", quote!(#item).to_string()),
183 }
184}
185
186fn introspect_features(attrs: &[syn::Attribute]) -> types::Features {
187 let mut ret = types::Features::default();
188
189 for attr in attrs {
190 if !attr.path.is_ident("cfg") {
191 continue;
192 }
193
194 let features: types::Features = syn::parse2(attr.tts.clone()).unwrap();
David Tolnay440fe582019-02-15 20:23:14 -0800195
196 if ret.any.is_empty() {
197 ret = features;
198 } else if ret.any.len() < features.any.len() {
199 assert!(ret.any.iter().all(|f| features.any.contains(f)));
200 } else {
201 assert!(features.any.iter().all(|f| ret.any.contains(f)));
202 ret = features;
203 }
Carl Lerche058ff472019-02-13 16:23:52 -0800204 }
205
206 ret
207}
208
209fn is_pub(vis: &syn::Visibility) -> bool {
210 match vis {
211 syn::Visibility::Public(_) => true,
212 _ => false,
213 }
214}
215
216fn first_arg(params: &syn::PathArguments) -> &syn::Type {
217 let data = match *params {
218 syn::PathArguments::AngleBracketed(ref data) => data,
219 _ => panic!("Expected at least 1 type argument here"),
220 };
221
222 match **data
223 .args
224 .first()
225 .expect("Expected at least 1 type argument here")
226 .value()
227 {
228 syn::GenericArgument::Type(ref ty) => ty,
229 _ => panic!("Expected at least 1 type argument here"),
230 }
231}
232
233fn last_arg(params: &syn::PathArguments) -> &syn::Type {
234 let data = match *params {
235 syn::PathArguments::AngleBracketed(ref data) => data,
236 _ => panic!("Expected at least 1 type argument here"),
237 };
238
239 match **data
240 .args
241 .last()
242 .expect("Expected at least 1 type argument here")
243 .value()
244 {
245 syn::GenericArgument::Type(ref ty) => ty,
246 _ => panic!("Expected at least 1 type argument here"),
247 }
248}
249
250mod parsing {
251 use super::{AstItem, TokenLookup};
252 use crate::types;
253
254 use proc_macro2::TokenStream;
255 use syn;
256 use syn::parse::{Parse, ParseStream, Result};
257 use syn::*;
258
David Tolnay440fe582019-02-15 20:23:14 -0800259 use std::collections::{BTreeMap, BTreeSet};
Carl Lerche058ff472019-02-13 16:23:52 -0800260
261 fn peek_tag(input: ParseStream, tag: &str) -> bool {
262 let ahead = input.fork();
263 ahead.parse::<Token![#]>().is_ok()
264 && ahead
265 .parse::<Ident>()
266 .map(|ident| ident == tag)
267 .unwrap_or(false)
268 }
269
270 // Parses #full - returns #[cfg(feature = "full")] if it is present, and
271 // nothing otherwise.
272 fn full(input: ParseStream) -> Vec<syn::Attribute> {
273 if peek_tag(input, "full") {
274 input.parse::<Token![#]>().unwrap();
275 input.parse::<Ident>().unwrap();
276 vec![parse_quote!(#[cfg(feature = "full")])]
277 } else {
278 vec![]
279 }
280 }
281
282 fn skip_manual_extra_traits(input: ParseStream) {
283 if peek_tag(input, "manual_extra_traits") {
284 input.parse::<Token![#]>().unwrap();
285 input.parse::<Ident>().unwrap();
286 }
287 }
288
289 // Parses a simple AstStruct without the `pub struct` prefix.
290 fn ast_struct_inner(input: ParseStream) -> Result<AstItem> {
291 let ident: Ident = input.parse()?;
292 let features = full(input);
293 skip_manual_extra_traits(input);
294 let rest: TokenStream = input.parse()?;
295 Ok(AstItem {
296 ast: syn::parse2(quote! {
297 pub struct #ident #rest
298 })?,
299 features,
300 })
301 }
302
303 // ast_struct! parsing
304 pub struct AstStruct(pub(super) Vec<AstItem>);
305 impl Parse for AstStruct {
306 fn parse(input: ParseStream) -> Result<Self> {
307 input.call(Attribute::parse_outer)?;
308 input.parse::<Token![pub]>()?;
309 input.parse::<Token![struct]>()?;
310 let res = input.call(ast_struct_inner)?;
311 Ok(AstStruct(vec![res]))
312 }
313 }
314
315 fn no_visit(input: ParseStream) -> bool {
316 if peek_tag(input, "no_visit") {
317 input.parse::<Token![#]>().unwrap();
318 input.parse::<Ident>().unwrap();
319 true
320 } else {
321 false
322 }
323 }
324
325 // ast_enum! parsing
326 pub struct AstEnum(pub Vec<AstItem>);
327 impl Parse for AstEnum {
328 fn parse(input: ParseStream) -> Result<Self> {
329 input.call(Attribute::parse_outer)?;
330 input.parse::<Token![pub]>()?;
331 input.parse::<Token![enum]>()?;
332 let ident: Ident = input.parse()?;
333 let no_visit = no_visit(input);
334 let rest: TokenStream = input.parse()?;
335 Ok(AstEnum(if no_visit {
336 vec![]
337 } else {
338 vec![AstItem {
339 ast: syn::parse2(quote! {
340 pub enum #ident #rest
341 })?,
342 features: vec![],
343 }]
344 }))
345 }
346 }
347
348 // A single variant of an ast_enum_of_structs!
349 struct EosVariant {
350 name: Ident,
351 member: Option<Path>,
352 inner: Option<AstItem>,
353 }
354 fn eos_variant(input: ParseStream) -> Result<EosVariant> {
355 input.call(Attribute::parse_outer)?;
356 input.parse::<Token![pub]>()?;
357 let variant: Ident = input.parse()?;
358 let (member, inner) = if input.peek(token::Paren) {
359 let content;
360 parenthesized!(content in input);
361 if content.fork().call(ast_struct_inner).is_ok() {
362 let item = content.call(ast_struct_inner)?;
363 (Some(Path::from(item.ast.ident.clone())), Some(item))
364 } else {
365 let path: Path = content.parse()?;
366 (Some(path), None)
367 }
368 } else {
369 (None, None)
370 };
371 input.parse::<Token![,]>()?;
372 Ok(EosVariant {
373 name: variant,
374 member,
375 inner,
376 })
377 }
378
379 // ast_enum_of_structs! parsing
380 pub struct AstEnumOfStructs(pub Vec<AstItem>);
381 impl Parse for AstEnumOfStructs {
382 fn parse(input: ParseStream) -> Result<Self> {
383 input.call(Attribute::parse_outer)?;
384 input.parse::<Token![pub]>()?;
385 input.parse::<Token![enum]>()?;
386 let ident: Ident = input.parse()?;
387
388 let content;
389 braced!(content in input);
390 let mut variants = Vec::new();
391 while !content.is_empty() {
392 variants.push(content.call(eos_variant)?);
393 }
394
395 if let Some(ident) = input.parse::<Option<Ident>>()? {
396 assert_eq!(ident, "do_not_generate_to_tokens");
397 }
398
399 let enum_item = {
400 let variants = variants.iter().map(|v| {
401 let name = v.name.clone();
402 match v.member {
403 Some(ref member) => quote!(#name(#member)),
404 None => quote!(#name),
405 }
406 });
407 parse_quote! {
408 pub enum #ident {
409 #(#variants),*
410 }
411 }
412 };
413 let mut items = vec![AstItem {
414 ast: enum_item,
415 features: vec![],
416 }];
417 items.extend(variants.into_iter().filter_map(|v| v.inner));
418 Ok(AstEnumOfStructs(items))
419 }
420 }
421
422 pub struct TokenMacro(pub TokenLookup);
423 impl Parse for TokenMacro {
424 fn parse(input: ParseStream) -> Result<Self> {
425 let mut tokens = BTreeMap::new();
426 while !input.is_empty() {
427 let content;
428 parenthesized!(content in input);
429 let token = content.parse::<TokenStream>()?.to_string();
430 input.parse::<Token![=]>()?;
431 input.parse::<Token![>]>()?;
432 let content;
433 braced!(content in input);
434 input.parse::<Token![;]>()?;
435 content.parse::<token::Dollar>()?;
436 let path: Path = content.parse()?;
437 let ty = path.segments.last().unwrap().into_value().ident.to_string();
438 tokens.insert(token, ty.to_string());
439 }
440 Ok(TokenMacro(tokens))
441 }
442 }
443
444 fn parse_feature(input: ParseStream) -> Result<String> {
445 let i: syn::Ident = input.parse()?;
446 assert_eq!(i, "feature");
447
448 input.parse::<Token![=]>()?;
449 let s = input.parse::<syn::LitStr>()?;
450
451 Ok(s.value())
452 }
453
454 impl Parse for types::Features {
455 fn parse(input: ParseStream) -> Result<Self> {
David Tolnay440fe582019-02-15 20:23:14 -0800456 let mut features = BTreeSet::new();
Carl Lerche058ff472019-02-13 16:23:52 -0800457
458 let level_1;
459 parenthesized!(level_1 in input);
460
461 let i: syn::Ident = level_1.fork().parse()?;
462
463 if i == "any" {
464 level_1.parse::<syn::Ident>()?;
465
466 let level_2;
467 parenthesized!(level_2 in level_1);
468
469 while !level_2.is_empty() {
David Tolnay440fe582019-02-15 20:23:14 -0800470 features.insert(parse_feature(&level_2)?);
Carl Lerche058ff472019-02-13 16:23:52 -0800471
472 if !level_2.is_empty() {
473 level_2.parse::<Token![,]>()?;
474 }
475 }
476 } else if i == "feature" {
David Tolnay440fe582019-02-15 20:23:14 -0800477 features.insert(parse_feature(&level_1)?);
Carl Lerche058ff472019-02-13 16:23:52 -0800478 assert!(level_1.is_empty());
479 } else {
480 panic!("{:?}", i);
481 }
482
483 assert!(input.is_empty());
484
David Tolnayfa67ab02019-02-15 20:17:30 -0800485 Ok(types::Features { any: features })
Carl Lerche058ff472019-02-13 16:23:52 -0800486 }
487 }
488}
489
490fn get_features(attrs: &[syn::Attribute], base: &[syn::Attribute]) -> Vec<syn::Attribute> {
491 let mut ret = base.to_owned();
492
493 for attr in attrs {
494 if attr.path.is_ident("cfg") {
495 ret.push(attr.clone());
496 }
497 }
498
499 ret
500}
501
502type Error = Box<::std::error::Error>;
503
504fn load_file<P: AsRef<Path>>(
505 name: P,
506 features: &[syn::Attribute],
507 lookup: &mut ItemLookup,
508) -> Result<(), Error> {
509 let name = name.as_ref();
510 let parent = name.parent().ok_or("no parent path")?;
511
512 let mut f = File::open(name)?;
513 let mut src = String::new();
514 f.read_to_string(&mut src)?;
515
516 // Parse the file
517 let file = syn::parse_file(&src)?;
518
519 // Collect all of the interesting AstItems declared in this file or submodules.
520 'items: for item in file.items {
521 match item {
522 Item::Mod(item) => {
523 // Don't inspect inline modules.
524 if item.content.is_some() {
525 continue;
526 }
527
528 // We don't want to try to load the generated rust files and
529 // parse them, so we ignore them here.
530 for name in IGNORED_MODS {
531 if item.ident == name {
532 continue 'items;
533 }
534 }
535
536 // Lookup any #[cfg()] attributes on the module and add them to
537 // the feature set.
538 //
539 // The derive module is weird because it is built with either
540 // `full` or `derive` but exported only under `derive`.
541 let features = if item.ident == "derive" {
542 vec![parse_quote!(#[cfg(feature = "derive")])]
543 } else {
544 get_features(&item.attrs, features)
545 };
546
547 // Look up the submodule file, and recursively parse it.
548 // XXX: Only handles same-directory .rs file submodules.
549 let path = parent.join(&format!("{}.rs", item.ident));
550 load_file(path, &features, lookup)?;
551 }
552 Item::Macro(item) => {
553 // Lookip any #[cfg()] attributes directly on the macro
554 // invocation, and add them to the feature set.
555 let features = get_features(&item.attrs, features);
556
557 // Try to parse the AstItem declaration out of the item.
558 let tts = &item.mac.tts;
559 let found = if item.mac.path.is_ident("ast_struct") {
560 syn::parse2::<parsing::AstStruct>(quote!(#tts))?.0
561 } else if item.mac.path.is_ident("ast_enum") {
562 syn::parse2::<parsing::AstEnum>(quote!(#tts))?.0
563 } else if item.mac.path.is_ident("ast_enum_of_structs") {
564 syn::parse2::<parsing::AstEnumOfStructs>(quote!(#tts))?.0
565 } else {
566 continue;
567 };
568
569 // Record our features on the parsed AstItems.
570 for mut item in found {
571 item.features.extend(features.clone());
572 lookup.insert(item.ast.ident.clone(), item);
573 }
574 }
575 Item::Struct(item) => {
576 let ident = item.ident;
577 if EXTRA_TYPES.contains(&&ident.to_string()[..]) {
578 lookup.insert(
579 ident.clone(),
580 AstItem {
581 ast: DeriveInput {
582 ident,
583 vis: item.vis,
584 attrs: item.attrs,
585 generics: item.generics,
586 data: Data::Struct(DataStruct {
587 fields: item.fields,
588 struct_token: item.struct_token,
589 semi_token: item.semi_token,
590 }),
591 },
592 features: features.to_owned(),
593 },
594 );
595 }
596 }
597 _ => {}
598 }
599 }
600 Ok(())
601}
602
603fn load_token_file<P: AsRef<Path>>(name: P) -> Result<TokenLookup, Error> {
604 let name = name.as_ref();
605 let mut f = File::open(name)?;
606 let mut src = String::new();
607 f.read_to_string(&mut src)?;
608 let file = syn::parse_file(&src)?;
609 for item in file.items {
610 match item {
611 Item::Macro(item) => {
612 match item.ident {
613 Some(ref i) if i == "Token" => {}
614 _ => continue,
615 }
616 let tts = &item.mac.tts;
617 let tokens = syn::parse2::<parsing::TokenMacro>(quote!(#tts))?.0;
618 return Ok(tokens);
619 }
620 _ => {}
621 }
622 }
623
624 Err("failed to parse Token macro".into())
625}