Copy in documentation from libproc_macro
diff --git a/src/lib.rs b/src/lib.rs
index 00c1aaf..7576020 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -68,12 +68,20 @@
#[cfg(feature = "nightly")]
mod imp;
+/// An abstract stream of tokens, or more concretely a sequence of token trees.
+///
+/// This type provides interfaces for iterating over token trees and for
+/// collecting token trees into one stream.
+///
+/// Token stream is both the input and output of `#[proc_macro]`,
+/// `#[proc_macro_attribute]` and `#[proc_macro_derive]` definitions.
#[derive(Clone)]
pub struct TokenStream {
inner: imp::TokenStream,
_marker: marker::PhantomData<Rc<()>>,
}
+/// Error returned from `TokenStream::from_str`.
pub struct LexError {
inner: imp::LexError,
_marker: marker::PhantomData<Rc<()>>,
@@ -94,15 +102,25 @@
}
}
+ /// Returns an empty `TokenStream` containing no token trees.
pub fn empty() -> TokenStream {
TokenStream::_new(imp::TokenStream::empty())
}
+ /// Checks if this `TokenStream` is empty.
pub fn is_empty(&self) -> bool {
self.inner.is_empty()
}
}
+/// Attempts to break the string into tokens and parse those tokens into a token
+/// stream.
+///
+/// May fail for a number of reasons, for example, if the string contains
+/// unbalanced delimiters or characters not existing in the language.
+///
+/// NOTE: Some errors may cause panics instead of returning `LexError`. We
+/// reserve the right to change these errors into `LexError`s later.
impl FromStr for TokenStream {
type Err = LexError;
@@ -135,18 +153,24 @@
}
}
+/// Collects a number of token trees into a single stream.
impl FromIterator<TokenTree> for TokenStream {
fn from_iter<I: IntoIterator<Item = TokenTree>>(streams: I) -> Self {
TokenStream::_new(streams.into_iter().collect())
}
}
+/// Prints the token stream as a string that is supposed to be losslessly
+/// convertible back into the same token stream (modulo spans), except for
+/// possibly `TokenTree::Group`s with `Delimiter::None` delimiters and negative
+/// numeric literals.
impl fmt::Display for TokenStream {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.inner.fmt(f)
}
}
+/// Prints token in a form convenient for debugging.
impl fmt::Debug for TokenStream {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.inner.fmt(f)
@@ -163,17 +187,32 @@
#[cfg(procmacro2_semver_exempt)]
pub use imp::FileName;
+/// The source file of a given `Span`.
#[cfg(procmacro2_semver_exempt)]
#[derive(Clone, PartialEq, Eq)]
pub struct SourceFile(imp::SourceFile);
#[cfg(procmacro2_semver_exempt)]
impl SourceFile {
- /// Get the path to this source file as a string.
+ /// Get the path to this source file.
+ ///
+ /// ### Note
+ ///
+ /// If the code span associated with this `SourceFile` was generated by an
+ /// external macro, this may not be an actual path on the filesystem. Use
+ /// [`is_real`] to check.
+ ///
+ /// Also note that even if `is_real` returns `true`, if
+ /// `--remap-path-prefix` was passed on the command line, the path as given
+ /// may not actually be valid.
+ ///
+ /// [`is_real`]: #method.is_real
pub fn path(&self) -> &FileName {
self.0.path()
}
+ /// Returns `true` if this source file is a real source file, and not
+ /// generated by an external macro's expansion.
pub fn is_real(&self) -> bool {
self.0.is_real()
}
@@ -193,12 +232,18 @@
}
}
+/// A line-column pair representing the start or end of a `Span`.
#[cfg(procmacro2_semver_exempt)]
pub struct LineColumn {
+ /// The 1-indexed line in the source file on which the span starts or ends
+ /// (inclusive).
pub line: usize,
+ /// The 0-indexed column (in UTF-8 characters) in the source file on which
+ /// the span starts or ends (inclusive).
pub column: usize,
}
+/// A region of source code, along with macro expansion information.
#[derive(Copy, Clone)]
pub struct Span {
inner: imp::Span,
@@ -220,10 +265,16 @@
}
}
+ /// The span of the invocation of the current procedural macro.
+ ///
+ /// Identifiers created with this span will be resolved as if they were
+ /// written directly at the macro call location (call-site hygiene) and
+ /// other code at the macro call site will be able to refer to them as well.
pub fn call_site() -> Span {
Span::_new(imp::Span::call_site())
}
+ /// A span that resolves at the macro definition site.
#[cfg(procmacro2_semver_exempt)]
pub fn def_site() -> Span {
Span::_new(imp::Span::def_site())
@@ -249,11 +300,13 @@
self.inner.unstable()
}
+ /// The original source file into which this span points.
#[cfg(procmacro2_semver_exempt)]
pub fn source_file(&self) -> SourceFile {
SourceFile(self.inner.source_file())
}
+ /// Get the starting line/column in the source file for this span.
#[cfg(procmacro2_semver_exempt)]
pub fn start(&self) -> LineColumn {
let imp::LineColumn { line, column } = self.inner.start();
@@ -263,6 +316,7 @@
}
}
+ /// Get the ending line/column in the source file for this span.
#[cfg(procmacro2_semver_exempt)]
pub fn end(&self) -> LineColumn {
let imp::LineColumn { line, column } = self.inner.end();
@@ -272,32 +326,44 @@
}
}
+ /// Create a new span encompassing `self` and `other`.
+ ///
+ /// Returns `None` if `self` and `other` are from different files.
#[cfg(procmacro2_semver_exempt)]
pub fn join(&self, other: Span) -> Option<Span> {
self.inner.join(other.inner).map(Span::_new)
}
+ /// Compares to spans to see if they're equal.
#[cfg(procmacro2_semver_exempt)]
pub fn eq(&self, other: &Span) -> bool {
self.inner.eq(&other.inner)
}
}
+/// Prints a span in a form convenient for debugging.
impl fmt::Debug for Span {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.inner.fmt(f)
}
}
+/// A single token or a delimited sequence of token trees (e.g. `[1, (), ..]`).
#[derive(Clone)]
pub enum TokenTree {
+ /// A token stream surrounded by bracket delimiters.
Group(Group),
+ /// An identifier.
Ident(Ident),
+ /// A single punctuation character (`+`, `,`, `$`, etc.).
Punct(Punct),
+ /// A literal character (`'a'`), string (`"hello"`), number (`2.3`), etc.
Literal(Literal),
}
impl TokenTree {
+ /// Returns the span of this tree, delegating to the `span` method of
+ /// the contained token or a delimited stream.
pub fn span(&self) -> Span {
match *self {
TokenTree::Group(ref t) => t.span(),
@@ -307,6 +373,11 @@
}
}
+ /// Configures the span for *only this token*.
+ ///
+ /// Note that if this token is a `Group` then this method will not configure
+ /// the span of each of the internal tokens, this will simply delegate to
+ /// the `set_span` method of each variant.
pub fn set_span(&mut self, span: Span) {
match *self {
TokenTree::Group(ref mut t) => t.set_span(span),
@@ -341,6 +412,10 @@
}
}
+/// Prints the token tree as a string that is supposed to be losslessly
+/// convertible back into the same token tree (modulo spans), except for
+/// possibly `TokenTree::Group`s with `Delimiter::None` delimiters and negative
+/// numeric literals.
impl fmt::Display for TokenTree {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
@@ -352,6 +427,7 @@
}
}
+/// Prints token tree in a form convenient for debugging.
impl fmt::Debug for TokenTree {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
// Each of these has the name in the struct type in the derived debug,
@@ -365,6 +441,10 @@
}
}
+/// A delimited token stream.
+///
+/// A `Group` internally contains a `TokenStream` which is surrounded by
+/// `Delimiter`s.
#[derive(Clone)]
pub struct Group {
delimiter: Delimiter,
@@ -372,15 +452,31 @@
span: Span,
}
+/// Describes how a sequence of token trees is delimited.
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
pub enum Delimiter {
+ /// `( ... )`
Parenthesis,
+ /// `{ ... }`
Brace,
+ /// `[ ... ]`
Bracket,
+ /// `Ø ... Ø`
+ ///
+ /// An implicit delimiter, that may, for example, appear around tokens
+ /// coming from a "macro variable" `$var`. It is important to preserve
+ /// operator priorities in cases like `$var * 3` where `$var` is `1 + 2`.
+ /// Implicit delimiters may not survive roundtrip of a token stream through
+ /// a string.
None,
}
impl Group {
+ /// Creates a new `Group` with the given delimiter and token stream.
+ ///
+ /// This constructor will set the span for this group to
+ /// `Span::call_site()`. To change the span you can use the `set_span`
+ /// method below.
pub fn new(delimiter: Delimiter, stream: TokenStream) -> Group {
Group {
delimiter: delimiter,
@@ -389,23 +485,39 @@
}
}
+ /// Returns the delimiter of this `Group`
pub fn delimiter(&self) -> Delimiter {
self.delimiter
}
+ /// Returns the `TokenStream` of tokens that are delimited in this `Group`.
+ ///
+ /// Note that the returned token stream does not include the delimiter
+ /// returned above.
pub fn stream(&self) -> TokenStream {
self.stream.clone()
}
+ /// Returns the span for the delimiters of this token stream, spanning the
+ /// entire `Group`.
pub fn span(&self) -> Span {
self.span
}
+ /// Configures the span for this `Group`'s delimiters, but not its internal
+ /// tokens.
+ ///
+ /// This method will **not** set the span of all the internal tokens spanned
+ /// by this group, but rather it will only set the span of the delimiter
+ /// tokens at the level of the `Group`.
pub fn set_span(&mut self, span: Span) {
self.span = span;
}
}
+/// Prints the group as a string that should be losslessly convertible back
+/// into the same group (modulo spans), except for possibly `TokenTree::Group`s
+/// with `Delimiter::None` delimiters.
impl fmt::Display for Group {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.stream.fmt(f)
@@ -423,6 +535,10 @@
}
}
+/// An `Punct` is an single punctuation character like `+`, `-` or `#`.
+///
+/// Multicharacter operators like `+=` are represented as two instances of
+/// `Punct` with different forms of `Spacing` returned.
#[derive(Clone)]
pub struct Punct {
op: char,
@@ -430,13 +546,27 @@
span: Span,
}
+/// Whether an `Punct` is followed immediately by another `Punct` or followed by
+/// another token or whitespace.
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
pub enum Spacing {
+ /// E.g. `+` is `Alone` in `+ =`, `+ident` or `+()`.
Alone,
+ /// E.g. `+` is `Joint` in `+=` or `'#`.
+ ///
+ /// Additionally, single quote `'` can join with identifiers to form
+ /// lifetimes `'ident`.
Joint,
}
impl Punct {
+ /// Creates a new `Punct` from the given character and spacing.
+ ///
+ /// The `ch` argument must be a valid punctuation character permitted by the
+ /// language, otherwise the function will panic.
+ ///
+ /// The returned `Punct` will have the default span of `Span::call_site()`
+ /// which can be further configured with the `set_span` method below.
pub fn new(op: char, spacing: Spacing) -> Punct {
Punct {
op: op,
@@ -445,23 +575,33 @@
}
}
+ /// Returns the value of this punctuation character as `char`.
pub fn as_char(&self) -> char {
self.op
}
+ /// Returns the spacing of this punctuation character, indicating whether
+ /// it's immediately followed by another `Punct` in the token stream, so
+ /// they can potentially be combined into a multicharacter operator
+ /// (`Joint`), or it's followed by some other token or whitespace (`Alone`)
+ /// so the operator has certainly ended.
pub fn spacing(&self) -> Spacing {
self.spacing
}
+ /// Returns the span for this punctuation character.
pub fn span(&self) -> Span {
self.span
}
+ /// Configure the span for this punctuation character.
pub fn set_span(&mut self, span: Span) {
self.span = span;
}
}
+/// Prints the punctuation character as a string that should be losslessly
+/// convertible back into the same character.
impl fmt::Display for Punct {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.op.fmt(f)
@@ -479,6 +619,7 @@
}
}
+/// An identifier (`ident`).
#[derive(Clone)]
pub struct Ident {
inner: imp::Ident,
@@ -493,10 +634,32 @@
}
}
+ /// Creates a new `Ident` with the given `string` as well as the specified
+ /// `span`.
+ ///
+ /// The `string` argument must be a valid identifier permitted by the
+ /// language, otherwise the function will panic.
+ ///
+ /// Note that `span`, currently in rustc, configures the hygiene information
+ /// for this identifier.
+ ///
+ /// As of this time `Span::call_site()` explicitly opts-in to "call-site"
+ /// hygiene meaning that identifiers created with this span will be resolved
+ /// as if they were written directly at the location of the macro call, and
+ /// other code at the macro call site will be able to refer to them as well.
+ ///
+ /// Later spans like `Span::def_site()` will allow to opt-in to
+ /// "definition-site" hygiene meaning that identifiers created with this
+ /// span will be resolved at the location of the macro definition and other
+ /// code at the macro call site will not be able to refer to them.
+ ///
+ /// Due to the current importance of hygiene this constructor, unlike other
+ /// tokens, requires a `Span` to be specified at construction.
pub fn new(string: &str, span: Span) -> Ident {
Ident::_new(imp::Ident::new(string, span.inner))
}
+ /// Same as `Ident::new`, but creates a raw identifier (`r#ident`).
#[cfg(procmacro2_semver_exempt)]
pub fn new_raw(string: &str, span: Span) -> Ident {
Ident::_new_raw(string, span)
@@ -506,10 +669,13 @@
Ident::_new(imp::Ident::new_raw(string, span.inner))
}
+ /// Returns the span of this `Ident`.
pub fn span(&self) -> Span {
Span::_new(self.inner.span())
}
+ /// Configures the span of this `Ident`, possibly changing its hygiene
+ /// context.
pub fn set_span(&mut self, span: Span) {
self.inner.set_span(span.inner);
}
@@ -550,6 +716,8 @@
}
}
+/// Prints the identifier as a string that should be losslessly convertible back
+/// into the same identifier.
impl fmt::Display for Ident {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.inner.fmt(f)
@@ -562,14 +730,52 @@
}
}
+/// A literal string (`"hello"`), byte string (`b"hello"`), character (`'a'`),
+/// byte character (`b'a'`), an integer or floating point number with or without
+/// a suffix (`1`, `1u8`, `2.3`, `2.3f32`).
+///
+/// Boolean literals like `true` and `false` do not belong here, they are
+/// `Ident`s.
#[derive(Clone)]
pub struct Literal {
inner: imp::Literal,
_marker: marker::PhantomData<Rc<()>>,
}
-macro_rules! int_literals {
+macro_rules! suffixed_int_literals {
($($name:ident => $kind:ident,)*) => ($(
+ /// Creates a new suffixed integer literal with the specified value.
+ ///
+ /// This function will create an integer like `1u32` where the integer
+ /// value specified is the first part of the token and the integral is
+ /// also suffixed at the end. Literals created from negative numbers may
+ /// not survive rountrips through `TokenStream` or strings and may be
+ /// broken into two tokens (`-` and positive literal).
+ ///
+ /// Literals created through this method have the `Span::call_site()`
+ /// span by default, which can be configured with the `set_span` method
+ /// below.
+ pub fn $name(n: $kind) -> Literal {
+ Literal::_new(imp::Literal::$name(n))
+ }
+ )*)
+}
+
+macro_rules! unsuffixed_int_literals {
+ ($($name:ident => $kind:ident,)*) => ($(
+ /// Creates a new unsuffixed integer literal with the specified value.
+ ///
+ /// This function will create an integer like `1` where the integer
+ /// value specified is the first part of the token. No suffix is
+ /// specified on this token, meaning that invocations like
+ /// `Literal::i8_unsuffixed(1)` are equivalent to
+ /// `Literal::u32_unsuffixed(1)`. Literals created from negative numbers
+ /// may not survive rountrips through `TokenStream` or strings and may
+ /// be broken into two tokens (`-` and positive literal).
+ ///
+ /// Literals created through this method have the `Span::call_site()`
+ /// span by default, which can be configured with the `set_span` method
+ /// below.
pub fn $name(n: $kind) -> Literal {
Literal::_new(imp::Literal::$name(n))
}
@@ -591,7 +797,7 @@
}
}
- int_literals! {
+ suffixed_int_literals! {
u8_suffixed => u8,
u16_suffixed => u16,
u32_suffixed => u32,
@@ -602,7 +808,9 @@
i32_suffixed => i32,
i64_suffixed => i64,
isize_suffixed => isize,
+ }
+ unsuffixed_int_literals! {
u8_unsuffixed => u8,
u16_unsuffixed => u16,
u32_unsuffixed => u32,
@@ -625,6 +833,19 @@
Literal::_new(imp::Literal::f64_suffixed(f))
}
+ /// Creates a new unsuffixed floating-point literal.
+ ///
+ /// This constructor is similar to those like `Literal::i8_unsuffixed` where
+ /// the float's value is emitted directly into the token but no suffix is
+ /// used, so it may be inferred to be a `f64` later in the compiler.
+ /// Literals created from negative numbers may not survive rountrips through
+ /// `TokenStream` or strings and may be broken into two tokens (`-` and
+ /// positive literal).
+ ///
+ /// # Panics
+ ///
+ /// This function requires that the specified float is finite, for example
+ /// if it is infinity or NaN this function will panic.
pub fn f32_unsuffixed(f: f32) -> Literal {
assert!(f.is_finite());
Literal::_new(imp::Literal::f32_unsuffixed(f))
@@ -668,6 +889,7 @@
}
}
+/// Public implementation details for the `TokenStream` type, such as iterators.
pub mod token_stream {
use std::fmt;
use std::marker;
@@ -677,6 +899,10 @@
pub use TokenStream;
use TokenTree;
+ /// An iterator over `TokenStream`'s `TokenTree`s.
+ ///
+ /// The iteration is "shallow", e.g. the iterator doesn't recurse into
+ /// delimited groups, and returns whole groups as token trees.
pub struct IntoIter {
inner: imp::TokenTreeIter,
_marker: marker::PhantomData<Rc<()>>,