Struct proc_macro_utils::TokenParser

pub struct TokenParser<T: Iterator<Item = TokenTree>> { /* private fields */ }

Wrapper for TokenStream::into_iter allowing not only to iterate on tokens but also to parse simple structures like types or expressions, though it does not make any claims about their correctness.

let mut token_parser = TokenParser::from(quote! {a + b, c});
assert_tokens!(token_parser.next_expression().unwrap(), { a + b });

Implementations

impl<I> TokenParser<I>where I: Iterator<Item = TokenTree>,

pub fn new<T: IntoIterator<Item = TokenTree, IntoIter = I>>(value: T) -> Self

Creates a new TokenParser from a TokenTree iterator.

impl<T: Iterator<Item = TokenTree>> TokenParser<T>

Some Iterator utilities

pub fn is_empty(&mut self) -> bool

Checks if there are remaining tokens

pub fn peek(&mut self) -> Option<&TokenTree>

Peeks the next token without advancing the parser

pub fn peek_n(&mut self, n: usize) -> Option<&TokenTree>

Peeks the next token without advancing the parser

Panics

Will panic if n >= 3.

pub fn next_if( &mut self, test: impl FnOnce(&TokenTree) -> bool ) -> Option<TokenTree>

Returns the next token if it fulfills the condition otherwise returns None and doesn’t advance the parser

pub fn next_if_each<P: Peeker<T>>(&mut self, tests: P) -> Option<TokenStream>

Returns the next tokens (up to 3) if they fulfill the conditions otherwise returns None and doesn’t advance the parser

pub fn next_while( &mut self, test: impl FnMut(&TokenTree) -> bool ) -> Option<TokenStream>

Returns all tokens while test evaluates to true.

Returns None if empty or test(first_token) == false

pub fn next_until( &mut self, test: impl FnMut(&TokenTree) -> bool ) -> Option<TokenStream>

Returns all tokens while test evaluates to false.

Returns None if empty or test(first_token) == true.

impl<T: Iterator<Item = TokenTree>> TokenParser<T>

pub fn into_token_stream(self) -> TokenStream

Collects remaining tokens back into a TokenStream

pub fn next_punctuation_group(&mut self) -> Option<TokenStream>

Returns the next group of punctuation with Punct::spacing Spacing::Joint

pub fn next_type(&mut self) -> Option<TokenStream>

“Parses” a type expression

This just means it collects all the tokens that should belong to the type, until it reaches either:

• a ;
• a , or > and all <> pairs are closed
• the end of the token stream

If the token stream is empty, or starts with ,, > or ; None is returned otherwise, Some(TokenStream) containing every token up to but excluding the terminator.

let mut tokens = TokenParser::from(quote! {A<Test, B>, remainder});
assert_tokens!(tokens.next_type().unwrap(), { A<Test, B> });
assert!(tokens.next_type().is_none());
assert_tokens!(tokens, { , remainder });

pub fn next_expression(&mut self) -> Option<TokenStream>

“Parses” an expression

This just means it collects all the tokens that should belong to the expression, until it reaches either:

• a ;
• a , outside a type
• the end of the token stream

If the token stream is empty, or starts with , or ; None is returned otherwise, Some(TokenStream) containing every token up to but excluding the terminator.

let mut tokens = TokenParser::from(quote! {A + c ::<a, b>::a < b, next_token});
assert_tokens!(tokens.next_expression().unwrap(), { A + c::<a, b>::a < b });
assert!(tokens.next_expression().is_none());
assert_tokens!(tokens, { , next_token });

pub fn next_string(&mut self) -> Option<String>

Returns the next string literal

pub fn next_bool(&mut self) -> Option<bool>

Returns the next boolean literal

impl<T: Iterator<Item = TokenTree>> TokenParser<T>

pub fn next_group(&mut self) -> Option<Group>

Returns the next token if it is a Group

pub fn next_ident(&mut self) -> Option<Ident>

Returns the next token if it is an Ident

pub fn next_punct(&mut self) -> Option<Punct>

Returns the next token if it is a Punct

pub fn next_literal(&mut self) -> Option<Literal>

Returns the next token if it is a Literal

pub fn next_parenthesized(&mut self) -> Option<TokenStream>

Returns the next parenthesized group

pub fn next_braced(&mut self) -> Option<TokenStream>

Returns the next braced group

pub fn next_bracketed(&mut self) -> Option<TokenStream>

Returns the next bracketed group

impl<T: Iterator<Item = TokenTree>> TokenParser<T>

For now the naming of the tokens follow the names used in the rust reference even though they diverge from the names used at TokenTreePunct.

Note that they only match the token with correct spacing, i.e. next_plus will match + = and +a but not +=.

pub fn next_plus(&mut self) -> Option<TokenStream>

Returns the next token if it is a +

pub fn next_minus(&mut self) -> Option<TokenStream>

Returns the next token if it is a -

pub fn next_star(&mut self) -> Option<TokenStream>

Returns the next token if it is a *

pub fn next_slash(&mut self) -> Option<TokenStream>

Returns the next token if it is a /

pub fn next_percent(&mut self) -> Option<TokenStream>

Returns the next token if it is a %

pub fn next_caret(&mut self) -> Option<TokenStream>

Returns the next token if it is a ^

pub fn next_not(&mut self) -> Option<TokenStream>

Returns the next token if it is a !

pub fn next_and(&mut self) -> Option<TokenStream>

Returns the next token if it is a &

pub fn next_or(&mut self) -> Option<TokenStream>

Returns the next token if it is a |

pub fn next_and_and(&mut self) -> Option<TokenStream>

Returns the next token if it is a &&

pub fn next_or_or(&mut self) -> Option<TokenStream>

Returns the next token if it is a ||

pub fn next_shl(&mut self) -> Option<TokenStream>

Returns the next token if it is a <<

pub fn next_shr(&mut self) -> Option<TokenStream>

Returns the next token if it is a >>

pub fn next_plus_eq(&mut self) -> Option<TokenStream>

Returns the next token if it is a +=

pub fn next_minus_eq(&mut self) -> Option<TokenStream>

Returns the next token if it is a -=

pub fn next_star_eq(&mut self) -> Option<TokenStream>

Returns the next token if it is a *=

pub fn next_slash_eq(&mut self) -> Option<TokenStream>

Returns the next token if it is a /=

pub fn next_percent_eq(&mut self) -> Option<TokenStream>

Returns the next token if it is a %=

pub fn next_caret_eq(&mut self) -> Option<TokenStream>

Returns the next token if it is a ^=

pub fn next_and_eq(&mut self) -> Option<TokenStream>

Returns the next token if it is a &=

pub fn next_or_eq(&mut self) -> Option<TokenStream>

Returns the next token if it is a |=

pub fn next_shl_eq(&mut self) -> Option<TokenStream>

Returns the next token if it is a <<=

pub fn next_shr_eq(&mut self) -> Option<TokenStream>

Returns the next token if it is a >>=

pub fn next_eq(&mut self) -> Option<TokenStream>

Returns the next token if it is a =

pub fn next_eq_eq(&mut self) -> Option<TokenStream>

Returns the next token if it is a ==

pub fn next_ne(&mut self) -> Option<TokenStream>

Returns the next token if it is a !=

pub fn next_gt(&mut self) -> Option<TokenStream>

Returns the next token if it is a >

pub fn next_lt(&mut self) -> Option<TokenStream>

Returns the next token if it is a <

pub fn next_ge(&mut self) -> Option<TokenStream>

Returns the next token if it is a >=

pub fn next_le(&mut self) -> Option<TokenStream>

Returns the next token if it is a <=

pub fn next_at(&mut self) -> Option<TokenStream>

Returns the next token if it is a @

pub fn next_dot(&mut self) -> Option<TokenStream>

Returns the next token if it is a .

pub fn next_dot_dot(&mut self) -> Option<TokenStream>

Returns the next token if it is a ..

pub fn next_dot_dot_dot(&mut self) -> Option<TokenStream>

Returns the next token if it is a ...

pub fn next_dot_dot_eq(&mut self) -> Option<TokenStream>

Returns the next token if it is a ..=

pub fn next_comma(&mut self) -> Option<TokenStream>

Returns the next token if it is a ,

pub fn next_semi(&mut self) -> Option<TokenStream>

Returns the next token if it is a ;

pub fn next_colon(&mut self) -> Option<TokenStream>

Returns the next token if it is a :

pub fn next_path_sep(&mut self) -> Option<TokenStream>

Returns the next token if it is a ::

pub fn next_r_arrow(&mut self) -> Option<TokenStream>

Returns the next token if it is a ->

pub fn next_fat_arrow(&mut self) -> Option<TokenStream>

Returns the next token if it is a =>

pub fn next_pound(&mut self) -> Option<TokenStream>

Returns the next token if it is a #

pub fn next_dollar(&mut self) -> Option<TokenStream>

Returns the next token if it is a $

pub fn next_question(&mut self) -> Option<TokenStream>

Returns the next token if it is a ?

pub fn next_tilde(&mut self) -> Option<TokenStream>

Returns the next token if it is a ~

Trait Implementations

impl<T, I> From<T> for TokenParser<I>where T: IntoIterator<Item = TokenTree, IntoIter = I>, I: Iterator<Item = TokenTree>,

fn from(value: T) -> Self

Converts to this type from the input type.

impl<I: Iterator<Item = TokenTree>> From<TokenParser<I>> for TokenStream

fn from(value: TokenParser<I>) -> Self

Converts to this type from the input type.

impl<T: Iterator<Item = TokenTree>> Iterator for TokenParser<T>

type Item = TokenTree

The type of the elements being iterated over.

fn next(&mut self) -> Option<Self::Item>

Advances the iterator and returns the next value.

fn next_chunk<const N: usize>( &mut self ) -> Result<[Self::Item; N], IntoIter<Self::Item, N>>where Self: Sized,

🔬This is a nightly-only experimental API. (iter_next_chunk)

Advances the iterator and returns an array containing the next N values.

fn size_hint(&self) -> (usize, Option<usize>)

Returns the bounds on the remaining length of the iterator.

fn count(self) -> usizewhere Self: Sized,

Consumes the iterator, counting the number of iterations and returning it.

fn last(self) -> Option<Self::Item>where Self: Sized,

Consumes the iterator, returning the last element.

fn advance_by(&mut self, n: usize) -> Result<(), NonZeroUsize>

🔬This is a nightly-only experimental API. (iter_advance_by)

Advances the iterator by n elements.

fn nth(&mut self, n: usize) -> Option<Self::Item>

Returns the nth element of the iterator.

fn step_by(self, step: usize) -> StepBy<Self>where Self: Sized,

Creates an iterator starting at the same point, but stepping by the given amount at each iteration.

fn chain<U>(self, other: U) -> Chain<Self, <U as IntoIterator>::IntoIter>where Self: Sized, U: IntoIterator<Item = Self::Item>,

Takes two iterators and creates a new iterator over both in sequence.

fn zip<U>(self, other: U) -> Zip<Self, <U as IntoIterator>::IntoIter>where Self: Sized, U: IntoIterator,

‘Zips up’ two iterators into a single iterator of pairs.

fn intersperse_with<G>(self, separator: G) -> IntersperseWith<Self, G>where Self: Sized, G: FnMut() -> Self::Item,

🔬This is a nightly-only experimental API. (iter_intersperse)

Creates a new iterator which places an item generated by separator between adjacent items of the original iterator.

fn map<B, F>(self, f: F) -> Map<Self, F>where Self: Sized, F: FnMut(Self::Item) -> B,

Takes a closure and creates an iterator which calls that closure on each element.

fn for_each<F>(self, f: F)where Self: Sized, F: FnMut(Self::Item),

Calls a closure on each element of an iterator.

fn filter<P>(self, predicate: P) -> Filter<Self, P>where Self: Sized, P: FnMut(&Self::Item) -> bool,

Creates an iterator which uses a closure to determine if an element should be yielded.

fn filter_map<B, F>(self, f: F) -> FilterMap<Self, F>where Self: Sized, F: FnMut(Self::Item) -> Option<B>,

Creates an iterator that both filters and maps.

fn enumerate(self) -> Enumerate<Self>where Self: Sized,

Creates an iterator which gives the current iteration count as well as the next value.

fn peekable(self) -> Peekable<Self>where Self: Sized,

Creates an iterator which can use the peek and peek_mut methods to look at the next element of the iterator without consuming it. See their documentation for more information.

fn skip_while<P>(self, predicate: P) -> SkipWhile<Self, P>where Self: Sized, P: FnMut(&Self::Item) -> bool,

Creates an iterator that skips elements based on a predicate.

fn take_while<P>(self, predicate: P) -> TakeWhile<Self, P>where Self: Sized, P: FnMut(&Self::Item) -> bool,

Creates an iterator that yields elements based on a predicate.

fn map_while<B, P>(self, predicate: P) -> MapWhile<Self, P>where Self: Sized, P: FnMut(Self::Item) -> Option<B>,

Creates an iterator that both yields elements based on a predicate and maps.

fn skip(self, n: usize) -> Skip<Self>where Self: Sized,

Creates an iterator that skips the first n elements.

fn take(self, n: usize) -> Take<Self>where Self: Sized,

Creates an iterator that yields the first n elements, or fewer if the underlying iterator ends sooner.

fn scan<St, B, F>(self, initial_state: St, f: F) -> Scan<Self, St, F>where Self: Sized, F: FnMut(&mut St, Self::Item) -> Option<B>,

An iterator adapter which, like fold, holds internal state, but unlike fold, produces a new iterator.

fn flat_map<U, F>(self, f: F) -> FlatMap<Self, U, F>where Self: Sized, U: IntoIterator, F: FnMut(Self::Item) -> U,

Creates an iterator that works like map, but flattens nested structure.

fn fuse(self) -> Fuse<Self>where Self: Sized,

Creates an iterator which ends after the first None.

fn inspect<F>(self, f: F) -> Inspect<Self, F>where Self: Sized, F: FnMut(&Self::Item),

Does something with each element of an iterator, passing the value on.

fn by_ref(&mut self) -> &mut Selfwhere Self: Sized,

Borrows an iterator, rather than consuming it.

fn collect<B>(self) -> Bwhere B: FromIterator<Self::Item>, Self: Sized,

Transforms an iterator into a collection.

fn collect_into<E>(self, collection: &mut E) -> &mut Ewhere E: Extend<Self::Item>, Self: Sized,

🔬This is a nightly-only experimental API. (iter_collect_into)

Collects all the items from an iterator into a collection.

fn partition<B, F>(self, f: F) -> (B, B)where Self: Sized, B: Default + Extend<Self::Item>, F: FnMut(&Self::Item) -> bool,

Consumes an iterator, creating two collections from it.

fn is_partitioned<P>(self, predicate: P) -> boolwhere Self: Sized, P: FnMut(Self::Item) -> bool,

🔬This is a nightly-only experimental API. (iter_is_partitioned)

Checks if the elements of this iterator are partitioned according to the given predicate, such that all those that return true precede all those that return false.

fn try_fold<B, F, R>(&mut self, init: B, f: F) -> Rwhere Self: Sized, F: FnMut(B, Self::Item) -> R, R: Try<Output = B>,

An iterator method that applies a function as long as it returns successfully, producing a single, final value.

fn try_for_each<F, R>(&mut self, f: F) -> Rwhere Self: Sized, F: FnMut(Self::Item) -> R, R: Try<Output = ()>,

An iterator method that applies a fallible function to each item in the iterator, stopping at the first error and returning that error.

fn fold<B, F>(self, init: B, f: F) -> Bwhere Self: Sized, F: FnMut(B, Self::Item) -> B,

Folds every element into an accumulator by applying an operation, returning the final result.

fn reduce<F>(self, f: F) -> Option<Self::Item>where Self: Sized, F: FnMut(Self::Item, Self::Item) -> Self::Item,

Reduces the elements to a single one, by repeatedly applying a reducing operation.

fn try_reduce<F, R>( &mut self, f: F ) -> <<R as Try>::Residual as Residual<Option<<R as Try>::Output>>>::TryTypewhere Self: Sized, F: FnMut(Self::Item, Self::Item) -> R, R: Try<Output = Self::Item>, <R as Try>::Residual: Residual<Option<Self::Item>>,

🔬This is a nightly-only experimental API. (iterator_try_reduce)

Reduces the elements to a single one by repeatedly applying a reducing operation. If the closure returns a failure, the failure is propagated back to the caller immediately.

fn all<F>(&mut self, f: F) -> boolwhere Self: Sized, F: FnMut(Self::Item) -> bool,

Tests if every element of the iterator matches a predicate.

fn any<F>(&mut self, f: F) -> boolwhere Self: Sized, F: FnMut(Self::Item) -> bool,

Tests if any element of the iterator matches a predicate.

fn find<P>(&mut self, predicate: P) -> Option<Self::Item>where Self: Sized, P: FnMut(&Self::Item) -> bool,

Searches for an element of an iterator that satisfies a predicate.

fn find_map<B, F>(&mut self, f: F) -> Option<B>where Self: Sized, F: FnMut(Self::Item) -> Option<B>,

Applies function to the elements of iterator and returns the first non-none result.

fn try_find<F, R>( &mut self, f: F ) -> <<R as Try>::Residual as Residual<Option<Self::Item>>>::TryTypewhere Self: Sized, F: FnMut(&Self::Item) -> R, R: Try<Output = bool>, <R as Try>::Residual: Residual<Option<Self::Item>>,

🔬This is a nightly-only experimental API. (try_find)

Applies function to the elements of iterator and returns the first true result or the first error.

fn position<P>(&mut self, predicate: P) -> Option<usize>where Self: Sized, P: FnMut(Self::Item) -> bool,

Searches for an element in an iterator, returning its index.

fn max_by_key<B, F>(self, f: F) -> Option<Self::Item>where B: Ord, Self: Sized, F: FnMut(&Self::Item) -> B,

Returns the element that gives the maximum value from the specified function.

fn max_by<F>(self, compare: F) -> Option<Self::Item>where Self: Sized, F: FnMut(&Self::Item, &Self::Item) -> Ordering,

Returns the element that gives the maximum value with respect to the specified comparison function.

fn min_by_key<B, F>(self, f: F) -> Option<Self::Item>where B: Ord, Self: Sized, F: FnMut(&Self::Item) -> B,

Returns the element that gives the minimum value from the specified function.

fn min_by<F>(self, compare: F) -> Option<Self::Item>where Self: Sized, F: FnMut(&Self::Item, &Self::Item) -> Ordering,

Returns the element that gives the minimum value with respect to the specified comparison function.

fn unzip<A, B, FromA, FromB>(self) -> (FromA, FromB)where FromA: Default + Extend<A>, FromB: Default + Extend<B>, Self: Sized + Iterator<Item = (A, B)>,

Converts an iterator of pairs into a pair of containers.

fn copied<'a, T>(self) -> Copied<Self>where T: 'a + Copy, Self: Sized + Iterator<Item = &'a T>,

Creates an iterator which copies all of its elements.

fn cloned<'a, T>(self) -> Cloned<Self>where T: 'a + Clone, Self: Sized + Iterator<Item = &'a T>,

Creates an iterator which clones all of its elements.

fn array_chunks<const N: usize>(self) -> ArrayChunks<Self, N>where Self: Sized,

🔬This is a nightly-only experimental API. (iter_array_chunks)

Returns an iterator over N elements of the iterator at a time.

fn sum<S>(self) -> Swhere Self: Sized, S: Sum<Self::Item>,

Sums the elements of an iterator.

fn product<P>(self) -> Pwhere Self: Sized, P: Product<Self::Item>,

Iterates over the entire iterator, multiplying all the elements

fn cmp_by<I, F>(self, other: I, cmp: F) -> Orderingwhere Self: Sized, I: IntoIterator, F: FnMut(Self::Item, <I as IntoIterator>::Item) -> Ordering,

🔬This is a nightly-only experimental API. (iter_order_by)

Lexicographically compares the elements of this Iterator with those of another with respect to the specified comparison function.

fn partial_cmp<I>(self, other: I) -> Option<Ordering>where I: IntoIterator, Self::Item: PartialOrd<<I as IntoIterator>::Item>, Self: Sized,

Lexicographically compares the PartialOrd elements of this Iterator with those of another. The comparison works like short-circuit evaluation, returning a result without comparing the remaining elements. As soon as an order can be determined, the evaluation stops and a result is returned.

fn partial_cmp_by<I, F>(self, other: I, partial_cmp: F) -> Option<Ordering>where Self: Sized, I: IntoIterator, F: FnMut(Self::Item, <I as IntoIterator>::Item) -> Option<Ordering>,

🔬This is a nightly-only experimental API. (iter_order_by)

Lexicographically compares the elements of this Iterator with those of another with respect to the specified comparison function.

fn eq<I>(self, other: I) -> boolwhere I: IntoIterator, Self::Item: PartialEq<<I as IntoIterator>::Item>, Self: Sized,

Determines if the elements of this Iterator are equal to those of another.

fn eq_by<I, F>(self, other: I, eq: F) -> boolwhere Self: Sized, I: IntoIterator, F: FnMut(Self::Item, <I as IntoIterator>::Item) -> bool,

🔬This is a nightly-only experimental API. (iter_order_by)

Determines if the elements of this Iterator are equal to those of another with respect to the specified equality function.

fn ne<I>(self, other: I) -> boolwhere I: IntoIterator, Self::Item: PartialEq<<I as IntoIterator>::Item>, Self: Sized,

Determines if the elements of this Iterator are not equal to those of another.

fn lt<I>(self, other: I) -> boolwhere I: IntoIterator, Self::Item: PartialOrd<<I as IntoIterator>::Item>, Self: Sized,

Determines if the elements of this Iterator are lexicographically less than those of another.

fn le<I>(self, other: I) -> boolwhere I: IntoIterator, Self::Item: PartialOrd<<I as IntoIterator>::Item>, Self: Sized,

Determines if the elements of this Iterator are lexicographically less or equal to those of another.

fn gt<I>(self, other: I) -> boolwhere I: IntoIterator, Self::Item: PartialOrd<<I as IntoIterator>::Item>, Self: Sized,

Determines if the elements of this Iterator are lexicographically greater than those of another.

fn ge<I>(self, other: I) -> boolwhere I: IntoIterator, Self::Item: PartialOrd<<I as IntoIterator>::Item>, Self: Sized,

Determines if the elements of this Iterator are lexicographically greater than or equal to those of another.

fn is_sorted_by<F>(self, compare: F) -> boolwhere Self: Sized, F: FnMut(&Self::Item, &Self::Item) -> Option<Ordering>,

🔬This is a nightly-only experimental API. (is_sorted)

Checks if the elements of this iterator are sorted using the given comparator function.

fn is_sorted_by_key<F, K>(self, f: F) -> boolwhere Self: Sized, F: FnMut(Self::Item) -> K, K: PartialOrd<K>,

🔬This is a nightly-only experimental API. (is_sorted)

Checks if the elements of this iterator are sorted using the given key extraction function.