Trait Tokenizer

pub trait Tokenizer {
    type Token;
    type Position: Clone + Default;
    type Error;

    // Required method
    fn next_token(&mut self) -> Option<Result<TokenFor<Self>, Self::Error>>;
}

Required Associated Types

type Token

type Position: Clone + Default

type Error

Required Methods

fn next_token(&mut self) -> Option<Result<TokenFor<Self>, Self::Error>>

Implementations on Foreign Types

impl<T: Tokenizer> Tokenizer for &mut T

type Token = <T as Tokenizer>::Token

type Position = <T as Tokenizer>::Position

type Error = <T as Tokenizer>::Error

fn next_token(&mut self) -> Option<Result<TokenFor<Self>, Self::Error>>

Implementors

impl<I, T, Idx, E> Tokenizer for IterTokenizer<I>

where I: Iterator<Item = Result<Token<T, Idx>, E>>, Idx: Clone + Default,

type Token = T

type Position = Idx

type Error = E

impl<S: Source, C: CharSet<S::Char>> Tokenizer for CharSetTokenizer<S, C>

type Token = (<S as Source>::String, <C as CharSet<<S as Source>::Char>>::TokenKind)

type Position = usize

type Error = Either<<S as Source>::Error, <C as CharSet<<S as Source>::Char>>::Error>