expr_parser/
token.rs

1use std::fmt;
2
3use crate::Span;
4
5#[cfg(feature = "charset-tokenizer")]
6pub mod charset;
7
8pub trait Tokenizer {
9    type Token;
10    type Position: Clone + Default;
11    type Error;
12
13    fn next_token(&mut self) -> Option<Result<TokenFor<Self>, Self::Error>>;
14}
15
16impl<T: Tokenizer> Tokenizer for &mut T {
17    type Token = T::Token;
18    type Position = T::Position;
19    type Error = T::Error;
20
21    fn next_token(&mut self) -> Option<Result<TokenFor<Self>, Self::Error>> {
22        T::next_token(self)
23    }
24}
25
26pub struct IterTokenizer<I>(pub I);
27impl<I, T, Idx, E> Tokenizer for IterTokenizer<I>
28where
29    I: Iterator<Item = Result<Token<T, Idx>, E>>,
30    Idx: Clone + Default,
31{
32    type Token = T;
33    type Position = Idx;
34    type Error = E;
35
36    fn next_token(&mut self) -> Option<Result<Token<Self::Token, Self::Position>, Self::Error>> {
37        self.0.next()
38    }
39}
40
41#[derive(Clone, Copy, Debug, Eq, PartialEq)]
42pub struct Token<T, Idx> {
43    pub span: Span<Idx>,
44    pub kind: T,
45}
46
47pub type TokenFor<T> = Token<<T as Tokenizer>::Token, <T as Tokenizer>::Position>;
48
49impl<T: fmt::Display, Idx: fmt::Display> fmt::Display for Token<T, Idx> {
50    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
51        fmt::Display::fmt(&self.kind, f)
52    }
53}
expr_parser/token.rs

expr_parser/
token.rs
