Struct Tokenizer

struct Tokenizer {
    input: Vec<char>,
    policy: TokenizePolicy,
    token_list: Vec<Token>,
    index: usize,
    next_index: usize,
    code_point: Option<char>,
}

Fields

input: Vec<char>

policy: TokenizePolicy

token_list: Vec<Token>

index: usize

next_index: usize

code_point: Option<char>

Implementations

impl Tokenizer

fn get_next_codepoint(&mut self)

fn add_token_with_default_pos_and_len(&mut self, kind: TokenType)

fn add_token_with_default_len( &mut self, kind: TokenType, next_pos: usize, value_pos: usize, )

fn add_token( &mut self, kind: TokenType, next_pos: usize, value_pos: usize, value_len: usize, )

fn process_tokenizing_error( &mut self, next_pos: usize, value_pos: usize, error: TokenizerError, ) -> Result<(), Error>

fn seek_and_get_next_codepoint(&mut self, index: usize)