Struct html5ever::tokenizer::Tokenizer

pub struct Tokenizer<Sink> {

    opts: TokenizerOpts,
    pub sink: Sink,
    state: Cell<State>,
    at_eof: Cell<bool>,
    char_ref_tokenizer: RefCell<Option<Box<CharRefTokenizer>>>,
    current_char: Cell<char>,
    reconsume: Cell<bool>,
    ignore_lf: Cell<bool>,
    discard_bom: Cell<bool>,
    current_tag_kind: Cell<TagKind>,
    current_tag_name: RefCell<StrTendril>,
    current_tag_self_closing: Cell<bool>,
    current_tag_attrs: RefCell<Vec<Attribute>>,
    current_attr_name: RefCell<StrTendril>,
    current_attr_value: RefCell<StrTendril>,
    current_comment: RefCell<StrTendril>,
    current_doctype: RefCell<Doctype>,
    last_start_tag_name: RefCell<Option<LocalName>>,
    temp_buf: RefCell<StrTendril>,
    state_profile: RefCell<BTreeMap<State, u64>>,
    time_in_sink: Cell<u64>,
    current_line: Cell<u64>,
}

The HTML tokenizer.

Fields

opts: TokenizerOpts

Options controlling the behavior of the tokenizer.

sink: Sink

Destination for tokens we emit.

state: Cell<State>

The abstract machine state as described in the spec.

at_eof: Cell<bool>

Are we at the end of the file, once buffers have been processed completely? This affects whether we will wait for lookahead or not.

char_ref_tokenizer: RefCell<Option<Box<CharRefTokenizer>>>

Tokenizer for character references, if we’re tokenizing one at the moment.

current_char: Cell<char>

Current input character. Just consumed, may reconsume.

reconsume: Cell<bool>

Should we reconsume the current input character?

ignore_lf: Cell<bool>

Did we just consume \r, translating it to \n? In that case we need to ignore the next character if it’s \n.

discard_bom: Cell<bool>

Discard a U+FEFF BYTE ORDER MARK if we see one? Only done at the beginning of the stream.

current_tag_kind: Cell<TagKind>

Current tag kind.

current_tag_name: RefCell<StrTendril>

Current tag name.

current_tag_self_closing: Cell<bool>

Current tag is self-closing?

current_tag_attrs: RefCell<Vec<Attribute>>

Current tag attributes.

current_attr_name: RefCell<StrTendril>

Current attribute name.

current_attr_value: RefCell<StrTendril>

Current attribute value.

current_comment: RefCell<StrTendril>

Current comment.

current_doctype: RefCell<Doctype>

Current doctype token.

last_start_tag_name: RefCell<Option<LocalName>>

Last start tag name, for use in checking “appropriate end tag”.

temp_buf: RefCell<StrTendril>

The “temporary buffer” mentioned in the spec.

state_profile: RefCell<BTreeMap<State, u64>>

Record of how many ns we spent in each state, if profiling is enabled.

time_in_sink: Cell<u64>

Record of how many ns we spent in the token sink.

current_line: Cell<u64>

Track current line

Implementations

impl<Sink: TokenSink> Tokenizer<Sink>

pub fn new(sink: Sink, opts: TokenizerOpts) -> Tokenizer<Sink>

Create a new tokenizer which feeds tokens to a particular TokenSink.

pub fn feed(&self, input: &BufferQueue) -> TokenizerResult<Sink::Handle>

Feed an input string into the tokenizer.

pub fn set_plaintext_state(&self)

fn process_token(&self, token: Token) -> TokenSinkResult<Sink::Handle>

fn process_token_and_continue(&self, token: Token)

fn get_preprocessed_char(&self, c: char, input: &BufferQueue) -> Option<char>

fn get_char(&self, input: &BufferQueue) -> Option<char>

fn pop_except_from( &self, input: &BufferQueue, set: SmallCharSet, ) -> Option<SetResult>

fn eat( &self, input: &BufferQueue, pat: &str, eq: fn(_: &u8, _: &u8) -> bool, ) -> Option<bool>

fn run(&self, input: &BufferQueue) -> TokenizerResult<Sink::Handle>

Run the state machine for as long as we can.

fn bad_char_error(&self)

fn bad_eof_error(&self)

fn emit_char(&self, c: char)

fn emit_chars(&self, b: StrTendril)

fn emit_current_tag(&self) -> ProcessResult<Sink::Handle>

fn emit_temp_buf(&self)

fn clear_temp_buf(&self)

fn emit_current_comment(&self)

fn discard_tag(&self)

fn create_tag(&self, kind: TagKind, c: char)

fn have_appropriate_end_tag(&self) -> bool

fn create_attribute(&self, c: char)

fn finish_attribute(&self)

fn emit_current_doctype(&self)

fn doctype_id(&self, kind: DoctypeIdKind) -> RefMut<'_, Option<StrTendril>>

fn clear_doctype_id(&self, kind: DoctypeIdKind)

fn consume_char_ref(&self)

fn emit_eof(&self)

fn peek(&self, input: &BufferQueue) -> Option<char>

fn discard_char(&self, input: &BufferQueue)

fn emit_error(&self, error: Cow<'static, str>)

impl<Sink: TokenSink> Tokenizer<Sink>

fn step(&self, input: &BufferQueue) -> ProcessResult<Sink::Handle>

fn step_char_ref_tokenizer( &self, input: &BufferQueue, ) -> ProcessResult<Sink::Handle>

fn process_char_ref(&self, char_ref: CharRef)

pub fn end(&self)

Indicate that we have reached the end of the input.

fn dump_profile(&self)

fn eof_step(&self) -> ProcessResult<Sink::Handle>