Struct Tokenizer 

pub struct Tokenizer { /* private fields */ }

Implementations

impl Tokenizer

pub fn new() -> Self

pub fn register_token<C: 'static + Fn(&str) -> Atom>( &mut self, regex: Regex, constr: C, )

pub fn register_fallible_token<C: 'static + Fn(&str) -> Result<Atom, String>>( &mut self, regex: Regex, constr: C, )

pub fn register_token_with_regex_str<C: 'static + Fn(&str) -> Atom>( &mut self, regex: &str, constr: C, )

pub fn register_function<T: 'static + GroundedFunction>( &mut self, func: GroundedFunctionAtom<T>, )

pub fn move_front(&mut self, from: &mut Tokenizer)

Moves all tokenizer entries from from into self, leaving from empty

NOTE: Tokens are tried in reverse order, so move_front actually adds entries that will be tried last in the priority order

pub fn move_back(&mut self, from: &mut Tokenizer)

Moves all tokenizer entries from from into self, leaving from empty

NOTE: Tokens are tried in reverse order, so move_back actually adds entries that will be tried first in the priority order

pub fn find_token( &self, token: &str, ) -> Option<&dyn Fn(&str) -> Result<Atom, String>>

pub fn remove_token(&mut self, regex_str: &str)

Trait Implementations

impl Clone for Tokenizer

fn clone(&self) -> Tokenizer

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Tokenizer

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.