capreolus.tokenizer

Package Contents

Classes

Tokenizer()
AnseriniTokenizer()
BertTokenizer()
class capreolus.tokenizer.Tokenizer[source]

Bases: profane.ModuleBase

module_type = tokenizer[source]
class capreolus.tokenizer.AnseriniTokenizer[source]

Bases: capreolus.tokenizer.Tokenizer

module_name = anserini[source]
config_spec[source]
build(self)[source]
tokenize(self, sentences)[source]
class capreolus.tokenizer.BertTokenizer[source]

Bases: capreolus.tokenizer.Tokenizer

module_name = berttokenizer[source]
config_spec[source]
build(self)[source]
convert_tokens_to_ids(self, tokens)[source]
tokenize(self, sentences)[source]