capreolus.benchmark

Package Contents

Classes

Benchmark() the module base class
DummyBenchmark() the module base class
WSDM20Demo() the module base class
Robust04Yang19() the module base class
ANTIQUE() the module base class
MSMarcoPassage() the module base class
CodeSearchNetCorpus() the module base class
CodeSearchNetChallenge() CodeSearchNetChallenge can only be used for training but not for evaluation since qrels is not provided
COVID() the module base class
CovidQA() the module base class
capreolus.benchmark.logger[source]
capreolus.benchmark.PACKAGE_PATH[source]
class capreolus.benchmark.Benchmark[source]

Bases: profane.ModuleBase

the module base class

module_type = benchmark[source]
qrel_file[source]
topic_file[source]
fold_file[source]
query_type[source]
relevance_level = 1[source]
qrels[source]
topics[source]
folds[source]
class capreolus.benchmark.DummyBenchmark[source]

Bases: capreolus.benchmark.Benchmark

the module base class

module_name = dummy[source]
dependencies[source]
qrel_file[source]
topic_file[source]
fold_file[source]
query_type = title[source]
class capreolus.benchmark.WSDM20Demo[source]

Bases: capreolus.benchmark.Benchmark

the module base class

module_name = wsdm20demo[source]
dependencies[source]
qrel_file[source]
topic_file[source]
fold_file[source]
query_type = title[source]
class capreolus.benchmark.Robust04Yang19[source]

Bases: capreolus.benchmark.Benchmark

the module base class

module_name = robust04.yang19[source]
dependencies[source]
qrel_file[source]
topic_file[source]
fold_file[source]
query_type = title[source]
class capreolus.benchmark.ANTIQUE[source]

Bases: capreolus.benchmark.Benchmark

the module base class

module_name = antique[source]
dependencies[source]
qrel_file[source]
topic_file[source]
fold_file[source]
query_type = title[source]
relevance_level = 2[source]
class capreolus.benchmark.MSMarcoPassage[source]

Bases: capreolus.benchmark.Benchmark

the module base class

module_name = msmarcopassage[source]
dependencies[source]
qrel_file[source]
topic_file[source]
fold_file[source]
query_type = title[source]
class capreolus.benchmark.CodeSearchNetCorpus[source]

Bases: capreolus.benchmark.Benchmark

the module base class

module_name = codesearchnet_corpus[source]
dependencies[source]
url = https://s3.amazonaws.com/code-search-net/CodeSearchNet/v2[source]
query_type = title[source]
file_fn[source]
qrel_dir[source]
topic_dir[source]
fold_dir[source]
qidmap_dir[source]
docidmap_dir[source]
config_spec[source]
qid_map[source]
docid_map[source]
build(self)[source]
download_if_missing(self)[source]
get_docid(self, url, code_tokens)[source]

retrieve the doc id according to the doc dict

class capreolus.benchmark.CodeSearchNetChallenge[source]

Bases: capreolus.benchmark.Benchmark

CodeSearchNetChallenge can only be used for training but not for evaluation since qrels is not provided

module_name = codesearchnet_challenge[source]
dependencies[source]
config_spec[source]
url = https://raw.githubusercontent.com/github/CodeSearchNet/master/resources/queries.csv[source]
query_type = title[source]
file_fn[source]
topic_file[source]
qid_map_file[source]
download_if_missing(self)[source]

download query.csv and prepare queryid - query mapping file

class capreolus.benchmark.COVID[source]

Bases: capreolus.benchmark.Benchmark

the module base class

module_name = covid[source]
dependencies[source]
data_dir[source]
topic_url = https://ir.nist.gov/covidSubmit/data/topics-rnd%d.xml[source]
qrel_url = https://ir.nist.gov/covidSubmit/data/qrels-rnd%d.txt[source]
lastest_round = 3[source]
config_spec[source]
build(self)[source]
download_if_missing(self)[source]
xml2trectopic(self, xmlfile)[source]
class capreolus.benchmark.CovidQA[source]

Bases: capreolus.benchmark.Benchmark

the module base class

module_name = covidqa[source]
dependencies[source]
url = https://raw.githubusercontent.com/castorini/pygaggle/master/data/kaggle-lit-review-%s.json[source]
available_versions = ['0.1', '0.2'][source]
datadir[source]
config_spec[source]
build(self)[source]
download_if_missing(self)[source]