:mod:`capreolus.benchmark`
==========================

.. py:module:: capreolus.benchmark


Package Contents
----------------

Classes
~~~~~~~

.. autoapisummary::

   capreolus.benchmark.Benchmark
   capreolus.benchmark.DummyBenchmark
   capreolus.benchmark.WSDM20Demo
   capreolus.benchmark.Robust04Yang19
   capreolus.benchmark.ANTIQUE
   capreolus.benchmark.MSMarcoPassage
   capreolus.benchmark.CodeSearchNetCorpus
   capreolus.benchmark.CodeSearchNetChallenge
   capreolus.benchmark.COVID
   capreolus.benchmark.CovidQA



.. data:: logger
   

   

.. data:: PACKAGE_PATH
   

   

.. py:class:: Benchmark

   Bases: :class:`profane.ModuleBase`

   the module base class

   .. attribute:: module_type
      :annotation: = benchmark

      

   .. attribute:: qrel_file
      

      

   .. attribute:: topic_file
      

      

   .. attribute:: fold_file
      

      

   .. attribute:: query_type
      

      

   .. attribute:: relevance_level
      :annotation: = 1

      

   .. attribute:: qrels
      

      

   .. attribute:: topics
      

      

   .. attribute:: folds
      

      


.. py:class:: DummyBenchmark

   Bases: :class:`capreolus.benchmark.Benchmark`

   the module base class

   .. attribute:: module_name
      :annotation: = dummy

      

   .. attribute:: dependencies
      

      

   .. attribute:: qrel_file
      

      

   .. attribute:: topic_file
      

      

   .. attribute:: fold_file
      

      

   .. attribute:: query_type
      :annotation: = title

      


.. py:class:: WSDM20Demo

   Bases: :class:`capreolus.benchmark.Benchmark`

   the module base class

   .. attribute:: module_name
      :annotation: = wsdm20demo

      

   .. attribute:: dependencies
      

      

   .. attribute:: qrel_file
      

      

   .. attribute:: topic_file
      

      

   .. attribute:: fold_file
      

      

   .. attribute:: query_type
      :annotation: = title

      


.. py:class:: Robust04Yang19

   Bases: :class:`capreolus.benchmark.Benchmark`

   the module base class

   .. attribute:: module_name
      :annotation: = robust04.yang19

      

   .. attribute:: dependencies
      

      

   .. attribute:: qrel_file
      

      

   .. attribute:: topic_file
      

      

   .. attribute:: fold_file
      

      

   .. attribute:: query_type
      :annotation: = title

      


.. py:class:: ANTIQUE

   Bases: :class:`capreolus.benchmark.Benchmark`

   the module base class

   .. attribute:: module_name
      :annotation: = antique

      

   .. attribute:: dependencies
      

      

   .. attribute:: qrel_file
      

      

   .. attribute:: topic_file
      

      

   .. attribute:: fold_file
      

      

   .. attribute:: query_type
      :annotation: = title

      

   .. attribute:: relevance_level
      :annotation: = 2

      


.. py:class:: MSMarcoPassage

   Bases: :class:`capreolus.benchmark.Benchmark`

   the module base class

   .. attribute:: module_name
      :annotation: = msmarcopassage

      

   .. attribute:: dependencies
      

      

   .. attribute:: qrel_file
      

      

   .. attribute:: topic_file
      

      

   .. attribute:: fold_file
      

      

   .. attribute:: query_type
      :annotation: = title

      


.. py:class:: CodeSearchNetCorpus

   Bases: :class:`capreolus.benchmark.Benchmark`

   the module base class

   .. attribute:: module_name
      :annotation: = codesearchnet_corpus

      

   .. attribute:: dependencies
      

      

   .. attribute:: url
      :annotation: = https://s3.amazonaws.com/code-search-net/CodeSearchNet/v2

      

   .. attribute:: query_type
      :annotation: = title

      

   .. attribute:: file_fn
      

      

   .. attribute:: qrel_dir
      

      

   .. attribute:: topic_dir
      

      

   .. attribute:: fold_dir
      

      

   .. attribute:: qidmap_dir
      

      

   .. attribute:: docidmap_dir
      

      

   .. attribute:: config_spec
      

      

   .. attribute:: qid_map
      

      

   .. attribute:: docid_map
      

      

   .. method:: build(self)


   .. method:: download_if_missing(self)


   .. method:: get_docid(self, url, code_tokens)

      retrieve the doc id according to the doc dict



.. py:class:: CodeSearchNetChallenge

   Bases: :class:`capreolus.benchmark.Benchmark`

   CodeSearchNetChallenge can only be used for training but not for evaluation since qrels is not provided

   .. attribute:: module_name
      :annotation: = codesearchnet_challenge

      

   .. attribute:: dependencies
      

      

   .. attribute:: config_spec
      

      

   .. attribute:: url
      :annotation: = https://raw.githubusercontent.com/github/CodeSearchNet/master/resources/queries.csv

      

   .. attribute:: query_type
      :annotation: = title

      

   .. attribute:: file_fn
      

      

   .. attribute:: topic_file
      

      

   .. attribute:: qid_map_file
      

      

   .. method:: download_if_missing(self)

      download query.csv and prepare queryid - query mapping file



.. py:class:: COVID

   Bases: :class:`capreolus.benchmark.Benchmark`

   the module base class

   .. attribute:: module_name
      :annotation: = covid

      

   .. attribute:: dependencies
      

      

   .. attribute:: data_dir
      

      

   .. attribute:: topic_url
      :annotation: = https://ir.nist.gov/covidSubmit/data/topics-rnd%d.xml

      

   .. attribute:: qrel_url
      :annotation: = https://ir.nist.gov/covidSubmit/data/qrels-rnd%d.txt

      

   .. attribute:: lastest_round
      :annotation: = 3

      

   .. attribute:: config_spec
      

      

   .. method:: build(self)


   .. method:: download_if_missing(self)


   .. method:: xml2trectopic(self, xmlfile)



.. py:class:: CovidQA

   Bases: :class:`capreolus.benchmark.Benchmark`

   the module base class

   .. attribute:: module_name
      :annotation: = covidqa

      

   .. attribute:: dependencies
      

      

   .. attribute:: url
      :annotation: = https://raw.githubusercontent.com/castorini/pygaggle/master/data/kaggle-lit-review-%s.json

      

   .. attribute:: available_versions
      :annotation: = ['0.1', '0.2']

      

   .. attribute:: datadir
      

      

   .. attribute:: config_spec
      

      

   .. method:: build(self)


   .. method:: download_if_missing(self)