Source code for capreolus.benchmark.antique

from capreolus import Dependency, constants
from capreolus.utils.loginit import get_logger

from . import Benchmark

[docs]logger = get_logger(__name__)
[docs]PACKAGE_PATH = constants["PACKAGE_PATH"]
@Benchmark.register
[docs]class ANTIQUE(Benchmark): """A Non-factoid Question Answering Benchmark from Hashemi et al. [1] [1] Helia Hashemi, Mohammad Aliannejadi, Hamed Zamani, and W. Bruce Croft. 2020. ANTIQUE: A non-factoid question answering benchmark. ECIR 2020. """
[docs] module_name = "antique"
[docs] dependencies = [Dependency(key="collection", module="collection", name="antique")]
[docs] qrel_file = PACKAGE_PATH / "data" / "qrels.antique.txt"
[docs] topic_file = PACKAGE_PATH / "data" / "topics.antique.txt"
[docs] fold_file = PACKAGE_PATH / "data" / "antique.json"
[docs] query_type = "title"
[docs] relevance_level = 2