Add Trec dataset

test/asset/raw_datasets.json

@@ -45,3 +45,5 @@
 {"dataset_name": "SQuAD2", "split": "train", "NUM_LINES": 130319, "MD5": {"train": "62108c273c268d70893182d5cf8df740", "dev": "246adae8b7002f8679c027697b0b7cf8"}, "URL": {"train": "https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json", "dev": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json"}, "first_line": ["Beyonc\u00e9 Giselle Knowles-Carter (/bi\u02d0\u02c8j\u0252nse\u026a/ bee-YON-say) (born September 4, 1981) is an American singer, songwriter, record producer and actress. Born and raised in Houston, Texas, she performed in various singing and dancing competitions as a child, and rose to fame in the late 1990s as lead singer of R&B girl-group Destiny's Child. Managed by her father, Mathew Knowles, the group became one of the world's best-selling girl groups of all time. Their hiatus saw the release of Beyonc\u00e9's debut album, Dangerously in Love (2003), which established her as a solo artist worldwide, earned five Grammy Awards and featured the Billboard Hot 100 number-one singles \"Crazy in Love\" and \"Baby Boy\".", "When did Beyonce start becoming popular?", ["in the late 1990s"], [269]]}
 {"dataset_name": "SQuAD2", "split": "dev", "NUM_LINES": 11873, "MD5": {"train": "62108c273c268d70893182d5cf8df740", "dev": "246adae8b7002f8679c027697b0b7cf8"}, "URL": {"train": "https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json", "dev": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json"}, "first_line": ["The Normans (Norman: Nourmands; French: Normands; Latin: Normanni) were the people who in the 10th and 11th centuries gave their name to Normandy, a region in France. They were descended from Norse (\"Norman\" comes from \"Norseman\") raiders and pirates from Denmark, Iceland and Norway who, under their leader Rollo, agreed to swear fealty to King Charles III of West Francia. Through generations of assimilation and mixing with the native Frankish and Roman-Gaulish populations, their descendants would gradually merge with the Carolingian-based cultures of West Francia. The distinct cultural and ethnic identity of the Normans emerged initially in the first half of the 10th century, and it continued to evolve over the succeeding centuries.", "In what country is Normandy located?", ["France", "France", "France", "France"], [159, 159, 159, 159]]}
 {"dataset_name": "EnWik9", "split": "train", "NUM_LINES": 13147026, "MD5": "3e773f8a1577fda2e27f871ca17f31fd", "URL": "http://mattmahoney.net/dc/enwik9.zip", "first_line": "<mediawiki xmlns=\"http://www.mediawiki.org/xml/export-0.3/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.mediawiki.org/xml/export-0.3/ http://www.mediawiki.org/xml/export-0.3.xsd\" version=\"0.3\" xml:lang=\"en\">\n"}
+{"dataset_name": "Trec", "split": "train", "NUM_LINES": 5452, "MD5": {"train": "073462e3fcefaae31e00edb1f18d2d02", "test": "323a3554401d86e650717e2d2f942589"}, "URL": {"train": "http://cogcomp.org/Data/QA/QC/train_5500.label", "test": "http://cogcomp.org/Data/QA/QC/TREC_10.label"}, "first_line": ["manner", "How did serfdom develop in and then leave Russia ?\n"]}
+{"dataset_name": "Trec", "split": "test", "NUM_LINES": 500, "MD5": {"train": "073462e3fcefaae31e00edb1f18d2d02", "test": "323a3554401d86e650717e2d2f942589"}, "URL": {"train": "http://cogcomp.org/Data/QA/QC/train_5500.label", "test": "http://cogcomp.org/Data/QA/QC/TREC_10.label"}, "first_line": ["dist", "How far is it from Denver to Aspen ?\n"]}

torchtext/datasets/__init__.py

@@ -12,6 +12,7 @@
 from .sogounews import SogouNews
 from .squad1 import SQuAD1
 from .squad2 import SQuAD2
+from .trec import Trec
 from .udpos import UDPOS
 from .wikitext103 import WikiText103
 from .wikitext2 import WikiText2
@@ -35,6 +36,7 @@
     'SQuAD1': SQuAD1,
     'SQuAD2': SQuAD2,
     'SogouNews': SogouNews,
+    'Trec': Trec,
     'UDPOS': UDPOS,
     'WMT14': WMT14,
     'WMTNewsCrawl': WMTNewsCrawl,

torchtext/datasets/trec.py

@@ -0,0 +1,39 @@
+from torchtext.utils import download_from_url
+from torchtext.datasets.common import RawTextIterableDataset
+from torchtext.datasets.common import wrap_split_argument
+from torchtext.datasets.common import add_docstring_header
+import os
+
+URL = {
+    'train': 'http://cogcomp.org/Data/QA/QC/train_5500.label',
+    'test': 'http://cogcomp.org/Data/QA/QC/TREC_10.label',
+}
+
+MD5 = {
+    'train': '073462e3fcefaae31e00edb1f18d2d02',
+    'test': '323a3554401d86e650717e2d2f942589',
+}
+
+NUM_LINES = {
+    'train': 5452,
+    'test': 500,
+}
+
+
+@wrap_split_argument
+@add_docstring_header()
+def Trec(root='.data', split=('train', 'test'), offset=0):
+    def _create_data_from_file(data_path):
+        for line in open(os.path.expanduser(data_path), 'rb'):
+            # there is one non-ASCII byte: sisterBADBYTEcity; replaced with space
+            label, _, text = line.replace(b'\xf0', b' ').decode().partition(' ')
+            label = label.split(":")[1]
+            yield label, text
+
+    datasets = []
+    for item in split:
+        data_path = download_from_url(URL[item], root=root,
+                                      hash_value=MD5[item], hash_type='md5')
+        datasets.append(RawTextIterableDataset("Trec", NUM_LINES[item],
+                                               _create_data_from_file(data_path), offset=offset))
+    return datasets