feat: wikit_ua in config

lukasellinger · Oct 21, 2024 · 1635e9f · 1635e9f
1 parent 1885952
commit 1635e9f
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 2 deletions.
diff --git a/config.py.template b/config.py.template
@@ -10,3 +10,7 @@ HF_WRITE_TOKEN = ''
 HF_READ_TOKENS = ['', '', '']
 
 OPEN_AI_TOKEN = ''
+
+# please follow https://foundation.wikimedia.org/wiki/Policy:User-Agent_policy
+# e.g.: evaluating-factuality-word-definitions-bot (<your email>)
+WIKI_USER_AGENT = ''
diff --git a/fetchers/wikipedia.py b/fetchers/wikipedia.py
@@ -8,6 +8,7 @@
 from requests import Response
 from transformers import RobertaTokenizer
 
+from config import WIKI_USER_AGENT
 from fetchers.wiktionary_parser import WiktionaryParser
 from general_utils.spacy_utils import (split_into_passage_sentences,
                                        split_into_sentences)
@@ -18,7 +19,7 @@
 class Wikipedia:
     """Wrapper for wikipedia api calls."""
 
-    USER_AGENT = 'summaryBot ([email protected])'
+    USER_AGENT = WIKI_USER_AGENT
     BASE_URL = "https://{source_lang}.{site}.org/w/api.php"
 
     def __init__(self, source_lang: str = 'en', user_agent: str = None, use_dataset: str = ''):

diff --git a/general_utils/utils.py b/general_utils/utils.py
@@ -344,7 +344,7 @@ def get_openai_prediction(response) -> str:
     return 'UNKNOWN'
 
 
-def split_into_passages(text: str, tokenizer, max_length=256) -> List[str]:
+def split_into_passages(text: str | List[str], tokenizer, max_length=256) -> List[str]:
     """
     Splits text into passages of a specified token length using a tokenizer.