diff --git a/crawler_detect/resources/crawler_regex_list.txt b/crawler_detect/resources/crawler_regex_list.txt index 8fa8ec6..22fa8af 100644 --- a/crawler_detect/resources/crawler_regex_list.txt +++ b/crawler_detect/resources/crawler_regex_list.txt @@ -188,7 +188,6 @@ Embedly europarchive\.org evc-batch\/[0-9] EventMachine HttpClient -Evidon Evrinid ExactSearch ExaleadCloudview diff --git a/crawler_detect/resources/known_crawler_useragents.txt b/crawler_detect/resources/known_crawler_useragents.txt index ea2c07a..daf7314 100644 --- a/crawler_detect/resources/known_crawler_useragents.txt +++ b/crawler_detect/resources/known_crawler_useragents.txt @@ -170,7 +170,6 @@ iqdb/0.1 (+http://iqdb.org/) Mozilla/5.0 (compatible; GimmeUSAbot/1.0; +https://gimmeusa.com/pages/crawler) Motoricerca-Robots.txt-Checker/1.0 (http://tool.motoricerca.info/robots-checker.phtml) Mozilla/5.0 (compatible; Tagoobot/3.0; +http://www.tagoo.ru) -Mozilla/5.0 (Windows NT 6.1; WOW64; rv:26.0) Gecko/20100101 Firefox/26.0 Evidon (lab@evidon.com) Mozilla/5.0 (compatible; Nmap Scripting Engine; http://nmap.org/book/nse.html) yacybot (i386 Linux 2.6.32-22-generic; java 1.6.0_20; Europe/de) http://yacy.net/bot.html Mozilla/5.0 (Linux; Android 4.4.3; HTC One Build/KTU84L) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2125.111 Mobile Safari/537.36 DareBoost