Pull request: 3972-hostlists-registry

Updates AdguardTeam#3972. Squashed commit of the following: commit 8341c13 Author: Ainar Garipov <[email protected]> Date: Fri Oct 21 13:06:55 2022 +0300 all: imp docs commit b7f961c Merge: 607077c 68d13fc Author: Ainar Garipov <[email protected]> Date: Thu Oct 20 18:23:57 2022 +0300 Merge branch 'master' into 3972-hostlists-registry commit 607077c Author: Ainar Garipov <[email protected]> Date: Thu Oct 20 18:07:09 2022 +0300 all: use hostlists registry
ssrahul96 · Oct 21, 2022 · 2a85d7d · 2a85d7d
1 parent 68d13fc
commit 2a85d7d
Show file tree

Hide file tree

Showing 7 changed files with 358 additions and 112 deletions.
diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml
@@ -6,6 +6,11 @@
         website
     'name': 'AdGuard filters issues'
     'url': 'https://link.adtidy.org/forward.html?action=report&app=home&from=github'
+  - 'about': >
+        Please send requests for addition to the vetted filtering lists to the
+        Hostlists Registry repository.
+    'name': 'AdGuard Hostlists Registry'
+    'url': 'https://github.com/AdguardTeam/HostlistsRegistry'
   - 'about': >
         Please use GitHub Discussions for questions
     'name': 'Q&A Discussions'

diff --git a/client/src/components/Filters/DnsBlocklist.js b/client/src/components/Filters/DnsBlocklist.js
@@ -15,7 +15,7 @@ import {
     getObjDiff,
 } from '../../helpers/helpers';
 
-const filtersCatalog = require('../../helpers/filters/filters.json');
+import filtersCatalog from '../../helpers/filters/filters';
 
 class DnsBlocklist extends Component {
     componentDidMount() {

diff --git a/client/src/components/Filters/Form.js b/client/src/components/Filters/Form.js
@@ -7,8 +7,7 @@ import classNames from 'classnames';
 import { validatePath, validateRequiredValue } from '../../helpers/validators';
 import { CheckboxField, renderInputField } from '../../helpers/form';
 import { MODAL_OPEN_TIMEOUT, MODAL_TYPE, FORM_NAME } from '../../helpers/constants';
-
-const filtersCatalog = require('../../helpers/filters/filters.json');
+import filtersCatalog from '../../helpers/filters/filters';
 
 const getIconsData = (homepage, source) => ([
     {

diff --git a/client/src/helpers/filters/filters.json → client/src/helpers/filters/filters.js b/client/src/helpers/filters/filters.json → client/src/helpers/filters/filters.js
@@ -1,162 +1,220 @@
-{
+// Code generated by go run ./scripts/vetted-filters/main.go; DO NOT EDIT.
+
+/* eslint quote-props: 'off', quotes: 'off', comma-dangle: 'off', semi: 'off' */
+
+export default {
     "categories": {
         "general": {
             "name": "filter_category_general",
             "description": "filter_category_general_desc"
         },
-        "security": {
-            "name": "filter_category_security",
-            "description": "filter_category_security_desc"
+        "other": {
+            "name": "filter_category_other",
+            "description": "filter_category_other_desc"
         },
         "regional": {
             "name": "filter_category_regional",
             "description": "filter_category_regional_desc"
         },
-        "other": {
-            "name": "filter_category_other",
-            "description": "filter_category_other_desc"
+        "security": {
+            "name": "filter_category_security",
+            "description": "filter_category_security_desc"
         }
     },
     "filters": {
-        "adguard-dns-filter": {
+        "1hosts_lite": {
+            "name": "1Hosts (Lite)",
+            "categoryId": "general",
+            "homepage": "https://badmojr.github.io/1Hosts/",
+            "source": "https://badmojr.gitlab.io/1hosts/Lite/adblock.txt"
+        },
+        "CHN_adrules": {
+            "name": "CHN: AdRules DNS List",
+            "categoryId": "regional",
+            "homepage": "https://github.com/Cats-Team/AdRules",
+            "source": "https://raw.githubusercontent.com/Cats-Team/AdRules/main/dns.txt"
+        },
+        "CHN_anti_ad": {
+            "name": "CHN: anti-AD",
+            "categoryId": "regional",
+            "homepage": "https://anti-ad.net/",
+            "source": "https://anti-ad.net/easylist.txt"
+        },
+        "IDN_abpindo": {
+            "name": "IDN: ABPindo",
+            "categoryId": "regional",
+            "homepage": "https://github.com/ABPindo/indonesianadblockrules",
+            "source": "https://raw.githubusercontent.com/ABPindo/indonesianadblockrules/master/subscriptions/aghome.txt"
+        },
+        "IRN_unwanted_iranian_domains": {
+            "name": "IRN: PersianBlocker list",
+            "categoryId": "regional",
+            "homepage": "https://github.com/MasterKia/PersianBlocker",
+            "source": "https://raw.githubusercontent.com/MasterKia/PersianBlocker/main/PersianBlockerHosts.txt"
+        },
+        "ITA_filtri_dns": {
+            "name": "ITA: Filtri-DNS",
+            "categoryId": "regional",
+            "homepage": "https://filtri-dns.ga/",
+            "source": "https://filtri-dns.ga/filtri.txt"
+        },
+        "KOR_list_kr": {
+            "name": "KOR: List-KR DNS",
+            "categoryId": "regional",
+            "homepage": "https://github.com/List-KR/List-KR",
+            "source": "https://github.com/List-KR/List-KR"
+        },
+        "KOR_youslist": {
+            "name": "KOR: YousList",
+            "categoryId": "regional",
+            "homepage": "https://github.com/yous/YousList",
+            "source": "https://raw.githubusercontent.com/yous/YousList/master/hosts.txt"
+        },
+        "MKD_macedonian_pi_hole_blocklist": {
+            "name": "MKD: Macedonian Pi-hole Blocklist",
+            "categoryId": "regional",
+            "homepage": "https://github.com/cchevy/macedonian-pi-hole-blocklist",
+            "source": "https://raw.githubusercontent.com/cchevy/macedonian-pi-hole-blocklist/master/hosts.txt"
+        },
+        "NOR_dandelion_sprouts_anti_malware_list": {
+            "name": "NOR: Dandelion Sprouts nordiske filtre",
+            "categoryId": "regional",
+            "homepage": "https://github.com/DandelionSprout/adfilt",
+            "source": "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/NorwegianExperimentalList%20alternate%20versions/NordicFiltersAdGuardHome.txt"
+        },
+        "POL_polish_filters_for_pi_hole": {
+            "name": "POL: Polish filters for Pi hole",
+            "categoryId": "regional",
+            "homepage": "https://www.certyficate.it/",
+            "source": "https://raw.githubusercontent.com/MajkiIT/polish-ads-filter/master/polish-pihole-filters/hostfile.txt"
+        },
+        "SWE_frellwit_swedish_hosts_file": {
+            "name": "SWE: Frellwit's Swedish Hosts File",
+            "categoryId": "regional",
+            "homepage": "https://github.com/lassekongo83/Frellwits-filter-lists/",
+            "source": "https://raw.githubusercontent.com/lassekongo83/Frellwits-filter-lists/master/Frellwits-Swedish-Hosts-File.txt"
+        },
+        "TUR_turk_adlist": {
+            "name": "TUR: turk-adlist",
+            "categoryId": "regional",
+            "homepage": "https://github.com/bkrucarci/turk-adlist",
+            "source": "https://raw.githubusercontent.com/bkrucarci/turk-adlist/master/hosts"
+        },
+        "VNM_abpvn": {
+            "name": "VNM: ABPVN List",
+            "categoryId": "regional",
+            "homepage": "http://abpvn.com/",
+            "source": "https://abpvn.com/android/abpvn.txt"
+        },
+        "adguard_dns_filter": {
             "name": "AdGuard DNS filter",
             "categoryId": "general",
             "homepage": "https://github.com/AdguardTeam/AdGuardSDNSFilter",
             "source": "https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt"
         },
-        "adaway-default-blocklist": {
+        "adway_default_blocklist": {
             "name": "AdAway Default Blocklist",
             "categoryId": "general",
             "homepage": "https://github.com/AdAway/adaway.github.io/",
             "source": "https://adaway.org/hosts.txt"
         },
-        "peter-lowe-list": {
-            "name": "Peter Lowe's List",
-            "categoryId": "general",
-            "homepage": "https://pgl.yoyo.org/adservers/",
-            "source": "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=adblockplus&showintro=1&mimetype=plaintext"
+        "curben_phishing_filter": {
+            "name": "Phishing URL Blocklist (PhishTank and OpenPhish)",
+            "categoryId": "security",
+            "homepage": "https://gitlab.com/malware-filter/phishing-filter",
+            "source": "https://malware-filter.gitlab.io/malware-filter/phishing-filter-agh.txt"
         },
-        "dan-pollock-list": {
+        "dan_pollocks_list": {
             "name": "Dan Pollock's List",
             "categoryId": "general",
             "homepage": "https://someonewhocares.org/",
             "source": "https://someonewhocares.org/hosts/zero/hosts"
         },
-        "oisd": {
-            "name": "OISD Blocklist Basic",
-            "categoryId": "general",
-            "homepage": "https://oisd.nl/",
-            "source": "https://abp.oisd.nl/basic/"
+        "dandelion_sprouts_anti_malware_list": {
+            "name": "Dandelion Sprout's Anti-Malware List",
+            "categoryId": "security",
+            "homepage": "https://github.com/DandelionSprout/adfilt",
+            "source": "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareAdGuardHome.txt"
         },
-        "game-console-adblock-list": {
-            "name": "Game Console Adblock List",
-            "categoryId": "general",
+        "dandelion_sprouts_game_console_adblock_list": {
+            "name": "Dandelion Sprout's Game Console Adblock List",
+            "categoryId": "other",
             "homepage": "https://github.com/DandelionSprout/adfilt",
             "source": "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/GameConsoleAdblockList.txt"
         },
-        "perflyst-dandelion-sprout-smart-tv-blocklist-for-adguard-home": {
-            "name": "Perflyst and Dandelion Sprout's Smart-TV Blocklist",
+        "energized_spark": {
+            "name": "Energized Spark",
             "categoryId": "general",
-            "homepage": "https://github.com/Perflyst/PiHoleBlocklist",
-            "source": "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV-AGH.txt"
+            "homepage": "https://energized.pro/",
+            "source": "https://block.energized.pro/spark/formats/filter"
         },
-        "nocoin-filter-list": {
+        "nocoin_filter_list": {
             "name": "NoCoin Filter List",
             "categoryId": "security",
             "homepage": "https://github.com/hoshsadiq/adblock-nocoin-list/",
             "source": "https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt"
         },
-        "the-big-list-of-hacked-malware-web-sites": {
-            "name": "The Big List of Hacked Malware Web Sites",
-            "categoryId": "security",
-            "homepage": "https://github.com/mitchellkrogza/The-Big-List-of-Hacked-Malware-Web-Sites",
-            "source": "https://raw.githubusercontent.com/mitchellkrogza/The-Big-List-of-Hacked-Malware-Web-Sites/master/hosts"
+        "notracking_hosts_blocklists": {
+            "name": "The NoTracking blocklist",
+            "categoryId": "general",
+            "homepage": "https://github.com/notracking/hosts-blocklists",
+            "source": "https://raw.githubusercontent.com/notracking/hosts-blocklists/master/adblock/adblock.txt"
+        },
+        "oisd_basic": {
+            "name": "OISD Blocklist Basic",
+            "categoryId": "general",
+            "homepage": "https://oisd.nl/",
+            "source": "https://abp.oisd.nl/basic/"
+        },
+        "oisd_full": {
+            "name": "OISD Blocklist Full",
+            "categoryId": "general",
+            "homepage": "https://oisd.nl/",
+            "source": "https://abp.oisd.nl/"
+        },
+        "perflyst_dandelion_sprout_smart_tv_blocklist_for_adguard_home": {
+            "name": "Perflyst and Dandelion Sprout's Smart-TV Blocklist",
+            "categoryId": "other",
+            "homepage": "https://github.com/Perflyst/PiHoleBlocklist",
+            "source": "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV-AGH.txt"
+        },
+        "peter_lowe_list": {
+            "name": "Peter Lowe's Blocklist",
+            "categoryId": "general",
+            "homepage": "https://pgl.yoyo.org/adservers/",
+            "source": "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=adblockplus\u0026showintro=1\u0026mimetype=plaintext"
         },
-        "scam-blocklist-by-durable-napkin": {
+        "scam_blocklist_by_durablenapkin": {
             "name": "Scam Blocklist by DurableNapkin",
             "categoryId": "security",
             "homepage": "https://github.com/durablenapkin/scamblocklist",
             "source": "https://raw.githubusercontent.com/durablenapkin/scamblocklist/master/adguard.txt"
         },
-        "urlhaus-filter-online": {
-            "name": "Online Malicious URL Blocklist",
-            "categoryId": "security",
-            "homepage": "https://gitlab.com/malware-filter/urlhaus-filter",
-            "source": "https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-agh-online.txt"
-        },
-        "dandelion-sprouts-anti-malware-list": {
-            "name": "Dandelion Sprout's Anti-Malware List",
+        "staklerware_indicators_list": {
+            "name": "Stalkerware Indicators List",
             "categoryId": "security",
-            "homepage": "https://github.com/DandelionSprout/adfilt",
-            "source": "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareAdGuardHome.txt"
-        },
-        "NOR-dandelion-sprouts-nordiske-filtre": {
-            "name": "NOR: Dandelion Sprouts nordiske filtre",
-            "categoryId": "regional",
-            "homepage": "https://github.com/DandelionSprout/adfilt",
-            "source": "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/NorwegianExperimentalList%20alternate%20versions/NordicFiltersAdGuardHome.txt"
-        },
-        "POL-polish-filters-for-pihole": {
-            "name": "POL: Polish filters for Pi hole",
-            "categoryId": "regional",
-            "homepage": "https://www.certyficate.it/",
-            "source": "https://raw.githubusercontent.com/MajkiIT/polish-ads-filter/master/polish-pihole-filters/hostfile.txt"
+            "homepage": "https://github.com/AssoEchap/stalkerware-indicators",
+            "source": "https://raw.githubusercontent.com/AssoEchap/stalkerware-indicators/master/generated/hosts"
         },
-        "KOR-youslist": {
-            "name": "KOR: YousList",
-            "categoryId": "regional",
-            "homepage": "https://github.com/yous/YousList",
-            "source": "https://raw.githubusercontent.com/yous/YousList/master/hosts.txt"
-        },
-        "VNM-abpvn-list": {
-            "name": "VNM: ABPVN List",
-            "categoryId": "regional",
-            "homepage": "http://abpvn.com/",
-            "source": "https://abpvn.com/android/abpvn.txt"
-        },
-        "SWE-frellwit-swedish-hosts-file": {
-            "name": "SWE: Frellwit's Swedish Hosts File",
-            "categoryId": "regional",
-            "homepage": "https://github.com/lassekongo83/Frellwits-filter-lists/",
-            "source": "https://raw.githubusercontent.com/lassekongo83/Frellwits-filter-lists/master/Frellwits-Swedish-Hosts-File.txt"
-        },
-        "ITA-filtri-dns": {
-            "name": "ITA: Filtri-DNS",
-            "categoryId": "regional",
-            "homepage": "https://filtri-dns.ga/",
-            "source": "https://filtri-dns.ga/filtri.txt"
-        },
-        "IRN-unwanted-iranian-domains": {
-            "name": "IRN: Unwanted Iranian domains",
-            "categoryId": "regional",
-            "homepage": "https://github.com/DRSDavidSoft/additional-hosts",
-            "source": "https://raw.githubusercontent.com/DRSDavidSoft/additional-hosts/master/domains/blacklist/unwanted-iranian.txt"
-        },
-        "MKD-macedonian-pi-hole-blocklist": {
-            "name": "MKD: Macedonian Pi-hole Blocklist",
-            "categoryId": "regional",
-            "homepage": "https://github.com/cchevy/macedonian-pi-hole-blocklist",
-            "source": "https://raw.githubusercontent.com/cchevy/macedonian-pi-hole-blocklist/master/hosts.txt"
-        },
-        "CHN-anti-ad" : {
-            "name": "CHN: anti-AD",
-            "categoryId": "regional",
-            "homepage": "https://anti-ad.net/",
-            "source": "https://anti-ad.net/easylist.txt"
+        "steven_blacks_list": {
+            "name": "Steven Black's List",
+            "categoryId": "general",
+            "homepage": "https://github.com/StevenBlack/hosts",
+            "source": "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts"
         },
-        "IDN-abpindo": {
-            "name": "IDN: ABPindo",
-            "categoryId": "regional",
-            "homepage": "https://github.com/ABPindo/indonesianadblockrules/",
-            "source": "https://raw.githubusercontent.com/ABPindo/indonesianadblockrules/master/subscriptions/abpindo.txt"
+        "the_big_list_of_hacked_malware_web_sites": {
+            "name": "The Big List of Hacked Malware Web Sites",
+            "categoryId": "security",
+            "homepage": "https://github.com/mitchellkrogza/The-Big-List-of-Hacked-Malware-Web-Sites",
+            "source": "https://raw.githubusercontent.com/mitchellkrogza/The-Big-List-of-Hacked-Malware-Web-Sites/master/hosts"
         },
-        "NLD-Easylist": {
-            "name": "NLD: Easylist",
-            "categoryId": "regional",
-            "homepage": "https://forums.lanik.us/viewforum.php?f=100",
-            "source": "https://easylist-downloads.adblockplus.org/easylistdutch.txt"
+        "urlhaus_filter_online": {
+            "name": "Malicious URL Blocklist (URLHaus)",
+            "categoryId": "security",
+            "homepage": "https://gitlab.com/malware-filter/urlhaus-filter",
+            "source": "https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-agh.txt"
         },
-        "windows-spy-blocker" : {
+        "windowsspyblocker_hosts_spy_rules": {
             "name": "WindowsSpyBlocker - Hosts spy rules",
             "categoryId": "other",
             "homepage": "https://github.com/crazy-max/WindowsSpyBlocker",

diff --git a/scripts/README.md b/scripts/README.md
@@ -205,3 +205,23 @@ code from [the repo][companiesrepo].
 ```
 
 [companiesrepo]: https://github.com/AdguardTeam/companiesdb
+
+
+
+##  `vetted-filters/`: Vetted Filters Updater
+
+A simple script that downloads and updates the vetted filtering list data from
+AdGuard's [Hostlists Registry][reg].
+
+Optional environment:
+
+ *  `URL`: the URL of the index file.  By default it's
+    `https://adguardteam.github.io/HostlistsRegistry/assets/filters.json`.
+
+ ###  Usage
+
+```sh
+go run ./scripts/vetted-filters/main.go
+```
+
+[reg]: https://github.com/AdguardTeam/HostlistsRegistry
diff --git a/scripts/make/go-lint.sh b/scripts/make/go-lint.sh
@@ -229,7 +229,7 @@ gocyclo --over 13 ./internal/dhcpd ./internal/filtering/ ./internal/home/
 gocyclo --over 10 ./internal/aghio/ ./internal/aghnet/ ./internal/aghos/\
 	./internal/aghtest/ ./internal/dnsforward/ ./internal/stats/\
 	./internal/tools/ ./internal/updater/ ./internal/next/ ./internal/version/\
-	./main.go
+	./scripts/vetted-filters/ ./main.go
 
 ineffassign ./...