From 2d25fc0f348bdcf4a9d68c1166602f01ac1ec677 Mon Sep 17 00:00:00 2001 From: Anne Douwe Bouma Date: Thu, 21 Feb 2019 21:49:13 +0100 Subject: [PATCH] Allow custom sources, expand README and create Makefile --- Makefile | 4 ++++ README.md | 27 ++++++++++++++++++++++++++- begoneads.py | 41 ++++++++++++++++++++++++++++++++++++++--- collector.py | 25 ++++++++++--------------- exceptions.py | 8 ++++++++ helpers.py | 3 ++- 6 files changed, 88 insertions(+), 20 deletions(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0034945 --- /dev/null +++ b/Makefile @@ -0,0 +1,4 @@ +.PHONY: format + +format: *.py + autopep8 --in-place *.py diff --git a/README.md b/README.md index edbf41a..f71d8ce 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,10 @@ BeGoneAds is a script that puts some popular hosts file lists into the systems hosts file as a adblocker measure. +## Requirements + +- Python 3.6 or higher + ## Getting started Clone this repository: @@ -45,6 +49,26 @@ Commands: uninstall Uninstall BeGoneAds ``` +## Usage + +To install the hosts to your system hosts file: + +```shell +python begoneads.py install +``` + +To install the hosts to your system hosts file with custom sources: + +```shell +python begoneads.py install --sources https://www.custom.sources/hosts,http://www.and-another.one/hosts +``` + +To uninstall the hosts to your system hosts file: + +```shell +python begoneads.py uninstall +``` + ## Sources of hosts data unified in this variant Updated `hosts` files from the following locations are always unified and @@ -70,10 +94,11 @@ add.Risk | [link](https://github.com/FadeMind/hosts.extras) | ## TODO for 1.0.0 - [X] Windows support +- [X] Custom selection of host files - [ ] Systemd integration -- [ ] Custom selection of host files - [ ] Apply own hosts - [ ] Package it for Debian, Arch, CentOS, Fedora, etc. +- [ ] Setuptools ## Testing diff --git a/begoneads.py b/begoneads.py index fcbebcb..d5dd7fe 100644 --- a/begoneads.py +++ b/begoneads.py @@ -4,9 +4,26 @@ import click from collector import Collector from hostsmanager import HostsManager -from exceptions import NotElevatedException +from exceptions import * from helpers import is_admin +# Default sources +sources = [ + 'https://www.malwaredomainlist.com/hostslist/hosts.txt', + 'https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Dead/hosts', + 'https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Spam/hosts', + 'https://someonewhocares.org/hosts/zero/hosts', + 'http://winhelp2002.mvps.org/hosts.txt', + 'https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&mimetype=plaintext&useip=0.0.0.0', + 'https://raw.githubusercontent.com/mitchellkrogza/Badd-Boyz-Hosts/master/hosts', + 'https://zerodot1.gitlab.io/CoinBlockerLists/hosts_browser', + 'https://raw.githubusercontent.com/FadeMind/hosts.extras/master/UncheckyAds/hosts', + 'https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.2o7Net/hosts', + 'https://raw.githubusercontent.com/azet12/KADhosts/master/KADhosts.txt', + 'https://raw.githubusercontent.com/AdAway/adaway.github.io/master/hosts.txt', + 'https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Risk/hosts', +] + @click.group() def cli(): @@ -14,14 +31,32 @@ def cli(): @cli.command('install', short_help='Install or update BeGoneAds') -def install(): +@click.option('--sources', is_flag=False, default=','.join(sources), + type=click.STRING, help='Sets sources to fetch from, seperated by ,') +def install(sources): # Check if we have sufficient permissions if not is_admin(sys.platform.startswith('win')): raise NotElevatedException( 'This program needs to be run as root to work properly') + sources = [i.strip() for i in sources.split(',')] + + url_pattern = re.compile( + r'^(?:http|ftp)s?://' # http:// or https:// + # domain... + r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' + r'localhost|' # localhost... + r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip + r'(?::\d+)?' # optional port + r'(?:/?|[/?]\S+)$', re.IGNORECASE + ) + + for source in sources: + if not re.match(url_pattern, source): + raise InvalidSourceException(source) + # Collect hosts for hosts file - collector = Collector() + collector = Collector(sources) print('⋯ Collecting and parsing hosts') hosts = collector.get_result() diff --git a/collector.py b/collector.py index fe088d6..2b9ecda 100644 --- a/collector.py +++ b/collector.py @@ -10,23 +10,12 @@ class Collector(object): sources: list """ - sources = [ - 'https://www.malwaredomainlist.com/hostslist/hosts.txt', - 'https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Dead/hosts', - 'https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Spam/hosts', - 'https://someonewhocares.org/hosts/zero/hosts', - 'http://winhelp2002.mvps.org/hosts.txt', - 'https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&mimetype=plaintext&useip=0.0.0.0', - 'https://raw.githubusercontent.com/mitchellkrogza/Badd-Boyz-Hosts/master/hosts', - 'https://zerodot1.gitlab.io/CoinBlockerLists/hosts_browser', - 'https://raw.githubusercontent.com/FadeMind/hosts.extras/master/UncheckyAds/hosts', - 'https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.2o7Net/hosts', - 'https://raw.githubusercontent.com/azet12/KADhosts/master/KADhosts.txt', - 'https://raw.githubusercontent.com/AdAway/adaway.github.io/master/hosts.txt', - 'https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Risk/hosts', - ] + def __init__(self, sources): + self.sources = sources def try_get_sources(self, sources): + """Try and get each source, don't return them if the request was not succesful""" + filtered = [] for source in tqdm(sources): @@ -40,6 +29,8 @@ def try_get_sources(self, sources): return filtered def fix_ips(self, sources): + """Replace all IP addresses with 0.0.0.0""" + pattern = re.compile(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', re.MULTILINE) sources = re.sub(pattern, '0.0.0.0', sources) @@ -47,6 +38,8 @@ def fix_ips(self, sources): return sources def filter_hosts(self, hosts): + """Only keep meaningful lines""" + hosts = hosts.split('\n') hosts = list(set(hosts)) @@ -57,6 +50,8 @@ def filter_hosts(self, hosts): return hosts def get_result(self): + """Get usable result""" + sources = self.try_get_sources(self.sources) sources = '\n'.join(sources) hosts = self.fix_ips(sources) diff --git a/exceptions.py b/exceptions.py index e083b06..38937e8 100644 --- a/exceptions.py +++ b/exceptions.py @@ -1,2 +1,10 @@ class NotElevatedException(Exception): pass + + +class InvalidSourceException(Exception): + def __init__(self, source): + self.message = f'{source} is not a valid source' + + def __str__(self): + return self.message diff --git a/helpers.py b/helpers.py index 28263db..bd58087 100644 --- a/helpers.py +++ b/helpers.py @@ -5,7 +5,8 @@ def is_admin(is_windows): if is_windows: try: # only windows users with admin privileges can read the C:\windows\temp - temp = os.listdir(os.sep.join([os.environ.get('SystemRoot','C:\\windows'),'temp'])) + temp = os.listdir(os.sep.join( + [os.environ.get('SystemRoot', 'C:\\windows'), 'temp'])) except: return False else: