Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added Mastodon to blocked services #100

Merged
merged 7 commits into from
Nov 15, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ jobs:
- name: Use Node.js
uses: actions/setup-node@v1
with:
node-version: 14.x
node-version: 18.x

- name: yarn install
run: yarn install
Expand Down
131 changes: 130 additions & 1 deletion assets/services.json
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,135 @@
],
"icon_svg": "<svg xmlns=\"http://www.w3.org/2000/svg\" fill=\"currentColor\" viewBox=\"0 0 24 24\"><path d=\"M12 8.8A3.2 3.2 0 0 0 8.8 12a3.2 3.2 0 0 0 3.2 3.2 3.2 3.2 0 0 0 3.2-3.2A3.2 3.2 0 0 0 12 8.8zm0 0\" /><path d=\"M16 2.398H8A5.609 5.609 0 0 0 2.398 8v8A5.609 5.609 0 0 0 8 21.602h8A5.609 5.609 0 0 0 21.602 16V8A5.609 5.609 0 0 0 16 2.398zm-4 14.403A4.805 4.805 0 0 1 7.2 12c0-2.648 2.152-4.8 4.8-4.8 2.648 0 4.8 2.152 4.8 4.8 0 2.648-2.152 4.8-4.8 4.8zm5.602-9.602a.799.799 0 1 1 0 0zm0 0\" /></svg>"
},
{
"id": "mastodon",
"name": "Mastodon",
"rules": [
"||mas.to^",
"||mstdn.social^",
"||mastodonapp.uk^",
"||mstdn.jp^",
"||mastodon.lol^",
"||mastodon.sdf.org^",
"||fosstodon.org^",
"||mastodon.uno^",
"||sfba.social^",
"||mastodon.scot^",
"||techhub.social^",
"||mastodon.ie^",
"||mastodon.nl^",
"||troet.cafe^",
"||home.social^",
"||ravenation.club^",
"||universeodon.com^",
"||mstdn.party^",
"||mindly.social^",
"||glasgow.social^",
"||qoto.org^",
"||fedibird.com^",
"||mastodon.com.tr^",
"||m.cmx.im^",
"||mastodon.iriseden.eu^",
"||ioc.exchange^",
"||mastodontech.de^",
"||noc.social^",
"||mastodon.nu^",
"||mastodon.nz^",
"||socel.net^",
"||mastouille.fr^",
"||h4.io^",
"||norden.social^",
"||nrw.social^",
"||o3o.ca^",
"||tkz.one^",
"||sueden.social^",
"||kolektiva.social^",
"||wxw.moe^",
"||toot.wales^",
"||masto.pt^",
"||muenchen.social^",
"||nerdculture.de^",
"||lor.sh^",
"||social.linux.pizza^",
"||hessen.social^",
"||ruhr.social^",
"||mastodon.bida.im^",
"||indieweb.social^",
"||ohai.social^",
"||mathstodon.xyz^",
"||social.anoxinon.de^",
"||masto.nobigtech.es^",
"||social.cologne^",
"||sociale.network^",
"||dresden.network^",
"||social.dev-wiki.de^",
"||mastodontti.fi^",
"||metalhead.club^",
"||social.politicaconciencia.org^",
"||activism.openworlds.info^",
"||xarxa.cloud^",
"||mastodon.zaclys.com^",
"||rollenspiel.social^",
"||oslo.town^",
"||hostux.social^",
"||colorid.es^",
"||kurry.social^",
"||muenster.im^",
"||freiburg.social^",
"||phpc.social^",
"||pettingzoo.co^",
"||tilde.zone^",
"||snabelen.no^",
"||mastodon.eus^",
"||todon.eu^",
"||dizl.de^",
"||ludosphere.fr^",
"||libretooth.gr^",
"||ani.work^",
"||mastodon.arch-linux.cz^",
"||mastodon.uy^",
"||vocalodon.net^",
"||bologna.one^",
"||mast.dragon-fly.club^",
"||dju.social^",
"||peoplemaking.games^",
"||maly.io^",
"||red.niboe.info^",
"||graz.social^",
"||pewtix.com^",
"||stereodon.social^",
"||mastodon.ml^",
"||toot.si^",
"||social.diva.exchange^",
"||khiar.net^",
"||fulda.social^",
"||darmstadt.social^",
"||toot.kif.rocks^",
"||x0r.be^",
"||hispagatos.space^",
"||mastodon.bachgau.social^",
"||fikaverse.club^",
"||feuerwehr.social^",
"||augsburg.social^",
"||pol.social^",
"||mastodon.pirateparty.be^",
"||puntarella.party^",
"||mastodon.vlaanderen^",
"||toot.funami.tech^",
"||travelpandas.fr^",
"||seo.chat^",
"||primarycare.app^",
"||tyrol.social^",
"||learningdisability.social^",
"||occitania.social^",
"||social.bau-ha.us^",
"||newsie.social^",
"||tea.codes^",
"||mastodon.social^",
"||mastodon.online^"
],
"icon_svg": "<svg xmlns=\"http://www.w3.org/2000/svg\" fill=\"currentColor\" viewBox=\"0 0 512 512\"><path d=\"M433 179.11c0-97.2-63.71-125.7-63.71-125.7-62.52-28.7-228.56-28.4-290.48 0 0 0-63.72 28.5-63.72 125.7 0 115.7-6.6 259.4 105.63 289.1 40.51 10.7 75.32 13 103.33 11.4 50.81-2.8 79.32-18.1 79.32-18.1l-1.7-36.9s-36.31 11.4-77.12 10.1c-40.41-1.4-83-4.4-89.63-54a102.54 102.54 0 0 1-.9-13.9c85.63 20.9 158.65 9.1 178.75 6.7 56.12-6.7 105-41.3 111.23-72.9 9.8-49.8 9-121.5 9-121.5zm-75.12 125.2h-46.63v-114.2c0-49.7-64-51.6-64 6.9v62.5h-46.33V197c0-58.5-64-56.6-64-6.9v114.2H90.19c0-122.1-5.2-147.9 18.41-175 25.9-28.9 79.82-30.8 103.83 6.1l11.6 19.5 11.6-19.5c24.11-37.1 78.12-34.8 103.83-6.1 23.71 27.3 18.4 53 18.4 175z\"/></svg>"
},
{
"id": "mail_ru",
"name": "Mail.ru",
Expand Down Expand Up @@ -549,4 +678,4 @@
"icon_svg": "<svg xmlns=\"http://www.w3.org/2000/svg\" fill=\"currentColor\" viewBox=\"0 0 30 30\"><path d=\"M14.46 14.191H9.982c0-.471.033-.954.039-1.458v-5.5h5.106V5.935a1.352 1.352 0 00-.404-.957 1.378 1.378 0 00-.968-.396H5.783c.028-.088.056-.177.084-.255.274-.82 1.153-3.326 1.153-3.326a4.262 4.262 0 00-2.413.698c-.57.4-.912.682-1.371 1.946-.532 1.453-.997 2.856-1.31 3.693C1.444 8.674.28 11.025.28 11.025a5.85 5.85 0 002.52-.61c1.119-.593 1.679-1.502 2.054-2.883l.09-.3h2.334v5.5c0 .5-.045.982-.073 1.46h-4.12c-.71 0-1.39.278-1.893.775a2.638 2.638 0 00-.783 1.874h6.527a17.717 17.717 0 01-.778 3.649 16.796 16.796 0 01-3.012 5.273A33.104 33.104 0 010 28.74s3.13 1.175 5.425-.954c1.388-1.292 2.631-3.814 3.23-5.727a28.09 28.09 0 001.12-5.229h5.967v-1.37a1.254 1.254 0 00-.373-.899 1.279 1.279 0 00-.909-.37zm-3.19 5.484l-2.312 1.491 5.038 7.458a6.905 6.905 0 00.672-2.218 3.15 3.15 0 00-.28-2.168l-3.118-4.563zM29.05 4.582H16.733V25.94h3.018l.403 2.572 4.081-2.572h4.815V4.582zm-5.207 18.69l-2.396 1.509-.235-1.508h-1.724V7.233h6.78v16.04h-2.425z\" /></svg>"
}
]
}
}
24 changes: 24 additions & 0 deletions hostlists-builder/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,14 @@ const path = require('path');
const fs = require('fs');
const md5 = require('md5');
const hostlistCompiler = require('@adguard/hostlist-compiler');
const mastodonServerlistCompiler = require('adguard-hostlists-builder/mastodon');

const HOSTLISTS_URL = 'https://adguardteam.github.io/HostlistsRegistry/assets';

const CONFIGURATION_FILE = 'configuration.json';
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since Mastodon logic is not related to hostlists and for better clarity I'd prefer to have it all in a separate file (mastodon.js) imported into index.js

const REVISION_FILE = 'revision.json';
const METADATA_FILE = 'metadata.json';
const SERVICES_FILE = 'services.json';

const FILTERS_METADATA_FILE = 'filters.json';
const FILTERS_METADATA_DEV_FILE = "filters-dev.json";
Expand Down Expand Up @@ -253,6 +255,28 @@ async function build(filtersDir, tagsDir, localesDir, assetsDir) {
filtersMetadataDev.push(filterMetadata);
}

// Build Mastodon dynamic server list
let services = JSON.parse(readFile(path.join(assetsDir, SERVICES_FILE)));
const mastodonServers = await mastodonServerlistCompiler();

const mastodonIndex = services.blocked_services
.findIndex((el) => {
return el.id === 'mastodon';
});

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We need additional checks for mastodonRules:

  1. Not less than N elements (how many is okay)?
  2. Not more than X elements (how many is a good upper limit? 100?)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

100 seems like a fair limit.

Just as an additional check, you want to sort them by total user count ? So that only the top 100 servers are blocked?

if (mastodonIndex == -1) {
throw Error("Mastodon service not found")
}

// Set Mastodon server list to be blocked
const mastodonService = services.blocked_services[mastodonIndex];
mastodonService.rules = mastodonServers;
services.blocked_services[mastodonIndex] = mastodonService;

// Write Mastodon dynamic server list to service.json
const servicesFile = path.join(assetsDir, SERVICES_FILE);
writeFile(servicesFile, JSON.stringify(services, undefined, 2));

// copy tags as is
const tagsMetadata = JSON.parse(readFile(path.join(tagsDir, METADATA_FILE)));

Expand Down
65 changes: 65 additions & 0 deletions hostlists-builder/mastodon.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
/**
* Returns an array of objects containing the following fields;
* "domain": <string>,
* "version": <string>,
* "description": <string>,
* "languages": Array<string> - containing ISO 639-1 language codes (en, fr, de),
* "region": <string>,
* "categories": Array<string> - A list of catgories
* "proxied_thumbnail": <string>,
* "total_users": <int>,
* "last_week_users": <int>,
* "approval_required": <boolean>,
* "language": <string> - containing ISO 639-1 language code (en, fr, de),
* "category": <string>
*
* @returns {Promise<Array>}
*/
const serverList = async function () {
/**
*
* @type {Promise<Response>}
*/
let servers = await (await fetch('https://api.joinmastodon.org/servers'))
.json();

// Sort servers by total_users
servers = servers.sort((a, b) => {
if (a.total_users > b.total_users) {
return -1;
}

if (a.total_users < b.total_users) {
return 1;
}

return 0;
});

// Separately add the two Instances by Mastodon GmbH themselves.
servers.unshift({
domain: "mastodon.social"
}, {
domain: "mastodon.online"
});

return servers
.slice(0, 99);
}

// Compile server list
const compile = async function () {
const result = (await serverList());

const mastodonRules = result.map((element) => {
if (!element.hasOwnProperty('domain')) {
throw Error("Domain key not found in server list")
}

return `||${element.domain}^`
});

return mastodonRules;
}

module.exports = compile
Loading