Added Batching

CHANGELOG.md

@@ -1,7 +1,51 @@
 
-n.n.n / 2023-10-02
+n.n.n / 2024-07-27
 ==================
 
+  * Batch request handling for whitelist API
+  * Removed security from GH Actions
+
+v2.1.2 / 2024-07-22
+===================
+
+  * Fixe security suite
+  * Fix test suite
+
+v2.1.1 / 2024-07-22
+===================
+
+  * Updatd poetry to use 3.11
+  * Updated deps
+  * Bump Python version to 3.11
+
+2.1.0 / 2024-03-29
+==================
+
+  * Added dynamic number of servers
+  * Update README.md
+  * Added timeout to GET
+
+v2.0.1 / 2023-10-31
+===================
+
+  * Swapped out for requests
+  * Adjusted get limit
+
+v2.0.0 / 2023-10-31
+===================
+
+  * Merge branch 'feat/fediseer'
+  * doc: Updated readme
+  * hack: Swapped data sources
+  * Added Fediseer pull
+  * wip: Started on fediseer db injection
+  * wip: Started on API function
+  * Added Matrix Chat link
+
+v1.0.3 / 2023-10-02
+===================
+
+  * Updated title bar
   * doc: Added PeeWee Info
   * version bump
 
@@ -104,10 +148,6 @@ v0.1.0 / 2023-09-09
   * wip: module refactor
   * doc: Started on dependency graph
   * wip: Started on seperate docker workers
-
-orig / 2023-07-06
-=================
-
   * wip: Started on AIO repo
   * doc: Added todo note
   * wip: Started on DB for storage

docs/fedi_gatus/config_gen/gen.md

@@ -4,19 +4,15 @@ Module fedi_gatus.config_gen.gen
 Functions
 ---------
 
-
 `Generate_endpoints(endpoint_list: [<class 'dict'>])`
 :   
 
-
 `generate_full_config()`
 :   
 
-
 `generate_top_instances()`
 :   
 
-
 `generate_ui()`
 :   
 

docs/fedi_gatus/shared/db.md

@@ -4,7 +4,6 @@ Module fedi_gatus.shared.db
 Functions
 ---------
 
-
 `get_connection()`
 :   
 
@@ -66,12 +65,14 @@ Classes
     `stats_user_count`
     :
 
-    ### Methods
+    ### Static methods
 
-    `get_single_record(self) ‑> dict`
+    `get_top_instances(software_name='Lemmy', count=25) ‑> list[fedi_gatus.shared.db.Model]`
     :
 
-    `get_top_lemmy_instances(self, count=25) ‑> list[fedi_gatus.shared.db.Model]`
+    ### Methods
+
+    `get_single_record(self) ‑> dict`
     :
 
     `initialize(self)`

docs/fedi_gatus/updater/data.md

@@ -12,12 +12,14 @@ Classes
     `raw_data`
     :
 
-    ### Methods
+    ### Static methods
 
-    `get_raw_data(self)`
+    `initialize_db()`
     :
 
-    `initialize_db(self)`
+    ### Methods
+
+    `get_raw_data(self)`
     :
 
     `insert_data(self)`

fedi_gatus/api_server/__main__.py

@@ -34,7 +34,9 @@ async def root():
         async def healthcheck():
             pass
             # TODO Write health check endpoint
-        #TODO Add top instance query function and JSON return
+
+        # TODO Add top instance query function and JSON return
+
 
 class Server:
     port = 8888

fedi_gatus/config_gen/gen.py

@@ -3,7 +3,8 @@
 import requests
 import yaml
 
-from fedi_gatus.shared import db
+# import math
+# from fedi_gatus.shared import db
 
 
 class Endpoint:
@@ -64,24 +65,40 @@ def generate_top_instances():
     import os
 
     logging.info("Get top instances")
-    from pythonseer import Fediseer
+    # from pythonseer import Fediseer
 
-    f = Fediseer()
+    # f = Fediseer()
     # fediseer_data = f.whitelist.get(guarantors=3, endorsements=4)['instances']
     # TODO Ask dbo about adding params to library
     # https://github.com/Fediseer/pythonseer/issues/7
 
-    d = requests.get(
-        url="https://fediseer.com/api/v1/whitelist",
-        timeout=60,
-        params={
-            "endorsements": int(os.getenv("ENDORSEMENTS")),
-            "guarantors": int(os.getenv("GUARANTORS")),
-            "software_csv": "lemmy",
-            "limit": int(os.getenv("NUMBER_OF_SERVERS")),
-            "domains": True,
-        },
-    ).json()["domains"]
+    # Batch requests in 100 increments
+
+    d = []
+    i = 0
+    while True:
+        if i >= int(os.getenv("NUMBER_OF_SERVERS")):
+            break
+        next = max(min(100, int(os.getenv("NUMBER_OF_SERVERS")) - i), 0)
+        i += next
+        response = requests.get(
+            url="https://fediseer.com/api/v1/whitelist",
+            timeout=60,
+            params={
+                "endorsements": 1,
+                "guarantors": 1,
+                "software_csv": "lemmy",
+                "limit": next,
+                "domains": True,
+            },
+        )
+
+        if response.status_code == 200:
+            d += response.json()["domains"]
+        else:
+            logging.error(response.json())
+            logging.error(response.status_code)
+            break
 
     # d = db.DbAccess().get_top_instances() # FIXME Backend is only returning a very small set of data... funnnnnn
     instances = []

fedi_gatus/shared/db.py

@@ -74,7 +74,8 @@ def __init__(self, *args, **kwargs):
     def get_single_record(self) -> dict:
         return self.select().get()
 
-    def get_top_instances(self, software_name='Lemmy', count=25) -> list[Model]:
+    @staticmethod
+    def get_top_instances(software_name="Lemmy", count=25) -> list[Model]:
         # TODO Add env var for count
 
         if not os.getenv("TEST_MODE"):

fedi_gatus/updater/data.py

@@ -5,7 +5,9 @@
 import requests
 
 from fedi_gatus.shared import db
-from pythonseer import Fediseer
+
+# from pythonseer import Fediseer
+
 
 class Worker:
     raw_data = None
@@ -40,6 +42,7 @@ def insert_data(self):
             except peewee.IntegrityError as e:
                 logging.error(e)
 
-    def initialize_db(self):
+    @staticmethod
+    def initialize_db():
         d = db.DbAccess()
         d.initialize()

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "fedi_gatus"
-version = "v1.0.2"
+version = "v2.1.2"
 description = ""
 authors = ["Jesse Schoepfer <[email protected]>"]
 readme = "README.md"