Merge branch 'release/2.0.0' into main

.github/workflows/docker_publish.yml

@@ -44,7 +44,6 @@ jobs:
             type=semver,pattern={{major}}.{{minor}}
       -
         name: Login to DockerHub
-        if: github.event_name != 'pull_request'
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -55,7 +54,7 @@ jobs:
         with:
           context: .
           platforms: linux/amd64
-          push: ${{ github.event_name != 'pull_request' }}
+          push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha

.github/workflows/pypi_publish.yml

@@ -9,26 +9,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Build and publish to pypi (3.10)
-        uses: JRubics/poetry-publish@v1.12
+      - name: Build and publish to pypi
+        uses: JRubics/poetry-publish@v1.15
         with:
-          python_version: "3.10.4"
+          python_version: "3.11.0"
           ignore_dev_requirements: "yes"
           pypi_token: ${{ secrets.PYPI_TOKEN }}
-          extra_build_dependency_packages: "capnproto libzmq3-dev"
-      - name: Build and publish to pypi (3.9)
-        uses: JRubics/[email protected]
-        with:
-          python_version: "3.9.13"
-          build_format: wheel
-          ignore_dev_requirements: "yes"
-          pypi_token: ${{ secrets.PYPI_TOKEN }}
-          extra_build_dependency_packages: "capnproto libzmq3-dev"
-      - name: Build and publish to pypi (3.8)
-        uses: JRubics/[email protected]
-        with:
-          python_version: "3.8.13"
-          build_format: wheel
-          ignore_dev_requirements: "yes"
-          pypi_token: ${{ secrets.PYPI_TOKEN }}
-          extra_build_dependency_packages: "capnproto libzmq3-dev"

.github/workflows/pytest.yml

@@ -29,6 +29,30 @@ jobs:
     strategy:
       matrix:
         python-version: [3.8, 3.9, "3.10"]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          sudo apt install capnproto
+          python -m pip install --upgrade pip
+          pip install poetry
+          poetry config virtualenvs.create false
+          poetry install --no-interaction --no-ansi
+      - name: Test with pytest
+        env:
+          PODPING_HIVE_ACCOUNT: ${{ secrets.PODPING_HIVE_ACCOUNT }}
+          PODPING_HIVE_POSTING_KEY: ${{ secrets.PODPING_HIVE_POSTING_KEY }}
+        run: |
+          pytest
+  test-runslow:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ "3.11" ]
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}

CLI.md

@@ -8,7 +8,7 @@ $ podping [OPTIONS] COMMAND [ARGS]...
 
 **Options**:
 
-* `--medium TEXT`: The medium of the feed being updated. If used in combination with the 'server', this sets the default medium only. Must be one of the following: blog music audiobook newsletter video film podcast  [env var: PODPING_MEDIUM; default: podcast]
+* `--medium TEXT`: The medium of the feed being updated. If used in combination with the 'server', this sets the default medium only. Must be one of the following: mixed podcast podcastL music musicL video videoL film filmL audiobook audiobookL newsletter newsletterL blog blogL  [env var: PODPING_MEDIUM; default: podcast]
 * `--reason TEXT`: The reason the feed is being updated. If used in combination with the 'server', this sets the default reason only. Must be one of the following: update live liveEnd  [env var: PODPING_REASON; default: update]
 * `--hive-account TEXT`: Hive account used to post  [env var: PODPING_HIVE_ACCOUNT, HIVE_ACCOUNT, HIVE_SERVER_ACCOUNT; required]
 * `--hive-posting-key TEXT`: Hive account used to post  [env var: PODPING_HIVE_POSTING_KEY, HIVE_POSTING_KEY; required]

Dockerfile

@@ -1,33 +1,4 @@
-FROM docker.io/python:3.10-bullseye AS compile
-
-ENV PYTHONFAULTHANDLER=1 \
-    PYTHONHASHSEED=random \
-    PYTHONDONTWRITEBYTECODE=1 \
-    PYTHONUNBUFFERED=1 \
-    PIP_DEFAULT_TIMEOUT=100 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    PIP_NO_CACHE_DIR=1 \
-    PATH="/home/podping/.local/bin:${PATH}"
-
-RUN useradd --create-home podping && mkdir /home/podping/app && chown -R podping:podping /home/podping
-
-RUN apt-get update \
-    && apt-get -y upgrade \
-    # rustc, cargo for armhf "cryptography"
-    # libzmq3-dev for armhf "pyzmq"
-    && apt-get -y install --no-install-recommends capnproto cargo libzmq3-dev rustc build-essential libssl-dev libffi-dev
-
-USER podping
-WORKDIR /home/podping/app
-
-COPY pyproject.toml poetry.lock ./
-
-RUN pip install --upgrade pip \
-    && pip install --user poetry \
-    && poetry config virtualenvs.in-project true \
-    && poetry install --no-root --only main --no-interaction --no-ansi
-
-FROM docker.io/python:3.10-slim-bullseye AS app
+FROM docker.io/python:3.11-slim-bullseye
 
 ENV PYTHONFAULTHANDLER=1 \
     PYTHONHASHSEED=random \
@@ -37,23 +8,20 @@ ENV PYTHONFAULTHANDLER=1 \
     PIP_DISABLE_PIP_VERSION_CHECK=1 \
     PIP_NO_CACHE_DIR=1
 
-RUN useradd --create-home podping && mkdir /home/podping/app && chown -R podping:podping /home/podping
+RUN useradd --create-home podping && mkdir /home/podping/app && mkdir /home/podping/.config && chown -R podping:podping /home/podping
 
 COPY install-packages.sh .
 RUN ./install-packages.sh
 
-COPY --from=compile --chown=podping:podping /home/podping/.local /home/podping/.local
-COPY --from=compile --chown=podping:podping /home/podping/app/.venv /home/podping/app/.venv
 USER podping
 WORKDIR /home/podping/app
 # poetry command installs here from pip
 ENV PATH="/home/podping/.local/bin:${PATH}"
 
 COPY --chown=podping:podping . .
-RUN pip install --upgrade pip \
-    && pip install poetry \
+RUN pip install --user poetry \
     && poetry config virtualenvs.in-project true \
-    && poetry install --only main --no-interaction --no-ansi
+    && poetry install --no-interaction --no-ansi --only main
 
 # podping command installs here
 ENV PATH="/home/podping/app/.venv/bin:${PATH}"

examples/memory_profile/long_running_zmq.py

@@ -2,24 +2,36 @@
 import linecache
 import logging
 import os
+import random
+import uuid
+from ipaddress import IPv4Address
+
+from plexo.ganglion.tcp_pair import GanglionZmqTcpPair
+from plexo.plexus import Plexus
+from podping_schemas.org.podcastindex.podping.hivewriter.podping_hive_transaction import (
+    PodpingHiveTransaction,
+)
+from podping_schemas.org.podcastindex.podping.podping_medium import PodpingMedium
+from podping_schemas.org.podcastindex.podping.podping_reason import PodpingReason
+from podping_schemas.org.podcastindex.podping.podping_write import PodpingWrite
+
+from podping_hivewriter.models.medium import mediums
+from podping_hivewriter.models.reason import reasons
+from podping_hivewriter.neuron import (
+    podping_hive_transaction_neuron,
+    podping_write_neuron,
+)
 
 try:
     import tracemalloc
 except ModuleNotFoundError:
     tracemalloc = False
-import uuid
-from random import randint
-from platform import python_version as pv, python_implementation as pi
 from timeit import default_timer as timer
 
-import zmq
-import zmq.asyncio
-
-from podping_hivewriter.constants import LIVETEST_OPERATION_ID
-from podping_hivewriter.models.medium import Medium
-from podping_hivewriter.models.reason import Reason
-from podping_hivewriter.podping_hivewriter import PodpingHivewriter
-from podping_hivewriter.podping_settings_manager import PodpingSettingsManager
+host = "127.0.0.1"
+port = 9979
+metrics = {"iris_sent": 0, "ops_received": 0, "iris_received": 0, "txs_received": 0}
+txs_received_lock = asyncio.Lock()
 
 
 def display_top(snapshot, key_type="lineno", limit=3):
@@ -51,57 +63,71 @@ def display_top(snapshot, key_type="lineno", limit=3):
     logging.info("Total allocated size: %.1f KiB" % (total / 1024))
 
 
-async def endless_send_loop(event_loop):
-    context = zmq.asyncio.Context()
-    socket = context.socket(zmq.REQ, io_loop=event_loop)
-    socket.connect(f"tcp://{host}:{port}")
+async def podping_hive_transaction_reaction(transaction: PodpingHiveTransaction, _, _2):
+    num_iris = sum(len(podping.iris) for podping in transaction.podpings)
+
+    async with txs_received_lock:
+        metrics["ops_received"] = metrics["ops_received"] + len(transaction.podpings)
+        metrics["iris_received"] = metrics["iris_received"] + num_iris
+        metrics["txs_received"] = metrics["txs_received"] + 1
+
+
+async def endless_send_loop():
+    tcp_pair_ganglion = GanglionZmqTcpPair(
+        peer=(IPv4Address(host), port),
+        relevant_neurons=(
+            podping_hive_transaction_neuron,
+            podping_write_neuron,
+        ),
+    )
+    plexus = Plexus(ganglia=(tcp_pair_ganglion,))
+    await plexus.adapt(
+        podping_hive_transaction_neuron,
+        reactants=(podping_hive_transaction_reaction,),
+    )
+    await plexus.adapt(podping_write_neuron)
 
-    test_name = "long_running_zmq"
-    python_version = pv()
-    python_implementation = pi()
     start_time = timer()
+    diag_time = timer()
 
     while True:
+        loop_start = timer()
         session_uuid = uuid.uuid4()
         session_uuid_str = str(session_uuid)
 
-        num_iris = randint(1, 10)
+        for i in range(1000):
+            iri = f"https://example.com?t=agates_test&i={i}&s={session_uuid_str}"
+            medium: PodpingMedium = random.sample(sorted(mediums), 1)[0]
+            reason: PodpingReason = random.sample(sorted(reasons), 1)[0]
+            podping_write = PodpingWrite(medium=medium, reason=reason, iri=iri)
 
-        for i in range(num_iris):
-            await socket.send_string(
-                f"https://example.com?t={test_name}&i={i}&v={python_version}&pi={python_implementation}&s={session_uuid_str}"
-            )
-            response = await socket.recv_string()
-            assert response == "OK"
+            await plexus.transmit(podping_write)
+
+        metrics["iris_sent"] = metrics["iris_sent"] + 1000
 
-        if tracemalloc and (timer() - start_time) >= 60:
+        await asyncio.sleep(3 - (timer() - loop_start))
+        if tracemalloc and (timer() - diag_time) >= 60:
             snapshot = tracemalloc.take_snapshot()
             display_top(snapshot)
-            start_time = timer()
-        await asyncio.sleep(3)
+            diag_time = timer()
+            logging.info(
+                f"IRIs sent: {metrics['iris_sent']} - {metrics['iris_sent'] / (diag_time - start_time)}s"
+            )
+            logging.info(
+                f"TXs received: {metrics['txs_received']} - {metrics['txs_received'] / (diag_time - start_time)}s"
+            )
+            logging.info(
+                f"OPs received: {metrics['ops_received']} - {metrics['ops_received'] / (diag_time - start_time)}s"
+            )
+            logging.info(
+                f"IRIs received: {metrics['iris_received']} - {metrics['iris_received'] / (diag_time - start_time)}s"
+            )
 
 
 if __name__ == "__main__":
     if tracemalloc:
         tracemalloc.start()
     loop = asyncio.get_event_loop()
     logging.getLogger().setLevel(level=logging.INFO)
-    settings_manager = PodpingSettingsManager()
-
-    host = "127.0.0.1"
-    port = 9979
-    podping_hivewriter = PodpingHivewriter(
-        os.environ["PODPING_HIVE_ACCOUNT"],
-        [os.environ["PODPING_HIVE_POSTING_KEY"]],
-        settings_manager,
-        medium=Medium.podcast,
-        reason=Reason.update,
-        listen_ip=host,
-        listen_port=port,
-        resource_test=True,
-        operation_id=LIVETEST_OPERATION_ID,
-    )
-    loop.run_until_complete(podping_hivewriter.wait_startup())
-    loop.run_until_complete(endless_send_loop(loop))
 
-    podping_hivewriter.close()
+    loop.run_until_complete(endless_send_loop())

install-packages.sh

@@ -17,7 +17,7 @@ apt-get update
 apt-get -y upgrade
 
 # Install application dependencies
-apt-get -y install --no-install-recommends capnproto libffi7 libssl1.1 libzmq5 zlib1g gcc libstdc++-10-dev
+apt-get -y install --no-install-recommends capnproto libffi7 zlib1g
 
 # Delete cached files we don't need anymore (note that if you're
 # using official Docker images for Debian or Ubuntu, this happens