diff --git a/.github/workflows/mac10-py37.yml b/.github/workflows/mac10-py37.yml deleted file mode 100644 index 48b6248..0000000 --- a/.github/workflows/mac10-py37.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: Mac 10 Python 3.7 - -on: - push: - branches: [ smolsky/testing ] - -jobs: - build: - runs-on: macos-10.15 - strategy: - # Add a list of python versions we want to use for testing. - matrix: - python-version: ['3.7'] - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - pip install flake8 - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - - name: Install librdkafka - run: brew install librdkafka - # - name: Install hop-client - # run: conda install -c conda-forge hop-client - - - name: Install requirements - run: | - python -m pip install --upgrade pip - pip install wheel - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - - shell: bash - env: - USERNAME: ${{ secrets.username }} - PASSWORD: ${{ secrets.password }} - run: | - /usr/bin/expect << HOP - spawn hop auth add - expect "Username:" - send "$USERNAME\n" - expect "Password:" - send "$PASSWORD\n" - expect "Hostname (may be empty):" - send "kafka.scimma.org\n" - expect eof - HOP - hop auth locate - - - name: Install snews-pt - run: pip install . - - - name: Check version - run: snews_pt --version - - - name: Run pytest - run: | - pip install pytest - pytest \ No newline at end of file diff --git a/.github/workflows/mac11-py37.yml b/.github/workflows/mac11-py37.yml deleted file mode 100644 index 1b61fc9..0000000 --- a/.github/workflows/mac11-py37.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: Mac 11 Python 3.7 - -on: - push: - branches: [ smolsky/testing ] - -jobs: - build: - runs-on: macos-11 - strategy: - # Add a list of python versions we want to use for testing. - matrix: - python-version: ['3.7'] - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - pip install flake8 - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - - name: Install librdkafka - run: brew install librdkafka - # - name: Install hop-client - # run: conda install -c conda-forge hop-client - - - name: Install requirements - run: | - python -m pip install --upgrade pip - pip install wheel - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - - shell: bash - env: - USERNAME: ${{ secrets.username }} - PASSWORD: ${{ secrets.password }} - run: | - /usr/bin/expect << HOP - spawn hop auth add - expect "Username:" - send "$USERNAME\n" - expect "Password:" - send "$PASSWORD\n" - expect "Hostname (may be empty):" - send "kafka.scimma.org\n" - expect eof - HOP - hop auth locate - - - name: Install snews-pt - run: pip install . - - - name: Check version - run: snews_pt --version - - - name: Run pytest - run: | - pip install pytest - pytest \ No newline at end of file diff --git a/.github/workflows/ubuntu20-py37-310.yml b/.github/workflows/ubuntu20-py37-310.yml deleted file mode 100644 index 926acfe..0000000 --- a/.github/workflows/ubuntu20-py37-310.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: testing - -on: - push: - branches: - - main - - pull_request: - branches: - - main - -jobs: - # Copied from snewpy. ;) - build: - # The type of runner that the job will run on. - runs-on: ubuntu-latest - strategy: - # Add a list of python versions we want to use for testing. - max-parallel: 1 - matrix: - python-version: ['3.7', '3.8', '3.9', '3.10'] - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Check python version - run: python --version - - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - pip install flake8 - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - - name: Install hop-client - run: | - pip install -U hop-client - - - name: Install requirements - run: | - python -m pip install --upgrade pip - pip install wheel - pip install python-dotenv click tabulate - - - name: Add hop authorization - shell: bash - env: - USERNAME: ${{ secrets.username }} - PASSWORD: ${{ secrets.password }} - run: | - sudo apt-get install -y expect - which expect - /usr/bin/expect << HOP - spawn hop auth add - expect "Username:" - send "$USERNAME\n" - expect "Password:" - send "$PASSWORD\n" - expect "Hostname (may be empty):" - send "kafka.scimma.org\n" - expect "Token endpoint (empty if not applicable):" - send "\n" - expect eof - HOP - hop auth locate - - - name: Install snews-pt - run: pip install . - - - name: Run pytest - run: | - pip install pytest - pytest snews_pt diff --git a/.github/workflows/ubuntu20-py37.yml b/.github/workflows/ubuntu20-py37.yml deleted file mode 100644 index 87ab993..0000000 --- a/.github/workflows/ubuntu20-py37.yml +++ /dev/null @@ -1,85 +0,0 @@ -name: Ubuntu 20.04 Python 3.7 - -on: - push: - branches: - - smolsky/testing - - main - - pull_request: - branches: - - main - -jobs: - # Copied from snewpy. ;) - build: - # The type of runner that the job will run on. - runs-on: ubuntu-latest - strategy: - # Add a list of python versions we want to use for testing. - matrix: - python-version: ['3.7'] - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Check python version - run: python --version - - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - pip install flake8 - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - - name: Install hop-client - run: | - pwd - wget https://files.pythonhosted.org/packages/de/a2/1f663c824b7e6ac139110005888cda7e1aea74e3bb8a57d774a00a1802f5/hop-client-0.5.0.tar.gz - tar -xzf hop-client-0.5.0.tar.gz - cd hop-client-0.5.0 - python setup.py install - cd /home/runner/work/SNEWS_Publishing_Tools/SNEWS_Publishing_Tools - - - name: Install requirements - run: | - python -m pip install --upgrade pip - pip install wheel - pip install python-dotenv click tabulate - - - shell: bash - env: - USERNAME: ${{ secrets.username }} - PASSWORD: ${{ secrets.password }} - run: | - sudo apt-get install -y expect - which expect - /usr/bin/expect << HOP - spawn hop auth add - expect "Username:" - send "$USERNAME\n" - expect "Password:" - send "$PASSWORD\n" - expect "Hostname (may be empty):" - send "kafka.scimma.org\n" - expect eof - HOP - hop auth locate - - - name: Install snews-pt - run: pip install . - - - name: Check version - run: snews_pt --version - - - name: Run pytest - run: | - pip install pytest - pytest snews_pt \ No newline at end of file diff --git a/docs/user/firedrills.md b/docs/user/firedrills.md index 5ca10e3..76b642f 100644 --- a/docs/user/firedrills.md +++ b/docs/user/firedrills.md @@ -44,24 +44,24 @@ We would like to test two main interactions; **subscribing** & **publishing** t - API: ```python - from snews_pt.snews_pub import SNEWSTiersPublisher - SNEWSTiersPublisher(detector_name='KamLAND', + from snews_pt.messages import SNEWSMessageBuilder + SNEWSMessageBuilder(detector_name='KamLAND', neutrino_time="2022-02-28T04:31:08.678999", p_val=0.000007, machine_time="2022-02-28T04:31:09.778859", firedrill_mode=True, is_test=True, - ).send_to_snews() + ).send_messages() ``` - or + or ```python - from snews_pt.snews_pub import SNEWSTiersPublisher - observation = SNEWSTiersPublisher.from_json('somejsonfile.json', + from snews_pt.messages import SNEWSMessageBuilder + observation = SNEWSMessageBuilder.from_json('somejsonfile.json', detector_name='XENONnT', firedrill_mode=True, is_test=True, comment="This is submitted from a json file") - observation.send_to_snews() + observation.send_messages() ``` Notice that `SNEWSTiersPublisher` returns an object which actually contains the decided tiers, and formatted messages. One can play with this object before finally `send_to_snews()`. diff --git a/docs/user/publishing_protocols.md b/docs/user/publishing_protocols.md index 5a8ee3f..4250ab3 100644 --- a/docs/user/publishing_protocols.md +++ b/docs/user/publishing_protocols.md @@ -18,10 +18,12 @@ The user can share as much information as they desire with one simple function c sets the "Tier" and any additional information is passed under `meta` field. The user can use `SNEWSTiersPublisher` to create observation message(s) and send them to snews. + ```python -from snews_pt.snews_pub import SNEWSTiersPublisher -messages = SNEWSTiersPublisher(neutrino_time="2022-02-28T04:31:08.678999") -messages.send_to_snews() +from snews_pt.messages import SNEWSMessageBuilder + +messages = SNEWSMessageBuilder(neutrino_time="2022-02-28T04:31:08.678999") +messages.send_messages() ``` User can also investigate their messages before sending it to SNEWS. The `SNEWSTiersPublisher` creates an object which contains the generated and formatted messages. It can also tell you what "Tiers" are selected based on the input that is given. @@ -92,7 +94,7 @@ Notice that your message can contain fields that correspond to several tiers e.g In the example below `SNEWSTierPublisher` creates a message for "CoincidenceTier" because the `neutrino_time` is passed, and it creates another message for the "Significance Tier" because the `p_values` together with the `t_bin_width` is passed. -Here the `p_val` is the p value of the detection and it is optional. The `detector_name` can be passed manually, however, if the name is initially set, this is also not needed. +Here the `p_val` is the p value of the detection, and it is optional. The `detector_name` can be passed manually, however, if the name is initially set, this is also not needed. Publication example diff --git a/examples.ipynb b/examples.ipynb index 69f4cd9..1804fbf 100644 --- a/examples.ipynb +++ b/examples.ipynb @@ -13,21 +13,46 @@ "id": "935f417b", "metadata": {}, "source": [ - "Last updated: 02/06/2023
\n", + "Last updated: 21/07/2023
\n", "This notebook contains the most basic publication and subscription examples." ] }, { "cell_type": "markdown", - "id": "af067f1c", + "id": "f6d67ae3", + "metadata": {}, + "source": [ + "## Subscription" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0ea6d89a", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "from snews_pt.snews_sub import Subscriber\n", + "\n", + "Subscriber().subscribe()" + ] + }, + { + "cell_type": "markdown", + "id": "5993f74e", "metadata": {}, "source": [ - "## Publication" + "---\n", + "# Publication" ] }, { "cell_type": "markdown", - "id": "a140a24c", + "id": "6a557d61", "metadata": {}, "source": [ "Here we create a message to which we pass the following arguments;
\n", @@ -39,100 +64,537 @@ "Similarly, the input arguments; `p_values` and `t_bin_width` indicates that the message should go to \"Significance Tier\" for combined significance computations using the " ] }, + { + "cell_type": "markdown", + "id": "dafbf99a", + "metadata": {}, + "source": [ + "## Old way" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "1709a9db", + "metadata": {}, + "outputs": [], + "source": [ + "# from snews_pt.snews_pub import SNEWSTiersPublisher\n", + "# from datetime import datetime\n", + "\n", + "# test_time = datetime.utcnow().isoformat()\n", + "# message = SNEWSTiersPublisher(detector_name='LZ', machine_time=test_time, neutrino_time=test_time, p_val=0.0007, p_values=[0.0007,0.0008,0.0009],t_bin_width=0.07)\n", + "# # print(message.message_data)\n", + "# # print(message.tiernames)\n", + "# # print('Sending message')\n", + "# message.send_to_snews()" + ] + }, + { + "cell_type": "markdown", + "id": "eae871f7", + "metadata": {}, + "source": [ + "## New construction" + ] + }, { "cell_type": "code", - "execution_count": 3, - "id": "34bebc1c-8272-4a87-b877-31b77700907d", + "execution_count": 113, + "id": "81f0b707", "metadata": { - "tags": [] + "pycharm": { + "name": "#%%\n" + } }, + "outputs": [], + "source": [ + "from snews_pt import messages\n", + "import importlib\n", + "importlib.reload(messages)\n", + "from datetime import datetime\n", + "\n", + "test_time = datetime.utcnow().isoformat()" + ] + }, + { + "cell_type": "markdown", + "id": "4b0d51ca", + "metadata": {}, + "source": [ + "## Valid messages" + ] + }, + { + "cell_type": "markdown", + "id": "8150844a", + "metadata": {}, + "source": [ + "**1) message from a valid detector with valid neutrino times and valid p values.**\n", + "This creates two messages; CoincidenceTierMessage and SignificanceTierMessage" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "5a27a67e", + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[\u001b[1mSNEWSCoincidenceTierMessage\n", + " \u001b[0m---------------------------\n", + " \u001b[91m _id : LZ_CoincidenceTier_2023-07-31T07:27:00.410593\n", + " \u001b[0m\u001b[91m schema_version : 1.3.0\n", + " \u001b[0m\u001b[91m detector_name : LZ\n", + " \u001b[0m\u001b[94m neutrino_time : 2023-07-31T07:27:00.410593\n", + " \u001b[0m---------------------------\n", + " \u001b[96m p_values : [0.0007, 0.0008, 0.0009]\n", + " \u001b[0m\u001b[96m t_bin_width : 0.07\n", + " \u001b[0m\u001b[96m is_test : True\n", + " \u001b[0m,\n", + " \u001b[1mSNEWSSignificanceTierMessage\n", + " \u001b[0m----------------------------\n", + " \u001b[91m _id : LZ_SignificanceTier_2023-07-31T07:27:00.410593\n", + " \u001b[0m\u001b[91m schema_version : 1.3.0\n", + " \u001b[0m\u001b[91m detector_name : LZ\n", + " \u001b[0m\u001b[94m p_values : [0.0007, 0.0008, 0.0009]\n", + " \u001b[0m\u001b[94m t_bin_width : 0.07\n", + " \u001b[0m----------------------------\n", + " \u001b[96m neutrino_time : 2023-07-31T07:27:00.410593\n", + " \u001b[0m\u001b[96m p_val : 0.000\n", + " \u001b[0m\u001b[96m is_test : True\n", + " \u001b[0m]" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "msg = messages.SNEWSMessageBuilder(detector_name='LZ', \n", + " machine_time=test_time, \n", + " neutrino_time=test_time, \n", + " p_val=\"0.000\", \n", + " p_values=[0.0007,0.0008,0.0009],\n", + " t_bin_width=0.07, \n", + " is_test=True)\n", + "\n", + "msg.messages" + ] + }, + { + "cell_type": "markdown", + "id": "fedb9b1b", + "metadata": {}, + "source": [ + "**2) Similarly, as long as the neutrino time iso-formattable**
\n", + "Notice the neutrino time comes from the past but it is a test, and therefore ignored" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "051f5029", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[\u001b[1mSNEWSCoincidenceTierMessage\n", + " \u001b[0m---------------------------\n", + " \u001b[91m _id : LZ_CoincidenceTier_2023-07-21T14:53:51.016430\n", + " \u001b[0m\u001b[91m schema_version : 1.3.0\n", + " \u001b[0m\u001b[91m detector_name : LZ\n", + " \u001b[0m\u001b[94m neutrino_time : 2023-06-12T18:30:00\n", + " \u001b[0m---------------------------\n", + " \u001b[96m is_test : True\n", + " \u001b[0m]" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "msg = messages.SNEWSMessageBuilder(detector_name='LZ', \n", + " machine_time=test_time, \n", + " neutrino_time=\"2023-06-12 18:30\", \n", + " is_test=True)\n", + "\n", + "msg.messages" + ] + }, + { + "cell_type": "markdown", + "id": "32f807b9", + "metadata": {}, + "source": [ + "## Invalid Messages" + ] + }, + { + "cell_type": "markdown", + "id": "e6327d9b", + "metadata": {}, + "source": [ + "**1) With no valid argument**" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "43856ed4", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "SNEWSMessageBuilder:\n", + "No messages have been built." + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "msg = messages.SNEWSMessageBuilder(test=1)\n", + "msg" + ] + }, + { + "cell_type": "markdown", + "id": "8c29fb5d", + "metadata": {}, + "source": [ + "**2) unknown detector name**" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "2ac8942a", + "metadata": { + "collapsed": true + }, + "outputs": [ + { + "ename": "KeyError", + "evalue": "\"aseyhsreshdshdshsdgsdg is not a valid detector. \\nChoose from ['Baksan', 'Borexino', 'DS-20K', 'DUNE', 'HALO', 'HALO-1kT', 'Hyper-K', 'IceCube', 'JUNO', 'KM3NeT', 'KamLAND', 'LVD', 'LZ', 'MicroBooNe', 'NOvA', 'PandaX-4T', 'SBND', 'SNO+', 'Super-K', 'XENONnT']\"", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[10], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m msg \u001b[38;5;241m=\u001b[39m \u001b[43mmessages\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mSNEWSMessageBuilder\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdetector_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43maseyhsreshdshdshsdgsdg\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 2\u001b[0m \u001b[43m \u001b[49m\u001b[43mmachine_time\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtest_time\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 3\u001b[0m \u001b[43m \u001b[49m\u001b[43mneutrino_time\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtest_time\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 4\u001b[0m \u001b[43m \u001b[49m\u001b[43mp_val\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0.0007\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 5\u001b[0m \u001b[43m \u001b[49m\u001b[43mp_values\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0.0007\u001b[39;49m\u001b[43m,\u001b[49m\u001b[38;5;241;43m0.0008\u001b[39;49m\u001b[43m,\u001b[49m\u001b[38;5;241;43m0.0009\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6\u001b[0m \u001b[43m \u001b[49m\u001b[43mt_bin_width\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0.07\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 7\u001b[0m \u001b[43m \u001b[49m\u001b[43mis_test\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/mnt/c/Users/bj7780/Desktop/Kara/GitHub/SNEWS/SNEWS_Publishing_Tools/snews_pt/messages.py:377\u001b[0m, in \u001b[0;36mSNEWSMessageBuilder.__init__\u001b[0;34m(self, env_file, detector_name, machine_time, neutrino_time, p_val, p_values, t_bin_width, timing_series, retract_latest, retraction_reason, detector_status, is_test, **kwargs)\u001b[0m\n\u001b[1;32m 361\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, env_file\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 362\u001b[0m detector_name\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mTEST\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[1;32m 363\u001b[0m machine_time\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 372\u001b[0m is_test\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m 373\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 375\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmessages \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m--> 377\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_build_messages\u001b[49m\u001b[43m(\u001b[49m\u001b[43menv_file\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43menv_file\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 378\u001b[0m \u001b[43m \u001b[49m\u001b[43mdetector_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdetector_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 379\u001b[0m \u001b[43m \u001b[49m\u001b[43mmachine_time\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmachine_time\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 380\u001b[0m \u001b[43m \u001b[49m\u001b[43mneutrino_time\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mneutrino_time\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 381\u001b[0m \u001b[43m \u001b[49m\u001b[43mp_val\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mp_val\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 382\u001b[0m \u001b[43m \u001b[49m\u001b[43mp_values\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mp_values\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 383\u001b[0m \u001b[43m \u001b[49m\u001b[43mt_bin_width\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mt_bin_width\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 384\u001b[0m \u001b[43m \u001b[49m\u001b[43mtiming_series\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtiming_series\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 385\u001b[0m \u001b[43m \u001b[49m\u001b[43mretract_latest\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mretract_latest\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 386\u001b[0m \u001b[43m \u001b[49m\u001b[43mretraction_reason\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mretraction_reason\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 387\u001b[0m \u001b[43m \u001b[49m\u001b[43mdetector_status\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdetector_status\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 388\u001b[0m \u001b[43m \u001b[49m\u001b[43mis_test\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mis_test\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 389\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/mnt/c/Users/bj7780/Desktop/Kara/GitHub/SNEWS/SNEWS_Publishing_Tools/snews_pt/messages.py:439\u001b[0m, in \u001b[0;36mSNEWSMessageBuilder._build_messages\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m 436\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m hasreqfields:\n\u001b[1;32m 437\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmessages \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 438\u001b[0m \u001b[38;5;66;03m# check is valid at creation\u001b[39;00m\n\u001b[0;32m--> 439\u001b[0m \u001b[43msmc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mnonull_kwargs\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mis_valid()\n\u001b[1;32m 440\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmessages \u001b[38;5;241m=\u001b[39m [smc(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mnonull_kwargs)]\n\u001b[1;32m 441\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", + "File \u001b[0;32m/mnt/c/Users/bj7780/Desktop/Kara/GitHub/SNEWS/SNEWS_Publishing_Tools/snews_pt/messages.py:243\u001b[0m, in \u001b[0;36mSNEWSCoincidenceTierMessage.__init__\u001b[0;34m(self, neutrino_time, p_val, **kwargs)\u001b[0m\n\u001b[1;32m 242\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, neutrino_time\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, p_val\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m--> 243\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfields\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 244\u001b[0m \u001b[43m \u001b[49m\u001b[43mneutrino_time\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mclean_time_input\u001b[49m\u001b[43m(\u001b[49m\u001b[43mneutrino_time\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 245\u001b[0m \u001b[43m \u001b[49m\u001b[43mp_val\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mp_val\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 246\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/mnt/c/Users/bj7780/Desktop/Kara/GitHub/SNEWS/SNEWS_Publishing_Tools/snews_pt/messages.py:119\u001b[0m, in \u001b[0;36mSNEWSMessage.__init__\u001b[0;34m(self, fields, detector_name, machine_time, **kwargs)\u001b[0m\n\u001b[1;32m 115\u001b[0m tier \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;241m.\u001b[39mreplace(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mSNEWS\u001b[39m\u001b[38;5;124m'\u001b[39m,\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m)\u001b[38;5;241m.\u001b[39mreplace(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mMessage\u001b[39m\u001b[38;5;124m'\u001b[39m,\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 117\u001b[0m \u001b[38;5;66;03m# Get the detector name from the input.\u001b[39;00m\n\u001b[1;32m 118\u001b[0m \u001b[38;5;66;03m# det = self.get_detector_name(detector_name)\u001b[39;00m\n\u001b[0;32m--> 119\u001b[0m det \u001b[38;5;241m=\u001b[39m \u001b[43msnews_pt_utils\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mset_name\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdetector_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_return\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m 120\u001b[0m mt \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclean_time_input(machine_time)\n\u001b[1;32m 122\u001b[0m \u001b[38;5;66;03m# Store basic message ID, detector name, and schema in a dictionary.\u001b[39;00m\n", + "File \u001b[0;32m/mnt/c/Users/bj7780/Desktop/Kara/GitHub/SNEWS/SNEWS_Publishing_Tools/snews_pt/snews_pt_utils.py:319\u001b[0m, in \u001b[0;36mset_name\u001b[0;34m(detector_name, _return)\u001b[0m\n\u001b[1;32m 317\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 318\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m detector_name \u001b[38;5;129;01min\u001b[39;00m detectors:\n\u001b[0;32m--> 319\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdetector_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m is not a valid detector. \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124mChoose from \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdetectors\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 320\u001b[0m os\u001b[38;5;241m.\u001b[39menviron[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDETECTOR_NAME\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m detector_name\n\u001b[1;32m 321\u001b[0m os\u001b[38;5;241m.\u001b[39menviron[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mHAS_NAME_CHANGED\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m1\u001b[39m\u001b[38;5;124m\"\u001b[39m\n", + "\u001b[0;31mKeyError\u001b[0m: \"aseyhsreshdshdshsdgsdg is not a valid detector. \\nChoose from ['Baksan', 'Borexino', 'DS-20K', 'DUNE', 'HALO', 'HALO-1kT', 'Hyper-K', 'IceCube', 'JUNO', 'KM3NeT', 'KamLAND', 'LVD', 'LZ', 'MicroBooNe', 'NOvA', 'PandaX-4T', 'SBND', 'SNO+', 'Super-K', 'XENONnT']\"" + ] + } + ], + "source": [ + "msg = messages.SNEWSMessageBuilder(detector_name='aseyhsreshdshdshsdgsdg', \n", + " machine_time=test_time, \n", + " neutrino_time=test_time, \n", + " p_val=0.0007, \n", + " p_values=[0.0007,0.0008,0.0009],\n", + " t_bin_width=0.07, \n", + " is_test=True)" + ] + }, + { + "cell_type": "markdown", + "id": "259152f5", + "metadata": {}, + "source": [ + "**3) Neutrino time in the past and it is not a test message.**" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "0cbbf351", + "metadata": {}, + "outputs": [], + "source": [ + "# msg = messages.SNEWSMessageBuilder(detector_name='LZ', \n", + "# machine_time=test_time, \n", + "# neutrino_time=\"2023-06-12 18:30\", \n", + "# is_test=False)\n", + "\n", + "# msg.messages" + ] + }, + { + "cell_type": "markdown", + "id": "8ba01c1d", + "metadata": {}, + "source": [ + "**4) p values beyond 0 and 1**" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "31a36522", + "metadata": {}, + "outputs": [], + "source": [ + "# msg = messages.SNEWSMessageBuilder(detector_name='LZ', \n", + "# machine_time=test_time, \n", + "# p_values=[-0.0007,0.0008,0.0009],\n", + "# t_bin_width=0.07, \n", + "# is_test=False)\n", + "\n", + "# msg.messages" + ] + }, + { + "cell_type": "markdown", + "id": "ec563e38", + "metadata": {}, + "source": [ + "## Other attributes and functions" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "45f72227", + "metadata": {}, + "outputs": [], + "source": [ + "msg = messages.SNEWSMessageBuilder(detector_name='LZ', \n", + " machine_time=test_time, \n", + " neutrino_time=test_time, \n", + " p_val=\"0.000\", \n", + " p_values=[0.0007,0.0008,0.0009],\n", + " t_bin_width=0.07, \n", + " is_test=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "cfe4b55a", + "metadata": { + "collapsed": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "\u001b[1mSNEWSCoincidenceTierMessage\n", + "\u001b[0m---------------------------\n", + "\u001b[91m _id : LZ_CoincidenceTier_2023-07-21T14:53:51.016430\n", + "\u001b[0m\u001b[91m schema_version : 1.3.0\n", + "\u001b[0m\u001b[91m detector_name : LZ\n", + "\u001b[0m\u001b[94m neutrino_time : 2023-07-21T14:53:51.016430\n", + "\u001b[0m---------------------------\n", + "\u001b[96m p_values : [0.0007, 0.0008, 0.0009]\n", + "\u001b[0m\u001b[96m t_bin_width : 0.07\n", + "\u001b[0m\u001b[96m is_test : True\n", + "\u001b[0m" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "msg.messages[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "09d73658", + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Message Generated for CoincidenceTier\n", - "Message Generated for SigTier\n", "\u001b[94m----------------------------------------------------------------\u001b[0m\n", "\u001b[91mSending message to CoincidenceTier on kafka://kafka.scimma.org/snews.experiments-firedrill\u001b[0m\n", - "_id :14_CoincidenceTier_2023-06-02T09:27:40.882808\n", - "detector_name :LZ\n", - "machine_time :2023-06-02T09:27:40.882808\n", - "neutrino_time :2023-06-02T09:27:40.882808\n", - "p_val :0.0007\n", - "meta :\n", - "\tis_test :False\n", + "_id :LZ_CoincidenceTier_2023-07-21T14:53:51.016430\n", "schema_version :1.3.0\n", - "sent_time :2023-06-02T09:27:41.016506\n", + "detector_name :LZ\n", + "neutrino_time :2023-07-21T14:53:51.016430\n", + "p_val :0.000\n", + "is_test :True\n", + "sent_time :2023-07-21T14:54:09.738349\n", "\u001b[94m----------------------------------------------------------------\u001b[0m\n", - "\u001b[91mSending message to SigTier on kafka://kafka.scimma.org/snews.experiments-firedrill\u001b[0m\n", - "_id :14_SigTier_2023-06-02T09:27:40.882808\n", + "\u001b[91mSending message to SignificanceTier on kafka://kafka.scimma.org/snews.experiments-firedrill\u001b[0m\n", + "_id :LZ_SignificanceTier_2023-07-21T14:53:51.016430\n", + "schema_version :1.3.0\n", "detector_name :LZ\n", - "machine_time :2023-06-02T09:27:40.882808\n", - "neutrino_time :2023-06-02T09:27:40.882808\n", "p_values :[0.0007, 0.0008, 0.0009]\n", "t_bin_width :0.07\n", - "meta :\n", - "\tis_test :False\n", - "schema_version :1.3.0\n", - "sent_time :2023-06-02T09:27:41.019664\n" + "is_test :True\n", + "sent_time :2023-07-21T14:54:09.741139\n" ] } ], "source": [ - "from snews_pt.snews_pub import SNEWSTiersPublisher\n", - "from datetime import datetime\n", - "\n", - "test_time = datetime.utcnow().isoformat()\n", - "message = SNEWSTiersPublisher(detector_name='LZ', machine_time=test_time, neutrino_time=test_time, p_val=0.0007, p_values=[0.0007,0.0008,0.0009],t_bin_width=0.07)\n", - "# print(message.message_data)\n", - "# print(message.tiernames)\n", - "# print('Sending message')\n", - "message.send_to_snews()" + "msg.send_messages()" ] }, { "cell_type": "markdown", - "id": "f6d67ae3", + "id": "541457ba", "metadata": {}, "source": [ - "## Subscription" + "Notice below, the fields that do not belong to the tier in question are initially appended as \"meta\" fields.
\n", + "However, when sending the messages to SNEWS, the program checks this and avoids redundancies by only keeping the \"meta\" fields, if the keys are not used in any other valid message." ] }, { "cell_type": "code", - "execution_count": 4, - "id": "0ea6d89a", - "metadata": { - "pycharm": { - "name": "#%%\n" + "execution_count": 12, + "id": "08e5425c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "({'_id': 'LZ_CoincidenceTier_2023-07-21T14:53:51.016430',\n", + " 'schema_version': '1.3.0',\n", + " 'detector_name': 'LZ',\n", + " 'neutrino_time': '2023-07-21T14:53:51.016430',\n", + " 'p_val': '0.000',\n", + " 'is_test': True,\n", + " 'sent_time': '2023-07-21T14:54:09.738349'},\n", + " {'p_values': [0.0007, 0.0008, 0.0009], 't_bin_width': 0.07, 'is_test': True})" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" } - }, + ], + "source": [ + "msg.messages[0].message_data, msg.messages[0].meta" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "eea25d44", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "({'_id': 'LZ_SignificanceTier_2023-07-21T14:53:51.016430',\n", + " 'schema_version': '1.3.0',\n", + " 'detector_name': 'LZ',\n", + " 'p_values': [0.0007, 0.0008, 0.0009],\n", + " 't_bin_width': 0.07,\n", + " 'is_test': True,\n", + " 'sent_time': '2023-07-21T14:54:09.741139'},\n", + " {'neutrino_time': '2023-07-21T14:53:51.016430',\n", + " 'p_val': '0.000',\n", + " 'is_test': True})" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "msg.messages[1].message_data, msg.messages[1].meta" + ] + }, + { + "cell_type": "markdown", + "id": "3413d944", + "metadata": {}, + "source": [ + "### See the message schemas." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0625292e", + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "You are subscribing to \u001b[41m\u001b[1mALERT\u001b[0m\n", - "Broker:\u001b[42mkafka://kafka.scimma.org/snews.alert-firedrill\u001b[0m\n" + "\u001b[34m\u001b[47mMessage schema for SNEWSCoincidenceTierMessage\u001b[0m\n", + "\u001b[91m_id : (SET AUTOMATICALLY)\u001b[0m\n", + "\u001b[91mschema_version : (SET AUTOMATICALLY)\u001b[0m\n", + "\u001b[91mdetector_name : (SET AUTOMATICALLY)\u001b[0m\n", + "\u001b[94mneutrino_time : (REQUIRED USER INPUT)\u001b[0m\n", + "\u001b[96mmachine_time : (USER INPUT)\u001b[0m\n", + "\u001b[96mp_val : (USER INPUT)\u001b[0m\n" ] } ], "source": [ - "from snews_pt.snews_sub import Subscriber\n", - "\n", - "Subscriber().subscribe()" + "msg.messages[0].print_schema()" ] }, { "cell_type": "code", - "execution_count": null, - "id": "81f0b707", - "metadata": { - "pycharm": { - "name": "#%%\n" + "execution_count": 15, + "id": "4177e2c5", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[47mMessage schema for SNEWSSignificanceTierMessage\u001b[0m\n", + "\u001b[91m_id : (SET AUTOMATICALLY)\u001b[0m\n", + "\u001b[91mschema_version : (SET AUTOMATICALLY)\u001b[0m\n", + "\u001b[91mdetector_name : (SET AUTOMATICALLY)\u001b[0m\n", + "\u001b[94mp_values : (REQUIRED USER INPUT)\u001b[0m\n", + "\u001b[94mt_bin_width : (REQUIRED USER INPUT)\u001b[0m\n", + "\u001b[96mmachine_time : (USER INPUT)\u001b[0m\n" + ] } - }, + ], + "source": [ + "msg.messages[1].print_schema()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "70fff993", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c8d7ddf9", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13068238", + "metadata": {}, "outputs": [], "source": [] } diff --git a/firedrill.ipynb b/firedrill.ipynb index 47bda7f..de6987a 100644 --- a/firedrill.ipynb +++ b/firedrill.ipynb @@ -64,12 +64,13 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "from snews_pt.snews_pt_utils import set_name # to set experiment name\n", - "from snews_pt.snews_pub import SNEWSTiersPublisher # to publish observations & heartbeats\n", + "# from snews_pt.snews_pub import SNEWSTiersPublisher # to publish observations & heartbeats\n", + "from snews_pt.messages import SNEWSMessageBuilder\n", "from snews_pt.snews_sub import Subscriber # to listen alerts\n", "from snews_pt.remote_commands import test_connection, get_feedback # to test connection and get feedback\n", "from datetime import datetime\n", @@ -85,7 +86,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [ { @@ -112,7 +113,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -139,20 +140,12 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Message Generated for CoincidenceTier\n" - ] - } - ], + "outputs": [], "source": [ "observation_time = \"2023-06-14T12:45:45.100000\"\n", - "msg = SNEWSTiersPublisher(neutrino_time=observation_time, is_test=True, firedrill_mode=True)" + "msg = SNEWSMessageBuilder(neutrino_time=observation_time, is_test=True, firedrill_mode=True)" ] }, { @@ -164,49 +157,46 @@ }, { "cell_type": "code", - "execution_count": 6, - "metadata": {}, + "execution_count": 5, + "metadata": { + "scrolled": true + }, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "Message Generated for CoincidenceTier\n", - "\u001b[91m----------------------------------------------------------------\u001b[0m\n", - "Skipping message! Improper format! see \u001b[0m\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/mnt/c/Users/bj7780/Desktop/Kara/GitHub/SNEWS/SNEWS_Publishing_Tools/snews_pt/snews_format_checker.py:48: UserWarning: \n", - "\n", - " Following check failed: \u001b[31mneutrino_time not valid\u001b[0m \n", - "See the full logs \u001b[34m/mnt/c/Users/bj7780/Desktop/Kara/GitHub/SNEWS/SNEWS_Publishing_Tools/snews_pt/core/../../logs/2023-04-25.log\u001b[0m\n", - " warnings.warn(f\"\\n\\n Following check failed: {click.style(ae, fg='red')} \"\n" + "ename": "ValueError", + "evalue": "Invalid isoformat string: '2023-06-9999999:45:100000'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[5], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m observation_time2 \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m2023-06-9999999:45:100000\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m----> 2\u001b[0m msg2 \u001b[38;5;241m=\u001b[39m \u001b[43mSNEWSMessageBuilder\u001b[49m\u001b[43m(\u001b[49m\u001b[43mneutrino_time\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mobservation_time2\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mis_test\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfiredrill_mode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/mnt/c/Users/bj7780/Desktop/Kara/GitHub/SNEWS/SNEWS_Publishing_Tools/snews_pt/messages.py:385\u001b[0m, in \u001b[0;36mSNEWSMessageBuilder.__init__\u001b[0;34m(self, env_file, detector_name, machine_time, neutrino_time, p_val, p_values, t_bin_width, timing_series, retract_latest, retraction_reason, detector_status, is_test, **kwargs)\u001b[0m\n\u001b[1;32m 382\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmessages \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 383\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mselected_tiers \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m--> 385\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_build_messages\u001b[49m\u001b[43m(\u001b[49m\u001b[43menv_file\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43menv_file\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 386\u001b[0m \u001b[43m \u001b[49m\u001b[43mdetector_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdetector_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 387\u001b[0m \u001b[43m \u001b[49m\u001b[43mmachine_time\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmachine_time\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 388\u001b[0m \u001b[43m \u001b[49m\u001b[43mneutrino_time\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mneutrino_time\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 389\u001b[0m \u001b[43m \u001b[49m\u001b[43mp_val\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mp_val\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 390\u001b[0m \u001b[43m \u001b[49m\u001b[43mp_values\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mp_values\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 391\u001b[0m \u001b[43m \u001b[49m\u001b[43mt_bin_width\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mt_bin_width\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 392\u001b[0m \u001b[43m \u001b[49m\u001b[43mtiming_series\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtiming_series\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 393\u001b[0m \u001b[43m \u001b[49m\u001b[43mretract_latest\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mretract_latest\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 394\u001b[0m \u001b[43m \u001b[49m\u001b[43mretraction_reason\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mretraction_reason\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 395\u001b[0m \u001b[43m \u001b[49m\u001b[43mdetector_status\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdetector_status\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 396\u001b[0m \u001b[43m \u001b[49m\u001b[43mis_test\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mis_test\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 397\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/mnt/c/Users/bj7780/Desktop/Kara/GitHub/SNEWS/SNEWS_Publishing_Tools/snews_pt/messages.py:447\u001b[0m, in \u001b[0;36mSNEWSMessageBuilder._build_messages\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m 444\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m hasreqfields:\n\u001b[1;32m 445\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmessages \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 446\u001b[0m \u001b[38;5;66;03m# check is valid at creation\u001b[39;00m\n\u001b[0;32m--> 447\u001b[0m \u001b[43msmc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mnonull_kwargs\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mis_valid()\n\u001b[1;32m 448\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmessages \u001b[38;5;241m=\u001b[39m [smc(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mnonull_kwargs)]\n\u001b[1;32m 449\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", + "File \u001b[0;32m/mnt/c/Users/bj7780/Desktop/Kara/GitHub/SNEWS/SNEWS_Publishing_Tools/snews_pt/messages.py:247\u001b[0m, in \u001b[0;36mSNEWSCoincidenceTierMessage.__init__\u001b[0;34m(self, neutrino_time, p_val, **kwargs)\u001b[0m\n\u001b[1;32m 245\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, neutrino_time\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, p_val\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 246\u001b[0m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfields,\n\u001b[0;32m--> 247\u001b[0m neutrino_time\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mclean_time_input\u001b[49m\u001b[43m(\u001b[49m\u001b[43mneutrino_time\u001b[49m\u001b[43m)\u001b[49m,\n\u001b[1;32m 248\u001b[0m p_val\u001b[38;5;241m=\u001b[39mp_val,\n\u001b[1;32m 249\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", + "File \u001b[0;32m/mnt/c/Users/bj7780/Desktop/Kara/GitHub/SNEWS/SNEWS_Publishing_Tools/snews_pt/messages.py:188\u001b[0m, in \u001b[0;36mSNEWSMessage.clean_time_input\u001b[0;34m(self, time)\u001b[0m\n\u001b[1;32m 185\u001b[0m time \u001b[38;5;241m=\u001b[39m datetime\u001b[38;5;241m.\u001b[39mutcnow()\n\u001b[1;32m 187\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(time, \u001b[38;5;28mstr\u001b[39m):\n\u001b[0;32m--> 188\u001b[0m time \u001b[38;5;241m=\u001b[39m \u001b[43mfromisoformat\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtime\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 190\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m time\u001b[38;5;241m.\u001b[39misoformat()\n", + "\u001b[0;31mValueError\u001b[0m: Invalid isoformat string: '2023-06-9999999:45:100000'" ] } ], "source": [ "observation_time2 = \"2023-06-9999999:45:100000\"\n", - "msg2 = SNEWSTiersPublisher(neutrino_time=observation_time2, is_test=True, firedrill_mode=True)" + "msg2 = SNEWSMessageBuilder(neutrino_time=observation_time2, is_test=True, firedrill_mode=True)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "#### SNEWSTierPublisher\n", - "The `SNEWSTierPublisher` creates a message for you from your given keys and decides what to do with this message.\n", + "#### SNEWSMessageBuilder\n", + "The `SNEWSMessageBuilder` builds and validates messages for you from your given keys.\n", "\n", - "The `neutrino_time` is the most important argument. The Publisher decides that this message is intended for the `CoincidenceTier` based on this input.
\n", + "The `neutrino_time` is the most important argument. The Builder decides that this message is intended for the `CoincidenceTier` based on this input.
\n", "\n", - "Try calling the function with no argument -> `SNEWSTiersPublisher()`
\n", - "This returns `\"No valid message is created! Check your input and try again!\"`
\n", + "Try calling the function with no argument -> `SNEWSMessageBuilder()`
\n", + "This returns `\"SNEWSMessageBuilder: No messages have been built\"`
\n", "\n", "Another example would be the heartbeat messages
\n", - "If you call it with a detector status i.e. `SNEWSTiersPublisher(detector_status='ON')` this should create a message for the heartbeats. What happens if you pass both neutrino times and detector status? Try it out!\n", + "If you call it with a detector status i.e. `SNEWSMessageBuilder(detector_status='ON')` this should create a message for the heartbeats. What happens if you pass both neutrino times and detector status? Try it out!\n", "\n", "----\n", "\n", @@ -224,12 +214,12 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "## Here try and play with different arguments\n", - "# SNEWSTiersPublisher()" + "# SNEWSMessageBuilder()" ] }, { @@ -242,82 +232,98 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 12, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "Message Generated for CoincidenceTier\n", - "Message Generated for Heartbeat\n" - ] + "data": { + "text/plain": [ + "['SNEWSCoincidenceTierMessage', 'SNEWSHeartbeatMessage']" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "msg = SNEWSTiersPublisher(neutrino_time=observation_time, is_test=True, firedrill_mode=True, detector_status='ON')" + "msg = SNEWSMessageBuilder(neutrino_time=observation_time, is_test=True, firedrill_mode=True, detector_status='ON')\n", + "msg.selected_tiers" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 14, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "dict_keys(['CoincidenceTier', 'Heartbeat'])\n" + "['SNEWSCoincidenceTierMessage', 'SNEWSHeartbeatMessage']\n" ] }, { "data": { "text/plain": [ - "{'detector_name': 'XENONnT',\n", - " 'machine_time': None,\n", - " 'neutrino_time': '2023-06-14T12:45:45.100000',\n", - " 'p_val': None,\n", - " 'p_values': None,\n", - " 't_bin_width': None,\n", - " 'timing_series': None,\n", - " 'retract_latest': None,\n", - " 'retraction_reason': None,\n", - " 'detector_status': 'ON',\n", - " 'is_pre_sn': False,\n", - " 'is_test': True}" + "[\u001b[1mSNEWSCoincidenceTierMessage\n", + " \u001b[0m---------------------------\n", + " \u001b[91m _id : TEST_CoincidenceTier_2023-07-31T09:15:14.352389\n", + " \u001b[0m\u001b[91m schema_version : 1.3.0\n", + " \u001b[0m\u001b[91m detector_name : TEST\n", + " \u001b[0m\u001b[94m neutrino_time : 2023-06-14T12:45:45.100000\n", + " \u001b[0m---------------------------\n", + " \u001b[96mdetector_status : ON\n", + " \u001b[0m\u001b[96m is_test : True\n", + " \u001b[0m\u001b[96m firedrill_mode : True\n", + " \u001b[0m,\n", + " \u001b[1mSNEWSHeartbeatMessage\n", + " \u001b[0m---------------------\n", + " \u001b[91m _id : TEST_Heartbeat_2023-07-31T09:15:14.363811\n", + " \u001b[0m\u001b[91m schema_version : 1.3.0\n", + " \u001b[0m\u001b[91m detector_name : TEST\n", + " \u001b[0m\u001b[94mdetector_status : ON\n", + " \u001b[0m---------------------\n", + " \u001b[96m neutrino_time : 2023-06-14T12:45:45.100000\n", + " \u001b[0m\u001b[96m is_test : True\n", + " \u001b[0m\u001b[96m firedrill_mode : True\n", + " \u001b[0m]" ] }, - "execution_count": 9, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# see the selected tiers\n", - "print(msg.tiernames)\n", + "print(msg.selected_tiers)\n", "# see the message data\n", - "msg.message_data" + "msg.messages" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'_id': '19_CoincidenceTier_2023-04-25T13:01:34.859207',\n", - " 'detector_name': 'XENONnT',\n", - " 'machine_time': '2023-04-25T13:01:34.859207',\n", - " 'neutrino_time': '2023-06-14T12:45:45.100000',\n", - " 'p_val': None,\n", - " 'meta': {'is_test': True},\n", - " 'schema_version': '1.3.0',\n", - " 'sent_time': '2023-04-25T13:01:34.859207'}" + "\u001b[1mSNEWSCoincidenceTierMessage\n", + "\u001b[0m---------------------------\n", + "\u001b[91m _id : TEST_CoincidenceTier_2023-07-31T09:15:14.352389\n", + "\u001b[0m\u001b[91m schema_version : 1.3.0\n", + "\u001b[0m\u001b[91m detector_name : TEST\n", + "\u001b[0m\u001b[94m neutrino_time : 2023-06-14T12:45:45.100000\n", + "\u001b[0m---------------------------\n", + "\u001b[96mdetector_status : ON\n", + "\u001b[0m\u001b[96m is_test : True\n", + "\u001b[0m\u001b[96m firedrill_mode : True\n", + "\u001b[0m" ] }, - "execution_count": 10, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -329,22 +335,26 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'_id': '19_Heartbeat_2023-04-25T13:01:34.859207',\n", - " 'detector_name': 'XENONnT',\n", - " 'machine_time': '2023-04-25T13:01:34.859207',\n", - " 'detector_status': 'ON',\n", - " 'meta': {},\n", - " 'schema_version': '1.3.0',\n", - " 'sent_time': '2023-04-25T13:01:34.859207'}" + "\u001b[1mSNEWSHeartbeatMessage\n", + "\u001b[0m---------------------\n", + "\u001b[91m _id : TEST_Heartbeat_2023-07-31T09:15:14.363811\n", + "\u001b[0m\u001b[91m schema_version : 1.3.0\n", + "\u001b[0m\u001b[91m detector_name : TEST\n", + "\u001b[0m\u001b[94mdetector_status : ON\n", + "\u001b[0m---------------------\n", + "\u001b[96m neutrino_time : 2023-06-14T12:45:45.100000\n", + "\u001b[0m\u001b[96m is_test : True\n", + "\u001b[0m\u001b[96m firedrill_mode : True\n", + "\u001b[0m" ] }, - "execution_count": 11, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } @@ -370,7 +380,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -381,15 +391,25 @@ "> Testing your connection.\n", "> Sending to kafka://kafka.scimma.org/snews.experiments-firedrill\n", "> Expecting from kafka://kafka.scimma.org/snews.connection-testing. \n", - "> Should take ~10 seconds...\n", + "> Going to wait 8 seconds before checking for confirmation...\n", "\u001b[0m\n", - "\u001b[31m\u001b[1m\tCouldn't get a confirmation in 10 sec. \n", + "\u001b[31m\u001b[1m\tWaited for 8 sec and checked from LATEST, couldn't get a confirmation\n", "\tMaybe increase timeout and try again.\u001b[0m\n" ] + }, + { + "data": { + "text/plain": [ + "False" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "test_connection()" + "test_connection(firedrill=True)" ] }, { @@ -405,7 +425,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 22, "metadata": { "scrolled": true }, @@ -418,20 +438,30 @@ "> Testing your connection.\n", "> Sending to kafka://kafka.scimma.org/snews.experiments-test\n", "> Expecting from kafka://kafka.scimma.org/snews.connection-testing. \n", - "> Should take ~10 seconds...\n", + "> Going to wait 8 seconds before checking for confirmation...\n", "\u001b[0m\n", - "You (\u001b[32m\u001b[1mXENONnT\u001b[0m) have a connection to the server at \u001b[32m\u001b[1m2023-04-25T12:21:01.527410\u001b[0m\n" + "You (\u001b[32m\u001b[1mXENONnT\u001b[0m) have a connection to the server at \u001b[32m\u001b[1m2023-07-31T09:16:54.159806\u001b[0m\n" ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ "from snews_pt.remote_commands import test_connection\n", - "test_connection(firedrill=False, wait=10)" + "test_connection(firedrill=False, patience=8)" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 25, "metadata": {}, "outputs": [ { @@ -439,30 +469,31 @@ "output_type": "stream", "text": [ "\u001b[94m----------------------------------------------------------------\u001b[0m\n", - "\u001b[91mSending message to CoincidenceTier on kafka://kafka.scimma.org/snews.experiments-firedrill\u001b[0m\n", - "_id :19_CoincidenceTier_2023-04-25T13:01:34.859207\n", - "detector_name :XENONnT\n", - "machine_time :2023-04-25T13:01:34.859207\n", + "\u001b[91mSending message to CoincidenceTier on kafka://kafka.scimma.org/snews.experiments-test\u001b[0m\n", + "_id :TEST_CoincidenceTier_2023-07-31T09:15:14.352389\n", + "schema_version :1.3.0\n", + "detector_name :TEST\n", + "machine_time :None\n", "neutrino_time :2023-06-14T12:45:45.100000\n", "p_val :None\n", - "meta :\n", - "\tis_test :True\n", - "schema_version :1.3.0\n", - "sent_time :2023-04-25T13:01:51.769002\n", + "is_test :True\n", + "firedrill_mode :True\n", + "sent_time :2023-07-31T09:17:37.096038\n", "\u001b[94m----------------------------------------------------------------\u001b[0m\n", - "\u001b[91mSending message to Heartbeat on kafka://kafka.scimma.org/snews.experiments-firedrill\u001b[0m\n", - "_id :19_Heartbeat_2023-04-25T13:01:34.859207\n", - "detector_name :XENONnT\n", - "machine_time :2023-04-25T13:01:34.859207\n", - "detector_status :ON\n", - "meta :\n", + "\u001b[91mSending message to Heartbeat on kafka://kafka.scimma.org/snews.experiments-test\u001b[0m\n", + "_id :TEST_Heartbeat_2023-07-31T09:15:14.363811\n", "schema_version :1.3.0\n", - "sent_time :2023-04-25T13:01:51.774349\n" + "detector_name :TEST\n", + "machine_time :None\n", + "detector_status :ON\n", + "is_test :True\n", + "firedrill_mode :True\n", + "sent_time :2023-07-31T09:17:37.097456\n" ] } ], "source": [ - "msg.send_to_snews()" + "msg.send_messages(firedrill_mode=False)" ] }, { @@ -475,37 +506,33 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 26, "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Message Generated for CoincidenceTier\n" - ] - }, { "data": { "text/plain": [ - "{'_id': '19_CoincidenceTier_2023-04-25T13:02:00.818821',\n", - " 'detector_name': 'XENONnT',\n", - " 'machine_time': '2023-04-25T13:02:00.818821',\n", - " 'neutrino_time': '2023-06-14T12:45:45.100000',\n", - " 'p_val': None,\n", - " 'meta': {'is_test': True, 'comment': 'We were taking calibration data'},\n", - " 'schema_version': '1.3.0',\n", - " 'sent_time': '2023-04-25T13:02:00.818821'}" + "(\u001b[1mSNEWSCoincidenceTierMessage\n", + " \u001b[0m---------------------------\n", + " \u001b[91m _id : TEST_CoincidenceTier_2023-07-31T09:18:01.695375\n", + " \u001b[0m\u001b[91m schema_version : 1.3.0\n", + " \u001b[0m\u001b[91m detector_name : TEST\n", + " \u001b[0m\u001b[94m neutrino_time : 2023-06-14T12:45:45.100000\n", + " \u001b[0m---------------------------\n", + " \u001b[96m is_test : True\n", + " \u001b[0m\u001b[96m comment : We were taking calibration data\n", + " \u001b[0m,\n", + " {'is_test': True, 'comment': 'We were taking calibration data'})" ] }, - "execution_count": 13, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "msg2 = SNEWSTiersPublisher(neutrino_time=observation_time, is_test=True, comment='We were taking calibration data')\n", - "msg2.messages[0]" + "msg2 = SNEWSMessageBuilder(neutrino_time=observation_time, is_test=True, comment='We were taking calibration data')\n", + "msg2.messages[0], msg2.messages[0].meta" ] }, { @@ -543,12 +570,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 46, "metadata": {}, "outputs": [], "source": [ - "# json_message = SNEWSTiersPublisher.from_json(\"observation_file.json\")\n", - "# json_message.send_to_snews()" + "from snews_pt import messages\n", + "import importlib\n", + "importlib.reload(messages)\n", + "from datetime import datetime\n", + "\n", + "test_time = datetime.utcnow().isoformat()" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": {}, + "outputs": [], + "source": [ + "json_message = messages.SNEWSMessageBuilder.from_json(\"./snews_pt/test/example_coincidence_tier_message.json\")\n", + "# json_message.messages\n", + "# json_message.send_messages()" ] }, { @@ -565,27 +607,19 @@ "SNEWS tracks several time informations with different meanings and purposes;
\n", "- `neutrino_time` : this is used for searching coincidences and refers to exact time of the first neutrino detection
\n", "- `machine_time` : this is an optional argument to indicate the time labeled by the machine, ideally for the observation message this would be the same as the neutrino time. Howeever, for the heartbeat messages it should be the time that detector heartbeat was checked. \n", - "- `sent_time` : this is added by the `SNEWSTierPublisher().send_to_snews()` upon execution. We use this information for latency measurements.
\n", + "- `sent_time` : this is added by the `SNEWSMessageBuilder().send_messages()` upon execution. We use this information for latency measurements.
\n", "\n", "On the server side, we also label messages by their `received_time` which allows us (for example for the heartbeats) to calculate the latency between the time detector checks their status, the time they actually send it, and the time we receive it at the server. " ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 49, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Message Generated for Heartbeat\n" - ] - } - ], + "outputs": [], "source": [ "# simply create a message for the heartbeat\n", - "msg_hb = SNEWSTiersPublisher(detector_status=\"ON\")" + "msg_hb = SNEWSMessageBuilder(detector_status=\"ON\")" ] }, { @@ -597,22 +631,24 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 50, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'_id': '19_Heartbeat_2023-04-25T13:02:08.299389',\n", - " 'detector_name': 'XENONnT',\n", - " 'machine_time': '2023-04-25T13:02:08.299389',\n", - " 'detector_status': 'ON',\n", - " 'meta': {},\n", - " 'schema_version': '1.3.0',\n", - " 'sent_time': '2023-04-25T13:02:08.299389'}" + "\u001b[1mSNEWSHeartbeatMessage\n", + "\u001b[0m---------------------\n", + "\u001b[91m _id : TEST_Heartbeat_2023-07-31T09:23:50.092086\n", + "\u001b[0m\u001b[91m schema_version : 1.3.0\n", + "\u001b[0m\u001b[91m detector_name : TEST\n", + "\u001b[0m\u001b[94mdetector_status : ON\n", + "\u001b[0m---------------------\n", + "\u001b[96m is_test : False\n", + "\u001b[0m" ] }, - "execution_count": 15, + "execution_count": 50, "metadata": {}, "output_type": "execute_result" } @@ -623,7 +659,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 51, "metadata": {}, "outputs": [ { @@ -632,18 +668,18 @@ "text": [ "\u001b[94m----------------------------------------------------------------\u001b[0m\n", "\u001b[91mSending message to Heartbeat on kafka://kafka.scimma.org/snews.experiments-firedrill\u001b[0m\n", - "_id :19_Heartbeat_2023-04-25T13:02:08.299389\n", - "detector_name :XENONnT\n", - "machine_time :2023-04-25T13:02:08.299389\n", - "detector_status :ON\n", - "meta :\n", + "_id :TEST_Heartbeat_2023-07-31T09:23:50.092086\n", "schema_version :1.3.0\n", - "sent_time :2023-04-25T13:02:12.070869\n" + "detector_name :TEST\n", + "machine_time :None\n", + "detector_status :ON\n", + "is_test :False\n", + "sent_time :2023-07-31T09:23:57.784270\n" ] } ], "source": [ - "msg_hb.send_to_snews()" + "msg_hb.send_messages()" ] }, { @@ -703,7 +739,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 52, "metadata": {}, "outputs": [ { @@ -769,12 +805,12 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 53, "metadata": {}, "outputs": [], "source": [ "from snews_pt.remote_commands import test_connection\n", - "from snews_pt.snews_pub import SNEWSTiersPublisher" + "from snews_pt.messages import SNEWSMessageBuilder" ] }, { @@ -788,10 +824,8 @@ }, { "cell_type": "code", - "execution_count": 18, - "metadata": { - "collapsed": true - }, + "execution_count": 24, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -799,74 +833,74 @@ "text": [ "Message Generated for Heartbeat\n", "\u001b[94m----------------------------------------------------------------\u001b[0m\n", - "\u001b[91mSending message to Heartbeat on kafka://kafka.scimma.org/snews.experiments-firedrill\u001b[0m\n", - "_id :19_Heartbeat_2023-04-24T13:29:22.960664\n", + "\u001b[91mSending message to Heartbeat on kafka://kafka.scimma.org/snews.experiments-test\u001b[0m\n", + "_id :19_Heartbeat_2023-06-02T15:14:18.602056\n", "detector_name :XENONnT\n", - "machine_time :2023-04-24T13:29:22.960664\n", + "machine_time :2023-06-02T15:14:18.602056\n", "detector_status :ON\n", "meta :\n", "schema_version :1.3.0\n", - "sent_time :2023-04-24T13:29:22.984495\n", + "sent_time :2023-06-02T15:14:18.624390\n", "Message Generated for Heartbeat\n", "\u001b[94m----------------------------------------------------------------\u001b[0m\n", - "\u001b[91mSending message to Heartbeat on kafka://kafka.scimma.org/snews.experiments-firedrill\u001b[0m\n", - "_id :19_Heartbeat_2023-04-24T13:29:29.108253\n", + "\u001b[91mSending message to Heartbeat on kafka://kafka.scimma.org/snews.experiments-test\u001b[0m\n", + "_id :19_Heartbeat_2023-06-02T15:14:24.462710\n", "detector_name :XENONnT\n", - "machine_time :2023-04-24T13:29:29.108253\n", + "machine_time :2023-06-02T15:14:24.462710\n", "detector_status :ON\n", "meta :\n", "schema_version :1.3.0\n", - "sent_time :2023-04-24T13:29:29.170582\n", + "sent_time :2023-06-02T15:14:24.483085\n", "Message Generated for Heartbeat\n", "\u001b[94m----------------------------------------------------------------\u001b[0m\n", - "\u001b[91mSending message to Heartbeat on kafka://kafka.scimma.org/snews.experiments-firedrill\u001b[0m\n", - "_id :19_Heartbeat_2023-04-24T13:29:35.210759\n", + "\u001b[91mSending message to Heartbeat on kafka://kafka.scimma.org/snews.experiments-test\u001b[0m\n", + "_id :19_Heartbeat_2023-06-02T15:14:30.292382\n", "detector_name :XENONnT\n", - "machine_time :2023-04-24T13:29:35.210759\n", + "machine_time :2023-06-02T15:14:30.292382\n", "detector_status :ON\n", "meta :\n", "schema_version :1.3.0\n", - "sent_time :2023-04-24T13:29:35.468665\n", + "sent_time :2023-06-02T15:14:30.314934\n", "Message Generated for Heartbeat\n", "\u001b[94m----------------------------------------------------------------\u001b[0m\n", - "\u001b[91mSending message to Heartbeat on kafka://kafka.scimma.org/snews.experiments-firedrill\u001b[0m\n", - "_id :19_Heartbeat_2023-04-24T13:29:41.896350\n", + "\u001b[91mSending message to Heartbeat on kafka://kafka.scimma.org/snews.experiments-test\u001b[0m\n", + "_id :19_Heartbeat_2023-06-02T15:14:36.153385\n", "detector_name :XENONnT\n", - "machine_time :2023-04-24T13:29:41.896350\n", + "machine_time :2023-06-02T15:14:36.153385\n", "detector_status :ON\n", "meta :\n", "schema_version :1.3.0\n", - "sent_time :2023-04-24T13:29:41.921992\n", + "sent_time :2023-06-02T15:14:36.175475\n", "Message Generated for Heartbeat\n", "\u001b[94m----------------------------------------------------------------\u001b[0m\n", - "\u001b[91mSending message to Heartbeat on kafka://kafka.scimma.org/snews.experiments-firedrill\u001b[0m\n", - "_id :19_Heartbeat_2023-04-24T13:29:47.391542\n", + "\u001b[91mSending message to Heartbeat on kafka://kafka.scimma.org/snews.experiments-test\u001b[0m\n", + "_id :19_Heartbeat_2023-06-02T15:14:42.027191\n", "detector_name :XENONnT\n", - "machine_time :2023-04-24T13:29:47.391542\n", + "machine_time :2023-06-02T15:14:42.027191\n", "detector_status :ON\n", "meta :\n", "schema_version :1.3.0\n", - "sent_time :2023-04-24T13:29:47.423036\n", + "sent_time :2023-06-02T15:14:42.050468\n", "Message Generated for Heartbeat\n", "\u001b[94m----------------------------------------------------------------\u001b[0m\n", - "\u001b[91mSending message to Heartbeat on kafka://kafka.scimma.org/snews.experiments-firedrill\u001b[0m\n", - "_id :19_Heartbeat_2023-04-24T13:29:52.891700\n", + "\u001b[91mSending message to Heartbeat on kafka://kafka.scimma.org/snews.experiments-test\u001b[0m\n", + "_id :19_Heartbeat_2023-06-02T15:14:47.831672\n", "detector_name :XENONnT\n", - "machine_time :2023-04-24T13:29:52.891700\n", + "machine_time :2023-06-02T15:14:47.831672\n", "detector_status :ON\n", "meta :\n", "schema_version :1.3.0\n", - "sent_time :2023-04-24T13:29:52.951034\n", + "sent_time :2023-06-02T15:14:47.894849\n", "Message Generated for Heartbeat\n", "\u001b[94m----------------------------------------------------------------\u001b[0m\n", - "\u001b[91mSending message to Heartbeat on kafka://kafka.scimma.org/snews.experiments-firedrill\u001b[0m\n", - "_id :19_Heartbeat_2023-04-24T13:29:58.504245\n", + "\u001b[91mSending message to Heartbeat on kafka://kafka.scimma.org/snews.experiments-test\u001b[0m\n", + "_id :19_Heartbeat_2023-06-02T15:14:53.651984\n", "detector_name :XENONnT\n", - "machine_time :2023-04-24T13:29:58.504245\n", + "machine_time :2023-06-02T15:14:53.651984\n", "detector_status :ON\n", "meta :\n", "schema_version :1.3.0\n", - "sent_time :2023-04-24T13:29:58.528545\n" + "sent_time :2023-06-02T15:14:53.673362\n" ] } ], @@ -874,8 +908,8 @@ "import time\n", "for i in range(7):\n", " time.sleep(2)\n", - " msg = SNEWSTiersPublisher(detector_status='ON', firedrill_mode=True) \n", - " msg.send_to_snews()" + " msg = SNEWSMessageBuilder(detector_status='ON', firedrill_mode=False) \n", + " msg.send_messages()" ] }, { @@ -948,7 +982,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 27, "metadata": {}, "outputs": [ { @@ -960,12 +994,12 @@ } ], "source": [ - "msg = SNEWSTiersPublisher(neutrino_time=\"2023-02-07T12:00:00.026127\", is_test=True, firedrill_mode=False)" + "msg = SNEWSMessageBuilder(neutrino_time=\"2023-02-07T12:00:00.026127\", is_test=True, firedrill_mode=True)" ] }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 28, "metadata": {}, "outputs": [ { @@ -973,21 +1007,21 @@ "output_type": "stream", "text": [ "\u001b[94m----------------------------------------------------------------\u001b[0m\n", - "\u001b[91mSending message to CoincidenceTier on kafka://kafka.scimma.org/snews.experiments-test\u001b[0m\n", - "_id :19_CoincidenceTier_2023-04-24T14:42:04.221770\n", + "\u001b[91mSending message to CoincidenceTier on kafka://kafka.scimma.org/snews.experiments-firedrill\u001b[0m\n", + "_id :19_CoincidenceTier_2023-06-02T15:21:10.414950\n", "detector_name :XENONnT\n", - "machine_time :2023-04-24T14:42:04.221770\n", + "machine_time :2023-06-02T15:21:10.414950\n", "neutrino_time :2023-02-07T12:00:00.026127\n", "p_val :None\n", "meta :\n", "\tis_test :True\n", "schema_version :1.3.0\n", - "sent_time :2023-04-24T14:42:23.346740\n" + "sent_time :2023-06-02T15:21:12.108085\n" ] } ], "source": [ - "msg.send_to_snews()" + "msg.send_messages()" ] }, { @@ -1013,6 +1047,20 @@ "- 5. send your observation to snews" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": null, diff --git a/requirements.txt b/requirements.txt index b4fc32d..558dd91 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,3 +21,4 @@ sphinx_rtd_theme==1.0.0 sphinx-autoapi==1.8.4 sphinxcontrib-programoutput==0.17 pytest~=6.2.5 +numpy>=1.22.3 \ No newline at end of file diff --git a/snews_pt/__main__.py b/snews_pt/__main__.py index d329f0e..952dfd6 100644 --- a/snews_pt/__main__.py +++ b/snews_pt/__main__.py @@ -6,14 +6,10 @@ from . import __version__ from . import snews_pt_utils -from .snews_pub import SNEWSTiersPublisher +from .messages import SNEWSMessageBuilder from .snews_sub import Subscriber -from .snews_pt_utils import coincidence_tier_data, sig_tier_data, time_tier_data -from .snews_pt_utils import retraction_data, heartbeat_data -from hop import Stream import click import os -from inspect import signature @click.group(invoke_without_command=True) @@ -53,7 +49,7 @@ def publish(ctx, file, firedrill): click.clear() for f in file: if f.endswith('.json'): - SNEWSTiersPublisher.from_json(jsonfile=f, env_file=ctx.obj['env'], firedrill_mode=firedrill).send_to_snews() + SNEWSMessageBuilder.from_json(jsonfile=f, env_file=ctx.obj['env'], firedrill_mode=firedrill).send_messages() else: # maybe just print instead of raising @@ -70,11 +66,11 @@ def heartbeat(ctx, status, time, firedrill): :param time: (optional) Machine time is appended as the time of execution different time can be passed following the iso-format """ - message = SNEWSTiersPublisher(detector_name=ctx.obj['DETECTOR_NAME'], + message = SNEWSMessageBuilder(detector_name=ctx.obj['DETECTOR_NAME'], machine_time=time, detector_status=status, firedrill_mode=firedrill) - message.send_to_snews() + message.send_messages() @main.command() @@ -102,22 +98,20 @@ def subscribe(ctx, plugin, outputfolder, firedrill): except KeyboardInterrupt: pass - @main.command() @click.argument('requested_tier', nargs=-1) @click.pass_context def message_schema(ctx, requested_tier): """ Display the message format for `tier` if 'all' displays everything - """ - detector = ctx.obj['DETECTOR_NAME'] - detector_str = click.style(detector, fg='yellow') - tier_data_pairs = {'CoincidenceTier': (coincidence_tier_data, 'neutrino_time'), - 'SigTier': (sig_tier_data, 'p_values'), - 'TimeTier': (time_tier_data, 'timing_series'), - 'FalseOBS': (retraction_data, 'retract_latest'), - 'Heartbeat': (heartbeat_data, 'detector_status')} + from .messages import SNEWSHeartbeatMessage, SNEWSTimingTierMessage, SNEWSSignificanceTierMessage, \ + SNEWSCoincidenceTierMessage, SNEWSRetractionMessage + tier_data_pairs = {'CoincidenceTier': SNEWSCoincidenceTierMessage, + 'SigTier': SNEWSSignificanceTierMessage, + 'TimeTier': SNEWSTimingTierMessage, + 'FalseOBS': SNEWSRetractionMessage, + 'Heartbeat': SNEWSHeartbeatMessage,} if len(requested_tier)>1: tier = [] @@ -132,19 +126,15 @@ def message_schema(ctx, requested_tier): tier = snews_pt_utils._check_aliases(requested_tier[0]) for t in tier: - tier_keys = list(signature(tier_data_pairs[t][0]).parameters.keys()) - tier_keys.pop(tier_keys.index('meta')) - must_key = tier_data_pairs[t][1] - click.secho(f'\t >The Message Schema for {t}', bg='white', fg='blue') - click.secho(f"{'_id':<20s}:(SNEWS SETS)", fg='bright_red') - click.secho(f"{'schema_version':<20s}:(SNEWS SETS)", fg='bright_red') - click.echo(click.style(f"{'detector_name':<20s}:(FETCHED FROM ENV {detector_str})", fg='red')) - for key in tier_keys: - if key == must_key: - click.secho(f'{key:<20s}:(User Input*)', fg='bright_cyan') + TierMessage = tier_data_pairs[t] + click.secho(f'Message schema for {TierMessage.__name__}', bg='white', fg='blue') + for f in TierMessage.fields: + if f in TierMessage.basefields: + click.secho(f'{f:<20s} : (SET AUTOMATICALLY)', fg='bright_red') + elif f in TierMessage.reqfields: + click.secho(f'{f:<20s} : (REQUIRED USER INPUT)', fg='bright_blue') else: - click.secho(f'{key:<20s}:(User Input)', fg='bright_cyan') - click.secho(f"{'**kwargs':<20s}:(APPENDED AS 'META')\n", fg='red') + click.secho(f'{f:<20s} : (USER INPUT)', fg='bright_cyan') @main.command() diff --git a/snews_pt/auxiliary/try_scenarios.py b/snews_pt/auxiliary/try_scenarios.py index 5f7fa6d..2d5428e 100644 --- a/snews_pt/auxiliary/try_scenarios.py +++ b/snews_pt/auxiliary/try_scenarios.py @@ -1,7 +1,7 @@ import json, click, time, sys from os import path as osp -from snews_pt.snews_pub import SNEWSTiersPublisher, Publisher +from snews_pt.messages import SNEWSMessageBuilder, Publisher fd_mode = True if sys.argv[1].lower() == "true" else False with open(osp.join(osp.dirname(__file__), "scenarios.json")) as json_file: @@ -34,7 +34,7 @@ messages = data[scenario] for msg in messages: # send one by one and sleep in between print(msg, "\n\n") - SNEWSTiersPublisher(**msg, firedrill_mode=fd_mode).send_to_snews() + SNEWSMessageBuilder(**msg).send_messages(firedrill_mode=fd_mode) time.sleep(1) # clear cache after each scenario with Publisher(firedrill_mode=fd_mode, verbose=False) as pub: diff --git a/snews_pt/message_schema.py b/snews_pt/message_schema.py deleted file mode 100644 index 7f3cc4f..0000000 --- a/snews_pt/message_schema.py +++ /dev/null @@ -1,95 +0,0 @@ -from .snews_pt_utils import get_detector, get_name -from ._version import version as __version__ - - -class Message_Schema: - """ The Message scheme for the alert and observations - Parameters - ---------- - detector_key : str - Optional. The name of the detector. If "TEST", looks in the env file - - """ - def __init__(self, detector_key='TEST', is_pre_sn=False): - if detector_key == "TEST": - detector_key = get_name() - self.detector = get_detector(detector_key) - self.detector_name = self.detector.name - self.is_pre_sn = is_pre_sn - - def id_format(self, tier, machine_time): - """ Returns formatted message ID - time format should always be ISO-Format for all detectors. - Parameters - ---------- - tier : str - The decided tier name - machine_time: str - Machine time in ISO format - - Returns - ------- - formatted id : str - - """ - if self.is_pre_sn: - return f'{self.detector.id}_PRE-SN_{tier}_{machine_time}' - else: - return f'{self.detector.id}_{tier}_{machine_time}' - - def get_schema(self, tier, data, sent_time, version=__version__): - """ Create a message schema for given topic type. - Internally called in hop_pub - - Parameters - ---------- - tier : `str` - type of message to be published. Can be; - 'TimeTier', 'SigTier', 'CoincidenceTier' for - observation messages and, 'HeartBeat' for - heartbeat messages - data : `dict` - object with message information - - Returns - ------- - message : dict - Message using the correct schema - - """ - if data['machine_time'] is None: - machine_time = sent_time - else: - machine_time = data['machine_time'] - message = {"_id": self.id_format(tier=tier, machine_time=machine_time), - "detector_name": self.detector_name, - "machine_time": machine_time, - } - if tier == 'Heartbeat': - message['detector_status'] = data['detector_status'] - message['meta'] = data['meta'] - - if tier == 'TimeTier': - message['neutrino_time'] = data['neutrino_time'] - message['timing_series'] = data['timing_series'] - message['meta'] = data['meta'] - - if tier == 'SigTier': - message['neutrino_time'] = data['neutrino_time'] - message['p_values'] = data['p_values'] - message['t_bin_width'] = data['t_bin_width'] - message['meta'] = data['meta'] - - if tier == 'CoincidenceTier': - message['neutrino_time'] = data['neutrino_time'] - message['p_val'] = data['p_val'] - message['meta'] = data['meta'] - - if tier == 'Retraction': - message['retract_latest'] = data['retract_latest'] - message['retraction_reason'] = data['retraction_reason'] - message['meta'] = data['meta'] - - message["schema_version"] = version - message["sent_time"] = sent_time - return message diff --git a/snews_pt/messages.py b/snews_pt/messages.py new file mode 100644 index 0000000..e540226 --- /dev/null +++ b/snews_pt/messages.py @@ -0,0 +1,549 @@ +"""Interface for SNEWS messages. +""" + +import os +import click +import json +import numpy as np +from abc import ABC, abstractmethod + +from datetime import datetime +try: + fromisoformat = datetime.fromisoformat +except AttributeError as e: + from dateutil.parser import isoparse as fromisoformat + +from hop import Stream +try: + from hop.models import JSONBlob +except ImportError as e: + raise ImportError(f'{e}\nSNEWS Publishing Tools and Coincidence System requires op version >= 0.8.0') + +from snews_pt import snews_pt_utils +from snews_pt._version import version as __version__ + +class Publisher: + + def __init__(self, env_path=None, verbose=True, auth=True, firedrill_mode=True): + """Class in charge of publishing messages to SNEWS-hop sever. + This class acts as a context manager. + + Parameters + ---------- + env_path: str + path to SNEWS env file, defaults to tes_config.env if None is passed. + verbose: bool + Option to display message when publishing. + auth: bool + Option to run hop-Stream without authentication. Pass False to do so + firedrill_mode :bool + whether to use firedrill broker + + """ + snews_pt_utils.set_env(env_path) + self.auth = auth + self.verbose = verbose + + self.obs_broker = os.getenv("OBSERVATION_TOPIC") + if firedrill_mode: + self.obs_broker = os.getenv("FIREDRILL_OBSERVATION_TOPIC") + + + def __enter__(self): + self.stream = Stream(until_eos=True, auth=self.auth).open(self.obs_broker, 'w') + return self + + def __exit__(self, *args): + self.stream.close() + + def send(self, messages): + """ This method will set the sent_time and send the message to the hop broker. + + Parameters + ---------- + messages: `list` + list containing observation message. + + """ + if len(messages) == 0: + # None of the messages passed the format checker! + raise UserWarning("No valid message exists!") + + if type(messages) == dict: + messages = list(messages) + for message in messages: + message["sent_time"] = datetime.utcnow().isoformat() + self.stream.write(JSONBlob(message)) + self.display_message(message) + + def display_message(self, message): + if self.verbose: + tier = message['_id'].split('_')[1] + click.secho(f'{"-" * 64}', fg='bright_blue') + click.secho(f'Sending message to {tier} on {self.obs_broker}', fg='bright_red') + if tier == 'Retraction': + click.secho("It's okay, we all make mistakes".upper(), fg='magenta') + snews_pt_utils.prettyprint_dictionary(message) + + +class SNEWSMessage(ABC): + """SNEWS 2.0 message interface. Defines base fields common to all messages and performs validation on messages during construction. + + Base fields: + - id: string ID for the message (defined automatically for user). + - schema_version: message schema version (defined automatically for user). + - detector_name: name of detector (string), defined in snews_pt setup. + """ + # Fields used in every message. + basefields = [ '_id', 'schema_version', 'detector_name' ] + + def __init__(self, fields, detector_name='TEST', **kwargs): + """Build a generic abstract message object for SNEWS 2.0. + + Parameters + ---------- + fields : list + List of all fields expected in the message. + detector_name : str + Name of detector sending the message. + machine_time : datetime or str + Machine time. + """ + self.meta = {} + + # Set tier using the subclass name (CoincidenceTier, TimeTier, ...) + tier = self.__class__.__name__.replace('SNEWS','').replace('Message','') + + # Get the detector name from the input. + det = self.get_detector_name(detector_name) # just fetches from the env + # det = snews_pt_utils.set_name(detector_name, _return=True) # rewrites the env each time + raw_mt = kwargs.get('machine_time', None) + mt = self.clean_time_input(raw_mt) + raw_mt = mt if isinstance(raw_mt, str) else None + + # Store basic message ID, detector name, and schema in a dictionary. + self.message_data = dict( + _id = f'{det}_{tier}_{mt}', + schema_version = __version__, + detector_name = det, + machine_time = raw_mt + ) + + for kw in kwargs: + if kw in fields: + # Append all kwargs matching subclass fields in message_data. + self.message_data[kw] = kwargs[kw] + else: + # Append all non-matching kwargs to meta. + self.meta[kw] = kwargs[kw] + + self.is_test = kwargs.get('is_test', False) + # Check that required fields are present and valid. + self.has_required_fields() + + def print_schema(self): + click.secho(f'Message schema for {self.__class__.__name__}', bg='white', fg='blue') + for f in self.fields: + if f in self.basefields: + click.secho(f'{f:<20s} : (SET AUTOMATICALLY)', fg='bright_red') + elif f in self.reqfields: + click.secho(f'{f:<20s} : (REQUIRED USER INPUT)', fg='bright_blue') + else: + click.secho(f'{f:<20s} : (USER INPUT)', fg='bright_cyan') + + def get_detector_name(self, detector_name): + """Get formatted detector name. + + Parameters + ---------- + detector_name : str or None + Name of the detector. + + Returns + ------- + detector_name : str + Correctly formatted detector name. + """ + if detector_name == 'TEST' or detector_name is None: + detector_name = snews_pt_utils.get_name() + return detector_name + + def clean_time_input(self, time): + """Get cleaned time string from input. + + Parameters + ---------- + time : datetime or str + Input time. + + Returns + ------- + tmfmt : str + Time string in ISO format. + """ + if time is None: + time = datetime.utcnow() + + if isinstance(time, str): + time = fromisoformat(time) + + return time.isoformat() + + def has_required_fields(self): + """Validate the message on construction. + """ + missing = [] + for f in self.reqfields: + if f in self.message_data: + if self.message_data[f] is None: + missing.append(f) + else: + missing.append(f) + + if missing: + raise RuntimeError(f'{self.__class__.__name__} missing ' + f'required field(s): {", ".join(missing)}.') + + def __repr__(self): + _repr_str = click.style(f'{self.__class__.__name__}\n', bold=True) + _repr_str += f'{"-"*len(self.__class__.__name__)}\n' + for k, v in self.message_data.items(): + if k in self.basefields: + _repr_str += click.style(f"{k:>15} : {v}\n", fg='bright_red') + elif k in self.fields and k not in self.basefields and k not in self.reqfields: + _repr_str += click.style(f"{k:>15} : {v}\n", fg='black') + elif k in self.reqfields: + _repr_str += click.style(f"{k:>15} : {v}\n", fg='bright_blue', bold=True) + # _repr_str += f'{"-"*len(self.__class__.__name__)}\n' + for k, v in self.meta.items(): + _repr_str += click.style(f"{k:>15} : {v}\n", fg='bright_cyan') + return _repr_str + + def __repr_markdown__(self): + _repr_str = f'### {self.__class__.__name__}\n' + for k, v in self.message_data.items(): + _repr_str += f"**{k}** : {v}\n" + _repr_str += f'---\n' + for k, v in self.meta.items(): + _repr_str += f"**{k}** : {v}\n" + return _repr_str + + @abstractmethod + def is_valid(self): + """Check that parameter values are valid for this tier.""" + pass + + def to_json(self, jsonfile): + with open(jsonfile, 'w') as outfile: + json.dump(self.message_data, outfile, indent=4) + + +class SNEWSCoincidenceTierMessage(SNEWSMessage): + """Message for SNEWS 2.0 coincidence tier.""" + + reqfields = [ 'neutrino_time' ] + fields = SNEWSMessage.basefields + reqfields + [ 'machine_time', 'p_val' ] + + def __init__(self, neutrino_time=None, p_val=None, **kwargs): + super().__init__(self.fields, + neutrino_time=self.clean_time_input(neutrino_time), + p_val=p_val, + **kwargs) + + def is_valid(self): + """Check that parameter values are valid for this tier. + Detector name must be known, neutrino times must make sense, etc. if not test""" + if not self.is_test: + # time format is corrected at the base class, check if reasonable + dateobj = fromisoformat(self.message_data['neutrino_time']) + duration = (dateobj - datetime.utcnow()).total_seconds() + if (duration <= -172800.0) or (duration > 0.0): + raise ValueError(f'{self.__class__.__name__} neutrino_time must be within 48 hours of now.') + + # p_val must be a float between 0 and 1 + pv = self.message_data['p_val'] + if isinstance(pv, str): + pv = float(pv) + if not (0.0 <= pv <= 1.0): + raise ValueError(f'{self.__class__.__name__} p_value of the detection must be between 0 and 1.') + + return True + + +class SNEWSSignificanceTierMessage(SNEWSMessage): + """Message for SNEWS 2.0 significance tier.""" + + reqfields = [ 'p_values', 't_bin_width' ] + fields = SNEWSMessage.basefields + reqfields + [ 'machine_time' ] + + def __init__(self, p_values=None, t_bin_width=None, **kwargs): + # Type check for proper types. + if np.isscalar(p_values): + raise RuntimeError(f'{self.__class__.__name__} p_values must be a list.') + + super().__init__(self.fields, + p_values=p_values, + t_bin_width=t_bin_width, + **kwargs) + + def is_valid(self): + """Check that parameter values are valid for this tier.""" + if not self.is_test: + for pv in self.message_data['p_values']: + if isinstance(pv, str): + pv = float(pv) + if not (0.0 <= pv <= 1.0): + raise ValueError(f'{self.__class__.__name__} p_values must be between 0 and 1.') + if isinstance(self.message_data['t_bin_width'], str): + if not self.message_data['t_bin_width'].replace('.','',1).isdigit(): + raise ValueError(f'{self.__class__.__name__} t_bin_width must be a float.') + elif not isinstance(self.message_data['t_bin_width'], float): + raise ValueError(f'{self.__class__.__name__} t_bin_width must be a float.') + return True + + +class SNEWSTimingTierMessage(SNEWSMessage): + """Message for SNEWS 2.0 timing tier.""" + + reqfields = [ 'timing_series' ] + fields = SNEWSMessage.basefields + reqfields + [ 'machine_time', 'p_val' ] + + def __init__(self, p_val=None, timing_series=None, **kwargs): + super().__init__(self.fields, + p_val=p_val, + timing_series=[self.clean_time_input(t) for t in timing_series], + **kwargs) + + def is_valid(self): + """Check that parameter values are valid for this tier.""" + for time in self.message_data['timing_series']: + if isinstance(time, str): + time = fromisoformat(time) + time = time.isoformat() + if not self.is_test: + # time format is corrected at the base class, check if reasonable + timeobj = fromisoformat(time) + duration = (timeobj - datetime.utcnow()).total_seconds() + if (duration <= -172800.0) or (duration > 0.0): + raise ValueError(f'{self.__class__.__name__} neutrino_time must be within 48 hours of now.') + return True + + +class SNEWSRetractionMessage(SNEWSMessage): + """Message for SNEWS 2.0 retractions.""" + + reqfields = [ 'retract_latest' ] + fields = SNEWSMessage.basefields + reqfields + [ 'machine_time', 'retraction_reason' ] + + def __init__(self, retract_latest=None, retraction_reason=None, **kwargs): + super().__init__(self.fields, + retract_latest=retract_latest, + retraction_reason=retraction_reason, + **kwargs) + + def is_valid(self): + """Check that parameter values are valid for this tier.""" + if not isinstance(self.message_data['retract_latest'], int): + raise ValueError(f'{self.__class__.__name__} retract_latest must be a positive integer.') + return True + + +class SNEWSHeartbeatMessage(SNEWSMessage): + """Message for SNEWS 2.0 heartbeats.""" + + reqfields = [ 'detector_status' ] + fields = SNEWSMessage.basefields + reqfields + [ 'machine_time' ] + + def __init__(self, machine_time=None, detector_status=None, **kwargs): + super().__init__(self.fields, + machine_time=machine_time, + detector_status=detector_status, + **kwargs) + + def is_valid(self): + """Check that parameter values are valid for this tier.""" + status = self.message_data['detector_status'] + if not isinstance(status, str) or status not in ['ON', 'OFF']: + raise ValueError(f'{self.__class__.__name__} detector_status must be either "ON" or "OFF".') + return True + + +class SNEWSMessageBuilder: + """Builder class that takes a list of message fields and builds all + appropriate messages for SNEWS 2.0 tiers based on the child classes of + SNEWSMessage. The class contains a messages' parameter that stores the list + of SNEWSMessage child instances. + """ + + def __init__(self, env_file=None, + detector_name='TEST', + machine_time=None, + neutrino_time=None, + p_val=None, + p_values=None, + t_bin_width=None, + timing_series=None, + retract_latest=None, + retraction_reason=None, + detector_status=None, + is_test=False, + **kwargs): + + self.messages = None + self.selected_tiers = [] + + self._build_messages(env_file=env_file, + detector_name=detector_name, + machine_time=machine_time, + neutrino_time=neutrino_time, + p_val=p_val, + p_values=p_values, + t_bin_width=t_bin_width, + timing_series=timing_series, + retract_latest=retract_latest, + retraction_reason=retraction_reason, + detector_status=detector_status, + is_test=is_test, + **kwargs) + + def __repr__(self): + _repr_str = f'{self.__class__.__name__}:\n' + if self.messages is None: + _repr_str += 'No messages have been built.' + else: + for m in self.messages: + _repr_str += f'{m.__class__.__name__}:\n' + for k, v in m.message_data.items(): + _repr_str += f" **{k}** : {v}\n" + _repr_str += f'---\n' + for k, v in m.meta.items(): + _repr_str += f" **{k}** : {v}\n" + return _repr_str + + def __repr_markdown__(self): + _repr_str = click.style(f'{self.__class__.__name__}:\n', bold=True) + if self.messages is None: + _repr_str += 'No messages have been built.' + else: + for m in self.messages: + _repr_str += m.__repr_markdown__() + _repr_str += 30*'-'+'\n' + return _repr_str + + def _build_messages(self, **kwargs): + """Utility function to create messages for all appropriate tiers. + """ + # Initialize the message list. + self.messages = None + + # Identify all non-null keyword arguments passed to the class constructor. + nonull_keys = [k for k in kwargs if kwargs[k] is not None] + nonull_kwargs = {k: kwargs[k] for k in nonull_keys} + + # Loop through all message types. + # To do: create a message type registry in this module? + for smc in [SNEWSCoincidenceTierMessage, + SNEWSSignificanceTierMessage, + SNEWSTimingTierMessage, + SNEWSRetractionMessage, + SNEWSHeartbeatMessage]: + + # If the required fields for a given message tier are present + # in this class, create a list of message objects. + hasreqfields = all(_ in nonull_keys for _ in smc.reqfields) + if hasreqfields: + if self.messages is None: + # check is valid at creation + smc(**nonull_kwargs).is_valid() + self.messages = [smc(**nonull_kwargs)] + else: + self.messages.append(smc(**nonull_kwargs)) + self.selected_tiers.append(smc.__name__) + + @classmethod + def from_json(cls, jsonfile, **kwargs): + """Build SNEWSMessage instances using a message in JSON format. + + Parameters + ---------- + jsonfile : str + Name of JSON file. + """ + with open(jsonfile, 'r') as infile: + jdata = json.load(infile) + return cls(**jdata, **kwargs) + + def send_messages(self, firedrill_mode=True, env_file=None, verbose=True, auth=True): + """Send all messages in the messages list to the SNEWS server. + Merges the meta fields accordingly before sending.""" + # all req fields + fields_set = set({k: v for m in self.messages for k, v in m.message_data.items()}) + messages_to_send = [] + + # append meta fields if the meta field is not in the other messages + for m in self.messages: + mes = m.message_data + met = m.meta + for k, v in met.items(): + if k not in fields_set: + mes[k] = v + messages_to_send.append(mes) + + with Publisher(env_path=env_file, + verbose=verbose, + auth=auth, + firedrill_mode=firedrill_mode) as pub: + pub.send(messages_to_send) + + +if __name__ == '__main__': + try: + # Build the individual message classes. + sn = SNEWSCoincidenceTierMessage(neutrino_time=datetime.utcnow(), dude=5) + sn.print_schema() + print(f'{sn.message_data}\n\n') + + sn = SNEWSSignificanceTierMessage(p_values=[1], t_bin_width=1) + sn.print_schema() + print(f'{sn.message_data}\n\n') + + sn = SNEWSTimingTierMessage(timing_series=[1,2,3]) + sn.print_schema() + print(f'{sn.message_data}\n\n') + + sn = SNEWSRetractionMessage(retract_latest=1) + sn.print_schema() + print(f'{sn.message_data}\n\n') + + sn = SNEWSHeartbeatMessage(detector_status='ON') + sn.print_schema() + print(f'{sn.message_data}\n\n') + + # Use the builder class. + print('Exercise the SNEWSMessageBuilder:\n') + sm = SNEWSMessageBuilder( + neutrino_time=datetime.utcnow(), + p_values=[1], + t_bin_width=1, + timing_series=[1,2,3] + ) + + # Print generated messages in the SNEWSMessageBuilder, save to JSON. + print('Messages created and saved to JSON:') + for j, msg in enumerate(sm.messages): + jsonfile = f'{msg.__class__.__name__}.json' + print(j, msg.__class__.__name__, msg.message_data) + msg.to_json(jsonfile) + + # Instantiate messages from JSON. + from glob import glob + jsonfiles = sorted(glob('*.json')) + for jsonfile in jsonfiles: + print(f'\nMessages from JSON file {jsonfile}:\n') + sm.from_json(jsonfile) + for j, msg in enumerate(sm.messages): + jsonfile = f'{msg.__class__.__name__}.json' + print(j, msg.__class__.__name__, msg.message_data, '\n') + + except RuntimeError as e: + print(e) diff --git a/snews_pt/remote_commands.py b/snews_pt/remote_commands.py index 11b17b0..49f57aa 100644 --- a/snews_pt/remote_commands.py +++ b/snews_pt/remote_commands.py @@ -73,6 +73,8 @@ def test_connection(detector_name=None, firedrill=True, start_at="LATEST", patie click.secho(f"\tWaited for {patience} sec and checked from {start_at}," f" couldn't get a confirmation" f"\n\tMaybe increase timeout and try again.", fg='red', bold=True) + return False + return True def write_hb_logs(detector_name=None, admin_pass=None, firedrill=True): diff --git a/snews_pt/snews_format_checker.py b/snews_pt/snews_format_checker.py index b58390e..1cf1737 100644 --- a/snews_pt/snews_format_checker.py +++ b/snews_pt/snews_format_checker.py @@ -68,6 +68,9 @@ def check_if_test(self): ------- True if the message contains ['meta']['is_test'] = True, else False """ + if "is_test" in self.message_keys: + return self.message['is_test'] + if "meta" in self.message_keys: if "is_test" in self.message['meta'].keys(): return self.message["meta"]["is_test"] diff --git a/snews_pt/snews_pt_utils.py b/snews_pt/snews_pt_utils.py index 8084150..f01ba28 100644 --- a/snews_pt/snews_pt_utils.py +++ b/snews_pt/snews_pt_utils.py @@ -27,7 +27,6 @@ def set_env(env_path=None): load_dotenv(env) - def retrieve_detectors(detectors_path=default_detector_file): """ Retrieve the name-ID-location of the participating detectors. @@ -79,144 +78,6 @@ def get_detector(detector, detectors_path=default_detector_file): return detectors['TEST'] -def coincidence_tier_data(machine_time=None, neutrino_time=None, p_val=None, meta=None): - """ Formats data for CoincidenceTier as dict object - - Parameters - ---------- - machine_time : str - The machine time at the time of execution of command - neutrino_time : str - The neutrino arrival time - p_val : float - If determined, the p value of the observation - meta : dict - Any other key-value pair desired to be published. - - Returns - ------- - coincidence_tier_dict : dict - dictionary of the complete CoincidenceTier data - - """ - keys = ['machine_time', 'neutrino_time', 'p_val', 'meta'] - values = [machine_time, neutrino_time, p_val, meta] - zip_iterator = zip(keys, values) - coincidence_tier_dict = dict(zip_iterator) - return coincidence_tier_dict - - -def sig_tier_data(machine_time=None, neutrino_time=None, p_values=None, t_bin_width=None, meta=None): - """ Formats data for SigTier as dict object - - Parameters - ---------- - t_bin_width : float - machine_time : str - The machine time at the time of execution of command - neutrino_time : str - The neutrino arrival time - p_values : list - If determined, the p values of the observation - meta : dict - Any other key-value pair desired to be published. - - Returns - ------- - sig_tier_dict : dict - dictionary of the complete observation data - - """ - keys = ['machine_time', 'neutrino_time', 'p_values', 't_bin_width', 'meta'] - values = [machine_time, neutrino_time, p_values, t_bin_width, meta] - zip_iterator = zip(keys, values) - sig_tier_dict = dict(zip_iterator) - return sig_tier_dict - - -def time_tier_data(machine_time=None, neutrino_time=None, p_val=None, timing_series=None, meta=None): - """ Formats data for TimingTier as dict object - - Parameters - ---------- - - machine_time : str - The machine time at the time of execution of command - neutrino_time : str - The neutrino arrival time - p_val : int - The p value of the observation - timing_series : array-like - Time series of the detected signal - meta : dict - Any other key-value pair desired to be published. - - Returns - ------- - data_dict : dict - dictionary of the TimingTier data - - """ - keys = ['machine_time', 'neutrino_time', 'timing_series', 'p_val', 'meta'] - values = [machine_time, neutrino_time, timing_series, p_val, meta] - zip_iterator = zip(keys, values) - time_tier_dict = dict(zip_iterator) - return time_tier_dict - - -def retraction_data(machine_time=None, retract_latest=0, retraction_reason=None, meta=None): - """ Formats data for Retraction as dict object - - Parameters - ---------- - machine_time : str - The machine time at the time of execution of command - retract_latest: int or str - Tells retraction methods to look for N the latest message sent by a detector. - retraction_reason: str - Reason for message(s) retraction - meta : `dict` - Any other key-value pair desired to be published. - - Returns - ------- - retraction_dict : dict - dictionary of the retraction data - - """ - keys = ['machine_time', 'retract_latest', 'retraction_reason', 'meta'] - values = [machine_time, retract_latest, retraction_reason, meta] - zip_iterator = zip(keys, values) - retraction_dict = dict(zip_iterator) - return retraction_dict - - -def heartbeat_data(machine_time=None, detector_status=None, meta=None): - """ Formats data for Heartbeat as dict object - - Parameters - ---------- - machine_time : str - The machine time at the time of execution of command - detector_status : str - ON or OFF - meta : dict - Any other key-value pair desired to be published. - - Returns - ------- - heartbeat_dict : dict - dictionary of the Heartbeat data - - """ - keys = ['machine_time', 'detector_status', 'meta'] - values = [machine_time, detector_status, meta] - - zip_iterator = zip(keys, values) - heartbeat_dict = dict(zip_iterator) - return heartbeat_dict - - # used in message schema display, keep for now def _check_aliases(tier): tier = tier.lower() @@ -292,7 +153,7 @@ def display_gif(): display(HTML(f'')) -def set_name(detector_name='TEST'): +def set_name(detector_name='TEST', _return=False): """ set your detector's name. Messages sent with detector_name="TEST" will be ignored at the server Alerts can still be subscribed and listened as "TEST" @@ -307,13 +168,13 @@ def set_name(detector_name='TEST'): for i,d in enumerate(detectors): click.secho(f"[{i:2d}] {d}") inp = input(click.secho("Please put select your detector's index\n", bold=True)) - selected_name = detectors[int(inp)] - os.environ["DETECTOR_NAME"] = selected_name + detector_name = detectors[int(inp)] + os.environ["DETECTOR_NAME"] = detector_name os.environ["HAS_NAME_CHANGED"] = "1" dotenv.set_key(envpath, "DETECTOR_NAME", os.environ["DETECTOR_NAME"]) dotenv.set_key(envpath, "HAS_NAME_CHANGED", os.environ["HAS_NAME_CHANGED"]) else: - click.secho(f'You are {os.environ["DETECTOR_NAME"]}') + detector_name = os.environ["DETECTOR_NAME"] else: if not detector_name in detectors: raise KeyError(f"{detector_name} is not a valid detector. \nChoose from {detectors}") @@ -321,6 +182,10 @@ def set_name(detector_name='TEST'): os.environ["HAS_NAME_CHANGED"] = "1" dotenv.set_key(envpath, "DETECTOR_NAME", os.environ["DETECTOR_NAME"]) dotenv.set_key(envpath, "HAS_NAME_CHANGED", os.environ["HAS_NAME_CHANGED"]) + if _return: + return detector_name + else: + click.secho(f'You are {os.environ["DETECTOR_NAME"]}') def get_name(): @@ -329,6 +194,7 @@ def get_name(): """ return os.getenv("DETECTOR_NAME") + def prettyprint_dictionary(dictionary, indent=0): """ tabulate the message in prettier form """ diff --git a/snews_pt/snews_pub.py b/snews_pt/snews_pub.py deleted file mode 100644 index f72e540..0000000 --- a/snews_pt/snews_pub.py +++ /dev/null @@ -1,263 +0,0 @@ -""" -An interface for SNEWS member experiment -to publish their observation and heartbeat messages. - -Created: -August 2021 -Authors: -Melih Kara -Sebastian Torres-Lara -Joe Smolsky -""" -from datetime import datetime -try: - fromisoformat = datetime.fromisoformat -except AttributeError: - from dateutil.parser import isoparse as fromisoformat -import os, click -from hop import Stream -try: - from hop.models import JSONBlob -except ImportError: - raise ImportError(f"SNEWS Publishing Tools and Coincidence System requires hop version>=0.8.0\n" - f"Please upgrade your `pip install -U hop-client`") -from . import snews_pt_utils -from .snews_format_checker import SnewsFormat -from .snews_pt_utils import prettyprint_dictionary -from .tier_decider import TierDecider - -def homogenise_time_field(message): - """ Make sure all messages follow the same ISO format. - - """ - # convert to iso-formatted datetime object and revert - dateobj = fromisoformat(message["neutrino_time"]) - datestr = dateobj.isoformat() - message["neutrino_time"] = datestr - return message - -def check_format(messages): - """ Check if the messages are valid SnewsFormat - Parameters - ---------- - messages : dict - The generated messages - - Returns - ------- - valid, invalid : dict - The valid/invalid messages - - """ - valid = [] - invalid = [] - for message in messages: - if SnewsFormat(message)(): - if "neutrino_time" in message.keys(): - # homogenise the valid messages - valid.append(homogenise_time_field(message)) - else: - valid.append(message) - else: - invalid.append(message) - click.secho(f'{"-" * 64}', fg='bright_red') - click.secho(f'Skipping message! Improper format! see ') - return valid, invalid - -class Publisher: - - def __init__(self, env_path=None, verbose=True, auth=True, firedrill_mode=True): - """Class in charge of publishing messages to SNEWS-hop sever. - This class acts as a context manager. - - Parameters - ---------- - env_path: str - path to SNEWS env file, defaults to tes_config.env if None is passed. - verbose: bool - Option to display message when publishing. - auth: bool - Option to run hop-Stream without authentication. Pass False to do so - firedrill_mode :bool - whether to use firedrill broker - - """ - snews_pt_utils.set_env(env_path) - self.auth = auth - self.verbose = verbose - - self.obs_broker = os.getenv("OBSERVATION_TOPIC") - if firedrill_mode: - self.obs_broker = os.getenv("FIREDRILL_OBSERVATION_TOPIC") - - - def __enter__(self): - self.stream = Stream(until_eos=True, auth=self.auth).open(self.obs_broker, 'w') - return self - - def __exit__(self, *args): - self.stream.close() - - def send(self, messages): - """ This method will set the sent_time and send the message to the hop broker. - - Parameters - ---------- - messages: `list` - list containing observation message. - - """ - if len(messages) == 0: - # None of the messages passed the format checker! - raise UserWarning("No valid message exists!") - - if type(messages) == dict: - messages = list(messages) - for message in messages: - message["sent_time"] = datetime.utcnow().isoformat() - self.stream.write(JSONBlob(message)) - self.display_message(message) - - - def display_message(self, message): - if self.verbose: - tier = message['_id'].split('_')[1] - click.secho(f'{"-" * 64}', fg='bright_blue') - click.secho(f'Sending message to {tier} on {self.obs_broker}', fg='bright_red') - if tier == 'Retraction': - click.secho("It's okay, we all make mistakes".upper(), fg='magenta') - prettyprint_dictionary(message) - - -class SNEWSTiersPublisher: - - def __init__(self, env_file=None, - detector_name='TEST', - machine_time=None, - neutrino_time=None, - p_val=None, - p_values=None, - t_bin_width=None, - timing_series=None, - retract_latest=None, - retraction_reason=None, - detector_status=None, - is_pre_sn=False, - firedrill_mode=True, - is_test=False, - **kwargs): - """ SNEWS Publisher Instance, it lets you create message data and interact with them - before publishing to snews. Submitting JSON files is also possible see - SNEWSTierPublisher.from_json() - - Parameters - ---------- - env_file: str - path to env file, when None, uses the default env file - detector_name: str - Name of your detector,defaults to None. - See auxiliary/detector_properties.json for available detector names. - machine_time: str - time recorded by your detector, defaults to None - format: '%y/%m/%d %H:%M:%S:%f' - neutrino_time: str - time stamp of initial neutrino signal, defaults to None - format: '%y/%m/%d %H:%M:%S:%f' - p_val: float - p value of possible observation, defaults to None - p_values: `list` - p values of individual neutrino observations, defaults to None. - If passed, `t_bin_width` is also expected. - t_bin_width: float - width of time window [sec], required for significance tier, defaults to None. - If passed, `p_values` is also expected. - timing_series: list - list of strings with individual neutrino times following ISO format defaults to None - retract_latest: int - how many of your last messages do you want to retract, defaults to None - retraction_reason: str - (optional) share with SNEWS what caused your false observation, defaults to None. - We won't judge you :) - detector_status: str - Send "ON" or "OFF" heartbeats. - is_pre_sn: bool - Whether the message is triggered by pre-supernova neutrinos. - firedrill_mode : bool - tell Publisher to send messages to the firedrill hop broker, defaults to True - is_test : bool - True if the messages are meant for testing, then the time checks are ignored - kwargs: - Any additional key-word argument you want to send to SNEWS - """ - if detector_name == "TEST": - detector_name = snews_pt_utils.get_name() - self.message_data = {'detector_name': detector_name, - 'machine_time': machine_time, - 'neutrino_time': neutrino_time, - 'p_val': p_val, - 'p_values': p_values, - 't_bin_width': t_bin_width, - 'timing_series': timing_series, - 'retract_latest': retract_latest, - 'retraction_reason': retraction_reason, - 'detector_status': detector_status, - 'is_pre_sn': is_pre_sn, - 'is_test':is_test} - self.meta = dict(**kwargs) - self.message_data = {**self.message_data, ** self.meta} - self.env_file = env_file - self.firedrill_mode = firedrill_mode - # split messages for different tiers, and format properly - self.messages, self.tiernames = TierDecider(self.message_data).decide() - # check the message contents - self.messages, self.invalid_messages = check_format(self.messages) - - @classmethod - def from_json(cls, jsonfile, env_file=None, **kwargs): - """ Read the data from a json file - Additional data / overwrite is allowed, by passing the key-value pair as additional arguments - - Parameters - ---------- - jsonfile : str - Path to your local JSON file - env_file : str - Path to your local environment file. If none, uses the default. - **kwargs : key-value pairs - Any other key-value pairs you want to pass or overwrite the ones in your JSON file - - Examples - -------- - > read the data from "myheartbeat.json" and overwrite the detector_name - publisher = SNEWSTierPublisher.from_json("myheartbeat.json", detector_name="IceCube") - print(publisher.messages) - publisher.send_to_snews() - - """ - input_json = snews_pt_utils._parse_file(jsonfile) - output_data = {**input_json, **kwargs} - return cls(env_file=env_file, **output_data) - - def to_json(self, jsonfile): - """ Serialize message to JSON. - - Parameters - ---------- - jsonfile : str - path to JSON output - """ - snews_pt_utils._dump_json(jsonfile, self.message_data) - - def send_to_snews(self, auth=True, verbose=True): - """ Send the message to SNEWS - - Parameters - ---------- - verbose : bool - Whether to display the message sent - auth : bool - whether to authenticate with hop - - """ - with Publisher(env_path=self.env_file, verbose=verbose, auth=auth, firedrill_mode=self.firedrill_mode) as pub: - pub.send(self.messages) diff --git a/snews_pt/test/test_coincidence_tier.py b/snews_pt/test/test_coincidence_tier.py index f82a08b..9340245 100644 --- a/snews_pt/test/test_coincidence_tier.py +++ b/snews_pt/test/test_coincidence_tier.py @@ -1,32 +1,30 @@ """Test publishing coincidence tier messages.""" -from snews_pt.snews_pub import SNEWSTiersPublisher -from snews_pt.snews_format_checker import SnewsFormat +from snews_pt.messages import SNEWSMessageBuilder def test_coincidence_expected(): """Test with example of expected message type.""" # Create coincidence tier message. - coin = SNEWSTiersPublisher(detector_name='KamLAND', neutrino_time='2012-06-09T15:31:08.891011', + coin = SNEWSMessageBuilder(detector_name='KamLAND', machine_time='2012-06-09T15:30:00.000501', + neutrino_time='2012-06-09T15:31:08.891011', firedrill_mode=False, is_test=True) # Check that message has expected structure. - assert list(coin.tiernames) == ['CoincidenceTier'] + assert coin.selected_tiers == ['SNEWSCoincidenceTierMessage'] assert len(coin.messages) == 1, f"Expected 1 CoincidenceTier Message got {len(coin.messages)}!" - assert coin.message_data == {'detector_name': 'KamLAND', 'machine_time': None, - 'neutrino_time': '2012-06-09T15:31:08.891011', - 'p_val': None, 'p_values': None, 't_bin_width': None, 'timing_series': None, - 'retract_latest': None, 'retraction_reason': None, - 'detector_status': None, 'is_pre_sn': False, 'is_test':True} + assert coin.messages[0].message_data == {'_id': 'KamLAND_CoincidenceTier_2012-06-09T15:30:00.000501', + 'detector_name': 'KamLAND', + 'machine_time': '2012-06-09T15:30:00.000501', + 'schema_version': '1.3.0', + 'neutrino_time': '2012-06-09T15:31:08.891011', + 'p_val': None} + assert coin.messages[0].meta == {'is_test': True, 'firedrill_mode': False} - # the SNEWSTierPublisher already checks this in creation, but we can double check - for message in coin.messages: - format_checker = SnewsFormat(message) - assert format_checker() is True, "Message is not in the snews format" - - assert len(coin.invalid_messages) == 0, "There are invalid messages" + # check if valid snews format + assert coin.messages[0].is_valid() is True, "Message is not valid" # Try to send message to SNEWS 2.0 server. try: - coin.send_to_snews() + coin.send_messages() except Exception as exc: - print('SNEWSTiersPublisher.send_to_snews() test failed!\n') + print('SNEWSMessageBuilder.send_messages() test failed!\n') assert False, f"Exception raised:\n {exc}" diff --git a/snews_pt/test/test_connection_to_server.py b/snews_pt/test/test_connection_to_server.py new file mode 100644 index 0000000..1785373 --- /dev/null +++ b/snews_pt/test/test_connection_to_server.py @@ -0,0 +1,7 @@ + +"""Test connection with the server.""" +from snews_pt.remote_commands import test_connection + +def test_connection_to_server(): + """Test connection with the server.""" + assert test_connection(detector_name="XENONnT", firedrill=False, start_at="LATEST", patience=8) is True, "Connection to server failed on no-firedrill broker!" \ No newline at end of file diff --git a/snews_pt/test/test_heartbeat.py b/snews_pt/test/test_heartbeat.py new file mode 100644 index 0000000..2213fb2 --- /dev/null +++ b/snews_pt/test/test_heartbeat.py @@ -0,0 +1,29 @@ +"""Test publishing heartbeat messages.""" +from snews_pt.messages import SNEWSMessageBuilder + +def test_heartbeat_expected(): + """Test with example of expected message type.""" + # Create heartbeat tier message. + hb = SNEWSMessageBuilder(detector_name='XENONnT', machine_time='2012-06-09T15:30:00.000501', + detector_status='ON', + firedrill_mode=False, is_test=True) + # Check that message has expected structure. + assert hb.selected_tiers == ['SNEWSHeartbeatMessage'] + assert len(hb.messages) == 1, f"Expected 1 Heartbeat Message got {len(hb.messages)}!" + + assert hb.messages[0].message_data == {'_id': 'XENONnT_Heartbeat_2012-06-09T15:30:00.000501', + 'schema_version': '1.3.0', + 'detector_name': 'XENONnT', + 'machine_time': '2012-06-09T15:30:00.000501', + 'detector_status': 'ON'} + assert hb.messages[0].meta == {'is_test': True, 'firedrill_mode': False} + + # # check if valid snews format + assert hb.messages[0].is_valid() is True, "Message is not valid" + + # Try to send message to SNEWS 2.0 server. + try: + hb.send_messages() + except Exception as exc: + print('SNEWSMessageBuilder.send_messages() test failed!\n') + assert False, f"Exception raised:\n {exc}" \ No newline at end of file diff --git a/snews_pt/test/test_old_crashes.py b/snews_pt/test/test_old_crashes.py index 0af1e59..19ce08b 100644 --- a/snews_pt/test/test_old_crashes.py +++ b/snews_pt/test/test_old_crashes.py @@ -1,36 +1,30 @@ """Test publishing coincidence tier messages.""" -from snews_pt.snews_pub import SNEWSTiersPublisher -from snews_pt.snews_format_checker import SnewsFormat - +from snews_pt.messages import SNEWSMessageBuilder def test_colon_in_time(): """Test with example of expected message type. If works properly both format of times should pass """ # Create coincidence tier message. - coin = SNEWSTiersPublisher(detector_name='XENONnT', neutrino_time='2012-06-09T15:31:07.891011', p_val=0.4, + coin = SNEWSMessageBuilder(detector_name='XENONnT', neutrino_time='2012-06-09T15:31:07.891011', p_val=0.4, firedrill_mode=False, is_test=True) - # the SNEWSTierPublisher already checks this in creation, but we can double check - for message in coin.messages: - format_checker = SnewsFormat(message) - assert format_checker() is True, "Message is not in the snews format" + # the SNEWSTierPublisher already checks this in creation, but we can double-check + assert coin.messages[0].is_valid() is True, "dot separated neutrino time failed!" try: - coin.send_to_snews() + coin.send_messages() except Exception as exc: print('SNEWSTiersPublisher.send_to_snews() test failed!\n') assert False, f"Exception raised:\n {exc}" -# ------------------------------------------------------------------------------------------------------ - coin2 = SNEWSTiersPublisher(detector_name='KamLAND', neutrino_time='2012-06-09T15:31:08:891011', p_val=0.4, + # ------------------------------------------------------------------------------------------------------ + coin2 = SNEWSMessageBuilder(detector_name='KamLAND', neutrino_time='2012-06-09T15:31:08:891011', p_val=0.4, firedrill_mode=False, is_test=True) - # the SNEWSTierPublisher already checks this in creation, but we can double check - for message in coin2.messages: - format_checker = SnewsFormat(message) - assert format_checker() is True, "Message is not in the snews format" + # the SNEWSTierPublisher already checks this in creation, but we can double-check + assert coin2.messages[0].is_valid() is True, "Semicolon separated neutrino time failed!" try: - coin2.send_to_snews() + coin2.send_messages() except Exception as exc: print('SNEWSTiersPublisher.send_to_snews() test failed!\n') assert False, f"Exception raised:\n {exc}" diff --git a/snews_pt/test/test_retraction.py b/snews_pt/test/test_retraction.py index 51533e6..7569218 100644 --- a/snews_pt/test/test_retraction.py +++ b/snews_pt/test/test_retraction.py @@ -1,28 +1,39 @@ """Test publishing coincidence tier message and then retracting it""" -from snews_pt.snews_pub import SNEWSTiersPublisher +from snews_pt.messages import SNEWSMessageBuilder def test_retraction(): """Test with example of expected message type.""" - # Create coincidence tier message. - coin = SNEWSTiersPublisher(detector_name='KamLAND', neutrino_time='2012-06-09T15:31:08.891011', + coin = SNEWSMessageBuilder(detector_name='KamLAND', neutrino_time='2012-06-09T15:31:08.891011', firedrill_mode=False, is_test=True) # Check that message has expected structure. - assert list(coin.tiernames) == ['CoincidenceTier'] - assert len(coin.messages) == 1, f"Expected 1 CoincidenceTier Message got {len(coin.messages)}!" + assert coin.selected_tiers == ["SNEWSCoincidenceTierMessage"] + assert coin.messages[0].is_valid() is True, "Invalid coincidence message created" # Try to send message to SNEWS 2.0 server. try: - coin.send_to_snews() + coin.send_messages() except Exception as exc: print('SNEWSTiersPublisher.send_to_snews() test failed!\n') assert False, f"Exception raised:\n {exc}" # Now try to retract it - retraction_message = SNEWSTiersPublisher(detector_name='KamLAND', retract_latest=1, is_test=True, firedrill_mode=False) + retraction_message = SNEWSMessageBuilder(detector_name='KamLAND', retract_latest=1, + machine_time='2012-06-09T15:30:00.000501', + is_test=True, firedrill_mode=False) + assert retraction_message.selected_tiers == ["SNEWSRetractionMessage"] + assert retraction_message.messages[0].message_data == {'_id': 'KamLAND_Retraction_2012-06-09T15:30:00.000501', + 'schema_version': '1.3.0', + 'detector_name': 'KamLAND', + 'machine_time': '2012-06-09T15:30:00.000501', + 'retract_latest': 1, + 'retraction_reason': None}, "created message data is wrong" + assert retraction_message.messages[0].meta == {'is_test': True, 'firedrill_mode': False}, "created meta is wrong" + assert retraction_message.messages[0].is_valid() is True, "Invalid retraction message created" + try: # we can only test if the retraction message is send, not if it really retracted. - retraction_message.send_to_snews() + retraction_message.send_messages() except Exception as exc: print('Retraction test failed!\n') assert False, f"Exception raised:\n {exc}" \ No newline at end of file diff --git a/snews_pt/test/test_significance_tier.py b/snews_pt/test/test_significance_tier.py index a5db783..fc4576e 100644 --- a/snews_pt/test/test_significance_tier.py +++ b/snews_pt/test/test_significance_tier.py @@ -1,13 +1,13 @@ -"""Test publishing significane tier messages.""" -from snews_pt.snews_pub import SNEWSTiersPublisher -from snews_pt.snews_format_checker import SnewsFormat +"""Test publishing significant tier messages.""" +from snews_pt.messages import SNEWSMessageBuilder def test_significance_expected(): """Test with example of expected message type.""" # Create significance tier message. - sign = SNEWSTiersPublisher(detector_name='DS-20K', - neutrino_time = '2012-06-09T15:31:08.109876', - neutrino_times=['2012-06-09T15:31:08.109876', + sign = SNEWSMessageBuilder(detector_name='DS-20K', + machine_time='2012-06-09T15:30:00.000501', + neutrino_time='2012-06-09T15:31:08.109876', + neutrino_times=['2012-06-09T15:31:08.109876', '2012-06-09T15:33:07.891098'], p_values=[0.4, 0.5], t_bin_width=0.8, @@ -15,35 +15,40 @@ def test_significance_expected(): is_test=True) # Check that message has expected structure. - assert list(sign.tiernames) == ['CoincidenceTier', 'SigTier'] - assert sign.message_data == {'detector_name': 'DS-20K', - 'machine_time': None, - 'neutrino_time': '2012-06-09T15:31:08.109876', - 'p_val': None, - 'p_values': [0.4, 0.5], - 't_bin_width': 0.8, - 'timing_series': None, - 'retract_latest': None, - 'retraction_reason': None, - 'detector_status': None, - 'is_pre_sn': False, - 'neutrino_times': ['2012-06-09T15:31:08.109876', - '2012-06-09T15:33:07.891098'], - 'is_test':True} - assert sign.env_file is None + assert sign.selected_tiers == ['SNEWSCoincidenceTierMessage', 'SNEWSSignificanceTierMessage'] + assert sign.messages[0].message_data == {'_id': 'DS-20K_CoincidenceTier_2012-06-09T15:30:00.000501', + 'schema_version': '1.3.0', + 'detector_name': 'DS-20K', + 'machine_time': '2012-06-09T15:30:00.000501', + 'neutrino_time': '2012-06-09T15:31:08.109876', + 'p_val': None} - # the SNEWSTierPublisher already checks this in creation, but we can double check - for message in sign.messages: - format_checker = SnewsFormat(message) - assert format_checker() is True, "Message is not in the snews format" + assert sign.messages[0].meta == {'p_values': [0.4, 0.5], + 't_bin_width': 0.8, + 'is_test': True, + 'neutrino_times': ['2012-06-09T15:31:08.109876', + '2012-06-09T15:33:07.891098'], + 'firedrill_mode': False} - assert len(sign.invalid_messages)==0, "There are invalid messages" + assert sign.messages[1].message_data == {'_id': 'DS-20K_SignificanceTier_2012-06-09T15:30:00.000501', + 'schema_version': '1.3.0', + 'detector_name': 'DS-20K', + 'machine_time': '2012-06-09T15:30:00.000501', + 'p_values': [0.4, 0.5], + 't_bin_width': 0.8} + + assert sign.messages[1].meta == {'neutrino_time': '2012-06-09T15:31:08.109876', + 'is_test': True, + 'neutrino_times': ['2012-06-09T15:31:08.109876', + '2012-06-09T15:33:07.891098'], + 'firedrill_mode': False} + + assert sign.messages[0].is_valid() is True, "invalid coincidence tier message in 'test_significance_tier'" + assert sign.messages[1].is_valid() is True, "invalid significance tier message in 'test_significance_tier'" # Try to send message to SNEWS 2.0 server. try: - sign.send_to_snews() + sign.send_messages() except Exception as exc: - print('SNEWSTiersPublisher.send_to_snews() test failed!\n') + print('SNEWSMessageBuilder.send_messages() test failed!\n') assert False, f"Exception raised:\n {exc}" - -test_significance_expected() \ No newline at end of file diff --git a/snews_pt/test/test_timing_tier.py b/snews_pt/test/test_timing_tier.py index d30335f..208ce1c 100644 --- a/snews_pt/test/test_timing_tier.py +++ b/snews_pt/test/test_timing_tier.py @@ -1,34 +1,30 @@ """Test publishing timing tier messages.""" -from snews_pt.snews_pub import SNEWSTiersPublisher -from snews_pt.snews_format_checker import SnewsFormat - +from snews_pt.messages import SNEWSMessageBuilder def test_timing_expected(): """Test with example of expected message type.""" # Create timing tier message. - tims = SNEWSTiersPublisher(detector_name='XENONnT', neutrino_time='2012-06-09T15:31:08.109876', + tims = SNEWSMessageBuilder(detector_name='XENONnT', timing_series=['2012-06-09T15:31:08.109876', '2012-06-09T15:33:07.891011'], + machine_time='2012-06-09T15:30:00.009876', firedrill_mode=False, is_test=True) - # Check that message has expected structure. - assert list(tims.tiernames) == ['CoincidenceTier', 'TimeTier'] - assert tims.message_data == {'detector_name': 'XENONnT', 'machine_time': None, - 'neutrino_time': '2012-06-09T15:31:08.109876', 'p_val': None, - 'p_values': None, 't_bin_width': None, - 'timing_series': ['2012-06-09T15:31:08.109876', '2012-06-09T15:33:07.891011'], - 'retract_latest': None, 'retraction_reason': None, - 'detector_status': None, 'is_pre_sn': False, 'is_test':True} - assert tims.env_file == None + # # Check that message has expected structure. + assert tims.selected_tiers == ['SNEWSTimingTierMessage'] + assert tims.messages[0].message_data == {'_id': 'XENONnT_TimingTier_2012-06-09T15:30:00.009876', + 'schema_version': '1.3.0', + 'detector_name': 'XENONnT', + 'machine_time': '2012-06-09T15:30:00.009876', + 'p_val': None, + 'timing_series': ['2012-06-09T15:31:08.109876', + '2012-06-09T15:33:07.891011']} + assert tims.messages[0].meta == {'is_test': True, 'firedrill_mode': False} - # the SNEWSTierPublisher already checks this in creation, but we can double check - for message in tims.messages: - format_checker = SnewsFormat(message) - assert format_checker() is True, "Message is not in the snews format" - assert len(tims.invalid_messages) == 0, "There are invalid messages" + assert tims.messages[0].is_valid() is True, "There are invalid messages" # Try to send message to SNEWS 2.0 server. try: - tims.send_to_snews() + tims.send_messages() except Exception as exc: - print('SNEWSTiersPublisher.send_to_snews() test failed!\n') + print('SNEWSMessageBuilder.send_messages() test failed!\n') assert False, f"Exception raised:\n {exc}" diff --git a/snews_pt/tier_decider.py b/snews_pt/tier_decider.py deleted file mode 100644 index 5be93f2..0000000 --- a/snews_pt/tier_decider.py +++ /dev/null @@ -1,98 +0,0 @@ - -""" -The script to parse input data and decide on intended Tier(s) - -""" -from inspect import signature -from .snews_pt_utils import set_env -from .snews_pt_utils import coincidence_tier_data, sig_tier_data, time_tier_data, heartbeat_data, retraction_data -from .message_schema import Message_Schema -from datetime import datetime -import os, sys - -valid_keys = ["detector_name", "machine_time", "neutrino_time", "p_val", "p_values", "timing_series", - "retract_latest", "retraction_reason", "detector_status", "is_pre_sn", 't_bin_width'] - -class TierDecider: - """ Class to decide on the tiers based on the input arguments of SNEWSTierPublisher - - """ - def __init__(self, data, env_file=None): - set_env(env_file) - self.data = data - self.meta_data = {} - self.sent_time = datetime.utcnow().isoformat() - self.schema = self.get_schema() - self.decided_tiers = {} - - def decide(self): - """ Decide how many messages need to be generated based on the contents. - The key-value pairs that do not belong to any of the tiers are packed in the "meta" field - - Returns - ------- - list of dictionaries : message contents for each tier that is selected - list of strings : Decided tier names - - """ - # handle the meta data first - self.handle_meta_fields() - # CoincidenceTier if it has nu time - if (type(self.data['neutrino_time']) == str) & ~(type(self.data['retract_latest']) == int): - self.append_messages(coincidence_tier_data, 'CoincidenceTier') - - # SignificanceTier if it has p_values - if type(self.data['p_values']) == list and type(self.data['t_bin_width']) == float: - self.append_messages(sig_tier_data, 'SigTier') - - # TimingTier if timing_series exists - if type(self.data['timing_series']) == list: - self.append_messages(time_tier_data, 'TimeTier') - - # asking which tier doesn't make sense if the user doesn't know the tiers - if type(self.data['retract_latest']) == int: - self.append_messages(retraction_data, 'Retraction') - - # Heartbeat if there is detector status - if type(self.data['detector_status']) == str: - self.append_messages(heartbeat_data, 'Heartbeat') - - for t in self.decided_tiers.keys(): - print(f"Message Generated for {t}") - - if len(self.decided_tiers.keys())==0: - print(f"No valid message is created! Check your input and try again!") - # Return the names and messages generated - return self.decided_tiers.values(), self.decided_tiers.keys() - - def get_schema(self): - self.data["is_pre_sn"] = self.data.get("is_pre_sn", False) - self.data['detector_name'] = self.data.get("detector_name", os.getenv('DETECTOR_NAME')) - return Message_Schema(detector_key=self.data['detector_name'], is_pre_sn=self.data['is_pre_sn']) - - def handle_meta_fields(self): - """ make a meta dict for all key-value pairs that don't belong any known keys - - """ - # if there are keys that wouldn't belong to any tier/command pass them as meta - meta_keys = [key for key, value in self.data.items() if sys.getsizeof(value) < 2048] - meta_data = {k: self.data[k] for k in meta_keys if k not in valid_keys} - self.meta_data = meta_data - - def append_messages(self, tier_function, name): - """ This function takes the relevant inputs and formats - a message accordingly. The messages are collected in a list - - """ - tier_keys = list(signature(tier_function).parameters.keys()) - data_for_tier = {k: v for k, v in self.data.items() if k in tier_keys} - data_for_tier = tier_function(**data_for_tier) - if name not in ['Heartbeat']: # 'Retraction', - # don't append meta field to Retraction and Heartbeat - data_for_tier['meta'] = self.meta_data - else: - data_for_tier['meta'] = {} - - msg = self.schema.get_schema(tier=name, data=data_for_tier, sent_time=self.sent_time) - - self.decided_tiers[name] = msg