Iso time

.gitignore

@@ -1,3 +1,7 @@
+# Alerts and logs
+SNEWS_ALERTS/
+logs/
+
 # Visual Studio Code
 .vscode/
 

examples.ipynb

@@ -16,9 +16,15 @@
     "## Publication"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "3bacfedc",
+   "metadata": {},
+   "source": []
+  },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 1,
    "id": "34bebc1c-8272-4a87-b877-31b77700907d",
    "metadata": {
     "tags": []
@@ -28,36 +34,53 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "{'detector_name': 'LZ', 'machine_time': '22/03/07 16:18:09:185459', 'neutrino_time': '22/03/07 16:18:09:185459', 'p_val': 0.0007, 'p_values': [0.0007, 0.0008, 0.0009], 't_bin_width': 0.07, 'timing_series': None, 'which_tier': None, 'n_retract_latest': None, 'retraction_reason': None, 'detector_status': None, 'is_pre_sn': False, 'meta': {}}\n",
+      "{'detector_name': 'LZ', 'machine_time': '2022-08-05T16:01:00.786929', 'neutrino_time': '2022-08-05T16:01:00.786929', 'p_val': 0.0007, 'p_values': [0.0007, 0.0008, 0.0009], 't_bin_width': 0.07, 'timing_series': None, 'which_tier': None, 'n_retract_latest': None, 'retraction_reason': None, 'detector_status': None, 'is_pre_sn': False}\n",
       "['CoincidenceTier', 'SigTier']\n",
       "Sending message\n",
-      "\u001B[94m----------------------------------------------------------------\u001B[0m\n",
-      "\u001B[91mSending message to CoincidenceTier\u001B[0m\n",
-      "_id                 :14_CoincidenceTier_22/03/07 16:18:09:185459\n",
+      "\u001b[94m----------------------------------------------------------------\u001b[0m\n",
+      "\u001b[91mSending message to CoincidenceTier on kafka://kafka.scimma.org/snews.experiments-firedrill\u001b[0m\n",
+      "_id                 :14_CoincidenceTier_2022-08-05T16:01:00.786929\n",
       "detector_name       :LZ\n",
-      "machine_time        :22/03/07 16:18:09:185459\n",
-      "neutrino_time       :22/03/07 16:18:09:185459\n",
+      "machine_time        :2022-08-05T16:01:00.786929\n",
+      "neutrino_time       :2022-08-05T16:01:00.786929\n",
       "p_val               :0.0007\n",
-      "meta                :{'meta': {}}\n",
-      "schema_version      :1.1.0\n",
-      "\u001B[94m----------------------------------------------------------------\u001B[0m\n",
-      "\u001B[91mSending message to SigTier\u001B[0m\n",
-      "_id                 :14_SigTier_22/03/07 16:18:09:185459\n",
+      "meta                :{}\n",
+      "schema_version      :1.1.1\n",
+      "sent_time           :2022-08-05T16:01:00.786982\n",
+      "\u001b[94m----------------------------------------------------------------\u001b[0m\n",
+      "\u001b[91mSending message to SigTier on kafka://kafka.scimma.org/snews.experiments-firedrill\u001b[0m\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/joe/src/gitjoe/SNEWS_Publishing_Tools/snews_pt/__init__.py:19: UserWarning: \u001b[31mYou are using default detector name \"TEST\"\n",
+      "Please change this by snews_pt.snews_pt_utils.set_name()\u001b[0m\n",
+      "  warnings.warn(warning_text, UserWarning)\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "_id                 :14_SigTier_2022-08-05T16:01:00.786929\n",
       "detector_name       :LZ\n",
-      "machine_time        :22/03/07 16:18:09:185459\n",
-      "neutrino_time       :22/03/07 16:18:09:185459\n",
+      "machine_time        :2022-08-05T16:01:00.786929\n",
+      "neutrino_time       :2022-08-05T16:01:00.786929\n",
       "p_values            :[0.0007, 0.0008, 0.0009]\n",
       "t_bin_width         :0.07\n",
-      "meta                :{'meta': {}}\n",
-      "schema_version      :1.1.0\n"
+      "meta                :{}\n",
+      "schema_version      :1.1.1\n",
+      "sent_time           :2022-08-05T16:01:00.786982\n"
      ]
     }
    ],
    "source": [
     "from snews_pt.snews_pub import SNEWSTiersPublisher\n",
     "from datetime import  datetime\n",
     "\n",
-    "test_time = datetime.utcnow().strftime(\"%y/%m/%d %H:%M:%S:%f\")\n",
+    "test_time = datetime.utcnow().isoformat()\n",
     "message = SNEWSTiersPublisher(detector_name='LZ',machine_time=test_time ,neutrino_time=test_time, p_val=0.0007, p_values=[0.0007,0.0008,0.0009],t_bin_width=0.07)\n",
     "print(message.message_data)\n",
     "print(message.tiernames)\n",
@@ -75,67 +98,47 @@
   },
   {
    "cell_type": "code",
+   "execution_count": 4,
    "id": "0ea6d89a",
    "metadata": {
     "pycharm": {
      "name": "#%%\n"
     }
    },
-   "source": [
-    "from snews_pt.snews_sub import Subscriber\n",
-    "\n",
-    "Subscriber().subscribe()"
-   ],
-   "execution_count": 2,
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "You are subscribing to \u001B[41m\u001B[1mALERT\u001B[0m\n",
-      "Broker:\u001B[42mkafka://kafka.scimma.org/snews.alert-test\u001B[0m\n"
-     ]
-    },
-    {
-     "ename": "ValueError",
-     "evalue": "topic snews.alert-test does not exist on the broker, so can't subscribe",
-     "output_type": "error",
-     "traceback": [
-      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
-      "\u001B[0;31mKeyError\u001B[0m                                  Traceback (most recent call last)",
-      "\u001B[0;32m~/opt/anaconda3/lib/python3.9/site-packages/adc/consumer.py\u001B[0m in \u001B[0;36msubscribe\u001B[0;34m(self, topics, timeout)\u001B[0m\n\u001B[1;32m     40\u001B[0m             \u001B[0;32mtry\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 41\u001B[0;31m                 \u001B[0mtopic_meta\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mdescribe_topic\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mtopic\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mtimeout\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m     42\u001B[0m             \u001B[0;32mexcept\u001B[0m \u001B[0mKeyError\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
-      "\u001B[0;32m~/opt/anaconda3/lib/python3.9/site-packages/adc/consumer.py\u001B[0m in \u001B[0;36mdescribe_topic\u001B[0;34m(self, topic, timeout)\u001B[0m\n\u001B[1;32m     64\u001B[0m         \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mlogger\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mdebug\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34mf\"cluster metadata: {cluster_meta.topics}\"\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 65\u001B[0;31m         \u001B[0;32mreturn\u001B[0m \u001B[0mcluster_meta\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mtopics\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mtopic\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m     66\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n",
-      "\u001B[0;31mKeyError\u001B[0m: 'snews.alert-test'",
-      "\nDuring handling of the above exception, another exception occurred:\n",
-      "\u001B[0;31mValueError\u001B[0m                                Traceback (most recent call last)",
-      "\u001B[0;32m/var/folders/mr/0p5djr157hxdh7bb2gmzywyr0000gn/T/ipykernel_4462/1770358198.py\u001B[0m in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[1;32m      1\u001B[0m \u001B[0;32mfrom\u001B[0m \u001B[0mSNEWS_PT\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0msnews_sub\u001B[0m \u001B[0;32mimport\u001B[0m \u001B[0mSubscriber\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m      2\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m----> 3\u001B[0;31m \u001B[0mSubscriber\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0msubscribe\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m",
-      "\u001B[0;32m~/work/SNEWS_Publishing_Tools/SNEWS_PT/snews_sub.py\u001B[0m in \u001B[0;36msubscribe\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m     91\u001B[0m         \u001B[0mstream\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mStream\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0muntil_eos\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mFalse\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m     92\u001B[0m         \u001B[0;32mtry\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 93\u001B[0;31m             \u001B[0;32mwith\u001B[0m \u001B[0mstream\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mopen\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0malert_topic\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m\"r\"\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;32mas\u001B[0m \u001B[0ms\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m     94\u001B[0m                 \u001B[0;32mfor\u001B[0m \u001B[0mmessage\u001B[0m \u001B[0;32min\u001B[0m \u001B[0ms\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m     95\u001B[0m                     \u001B[0msave_message\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmessage\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
-      "\u001B[0;32m~/opt/anaconda3/lib/python3.9/site-packages/hop/io.py\u001B[0m in \u001B[0;36mopen\u001B[0;34m(self, url, mode, group_id, **kwargs)\u001B[0m\n\u001B[1;32m    114\u001B[0m                 \u001B[0mgroup_id\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0m_generate_group_id\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0musername\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;36m10\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    115\u001B[0m                 \u001B[0mlogger\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0minfo\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34mf\"group ID not specified, generating a random group ID: {group_id}\"\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 116\u001B[0;31m             return Consumer(\n\u001B[0m\u001B[1;32m    117\u001B[0m                 \u001B[0mgroup_id\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    118\u001B[0m                 \u001B[0mbroker_addresses\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
-      "\u001B[0;32m~/opt/anaconda3/lib/python3.9/site-packages/hop/io.py\u001B[0m in \u001B[0;36m__init__\u001B[0;34m(self, group_id, broker_addresses, topics, **kwargs)\u001B[0m\n\u001B[1;32m    291\u001B[0m         ))\n\u001B[1;32m    292\u001B[0m         \u001B[0mlogger\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0minfo\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34mf\"subscribing to topics: {topics}\"\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 293\u001B[0;31m         \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_consumer\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0msubscribe\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mtopics\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    294\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    295\u001B[0m     \u001B[0;32mdef\u001B[0m \u001B[0mread\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mmetadata\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mFalse\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mautocommit\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mTrue\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
-      "\u001B[0;32m~/opt/anaconda3/lib/python3.9/site-packages/adc/consumer.py\u001B[0m in \u001B[0;36msubscribe\u001B[0;34m(self, topics, timeout)\u001B[0m\n\u001B[1;32m     41\u001B[0m                 \u001B[0mtopic_meta\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mdescribe_topic\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mtopic\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mtimeout\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m     42\u001B[0m             \u001B[0;32mexcept\u001B[0m \u001B[0mKeyError\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 43\u001B[0;31m                 \u001B[0;32mraise\u001B[0m \u001B[0mValueError\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34mf\"topic {topic} does not exist on the broker, so can't subscribe\"\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m     44\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m     45\u001B[0m             \u001B[0;32mfor\u001B[0m \u001B[0mpartition_id\u001B[0m \u001B[0;32min\u001B[0m \u001B[0mtopic_meta\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mpartitions\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mkeys\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
-      "\u001B[0;31mValueError\u001B[0m: topic snews.alert-test does not exist on the broker, so can't subscribe"
+      "You are subscribing to \u001b[41m\u001b[1mALERT\u001b[0m\n",
+      "Broker:\u001b[42mkafka://kafka.scimma.org/snews.alert-firedrill\u001b[0m\n"
      ]
     }
+   ],
+   "source": [
+    "from snews_pt.snews_sub import Subscriber\n",
+    "\n",
+    "Subscriber().subscribe()"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "outputs": [],
-   "source": [],
    "metadata": {
     "collapsed": false,
     "pycharm": {
      "name": "#%%\n"
     }
-   }
+   },
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
   "kernelspec": {
-   "name": "python3",
+   "display_name": "Python 3.9.9 ('snews-pt')",
    "language": "python",
-   "display_name": "Python 3 (ipykernel)"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
@@ -147,9 +150,14 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.11"
+   "version": "3.9.9"
+  },
+  "vscode": {
+   "interpreter": {
+    "hash": "f581a8651b7bc0666f0167af942e3b60152ddbb51a5d7dbac8fd8281a3c56e84"
+   }
   }
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}

snews_pt/__init__.py

@@ -1,3 +1,7 @@
+from .core.logging import initialize_logging
+
+initialize_logging("debug")
+
 try:
     from ._version import version as __version__
 except ImportError:

snews_pt/__main__.py

@@ -3,7 +3,7 @@
     Notes to dev team
     https://stackoverflow.com/questions/55099243/python3-dataclass-with-kwargsasterisk
 """
-import time
+from datetime import datetime
 
 from . import __version__
 from . import snews_pt_utils
@@ -147,8 +147,7 @@ def test_connection(ctx, firedrill, start_at):
     """
     from hop import Stream
     name = ctx.obj['DETECTOR_NAME']
-    snewstimes = snews_pt_utils.TimeStuff()
-    stamp_time = snewstimes.get_utcnow()
+    stamp_time = datetime.utcnow().isoformat()
     message = {'_id': '0_test-connection',
                'detector_name': name,
                'time': stamp_time,

snews_pt/_version.py

@@ -1 +1 @@
-version = '1.1.0'
+version = '1.1.1'

snews_pt/auxiliary/make_scenarios.py

@@ -1,56 +1,56 @@
 import json
 
 msg1 = {"detector_name": "XENONnT",
-        "neutrino_time": "30/01/01 12:34:45:678999",
+        "neutrino_time": "2030-01-01T12:34:45.678999",
         "p_val": 0.98}
 
 msg2 = {"detector_name": "DS-20K",
-        "neutrino_time": "30/01/01 12:34:47:678999",
+        "neutrino_time": "2030-01-01T12:34:47.678999",
         "p_val": 0.98}
 
 msg3 = {"detector_name": "DUNE",
-        "neutrino_time": "30/01/01 12:34:55:678999",
+        "neutrino_time": "2030-01-01T12:34:55.678999",
         "p_val": 0.98}
 
 msg4 = {"detector_name": "JUNO",
-        "neutrino_time": "30/01/01 12:30:55:678999",
+        "neutrino_time": "2030-01-01T12:30:55.678999",
         "p_val": 0.98}
 
 msg5 = {"detector_name": "Baksan",
-        "neutrino_time": "30/01/01 12:30:52:678999",
+        "neutrino_time": "2030-01-01T12:30:52.678999",
         "p_val": 0.98}
 
 msg6 = {"detector_name": "JUNO",
-        "neutrino_time": "30/01/01 12:30:56:678999",
+        "neutrino_time": "2030-01-01T12:30:56.678999",
         "p_val": 0.98}
 
 ####
 msg7 = {"detector_name": "Borexino",
-        "neutrino_time": "30/01/01 12:30:10:678999",
+        "neutrino_time": "2030-01-01T12:30:10.678999",
         "p_val": 0.98}
 
 msg8 = {"detector_name": "ICE",
-        "neutrino_time": "30/01/01 12:30:19:678999",
+        "neutrino_time": "2030-01-01T12:30:19.678999",
         "p_val": 0.98}
 
 msg9 = {"detector_name": "NOvA",
-        "neutrino_time": "30/01/01 12:30:21:678999",
+        "neutrino_time": "2030-01-01T12:30:21.678999",
         "p_val": 0.98}
 
 msg10 = {"detector_name": "DS-20K",
-        "neutrino_time": "30/01/01 12:30:22:678999",
+        "neutrino_time": "2030-01-01T12:30:22.678999",
         "p_val": 0.98}
 
 msg11 = {"detector_name": "HALO",
-        "neutrino_time": "30/01/01 12:30:20:678999",
+        "neutrino_time": "2030-01-01T12:30:20.678999",
         "p_val": 0.98}
 
 msg12 = {"detector_name": "XENONnT",
-        "neutrino_time": "30/01/01 12:30:29:678999",
+        "neutrino_time": "2030-01-01T12:30:29.678999",
         "p_val": 0.98}
 
 msg13 = {"detector_name": "JUNO",
-        "neutrino_time": "30/01/01 12:30:18:678999",
+        "neutrino_time": "2030-01-01T12:30:18.678999",
         "p_val": 0.98}