Merge pull request #158 from CuriBio/serial-comm-set-time

Serial comm set time

src/mantarray_desktop_app/__init__.py

@@ -104,13 +104,15 @@
 from .constants import SERIAL_COMM_REGISTRATION_TIMEOUT_SECONDS
 from .constants import SERIAL_COMM_RESPONSE_TIMEOUT_SECONDS
 from .constants import SERIAL_COMM_SET_NICKNAME_COMMAND_BYTE
+from .constants import SERIAL_COMM_SET_TIME_COMMAND_BYTE
 from .constants import SERIAL_COMM_SIMPLE_COMMAND_PACKET_TYPE
 from .constants import SERIAL_COMM_STATUS_BEACON_PACKET_TYPE
 from .constants import SERIAL_COMM_STATUS_BEACON_PERIOD_SECONDS
 from .constants import SERIAL_COMM_STATUS_BEACON_TIMEOUT_SECONDS
 from .constants import SERIAL_COMM_STATUS_CODE_LENGTH_BYTES
 from .constants import SERIAL_COMM_TIME_SYNC_READY_CODE
 from .constants import SERIAL_COMM_TIMESTAMP_BYTES_INDEX
+from .constants import SERIAL_COMM_TIMESTAMP_EPOCH
 from .constants import SERIAL_COMM_TIMESTAMP_LENGTH_BYTES
 from .constants import SERVER_INITIALIZING_STATE
 from .constants import SERVER_READY_STATE
@@ -208,7 +210,9 @@
 from .serial_comm_utils import convert_metadata_bytes_to_str
 from .serial_comm_utils import convert_to_metadata_bytes
 from .serial_comm_utils import convert_to_status_code_bytes
+from .serial_comm_utils import convert_to_timestamp_bytes
 from .serial_comm_utils import create_data_packet
+from .serial_comm_utils import get_serial_comm_timestamp
 from .serial_comm_utils import parse_metadata_bytes
 from .serial_comm_utils import validate_checksum
 from .server import clear_the_server_thread
@@ -453,4 +457,8 @@
     "SerialCommHandshakeTimeoutError",
     "convert_to_status_code_bytes",
     "SERIAL_COMM_MAX_DATA_LENGTH_BYTES",
+    "SERIAL_COMM_SET_TIME_COMMAND_BYTE",
+    "convert_to_timestamp_bytes",
+    "get_serial_comm_timestamp",
+    "SERIAL_COMM_TIMESTAMP_EPOCH",
 ]

src/mantarray_desktop_app/constants.py

@@ -9,6 +9,7 @@
 * ADC_CH_TO_IS_REF_SENSOR
 * WELL_24_INDEX_TO_ADC_AND_CH_INDEX
 """
+import datetime
 from typing import Dict
 from typing import Tuple
 import uuid
@@ -237,6 +238,10 @@
 
 SERIAL_COMM_NUM_ALLOWED_MISSED_HANDSHAKES = 3
 
+SERIAL_COMM_TIMESTAMP_EPOCH = datetime.datetime(
+    year=2021, month=1, day=1, tzinfo=datetime.timezone.utc
+)
+
 SERIAL_COMM_STATUS_BEACON_PERIOD_SECONDS = 5
 SERIAL_COMM_HANDSHAKE_PERIOD_SECONDS = 5
 SERIAL_COMM_RESPONSE_TIMEOUT_SECONDS = 5
@@ -272,15 +277,20 @@
 SERIAL_COMM_ADDITIONAL_BYTES_INDEX = 20
 
 SERIAL_COMM_MAIN_MODULE_ID = 0
-SERIAL_COMM_STATUS_BEACON_PACKET_TYPE = 0
+
+# PC to Mantarray Packet Types
 SERIAL_COMM_SIMPLE_COMMAND_PACKET_TYPE = 3
-SERIAL_COMM_COMMAND_RESPONSE_PACKET_TYPE = 4
 SERIAL_COMM_HANDSHAKE_PACKET_TYPE = 4
+# Mantarray to PC Packet Types
+SERIAL_COMM_STATUS_BEACON_PACKET_TYPE = 0
+SERIAL_COMM_COMMAND_RESPONSE_PACKET_TYPE = 4
 SERIAL_COMM_CHECKSUM_FAILURE_PACKET_TYPE = 255
+# Simple Command Codes
 SERIAL_COMM_REBOOT_COMMAND_BYTE = 0
 SERIAL_COMM_GET_METADATA_COMMAND_BYTE = 6
+SERIAL_COMM_SET_TIME_COMMAND_BYTE = 8
 SERIAL_COMM_SET_NICKNAME_COMMAND_BYTE = 9
-
+# Mantarray Status Codes
 SERIAL_COMM_IDLE_READY_CODE = 0
 SERIAL_COMM_TIME_SYNC_READY_CODE = 1
 SERIAL_COMM_HANDSHAKE_TIMEOUT_CODE = 2

src/mantarray_desktop_app/mc_comm.py

@@ -22,6 +22,7 @@
 from stdlib_utils import put_log_message_into_queue
 
 from .constants import MAX_MC_REBOOT_DURATION_SECONDS
+from .constants import MICROSECONDS_PER_CENTIMILLISECOND
 from .constants import SERIAL_COMM_ADDITIONAL_BYTES_INDEX
 from .constants import SERIAL_COMM_BAUD_RATE
 from .constants import SERIAL_COMM_CHECKSUM_FAILURE_PACKET_TYPE
@@ -42,10 +43,12 @@
 from .constants import SERIAL_COMM_REGISTRATION_TIMEOUT_SECONDS
 from .constants import SERIAL_COMM_RESPONSE_TIMEOUT_SECONDS
 from .constants import SERIAL_COMM_SET_NICKNAME_COMMAND_BYTE
+from .constants import SERIAL_COMM_SET_TIME_COMMAND_BYTE
 from .constants import SERIAL_COMM_SIMPLE_COMMAND_PACKET_TYPE
 from .constants import SERIAL_COMM_STATUS_BEACON_PACKET_TYPE
 from .constants import SERIAL_COMM_STATUS_BEACON_PERIOD_SECONDS
 from .constants import SERIAL_COMM_STATUS_BEACON_TIMEOUT_SECONDS
+from .constants import SERIAL_COMM_TIME_SYNC_READY_CODE
 from .constants import SERIAL_COMM_TIMESTAMP_LENGTH_BYTES
 from .exceptions import InstrumentRebootTimeoutError
 from .exceptions import SerialCommCommandResponseTimeoutError
@@ -65,7 +68,9 @@
 from .instrument_comm import InstrumentCommProcess
 from .mc_simulator import MantarrayMcSimulator
 from .serial_comm_utils import convert_to_metadata_bytes
+from .serial_comm_utils import convert_to_timestamp_bytes
 from .serial_comm_utils import create_data_packet
+from .serial_comm_utils import get_serial_comm_timestamp
 from .serial_comm_utils import parse_metadata_bytes
 from .serial_comm_utils import validate_checksum
 
@@ -213,8 +218,12 @@ def _send_data_packet(
         packet_type: int,
         data_to_send: bytes = bytes(0),
     ) -> None:
+        # TODO Tanner (4/7/21): change timestamp to microseconds when the real Mantarray makes the switch
         data_packet = create_data_packet(
-            self.get_cms_since_init(), module_id, packet_type, data_to_send
+            get_serial_comm_timestamp() // MICROSECONDS_PER_CENTIMILLISECOND,
+            module_id,
+            packet_type,
+            data_to_send,
         )
         board = self._board_connections[board_idx]
         if board is None:
@@ -382,6 +391,7 @@ def _handle_incoming_data(self) -> None:
             raise UnrecognizedSerialCommModuleIdError(module_id)
 
     def _process_main_module_comm(self, comm_from_instrument: bytes) -> None:
+        board_idx = 0
         packet_body = comm_from_instrument[
             SERIAL_COMM_ADDITIONAL_BYTES_INDEX:-SERIAL_COMM_CHECKSUM_LENGTH_BYTES
         ]
@@ -398,7 +408,7 @@ def _process_main_module_comm(self, comm_from_instrument: bytes) -> None:
             ):  # Tanner (4/1/21): want to check that reboot has actually started before considering a status beacon to mean that reboot has completed. It is possible (and has happened in unit tests) where a beacon is received in between sending the reboot command and the instrument actually beginning to reboot
                 self._is_waiting_for_reboot = False
                 self._time_of_reboot_start = None
-                self._board_queues[0][1].put_nowait(
+                self._board_queues[board_idx][1].put_nowait(
                     {
                         "communication_type": "to_instrument",
                         "command": "reboot",
@@ -409,6 +419,17 @@ def _process_main_module_comm(self, comm_from_instrument: bytes) -> None:
             self._log_status_code(status_code, "Status Beacon")
             if status_code == SERIAL_COMM_HANDSHAKE_TIMEOUT_CODE:
                 raise SerialCommHandshakeTimeoutError()
+            if status_code == SERIAL_COMM_TIME_SYNC_READY_CODE:
+                self._send_data_packet(
+                    board_idx,
+                    SERIAL_COMM_MAIN_MODULE_ID,
+                    SERIAL_COMM_SIMPLE_COMMAND_PACKET_TYPE,
+                    bytes([SERIAL_COMM_SET_TIME_COMMAND_BYTE])
+                    + convert_to_timestamp_bytes(get_serial_comm_timestamp()),
+                )
+                self._commands_awaiting_response.append(
+                    {"command": "set_time", "timepoint": perf_counter()}
+                )
         elif packet_type == SERIAL_COMM_COMMAND_RESPONSE_PACKET_TYPE:
             response_data = packet_body[SERIAL_COMM_TIMESTAMP_LENGTH_BYTES:]
             if not self._commands_awaiting_response:
@@ -425,10 +446,12 @@ def _process_main_module_comm(self, comm_from_instrument: bytes) -> None:
             elif prev_command["command"] == "reboot":
                 prev_command["message"] = "Instrument beginning reboot"
                 self._time_of_reboot_start = perf_counter()
+            elif prev_command["command"] == "set_time":
+                prev_command["message"] = "Instrument time synced with PC"
             del prev_command[
                 "timepoint"
             ]  # main process does not need to know the timepoint and is not expecting this key in the dictionary returned to it
-            self._board_queues[0][1].put_nowait(
+            self._board_queues[board_idx][1].put_nowait(
                 prev_command
             )  # Tanner (3/17/21): to be consistent with OkComm, command responses will be sent back to main after the command is acknowledged by the Mantarray
         else:

src/mantarray_desktop_app/mc_simulator.py

@@ -8,6 +8,7 @@
 import random
 import time
 from time import perf_counter
+from time import perf_counter_ns
 from typing import Any
 from typing import Dict
 from typing import Optional
@@ -24,6 +25,7 @@
 
 from .constants import BOOTUP_COUNTER_UUID
 from .constants import MAX_MC_REBOOT_DURATION_SECONDS
+from .constants import MICROSECONDS_PER_CENTIMILLISECOND
 from .constants import PCB_SERIAL_NUMBER_UUID
 from .constants import SERIAL_COMM_ADDITIONAL_BYTES_INDEX
 from .constants import SERIAL_COMM_BOOT_UP_CODE
@@ -34,6 +36,7 @@
 from .constants import SERIAL_COMM_HANDSHAKE_PERIOD_SECONDS
 from .constants import SERIAL_COMM_HANDSHAKE_TIMEOUT_CODE
 from .constants import SERIAL_COMM_HANDSHAKE_TIMEOUT_SECONDS
+from .constants import SERIAL_COMM_IDLE_READY_CODE
 from .constants import SERIAL_COMM_MAGIC_WORD_BYTES
 from .constants import SERIAL_COMM_MAIN_MODULE_ID
 from .constants import SERIAL_COMM_METADATA_BYTES_LENGTH
@@ -42,6 +45,7 @@
 from .constants import SERIAL_COMM_PACKET_TYPE_INDEX
 from .constants import SERIAL_COMM_REBOOT_COMMAND_BYTE
 from .constants import SERIAL_COMM_SET_NICKNAME_COMMAND_BYTE
+from .constants import SERIAL_COMM_SET_TIME_COMMAND_BYTE
 from .constants import SERIAL_COMM_SIMPLE_COMMAND_PACKET_TYPE
 from .constants import SERIAL_COMM_STATUS_BEACON_PACKET_TYPE
 from .constants import SERIAL_COMM_STATUS_BEACON_PERIOD_SECONDS
@@ -65,6 +69,11 @@
 MC_SIMULATOR_BOOT_UP_DURATION_SECONDS = 3
 
 
+def _perf_counter_us() -> int:
+    """Return perf_counter value as microseconds."""
+    return perf_counter_ns() // 10 ** 3
+
+
 def _get_secs_since_last_handshake(last_time: float) -> float:
     return perf_counter() - last_time
 
@@ -89,6 +98,8 @@ def _get_secs_since_last_comm_from_pc(last_time: float) -> float:
 class MantarrayMcSimulator(InfiniteProcess):
     """Simulate a running Mantarray instrument with Microcontroller.
 
+    If a command from the PC triggers an update to the status code, the updated status beacon will be sent after the command response
+
     Args:
         input_queue: queue bytes sent to the simulator using the `write` method
         output_queue: queue bytes sent from the simulator using the `read` method
@@ -132,6 +143,8 @@ def __init__(
         self._output_queue = output_queue
         self._input_queue = input_queue
         self._testing_queue = testing_queue
+        self._baseline_time_usec: Optional[int] = None
+        self._timepoint_of_time_sync_us: Optional[int] = None
         self._time_of_last_status_beacon_secs: Optional[float] = None
         self._time_of_last_handshake_secs: Optional[float] = None
         self._time_of_last_comm_from_pc_secs: Optional[float] = None
@@ -168,6 +181,13 @@ def _reset_metadata_dict(self) -> None:
                 metadata_value
             )
 
+    def _get_us_since_time_sync(self) -> int:
+        return (
+            0
+            if self._timepoint_of_time_sync_us is None
+            else _perf_counter_us() - self._timepoint_of_time_sync_us
+        )
+
     def get_metadata_dict(self) -> Dict[bytes, bytes]:
         """Mainly for use in unit tests."""
         return self._metadata_dict
@@ -183,8 +203,15 @@ def _send_data_packet(
         data_to_send: bytes = bytes(0),
         truncate: bool = False,
     ) -> None:
+        # TODO Tanner (4/7/21): convert timestamp to microseconds once real board makes the switch
+        timestamp = (
+            self.get_cms_since_init()
+            if self._baseline_time_usec is None
+            else (self._baseline_time_usec + self._get_us_since_time_sync())
+            // MICROSECONDS_PER_CENTIMILLISECOND
+        )
         data_packet = create_data_packet(
-            self.get_cms_since_init(), module_id, packet_type, data_to_send
+            timestamp, module_id, packet_type, data_to_send
         )
         if truncate:
             trunc_index = random.randint(  # nosec B311 # Tanner (2/4/21): Bandit blacklisted this psuedo-random generator for cryptographic security reasons that do not apply to the desktop app.
@@ -222,6 +249,8 @@ def _handle_reboot_completion(self) -> None:
         self._reset_status_code()
         self._send_status_beacon(truncate=False)
         self._boot_up_time_secs = perf_counter()
+        self._baseline_time_usec = None
+        self._timepoint_of_time_sync_us = None
 
     def _handle_comm_from_pc(self) -> None:
         try:
@@ -262,28 +291,35 @@ def _check_handshake_timeout(self) -> None:
             self._update_status_code(SERIAL_COMM_HANDSHAKE_TIMEOUT_CODE)
 
     def _process_main_module_command(self, comm_from_pc: bytes) -> None:
+        status_code_update: Optional[int] = None
+
         timestamp_from_pc_bytes = comm_from_pc[
             SERIAL_COMM_TIMESTAMP_BYTES_INDEX : SERIAL_COMM_TIMESTAMP_BYTES_INDEX
             + SERIAL_COMM_TIMESTAMP_LENGTH_BYTES
         ]
         response_body = timestamp_from_pc_bytes
-
         packet_type = comm_from_pc[SERIAL_COMM_PACKET_TYPE_INDEX]
         if packet_type == SERIAL_COMM_SIMPLE_COMMAND_PACKET_TYPE:
             command_byte = comm_from_pc[SERIAL_COMM_ADDITIONAL_BYTES_INDEX]
             if command_byte == SERIAL_COMM_REBOOT_COMMAND_BYTE:
                 self._reboot_time_secs = perf_counter()
+            elif command_byte == SERIAL_COMM_GET_METADATA_COMMAND_BYTE:
+                metadata_bytes = bytes(0)
+                for key, value in self._metadata_dict.items():
+                    metadata_bytes += key + value
+                response_body += metadata_bytes
+            elif command_byte == SERIAL_COMM_SET_TIME_COMMAND_BYTE:
+                self._baseline_time_usec = int.from_bytes(
+                    response_body, byteorder="little"
+                )
+                self._timepoint_of_time_sync_us = _perf_counter_us()
+                status_code_update = SERIAL_COMM_IDLE_READY_CODE
             elif command_byte == SERIAL_COMM_SET_NICKNAME_COMMAND_BYTE:
                 start_idx = SERIAL_COMM_ADDITIONAL_BYTES_INDEX + 1
                 nickname_bytes = comm_from_pc[
                     start_idx : start_idx + SERIAL_COMM_METADATA_BYTES_LENGTH
                 ]
                 self._metadata_dict[MANTARRAY_NICKNAME_UUID.bytes] = nickname_bytes
-            elif command_byte == SERIAL_COMM_GET_METADATA_COMMAND_BYTE:
-                metadata_bytes = bytes(0)
-                for key, value in self._metadata_dict.items():
-                    metadata_bytes += key + value
-                response_body += metadata_bytes
             else:
                 # TODO Tanner (3/4/21): Determine what to do if command_byte, module_id, or packet_type are incorrect. It may make more sense to respond with a message rather than raising an error
                 raise NotImplementedError(command_byte)
@@ -300,6 +336,9 @@ def _process_main_module_command(self, comm_from_pc: bytes) -> None:
             SERIAL_COMM_COMMAND_RESPONSE_PACKET_TYPE,
             response_body,
         )
+        # update status code (if an update is necessary) after sending command response
+        if status_code_update is not None:
+            self._update_status_code(status_code_update)
 
     def _update_status_code(self, new_code: int) -> None:
         self._status_code = new_code
@@ -357,7 +396,19 @@ def _handle_test_comm(self) -> None:
             for read in read_bytes:
                 self._output_queue.put_nowait(read)
         elif command == "set_status_code":
-            self._status_code = test_comm["status_code"]
+            status_code = test_comm["status_code"]
+            self._status_code = status_code
+            baseline_time = test_comm.get("baseline_time", None)
+            if baseline_time is not None:
+                if status_code in (
+                    SERIAL_COMM_BOOT_UP_CODE,
+                    SERIAL_COMM_TIME_SYNC_READY_CODE,
+                ):
+                    raise NotImplementedError(
+                        "baseline_time cannot be set through testing queue in boot up or time sync state"
+                    )
+                self._baseline_time_usec = baseline_time
+                self._timepoint_of_time_sync_us = _perf_counter_us()
         elif command == "set_metadata":
             for key, value in test_comm["metadata_values"].items():
                 value_bytes = convert_to_metadata_bytes(value)

src/mantarray_desktop_app/serial_comm_utils.py

@@ -2,6 +2,7 @@
 """Utility functions for Serial Communication."""
 from __future__ import annotations
 
+import datetime
 from typing import Any
 from typing import Dict
 from typing import Union
@@ -20,6 +21,7 @@
 from .constants import SERIAL_COMM_METADATA_BYTES_LENGTH
 from .constants import SERIAL_COMM_PACKET_INFO_LENGTH_BYTES
 from .constants import SERIAL_COMM_STATUS_CODE_LENGTH_BYTES
+from .constants import SERIAL_COMM_TIMESTAMP_EPOCH
 from .constants import SERIAL_COMM_TIMESTAMP_LENGTH_BYTES
 from .constants import TAMPER_FLAG_UUID
 from .constants import TOTAL_WORKING_HOURS_UUID
@@ -53,9 +55,7 @@ def create_data_packet(
     packet_data: bytes,
 ) -> bytes:
     """Create a data packet to send to the PC."""
-    packet_body = timestamp.to_bytes(
-        SERIAL_COMM_TIMESTAMP_LENGTH_BYTES, byteorder="little"
-    )
+    packet_body = convert_to_timestamp_bytes(timestamp)
     packet_body += bytes([module_id, packet_type])
     packet_body += packet_data
     packet_length = len(packet_body) + SERIAL_COMM_CHECKSUM_LENGTH_BYTES
@@ -143,3 +143,14 @@ def convert_to_status_code_bytes(status_code: int) -> bytes:
     return status_code.to_bytes(
         SERIAL_COMM_STATUS_CODE_LENGTH_BYTES, byteorder="little"
     )
+
+
+def convert_to_timestamp_bytes(timestamp: int) -> bytes:
+    return timestamp.to_bytes(SERIAL_COMM_TIMESTAMP_LENGTH_BYTES, byteorder="little")
+
+
+# Tanner (4/7/21): This method should not be used in the simulator. It has its own way of determining the timestamp to send in order to behave more accurately like the real Mantarray instrument
+def get_serial_comm_timestamp() -> int:
+    return (
+        datetime.datetime.now(tz=datetime.timezone.utc) - SERIAL_COMM_TIMESTAMP_EPOCH
+    ) // datetime.timedelta(microseconds=1)