From 0af50a57431b549fc0abbfa64b55940be7bb2d68 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Fri, 20 Dec 2024 15:52:26 -0700
Subject: [PATCH 01/84] Add shutdown_socket()

---
 core/federated/network/socket_common.c        | 31 +++++++++++++++++++
 .../core/federated/network/socket_common.h    |  9 ++++++
 2 files changed, 40 insertions(+)

diff --git a/core/federated/network/socket_common.c b/core/federated/network/socket_common.c
index 20fff0585..4fd5f319b 100644
--- a/core/federated/network/socket_common.c
+++ b/core/federated/network/socket_common.c
@@ -410,3 +410,34 @@ void write_to_socket_fail_on_error(int* socket, size_t num_bytes, unsigned char*
     }
   }
 }
+
+int shutdown_socket(int* socket, bool read_before_closing) {
+  if (!read_before_closing) {
+    if (shutdown(*socket, SHUT_RDWR)) {
+      lf_print_log("On shut down TCP socket, received reply: %s", strerror(errno));
+      return -1;
+    }
+  } else {
+    // According to this: https://stackoverflow.com/questions/4160347/close-vs-shutdown-socket,
+    // the close should happen when receiving a 0 length message from the other end.
+    // Here, we just signal the other side that no further writes to the socket are
+    // forthcoming, which should result in the other end getting a zero-length reception.
+    if (shutdown(*socket, SHUT_WR)) {
+      lf_print_log("On shut down TCP socket, received reply: %s", strerror(errno));
+      return -1;
+    }
+
+    // Wait for the other end to send an EOF or a socket error to occur.
+    // Discard any incoming bytes. Normally, this read should return 0 because
+    // the federate is resigning and should itself invoke shutdown.
+    unsigned char buffer[10];
+    while (read(*socket, buffer, 10) > 0)
+      ;
+  }
+  if (close(*socket)) {
+    lf_print_log("Error while closing socket: %s\n", strerror(errno));
+    return -1;
+  }
+  *socket = -1;
+  return 0;
+}
diff --git a/include/core/federated/network/socket_common.h b/include/core/federated/network/socket_common.h
index 71570c125..de22bf4b3 100644
--- a/include/core/federated/network/socket_common.h
+++ b/include/core/federated/network/socket_common.h
@@ -240,4 +240,13 @@ int write_to_socket_close_on_error(int* socket, size_t num_bytes, unsigned char*
 void write_to_socket_fail_on_error(int* socket, size_t num_bytes, unsigned char* buffer, lf_mutex_t* mutex,
                                    char* format, ...);
 
+/**
+ * @brief Gracefully shuts down and closes a socket, optionally reading until EOF.
+ * 
+ * @param socket Pointer to the socket descriptor to shutdown and close.
+ * @param read_before_closing If true, read until EOF before closing the socket.
+ * @return int Returns 0 on success, -1 on failure (errno will indicate the error).
+ */
+int shutdown_socket(int* socket, bool read_before_closing);
+
 #endif /* SOCKET_COMMON_H */

From e5129c574db2fa98834f8fdc82c6a6f056363155 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Fri, 20 Dec 2024 16:17:50 -0700
Subject: [PATCH 02/84] Use shutdown_socket() in rti_remote.c and federate.c

---
 core/federated/RTI/rti_remote.c        | 51 ++++----------------------
 core/federated/federate.c              | 21 ++---------
 core/federated/network/socket_common.c | 28 ++++++++------
 3 files changed, 28 insertions(+), 72 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index c5bd02955..34075d917 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -871,14 +871,7 @@ static void handle_federate_failed(federate_info_t* my_fed) {
   // Indicate that there will no further events from this federate.
   my_fed->enclave.next_event = FOREVER_TAG;
 
-  // According to this: https://stackoverflow.com/questions/4160347/close-vs-shutdown-socket,
-  // the close should happen when receiving a 0 length message from the other end.
-  // Here, we just signal the other side that no further writes to the socket are
-  // forthcoming, which should result in the other end getting a zero-length reception.
-  shutdown(my_fed->socket, SHUT_RDWR);
-
-  // We can now safely close the socket.
-  close(my_fed->socket); //  from unistd.h
+  shutdown_socket(&my_fed->socket, false);
 
   // Check downstream federates to see whether they should now be granted a TAG.
   // To handle cycles, need to create a boolean array to keep
@@ -917,21 +910,7 @@ static void handle_federate_resign(federate_info_t* my_fed) {
   // Indicate that there will no further events from this federate.
   my_fed->enclave.next_event = FOREVER_TAG;
 
-  // According to this: https://stackoverflow.com/questions/4160347/close-vs-shutdown-socket,
-  // the close should happen when receiving a 0 length message from the other end.
-  // Here, we just signal the other side that no further writes to the socket are
-  // forthcoming, which should result in the other end getting a zero-length reception.
-  shutdown(my_fed->socket, SHUT_WR);
-
-  // Wait for the federate to send an EOF or a socket error to occur.
-  // Discard any incoming bytes. Normally, this read should return 0 because
-  // the federate is resigning and should itself invoke shutdown.
-  unsigned char buffer[10];
-  while (read(my_fed->socket, buffer, 10) > 0)
-    ;
-
-  // We can now safely close the socket.
-  close(my_fed->socket); //  from unistd.h
+  shutdown_socket(&my_fed->socket, true);
 
   // Check downstream federates to see whether they should now be granted a TAG.
   // To handle cycles, need to create a boolean array to keep
@@ -1030,9 +1009,7 @@ void send_reject(int* socket_id, unsigned char error_code) {
     lf_print_warning("RTI failed to write MSG_TYPE_REJECT message on the socket.");
   }
   // Close the socket.
-  shutdown(*socket_id, SHUT_RDWR);
-  close(*socket_id);
-  *socket_id = -1;
+  shutdown_socket(socket_id, false);
   LF_MUTEX_UNLOCK(&rti_mutex);
 }
 
@@ -1420,9 +1397,7 @@ void lf_connect_to_federates(int socket_descriptor) {
       if (!authenticate_federate(&socket_id)) {
         lf_print_warning("RTI failed to authenticate the incoming federate.");
         // Close the socket.
-        shutdown(socket_id, SHUT_RDWR);
-        close(socket_id);
-        socket_id = -1;
+        shutdown_socket(&socket_id, false);
         // Ignore the federate that failed authentication.
         i--;
         continue;
@@ -1490,8 +1465,7 @@ void* respond_to_erroneous_connections(void* nothing) {
       lf_print_warning("RTI failed to write FEDERATION_ID_DOES_NOT_MATCH to erroneous incoming connection.");
     }
     // Close the socket.
-    shutdown(socket_id, SHUT_RDWR);
-    close(socket_id);
+    shutdown_socket(&socket_id, false);
   }
   return NULL;
 }
@@ -1554,21 +1528,10 @@ void wait_for_federates(int socket_descriptor) {
   // Shutdown and close the socket that is listening for incoming connections
   // so that the accept() call in respond_to_erroneous_connections returns.
   // That thread should then check rti->all_federates_exited and it should exit.
-  if (shutdown(socket_descriptor, SHUT_RDWR)) {
-    LF_PRINT_LOG("On shut down TCP socket, received reply: %s", strerror(errno));
-  }
-  // NOTE: In all common TCP/IP stacks, there is a time period,
-  // typically between 30 and 120 seconds, called the TIME_WAIT period,
-  // before the port is released after this close. This is because
-  // the OS is preventing another program from accidentally receiving
-  // duplicated packets intended for this program.
-  close(socket_descriptor);
+  shutdown_socket(&socket_descriptor, false);
 
   if (rti_remote->socket_descriptor_UDP > 0) {
-    if (shutdown(rti_remote->socket_descriptor_UDP, SHUT_RDWR)) {
-      LF_PRINT_LOG("On shut down UDP socket, received reply: %s", strerror(errno));
-    }
-    close(rti_remote->socket_descriptor_UDP);
+    shutdown_socket(&rti_remote->socket_descriptor_UDP, false);
   }
 }
 
diff --git a/core/federated/federate.c b/core/federated/federate.c
index 4b12f8b53..dbdb691c1 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -417,14 +417,12 @@ static void close_inbound_socket(int fed_id, int flag) {
   if (_fed.sockets_for_inbound_p2p_connections[fed_id] >= 0) {
     if (flag >= 0) {
       if (flag > 0) {
-        shutdown(_fed.sockets_for_inbound_p2p_connections[fed_id], SHUT_RDWR);
+        shutdown_socket(&_fed.sockets_for_inbound_p2p_connections[fed_id], false);
       } else {
         // Have received EOF from the other end. Send EOF to the other end.
-        shutdown(_fed.sockets_for_inbound_p2p_connections[fed_id], SHUT_WR);
+        shutdown_socket(&_fed.sockets_for_inbound_p2p_connections[fed_id], true);
       }
     }
-    close(_fed.sockets_for_inbound_p2p_connections[fed_id]);
-    _fed.sockets_for_inbound_p2p_connections[fed_id] = -1;
   }
   LF_MUTEX_UNLOCK(&socket_mutex);
 }
@@ -837,20 +835,9 @@ static void close_outbound_socket(int fed_id, int flag) {
   if (_fed.sockets_for_outbound_p2p_connections[fed_id] >= 0) {
     // Close the socket by sending a FIN packet indicating that no further writes
     // are expected.  Then read until we get an EOF indication.
-    if (flag >= 0) {
-      // SHUT_WR indicates no further outgoing messages.
-      shutdown(_fed.sockets_for_outbound_p2p_connections[fed_id], SHUT_WR);
-      if (flag > 0) {
-        // Have not received EOF yet. read until we get an EOF or error indication.
-        // This compensates for delayed ACKs and disabling of Nagles algorithm
-        // by delaying exiting until the shutdown is complete.
-        unsigned char message[32];
-        while (read(_fed.sockets_for_outbound_p2p_connections[fed_id], &message, 32) > 0)
-          ;
-      }
+    if (flag > 0) {
+      shutdown_socket(&_fed.sockets_for_outbound_p2p_connections[fed_id], true);
     }
-    close(_fed.sockets_for_outbound_p2p_connections[fed_id]);
-    _fed.sockets_for_outbound_p2p_connections[fed_id] = -1;
   }
   if (_lf_normal_termination) {
     LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex);
diff --git a/core/federated/network/socket_common.c b/core/federated/network/socket_common.c
index 4fd5f319b..6d12a8657 100644
--- a/core/federated/network/socket_common.c
+++ b/core/federated/network/socket_common.c
@@ -313,10 +313,7 @@ int read_from_socket_close_on_error(int* socket, size_t num_bytes, unsigned char
     // Read failed.
     // Socket has probably been closed from the other side.
     // Shut down and close the socket from this side.
-    shutdown(*socket, SHUT_RDWR);
-    close(*socket);
-    // Mark the socket closed.
-    *socket = -1;
+    shutdown_socket(socket, false);
     return -1;
   }
   return 0;
@@ -383,10 +380,7 @@ int write_to_socket_close_on_error(int* socket, size_t num_bytes, unsigned char*
     // Write failed.
     // Socket has probably been closed from the other side.
     // Shut down and close the socket from this side.
-    shutdown(*socket, SHUT_RDWR);
-    close(*socket);
-    // Mark the socket closed.
-    *socket = -1;
+    shutdown_socket(socket, false);
   }
   return result;
 }
@@ -414,7 +408,7 @@ void write_to_socket_fail_on_error(int* socket, size_t num_bytes, unsigned char*
 int shutdown_socket(int* socket, bool read_before_closing) {
   if (!read_before_closing) {
     if (shutdown(*socket, SHUT_RDWR)) {
-      lf_print_log("On shut down TCP socket, received reply: %s", strerror(errno));
+      lf_print_warning("On shut down TCP socket, received reply: %s", strerror(errno));
       return -1;
     }
   } else {
@@ -422,20 +416,32 @@ int shutdown_socket(int* socket, bool read_before_closing) {
     // the close should happen when receiving a 0 length message from the other end.
     // Here, we just signal the other side that no further writes to the socket are
     // forthcoming, which should result in the other end getting a zero-length reception.
+
+        // Close the socket by sending a FIN packet indicating that no further writes
+    // are expected.  Then read until we get an EOF indication.
     if (shutdown(*socket, SHUT_WR)) {
-      lf_print_log("On shut down TCP socket, received reply: %s", strerror(errno));
+      lf_print_warning("On shut down socket, received reply: %s", strerror(errno));
       return -1;
     }
 
     // Wait for the other end to send an EOF or a socket error to occur.
     // Discard any incoming bytes. Normally, this read should return 0 because
     // the federate is resigning and should itself invoke shutdown.
+
+            // Have not received EOF yet. read until we get an EOF or error indication.
+        // This compensates for delayed ACKs and disabling of Nagles algorithm
+        // by delaying exiting until the shutdown is complete.
     unsigned char buffer[10];
     while (read(*socket, buffer, 10) > 0)
       ;
   }
+  // NOTE: In all common TCP/IP stacks, there is a time period,
+  // typically between 30 and 120 seconds, called the TIME_WAIT period,
+  // before the port is released after this close. This is because
+  // the OS is preventing another program from accidentally receiving
+  // duplicated packets intended for this program.
   if (close(*socket)) {
-    lf_print_log("Error while closing socket: %s\n", strerror(errno));
+    lf_print_warning("Error while closing socket: %s\n", strerror(errno));
     return -1;
   }
   *socket = -1;

From 1b00ed37f915cb97ac33c6428a4045fc0d6ea8b7 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Fri, 20 Dec 2024 16:26:50 -0700
Subject: [PATCH 03/84] Fix comments.

---
 core/federated/network/socket_common.c | 24 +++++++++---------------
 1 file changed, 9 insertions(+), 15 deletions(-)

diff --git a/core/federated/network/socket_common.c b/core/federated/network/socket_common.c
index 6d12a8657..330db1b27 100644
--- a/core/federated/network/socket_common.c
+++ b/core/federated/network/socket_common.c
@@ -412,25 +412,19 @@ int shutdown_socket(int* socket, bool read_before_closing) {
       return -1;
     }
   } else {
-    // According to this: https://stackoverflow.com/questions/4160347/close-vs-shutdown-socket,
-    // the close should happen when receiving a 0 length message from the other end.
-    // Here, we just signal the other side that no further writes to the socket are
-    // forthcoming, which should result in the other end getting a zero-length reception.
-
-        // Close the socket by sending a FIN packet indicating that no further writes
-    // are expected.  Then read until we get an EOF indication.
+    // Signal the other side that no further writes are expected by sending a FIN packet.
+    // This indicates the write direction is closed. For more details, refer to:
+    // https://stackoverflow.com/questions/4160347/close-vs-shutdown-socket
     if (shutdown(*socket, SHUT_WR)) {
-      lf_print_warning("On shut down socket, received reply: %s", strerror(errno));
+      lf_print_warning("Failed to shut down socket: %s", strerror(errno));
       return -1;
     }
 
-    // Wait for the other end to send an EOF or a socket error to occur.
-    // Discard any incoming bytes. Normally, this read should return 0 because
-    // the federate is resigning and should itself invoke shutdown.
-
-            // Have not received EOF yet. read until we get an EOF or error indication.
-        // This compensates for delayed ACKs and disabling of Nagles algorithm
-        // by delaying exiting until the shutdown is complete.
+    // Wait for the other side to send an EOF or encounter a socket error.
+    // Discard any incoming bytes. Normally, this read should return 0, indicating the peer has also closed the
+    // connection.
+    // This compensates for delayed ACKs and scenarios where Nagle's algorithm is disabled, ensuring the shutdown
+    // completes gracefully.
     unsigned char buffer[10];
     while (read(*socket, buffer, 10) > 0)
       ;

From de2fbdeed88173fe3d2d761c497ce23be3a6a6d2 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Fri, 20 Dec 2024 17:11:55 -0700
Subject: [PATCH 04/84] Fix port type && change to print_log on shutdown
 failures && Refactor close_inbound_socket()

---
 core/federated/federate.c              | 25 ++++++++-----------------
 core/federated/network/socket_common.c |  8 ++++----
 2 files changed, 12 insertions(+), 21 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index dbdb691c1..2fcadda50 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -403,26 +403,15 @@ static trigger_handle_t schedule_message_received_from_network_locked(environmen
 
 /**
  * Close the socket that receives incoming messages from the
- * specified federate ID. This function should be called when a read
- * of incoming socket fails or when an EOF is received.
- * It can also be called when the receiving end wants to stop communication,
- * in which case, flag should be 1.
+ * specified federate ID.
  *
  * @param fed_id The ID of the peer federate sending messages to this
  *  federate.
- * @param flag 0 if an EOF was received, -1 if a socket error occurred, 1 otherwise.
  */
-static void close_inbound_socket(int fed_id, int flag) {
+static void close_inbound_socket(int fed_id) {
   LF_MUTEX_LOCK(&socket_mutex);
   if (_fed.sockets_for_inbound_p2p_connections[fed_id] >= 0) {
-    if (flag >= 0) {
-      if (flag > 0) {
-        shutdown_socket(&_fed.sockets_for_inbound_p2p_connections[fed_id], false);
-      } else {
-        // Have received EOF from the other end. Send EOF to the other end.
-        shutdown_socket(&_fed.sockets_for_inbound_p2p_connections[fed_id], true);
-      }
-    }
+    shutdown_socket(&_fed.sockets_for_inbound_p2p_connections[fed_id], false);
   }
   LF_MUTEX_UNLOCK(&socket_mutex);
 }
@@ -663,7 +652,7 @@ static int handle_tagged_message(int* socket, int fed_id) {
                      env->current_tag.time - start_time, env->current_tag.microstep, intended_tag.time - start_time,
                      intended_tag.microstep);
       // Close socket, reading any incoming data and discarding it.
-      close_inbound_socket(fed_id, 1);
+      close_inbound_socket(fed_id);
     } else {
       // Need to use intended_tag here, not actual_tag, so that STP violations are detected.
       // It will become actual_tag (that is when the reactions will be invoked).
@@ -1640,7 +1629,7 @@ void lf_terminate_execution(environment_t* env) {
   LF_PRINT_DEBUG("Closing incoming P2P sockets.");
   // Close any incoming P2P sockets that are still open.
   for (int i = 0; i < NUMBER_OF_FEDERATES; i++) {
-    close_inbound_socket(i, 1);
+    close_inbound_socket(i);
     // Ignore errors. Mark the socket closed.
     _fed.sockets_for_inbound_p2p_connections[i] = -1;
   }
@@ -1930,9 +1919,11 @@ void lf_connect_to_rti(const char* hostname, int port) {
 
 void lf_create_server(int specified_port) {
   assert(specified_port <= UINT16_MAX && specified_port >= 0);
-  if (create_TCP_server(specified_port, &_fed.server_socket, (uint16_t*)&_fed.server_port, false)) {
+  uint16_t port;
+  if (create_TCP_server(specified_port, &_fed.server_socket, &port, false)) {
     lf_print_error_system_failure("RTI failed to create TCP server: %s.", strerror(errno));
   };
+  _fed.server_port = (int)port;
   LF_PRINT_LOG("Server for communicating with other federates started using port %d.", _fed.server_port);
 
   // Send the server port number to the RTI
diff --git a/core/federated/network/socket_common.c b/core/federated/network/socket_common.c
index 330db1b27..92bfec84b 100644
--- a/core/federated/network/socket_common.c
+++ b/core/federated/network/socket_common.c
@@ -155,7 +155,7 @@ static int create_server(uint16_t port, int* final_socket, uint16_t* final_port,
     return -1;
   }
   set_socket_timeout_option(socket_descriptor, &timeout_time);
-  int used_port = set_socket_bind_option(socket_descriptor, port, increment_port_on_retry);
+  uint16_t used_port = set_socket_bind_option(socket_descriptor, port, increment_port_on_retry);
   if (sock_type == 0) {
     // Enable listening for socket connections.
     // The second argument is the maximum number of queued socket requests,
@@ -408,7 +408,7 @@ void write_to_socket_fail_on_error(int* socket, size_t num_bytes, unsigned char*
 int shutdown_socket(int* socket, bool read_before_closing) {
   if (!read_before_closing) {
     if (shutdown(*socket, SHUT_RDWR)) {
-      lf_print_warning("On shut down TCP socket, received reply: %s", strerror(errno));
+      lf_print_log("On shutdown socket, received reply: %s", strerror(errno));
       return -1;
     }
   } else {
@@ -416,7 +416,7 @@ int shutdown_socket(int* socket, bool read_before_closing) {
     // This indicates the write direction is closed. For more details, refer to:
     // https://stackoverflow.com/questions/4160347/close-vs-shutdown-socket
     if (shutdown(*socket, SHUT_WR)) {
-      lf_print_warning("Failed to shut down socket: %s", strerror(errno));
+      lf_print_log("Failed to shutdown socket: %s", strerror(errno));
       return -1;
     }
 
@@ -435,7 +435,7 @@ int shutdown_socket(int* socket, bool read_before_closing) {
   // the OS is preventing another program from accidentally receiving
   // duplicated packets intended for this program.
   if (close(*socket)) {
-    lf_print_warning("Error while closing socket: %s\n", strerror(errno));
+    lf_print_log("Error while closing socket: %s\n", strerror(errno));
     return -1;
   }
   *socket = -1;

From 9ea9f1d8484e92f1e9b673d99b645f89e49d1d0f Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Fri, 20 Dec 2024 17:16:52 -0700
Subject: [PATCH 05/84] Refactor close_outbound_socket().

---
 core/federated/federate.c | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index 2fcadda50..e7face46f 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -814,22 +814,22 @@ static void* listen_to_federates(void* _args) {
  * if _lf_normal_termination is true and otherwise proceeds without the lock.
  * @param fed_id The ID of the peer federate receiving messages from this
  *  federate, or -1 if the RTI (centralized coordination).
- * @param flag 0 if the socket has received EOF, 1 if not, -1 if abnormal termination.
  */
-static void close_outbound_socket(int fed_id, int flag) {
+static void close_outbound_socket(int fed_id) {
   assert(fed_id >= 0 && fed_id < NUMBER_OF_FEDERATES);
+  // Close outbound connections, in case they have not closed themselves.
+  // This will result in EOF being sent to the remote federate, except for
+  // abnormal termination, in which case it will just close the socket.
   if (_lf_normal_termination) {
     LF_MUTEX_LOCK(&lf_outbound_socket_mutex);
-  }
-  if (_fed.sockets_for_outbound_p2p_connections[fed_id] >= 0) {
-    // Close the socket by sending a FIN packet indicating that no further writes
-    // are expected.  Then read until we get an EOF indication.
-    if (flag > 0) {
+    if (_fed.sockets_for_outbound_p2p_connections[fed_id] >= 0) {
+      // Close the socket by sending a FIN packet indicating that no further writes
+      // are expected.  Then read until we get an EOF indication.
       shutdown_socket(&_fed.sockets_for_outbound_p2p_connections[fed_id], true);
     }
-  }
-  if (_lf_normal_termination) {
     LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex);
+  } else {
+    shutdown_socket(&_fed.sockets_for_outbound_p2p_connections[fed_id], false);
   }
 }
 
@@ -1643,8 +1643,7 @@ void lf_terminate_execution(environment_t* env) {
     // Close outbound connections, in case they have not closed themselves.
     // This will result in EOF being sent to the remote federate, except for
     // abnormal termination, in which case it will just close the socket.
-    int flag = _lf_normal_termination ? 1 : -1;
-    close_outbound_socket(i, flag);
+    close_outbound_socket(i);
   }
 
   LF_PRINT_DEBUG("Waiting for inbound p2p socket listener threads.");

From 996896c05bad0d601d87f02a10b8be6cb19b305a Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Fri, 20 Dec 2024 18:23:59 -0700
Subject: [PATCH 06/84] Add comment to shutdown_socket function.

---
 include/core/federated/network/socket_common.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/include/core/federated/network/socket_common.h b/include/core/federated/network/socket_common.h
index de22bf4b3..14452b65a 100644
--- a/include/core/federated/network/socket_common.h
+++ b/include/core/federated/network/socket_common.h
@@ -242,7 +242,9 @@ void write_to_socket_fail_on_error(int* socket, size_t num_bytes, unsigned char*
 
 /**
  * @brief Gracefully shuts down and closes a socket, optionally reading until EOF.
- * 
+ * Shutdown and close the socket. If read_before_closing is false, it just immediately calls shutdown() with SHUT_RDWR
+ * and close(). If read_before_closing is true, it calls shutdown with SHUT_WR, only disallowing further writing. Then,
+ * it calls read() until EOF is received, and discards all received bytes.
  * @param socket Pointer to the socket descriptor to shutdown and close.
  * @param read_before_closing If true, read until EOF before closing the socket.
  * @return int Returns 0 on success, -1 on failure (errno will indicate the error).

From e3b0ea1ffcdb85421e50a194034bc07c8cb6f178 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Fri, 20 Dec 2024 20:56:20 -0700
Subject: [PATCH 07/84] Add commnets.

---
 core/federated/RTI/rti_remote.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index cc654e3e0..4e5778b2e 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -1008,7 +1008,7 @@ void send_reject(int* socket_id, unsigned char error_code) {
   if (write_to_socket(*socket_id, 2, response)) {
     lf_print_warning("RTI failed to write MSG_TYPE_REJECT message on the socket.");
   }
-  // Close the socket.
+  // Close the socket without reading until EOF.
   shutdown_socket(socket_id, false);
   LF_MUTEX_UNLOCK(&rti_mutex);
 }
@@ -1394,7 +1394,7 @@ void lf_connect_to_federates(int socket_descriptor) {
     if (rti_remote->authentication_enabled) {
       if (!authenticate_federate(&socket_id)) {
         lf_print_warning("RTI failed to authenticate the incoming federate.");
-        // Close the socket.
+        // Close the socket without reading until EOF.
         shutdown_socket(&socket_id, false);
         // Ignore the federate that failed authentication.
         i--;
@@ -1462,7 +1462,7 @@ void* respond_to_erroneous_connections(void* nothing) {
     if (write_to_socket(socket_id, 2, response)) {
       lf_print_warning("RTI failed to write FEDERATION_ID_DOES_NOT_MATCH to erroneous incoming connection.");
     }
-    // Close the socket.
+    // Close the socket without reading until EOF.
     shutdown_socket(&socket_id, false);
   }
   return NULL;

From 30a94469b76205fc65f04bba36e4f1717c80e37f Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sat, 21 Dec 2024 14:27:51 -0700
Subject: [PATCH 08/84] Properly replace close() to shutdown_socket()

---
 core/federated/RTI/rti_remote.c | 14 ++++++--------
 core/federated/federate.c       |  6 +++---
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index 4e5778b2e..63d266f05 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -939,7 +939,11 @@ void* federate_info_thread_TCP(void* fed) {
       // Socket is closed
       lf_print_error("RTI: Socket to federate %d is closed. Exiting the thread.", my_fed->enclave.id);
       my_fed->enclave.state = NOT_CONNECTED;
-      my_fed->socket = -1;
+      // Nothing more to do. Close the socket and exit.
+      // Prevent multiple threads from closing the same socket at the same time.
+      LF_MUTEX_LOCK(&rti_mutex);
+      shutdown_socket(my_fed->socket, false); //  from unistd.h
+      LF_MUTEX_UNLOCK(&rti_mutex);
       // FIXME: We need better error handling here, but do not stop execution here.
       break;
     }
@@ -989,12 +993,6 @@ void* federate_info_thread_TCP(void* fed) {
       }
     }
   }
-
-  // Nothing more to do. Close the socket and exit.
-  // Prevent multiple threads from closing the same socket at the same time.
-  LF_MUTEX_LOCK(&rti_mutex);
-  close(my_fed->socket); //  from unistd.h
-  LF_MUTEX_UNLOCK(&rti_mutex);
   return NULL;
 }
 
@@ -1445,7 +1443,7 @@ void* respond_to_erroneous_connections(void* nothing) {
   while (true) {
     // Wait for an incoming connection request.
     // The following will block until either a federate attempts to connect
-    // or close(rti->socket_descriptor_TCP) is called.
+    // or shutdown_socket(rti->socket_descriptor_TCP) is called.
     int socket_id = accept_socket(rti_remote->socket_descriptor_TCP, -1);
     if (socket_id < 0) {
       return NULL;
diff --git a/core/federated/federate.c b/core/federated/federate.c
index 13da24aa0..3b135ea9f 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -1993,7 +1993,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
         // Ignore errors on this response.
         write_to_socket(socket_id, 2, response);
       }
-      close(socket_id);
+      shutdown_socket(socket_id, false);
       continue;
     }
 
@@ -2013,7 +2013,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
         // Ignore errors on this response.
         write_to_socket(socket_id, 2, response);
       }
-      close(socket_id);
+      shutdown_socket(socket_id, false);
       continue;
     }
 
@@ -2051,7 +2051,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
       // Failed to create a listening thread.
       LF_MUTEX_LOCK(&socket_mutex);
       if (_fed.sockets_for_inbound_p2p_connections[remote_fed_id] != -1) {
-        close(socket_id);
+        shutdown_socket(socket_id, false);
         _fed.sockets_for_inbound_p2p_connections[remote_fed_id] = -1;
       }
       LF_MUTEX_UNLOCK(&socket_mutex);

From 276d7b4d1e635e1f2c5360d68155389717d281aa Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sat, 21 Dec 2024 14:32:30 -0700
Subject: [PATCH 09/84] Minor fix.

---
 core/federated/RTI/rti_remote.c | 4 ++--
 core/federated/federate.c       | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index 63d266f05..42d2be3b7 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -942,7 +942,7 @@ void* federate_info_thread_TCP(void* fed) {
       // Nothing more to do. Close the socket and exit.
       // Prevent multiple threads from closing the same socket at the same time.
       LF_MUTEX_LOCK(&rti_mutex);
-      shutdown_socket(my_fed->socket, false); //  from unistd.h
+      shutdown_socket(&my_fed->socket, false); //  from unistd.h
       LF_MUTEX_UNLOCK(&rti_mutex);
       // FIXME: We need better error handling here, but do not stop execution here.
       break;
@@ -1007,7 +1007,7 @@ void send_reject(int* socket_id, unsigned char error_code) {
     lf_print_warning("RTI failed to write MSG_TYPE_REJECT message on the socket.");
   }
   // Close the socket without reading until EOF.
-  shutdown_socket(socket_id, false);
+  shutdown_socket(&socket_id, false);
   LF_MUTEX_UNLOCK(&rti_mutex);
 }
 
diff --git a/core/federated/federate.c b/core/federated/federate.c
index 3b135ea9f..5af2b1f51 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -1993,7 +1993,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
         // Ignore errors on this response.
         write_to_socket(socket_id, 2, response);
       }
-      shutdown_socket(socket_id, false);
+      shutdown_socket(&socket_id, false);
       continue;
     }
 
@@ -2013,7 +2013,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
         // Ignore errors on this response.
         write_to_socket(socket_id, 2, response);
       }
-      shutdown_socket(socket_id, false);
+      shutdown_socket(&socket_id, false);
       continue;
     }
 
@@ -2051,7 +2051,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
       // Failed to create a listening thread.
       LF_MUTEX_LOCK(&socket_mutex);
       if (_fed.sockets_for_inbound_p2p_connections[remote_fed_id] != -1) {
-        shutdown_socket(socket_id, false);
+        shutdown_socket(&socket_id, false);
         _fed.sockets_for_inbound_p2p_connections[remote_fed_id] = -1;
       }
       LF_MUTEX_UNLOCK(&socket_mutex);

From 93adf2ee3e8c0cedb2dd04656c05b91b144ce6e4 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sat, 21 Dec 2024 14:57:53 -0700
Subject: [PATCH 10/84] Minor fix.

---
 core/federated/RTI/rti_remote.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index 42d2be3b7..8e49c4c88 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -1007,7 +1007,7 @@ void send_reject(int* socket_id, unsigned char error_code) {
     lf_print_warning("RTI failed to write MSG_TYPE_REJECT message on the socket.");
   }
   // Close the socket without reading until EOF.
-  shutdown_socket(&socket_id, false);
+  shutdown_socket(socket_id, false);
   LF_MUTEX_UNLOCK(&rti_mutex);
 }
 

From d27c7430bcef0c37ee382aa3b48aa1c84cd0ef93 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sat, 21 Dec 2024 14:59:10 -0700
Subject: [PATCH 11/84] Properly close federate's socket connected to the RTI.

---
 core/federated/federate.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index 5af2b1f51..c277f54bb 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -1493,7 +1493,7 @@ static void* listen_to_rti_TCP(void* args) {
         lf_print_error("Socket connection to the RTI was closed by the RTI without"
                        " properly sending an EOF first. Considering this a soft error.");
         // FIXME: If this happens, possibly a new RTI must be elected.
-        _fed.socket_TCP_RTI = -1;
+        shutdown_socket(_fed.socket_TCP_RTI, false);
         return NULL;
       } else {
         lf_print_error("Socket connection to the RTI has been broken with error %d: %s."
@@ -1501,13 +1501,13 @@ static void* listen_to_rti_TCP(void* args) {
                        " Considering this a soft error.",
                        errno, strerror(errno));
         // FIXME: If this happens, possibly a new RTI must be elected.
-        _fed.socket_TCP_RTI = -1;
+        shutdown_socket(_fed.socket_TCP_RTI, false);
         return NULL;
       }
     } else if (read_failed > 0) {
       // EOF received.
       lf_print("Connection to the RTI closed with an EOF.");
-      _fed.socket_TCP_RTI = -1;
+      shutdown_socket(_fed.socket_TCP_RTI, false);
       return NULL;
     }
     switch (buffer[0]) {

From ef3345b9a7809616a40939f214ab06a5bb9ec61a Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sat, 21 Dec 2024 15:00:37 -0700
Subject: [PATCH 12/84] Minor fix.

---
 core/federated/federate.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index c277f54bb..18bd244f5 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -1493,7 +1493,7 @@ static void* listen_to_rti_TCP(void* args) {
         lf_print_error("Socket connection to the RTI was closed by the RTI without"
                        " properly sending an EOF first. Considering this a soft error.");
         // FIXME: If this happens, possibly a new RTI must be elected.
-        shutdown_socket(_fed.socket_TCP_RTI, false);
+        shutdown_socket(&_fed.socket_TCP_RTI, false);
         return NULL;
       } else {
         lf_print_error("Socket connection to the RTI has been broken with error %d: %s."
@@ -1501,13 +1501,13 @@ static void* listen_to_rti_TCP(void* args) {
                        " Considering this a soft error.",
                        errno, strerror(errno));
         // FIXME: If this happens, possibly a new RTI must be elected.
-        shutdown_socket(_fed.socket_TCP_RTI, false);
+        shutdown_socket(&_fed.socket_TCP_RTI, false);
         return NULL;
       }
     } else if (read_failed > 0) {
       // EOF received.
       lf_print("Connection to the RTI closed with an EOF.");
-      shutdown_socket(_fed.socket_TCP_RTI, false);
+      shutdown_socket(&_fed.socket_TCP_RTI, false);
       return NULL;
     }
     switch (buffer[0]) {

From c0d59dbb595eb13b6a40dbe157ba0c66f63ba3e2 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Mon, 13 Jan 2025 11:05:05 -0700
Subject: [PATCH 13/84] Fix merge error.

---
 core/federated/federate.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index 05339f331..51b9c2602 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -1920,7 +1920,6 @@ void lf_create_server(int specified_port) {
   if (create_server(specified_port, &_fed.server_socket, (uint16_t*)&_fed.server_port, TCP, false)) {
     lf_print_error_system_failure("RTI failed to create TCP server: %s.", strerror(errno));
   };
-  _fed.server_port = (int)port;
   LF_PRINT_LOG("Server for communicating with other federates started using port %d.", _fed.server_port);
 
   // Send the server port number to the RTI

From 45b7f48bc9acee8e81767ab5507b88d9ab86c4f0 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Wed, 15 Jan 2025 10:27:24 -0700
Subject: [PATCH 14/84] Move files to reactor-c/network/

---
 {include/core/federated/network => network/api}/net_common.h    | 0
 {include/core/federated/network => network/api}/net_util.h      | 0
 {include/core/federated/network => network/api}/socket_common.h | 0
 {core/federated/network => network/impl}/CMakeLists.txt         | 0
 {core/federated/network => network/impl/src}/net_util.c         | 0
 {core/federated/network => network/impl/src}/socket_common.c    | 0
 6 files changed, 0 insertions(+), 0 deletions(-)
 rename {include/core/federated/network => network/api}/net_common.h (100%)
 rename {include/core/federated/network => network/api}/net_util.h (100%)
 rename {include/core/federated/network => network/api}/socket_common.h (100%)
 rename {core/federated/network => network/impl}/CMakeLists.txt (100%)
 rename {core/federated/network => network/impl/src}/net_util.c (100%)
 rename {core/federated/network => network/impl/src}/socket_common.c (100%)

diff --git a/include/core/federated/network/net_common.h b/network/api/net_common.h
similarity index 100%
rename from include/core/federated/network/net_common.h
rename to network/api/net_common.h
diff --git a/include/core/federated/network/net_util.h b/network/api/net_util.h
similarity index 100%
rename from include/core/federated/network/net_util.h
rename to network/api/net_util.h
diff --git a/include/core/federated/network/socket_common.h b/network/api/socket_common.h
similarity index 100%
rename from include/core/federated/network/socket_common.h
rename to network/api/socket_common.h
diff --git a/core/federated/network/CMakeLists.txt b/network/impl/CMakeLists.txt
similarity index 100%
rename from core/federated/network/CMakeLists.txt
rename to network/impl/CMakeLists.txt
diff --git a/core/federated/network/net_util.c b/network/impl/src/net_util.c
similarity index 100%
rename from core/federated/network/net_util.c
rename to network/impl/src/net_util.c
diff --git a/core/federated/network/socket_common.c b/network/impl/src/socket_common.c
similarity index 100%
rename from core/federated/network/socket_common.c
rename to network/impl/src/socket_common.c

From 536cdc769f2aa79b9bc441afe7e3eb90c7d3b5b1 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Wed, 15 Jan 2025 10:28:30 -0700
Subject: [PATCH 15/84] Remove unnecessary #include on socket_common.h

---
 network/api/net_util.h | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/network/api/net_util.h b/network/api/net_util.h
index 1e9008816..7cd5faf65 100644
--- a/network/api/net_util.h
+++ b/network/api/net_util.h
@@ -51,9 +51,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "low_level_platform.h"
 #include "tag.h"
 
-#ifdef FEDERATED
-#include "socket_common.h"
-#endif
 
 #define HOST_LITTLE_ENDIAN 1
 #define HOST_BIG_ENDIAN 2

From 684ab06644db0eebe308f577b35659831ad2a770 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Wed, 15 Jan 2025 10:30:28 -0700
Subject: [PATCH 16/84] Add net_driver.

---
 core/federated/RTI/rti_remote.h      |  7 ++++++-
 core/federated/federate.c            |  1 +
 network/api/net_driver.h             | 13 +++++++++++++
 network/api/socket_common.h          |  1 +
 network/impl/src/lf_socket_support.c |  0
 network/impl/src/net_driver.c        |  1 +
 6 files changed, 22 insertions(+), 1 deletion(-)
 create mode 100644 network/api/net_driver.h
 create mode 100644 network/impl/src/lf_socket_support.c
 create mode 100644 network/impl/src/net_driver.c

diff --git a/core/federated/RTI/rti_remote.h b/core/federated/RTI/rti_remote.h
index de6b144aa..3dc56c180 100644
--- a/core/federated/RTI/rti_remote.h
+++ b/core/federated/RTI/rti_remote.h
@@ -31,7 +31,7 @@
 
 #include "lf_types.h"
 #include "pqueue_tag.h"
-#include "socket_common.h"
+#include "net_driver.h"
 
 /** Time allowed for federates to reply to stop request. */
 #define MAX_TIME_FOR_REPLY_TO_STOP_REQUEST SEC(30)
@@ -129,6 +129,11 @@ typedef struct rti_remote_t {
   /** The UDP socket descriptor for the socket server. */
   int socket_descriptor_UDP;
 
+  /**
+   * The rti's netdriver.
+   */
+  netdrv_t* rti_netdrv;
+
   /************* Clock synchronization information *************/
   /* Thread performing PTP clock sync sessions periodically. */
   lf_thread_t clock_thread;
diff --git a/core/federated/federate.c b/core/federated/federate.c
index 51b9c2602..a689ae090 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -30,6 +30,7 @@
 #include "federate.h"
 #include "net_common.h"
 #include "net_util.h"
+#include "net_driver.h"
 #include "reactor.h"
 #include "reactor_common.h"
 #include "reactor_threaded.h"
diff --git a/network/api/net_driver.h b/network/api/net_driver.h
new file mode 100644
index 000000000..993d22853
--- /dev/null
+++ b/network/api/net_driver.h
@@ -0,0 +1,13 @@
+#ifndef NET_DRIVER_H
+#define NET_DRIVER_H
+
+#include "socket_common.h"
+
+typedef struct netdrv_t {
+  void* priv;
+  //   unsigned int read_remaining_bytes;
+  //   int my_federate_id; // The RTI is -1, and unitialized is -2. This must be int not uint16_t
+  //   const char* federation_id;
+} netdrv_t;
+
+#endif /* NET_DRIVER_H */
diff --git a/network/api/socket_common.h b/network/api/socket_common.h
index c273b4b74..878f0b107 100644
--- a/network/api/socket_common.h
+++ b/network/api/socket_common.h
@@ -2,6 +2,7 @@
 #define SOCKET_COMMON_H
 
 #include "low_level_platform.h"
+#include <sys/types.h>
 
 /**
  * The amount of time to wait after a failed socket read or write before trying again. This defaults to 100 ms.
diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
new file mode 100644
index 000000000..e69de29bb
diff --git a/network/impl/src/net_driver.c b/network/impl/src/net_driver.c
new file mode 100644
index 000000000..4b6232348
--- /dev/null
+++ b/network/impl/src/net_driver.c
@@ -0,0 +1 @@
+#include "net_driver.h"
\ No newline at end of file

From 43960cfad2ac5652d5648e7ef786e13151d69d36 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Wed, 15 Jan 2025 10:31:23 -0700
Subject: [PATCH 17/84] Create CMake files to make the network related c files
 as a library.

---
 core/CMakeLists.txt               | 11 +++++++-
 core/federated/RTI/CMakeLists.txt | 17 +++++++++--
 network/api/CMakeLists.txt        | 14 +++++++++
 network/impl/CMakeLists.txt       | 47 +++++++++++++++++++++++++++++--
 4 files changed, 82 insertions(+), 7 deletions(-)
 create mode 100644 network/api/CMakeLists.txt

diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index 6d938ae0c..6f5f021c8 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -18,7 +18,6 @@ list(APPEND REACTORC_SOURCES ${GENERAL_SOURCES})
 # Add sources for either threaded or single-threaded runtime
 if(DEFINED FEDERATED)
     include(federated/CMakeLists.txt)
-    include(federated/network/CMakeLists.txt)
 endif()
 
 # Add sources for either threaded or single-threaded runtime
@@ -96,6 +95,16 @@ include(${LF_ROOT}/platform/impl/CMakeLists.txt)
 target_link_libraries(reactor-c PUBLIC lf::platform-api)
 target_link_libraries(reactor-c PRIVATE lf::platform-impl)
 
+option(COMM_TYPE "Communication type between RTI and federate(s)." ON)
+IF(COMM_TYPE MATCHES ON)
+   set(COMM_TYPE TCP)
+ENDIF()
+
+include(${LF_ROOT}/network/api/CMakeLists.txt)
+include(${LF_ROOT}/network/impl/CMakeLists.txt)
+target_link_libraries(reactor-c PUBLIC lf::network-api)
+target_link_libraries(reactor-c PRIVATE lf::network-impl)
+
 target_include_directories(reactor-c PUBLIC ../include)
 target_include_directories(reactor-c PUBLIC ../include/core)
 target_include_directories(reactor-c PUBLIC ../include/core/federated)
diff --git a/core/federated/RTI/CMakeLists.txt b/core/federated/RTI/CMakeLists.txt
index d9a93c246..fce1517a5 100644
--- a/core/federated/RTI/CMakeLists.txt
+++ b/core/federated/RTI/CMakeLists.txt
@@ -16,8 +16,6 @@ add_library(${RTI_LIB} STATIC
     ${CoreLib}/utils/util.c
     ${CoreLib}/tag.c
     ${CoreLib}/clock.c
-    ${CoreLib}/federated/network/net_util.c
-    ${CoreLib}/federated/network/socket_common.c
     ${CoreLib}/utils/pqueue_base.c
     ${CoreLib}/utils/pqueue_tag.c
     ${CoreLib}/utils/pqueue.c
@@ -29,7 +27,6 @@ add_executable(${RTI_MAIN} main.c)
 target_include_directories(${RTI_LIB} PUBLIC ../../../include)
 target_include_directories(${RTI_LIB} PUBLIC ${IncludeDir})
 target_include_directories(${RTI_LIB} PUBLIC ${IncludeDir}/federated)
-target_include_directories(${RTI_LIB} PUBLIC ${IncludeDir}/federated/network)
 target_include_directories(${RTI_LIB} PUBLIC ${IncludeDir}/modal_models)
 target_include_directories(${RTI_LIB} PUBLIC ${IncludeDir}/utils)
 
@@ -37,6 +34,11 @@ if (NOT DEFINED LOG_LEVEL)
     set(LOG_LEVEL 0)
 ENDIF(NOT DEFINED LOG_LEVEL)
 
+# option(COMM_TYPE "Communication type between RTI and federate(s)." ON)
+# IF(COMM_TYPE MATCHES ON)
+set(COMM_TYPE TCP)
+# ENDIF()
+
 IF(CMAKE_BUILD_TYPE MATCHES DEBUG)
     # Set the LOG_LEVEL to 4 to get DEBUG messages
     message("-- Building RTI with DEBUG messages enabled")
@@ -71,6 +73,12 @@ target_link_libraries(${RTI_LIB} PUBLIC  lf::low-level-platform-impl)
 include(${LF_ROOT}/low_level_platform/api/CMakeLists.txt)
 target_link_libraries(${RTI_LIB} PUBLIC  lf::low-level-platform-api)
 
+include(${LF_ROOT}/network/impl/CMakeLists.txt)
+target_link_libraries(${RTI_LIB} PUBLIC  lf::network-impl)
+
+include(${LF_ROOT}/network/api/CMakeLists.txt)
+target_link_libraries(${RTI_LIB} PUBLIC  lf::network-api)
+
 # Set the STANDALONE_RTI flag to include the rti_remote and rti_common.
 target_compile_definitions(${RTI_LIB} PUBLIC STANDALONE_RTI=1)
 
@@ -78,6 +86,9 @@ target_compile_definitions(${RTI_LIB} PUBLIC STANDALONE_RTI=1)
 target_compile_definitions(${RTI_LIB} PUBLIC FEDERATED=1)
 target_compile_definitions(${RTI_LIB} PUBLIC PLATFORM_${CMAKE_SYSTEM_NAME})
 
+# Set communication type.
+target_compile_definitions(${RTI_LIB} PUBLIC COMM_TYPE_${COMM_TYPE})
+
 # Set RTI Tracing
 target_compile_definitions(${RTI_LIB} PUBLIC RTI_TRACE)
 
diff --git a/network/api/CMakeLists.txt b/network/api/CMakeLists.txt
new file mode 100644
index 000000000..6f724fe1a
--- /dev/null
+++ b/network/api/CMakeLists.txt
@@ -0,0 +1,14 @@
+add_library(lf-network-api INTERFACE)
+target_include_directories(lf-network-api INTERFACE ${CMAKE_CURRENT_LIST_DIR})
+add_library(lf::network-api ALIAS lf-network-api)
+
+#Link necessary libraries.
+target_link_libraries(lf-network-api INTERFACE lf::tag-api)
+target_link_libraries(lf-network-api INTERFACE lf::low-level-platform-api)
+
+
+# #TODO:CHECK.
+# target_include_directories(lf-network-api INTERFACE ${CMAKE_CURRENT_LIST_DIR}/type)
+target_include_directories(lf-network-api INTERFACE ${CMAKE_CURRENT_LIST_DIR}/../../include/core/utils)
+
+target_compile_definitions(lf-network-api INTERFACE COMM_TYPE_${COMM_TYPE})
\ No newline at end of file
diff --git a/network/impl/CMakeLists.txt b/network/impl/CMakeLists.txt
index f61d69897..e54a9631a 100644
--- a/network/impl/CMakeLists.txt
+++ b/network/impl/CMakeLists.txt
@@ -1,4 +1,45 @@
-set(LF_NETWORK_FILES net_util.c socket_common.c)
+set(LF_ROOT ${CMAKE_CURRENT_LIST_DIR}/../..)
+include(${LF_ROOT}/core/lf_utils.cmake)
 
-list(TRANSFORM LF_NETWORK_FILES PREPEND federated/network/)
-list(APPEND REACTORC_SOURCES ${LF_NETWORK_FILES})
+add_library(lf-network-impl STATIC ${LF_NETWORK_FILES})
+add_library(lf::network-impl ALIAS lf-network-impl)
+target_sources(lf-network-impl PUBLIC
+    ${CMAKE_CURRENT_LIST_DIR}/src/net_util.c
+    ${CMAKE_CURRENT_LIST_DIR}/src/net_driver.c
+    ${CMAKE_CURRENT_LIST_DIR}/src/socket_common.c
+)
+
+if(COMM_TYPE MATCHES TCP)
+    target_sources(lf-network-impl PUBLIC ${CMAKE_CURRENT_LIST_DIR}/src/lf_socket_support.c)
+elseif(COMM_TYPE MATCHES MQTT)
+# # $ git clone https://github.com/eclipse/paho.mqtt.c.git
+# # $ cd paho.mqtt.c.git
+# # $ mkdir build.paho; cd build.paho
+# # $ cmake ../
+# # $ make
+# # $ sudo make install
+# # It will be installed in /usr/local/lib
+# find_package(eclipse-paho-mqtt-c REQUIRED)
+# target_link_libraries(lf-network-impl PRIVATE eclipse-paho-mqtt-c::paho-mqtt3a eclipse-paho-mqtt-c::paho-mqtt3c)
+# # $ apt-get install libssl-dev
+# # find_package(OpenSSL REQUIRED)
+
+# # target_link_libraries(lf-network-impl PUBLIC OpenSSL::SSL)
+# # target_sources(lf-network-impl PUBLIC ${CMAKE_CURRENT_LIST_DIR}/src/lf_mqtt_support.c)
+elseif(COMM_TYPE MATCHES SST)
+# target_link_libraries(lf-network-impl PUBLIC SSTLIB)
+# target_sources(lf-network-impl PUBLIC ${CMAKE_CURRENT_LIST_DIR}/src/lf_sst_support.c)
+# # add_compile_definitions(OPENSSL_REQUIRED)
+else()
+    message(FATAL_ERROR "Your communication type is not supported! The C target supports TCP, MQTT and SST.")
+endif()
+
+# Link necessary libraries
+target_link_libraries(lf-network-impl PUBLIC lf-logging-api)
+target_link_libraries(lf-network-impl PUBLIC lf-low-level-platform-api)
+target_link_libraries(lf-network-impl PUBLIC lf::tag-api)
+target_link_libraries(lf-network-impl PRIVATE lf-network-api)
+
+lf_enable_compiler_warnings(lf-network-impl)
+
+target_compile_definitions(lf-network-impl PUBLIC COMM_TYPE_${COMM_TYPE})

From 6b00c237d51e3c52062f8e565205c08861379f04 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Wed, 15 Jan 2025 11:58:49 -0700
Subject: [PATCH 18/84] Remove parameter from create_server()

---
 core/federated/RTI/main.c       | 6 +++++-
 core/federated/RTI/rti_remote.c | 6 ++++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/core/federated/RTI/main.c b/core/federated/RTI/main.c
index 17d73e93e..7a5d418cb 100644
--- a/core/federated/RTI/main.c
+++ b/core/federated/RTI/main.c
@@ -233,6 +233,7 @@ int process_args(int argc, const char* argv[]) {
       rti.base.number_of_scheduling_nodes = (int32_t)num_federates; // FIXME: Loses numbers on 64-bit machines
       lf_print("RTI: Number of federates: %d", rti.base.number_of_scheduling_nodes);
     } else if (strcmp(argv[i], "-p") == 0 || strcmp(argv[i], "--port") == 0) {
+#ifdef COMM_TYPE_TCP
       if (argc < i + 2) {
         lf_print_error("--port needs a short unsigned integer argument ( > 0 and < %d).", UINT16_MAX);
         usage(argc, argv);
@@ -246,6 +247,9 @@ int process_args(int argc, const char* argv[]) {
         return 0;
       }
       rti.user_specified_port = (uint16_t)RTI_port;
+#else
+      lf_print_error("--port is only available for TCP.");
+#endif
     } else if (strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "--clock_sync") == 0) {
       if (argc < i + 2) {
         lf_print_error("--clock-sync needs off|init|on.");
@@ -324,7 +328,7 @@ int main(int argc, const char* argv[]) {
     rti.base.scheduling_nodes[i] = (scheduling_node_t*)fed_info;
   }
 
-  int socket_descriptor = start_rti_server(rti.user_specified_port);
+  int socket_descriptor = start_rti_server();
   if (socket_descriptor >= 0) {
     wait_for_federates(socket_descriptor);
     normal_termination = true;
diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index d800dcaac..755912994 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -1477,10 +1477,10 @@ void initialize_federate(federate_info_t* fed, uint16_t id) {
   fed->server_port = -1;
 }
 
-int32_t start_rti_server(uint16_t port) {
+int32_t start_rti_server() {
   _lf_initialize_clock();
   // Create the TCP socket server
-  if (create_server(port, &rti_remote->socket_descriptor_TCP, &rti_remote->final_port_TCP, TCP, true)) {
+  if (create_server(rti_remote->user_specified_port, &rti_remote->socket_descriptor_TCP, &rti_remote->final_port_TCP, TCP, true)) {
     lf_print_error_system_failure("RTI failed to create TCP server: %s.", strerror(errno));
   };
   lf_print("RTI: Listening for federates.");
@@ -1548,8 +1548,10 @@ void initialize_RTI(rti_remote_t* rti) {
   rti_remote->all_federates_exited = false;
   rti_remote->federation_id = "Unidentified Federation";
   rti_remote->user_specified_port = 0;
+  // TODO: Erase
   rti_remote->final_port_TCP = 0;
   rti_remote->socket_descriptor_TCP = -1;
+
   rti_remote->final_port_UDP = UINT16_MAX;
   rti_remote->socket_descriptor_UDP = -1;
   rti_remote->clock_sync_global_status = clock_sync_init;

From d62bf0cf5a5ee0d8152ea60b8e4f6ebe573c4151 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Wed, 15 Jan 2025 13:30:51 -0700
Subject: [PATCH 19/84] Fix create_server to use netdriver. First only fix RTI.
 Separate the create_clock_server, because there are no plans using other
 network stacks rather than UDP.

---
 core/federated/RTI/rti_remote.c  |  6 ++--
 core/federated/RTI/rti_remote.h  | 11 ++++---
 network/api/net_driver.h         |  3 ++
 network/api/socket_common.h      | 22 ++++++++++++++
 network/impl/src/socket_common.c | 49 +++++++++++++++++++++++++++++++-
 5 files changed, 81 insertions(+), 10 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index 755912994..4eee736b2 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -1480,15 +1480,15 @@ void initialize_federate(federate_info_t* fed, uint16_t id) {
 int32_t start_rti_server() {
   _lf_initialize_clock();
   // Create the TCP socket server
-  if (create_server(rti_remote->user_specified_port, &rti_remote->socket_descriptor_TCP, &rti_remote->final_port_TCP, TCP, true)) {
+  if (create_server_(rti_remote->rti_netdrv, RTI)) {
     lf_print_error_system_failure("RTI failed to create TCP server: %s.", strerror(errno));
   };
   lf_print("RTI: Listening for federates.");
   // Create the UDP socket server
   // Try to get the rti_remote->final_port_TCP + 1 port
   if (rti_remote->clock_sync_global_status >= clock_sync_on) {
-    if (create_server(rti_remote->final_port_TCP + 1, &rti_remote->socket_descriptor_UDP, &rti_remote->final_port_UDP,
-                      UDP, true)) {
+    if (create_clock_server(rti_remote->final_port_TCP + 1, &rti_remote->socket_descriptor_UDP,
+                            &rti_remote->final_port_UDP)) {
       lf_print_error_system_failure("RTI failed to create UDP server: %s.", strerror(errno));
     }
   }
diff --git a/core/federated/RTI/rti_remote.h b/core/federated/RTI/rti_remote.h
index 3dc56c180..bb3000cfe 100644
--- a/core/federated/RTI/rti_remote.h
+++ b/core/federated/RTI/rti_remote.h
@@ -113,7 +113,9 @@ typedef struct rti_remote_t {
   const char* federation_id;
 
   /************* TCP server information *************/
-  /** The desired port specified by the user on the command line. */
+  /** The desired port specified by the user on the command line.
+   * This should be not moved to the net_driver, because the user can configure this as -p or --port.
+   */
   uint16_t user_specified_port;
 
   /** The final port number that the TCP socket server ends up using. */
@@ -130,7 +132,7 @@ typedef struct rti_remote_t {
   int socket_descriptor_UDP;
 
   /**
-   * The rti's netdriver.
+   * The rti's network driver.
    */
   netdrv_t* rti_netdrv;
 
@@ -375,11 +377,8 @@ void initialize_federate(federate_info_t* fed, uint16_t id);
 /**
  * Start the socket server for the runtime infrastructure (RTI) and
  * return the socket descriptor.
- * @param num_feds Number of federates.
- * @param port The port on which to listen for socket connections, or
- *  0 to use the default port range.
  */
-int32_t start_rti_server(uint16_t port);
+int32_t start_rti_server();
 
 /**
  * Start the runtime infrastructure (RTI) interaction with the federates
diff --git a/network/api/net_driver.h b/network/api/net_driver.h
index 993d22853..ce4d0b49d 100644
--- a/network/api/net_driver.h
+++ b/network/api/net_driver.h
@@ -3,6 +3,7 @@
 
 #include "socket_common.h"
 
+typedef enum server_type_t { RTI, FED} server_type_t;
 typedef struct netdrv_t {
   void* priv;
   //   unsigned int read_remaining_bytes;
@@ -10,4 +11,6 @@ typedef struct netdrv_t {
   //   const char* federation_id;
 } netdrv_t;
 
+int create_server_(netdrv_t* drv, server_type_t serv_type);
+
 #endif /* NET_DRIVER_H */
diff --git a/network/api/socket_common.h b/network/api/socket_common.h
index 878f0b107..7fd6c0f06 100644
--- a/network/api/socket_common.h
+++ b/network/api/socket_common.h
@@ -3,6 +3,7 @@
 
 #include "low_level_platform.h"
 #include <sys/types.h>
+#include <netinet/in.h>
 
 /**
  * The amount of time to wait after a failed socket read or write before trying again. This defaults to 100 ms.
@@ -74,6 +75,23 @@ typedef enum socket_type_t { TCP, UDP } socket_type_t;
  */
 extern lf_mutex_t socket_mutex;
 
+typedef struct socket_priv_t {
+  int socket_descriptor;
+  uint16_t port; // my port number // TODO: Only used in federate.c to send federate's port.
+  uint16_t user_specified_port;
+
+  // The connected other side's info.
+  char server_hostname[INET_ADDRSTRLEN]; // Human-readable IP address and
+  int32_t server_port;                   // port number of the socket server of the federate
+                                         // if it has any incoming direct connections from other federates.
+                                         // The port number will be -1 if there is no server or if the
+                                         // RTI has not been informed of the port number.
+  struct in_addr server_ip_addr;         // Information about the IP address of the socket
+                                         // server of the federate.
+
+  struct sockaddr_in UDP_addr; // The UDP address for the federate.
+} socket_priv_t;
+
 /**
  * @brief Create an IPv4 TCP socket with Nagle's algorithm disabled
  * (TCP_NODELAY) and Delayed ACKs disabled (TCP_QUICKACK). Exits application
@@ -106,6 +124,10 @@ int create_real_time_tcp_socket_errexit();
 int create_server(uint16_t port, int* final_socket, uint16_t* final_port, socket_type_t sock_type,
                   bool increment_port_on_retry);
 
+
+
+int create_clock_server(uint16_t port, int* final_socket, uint16_t* final_port);
+
 /**
  * Wait for an incoming connection request on the specified server socket.
  * This blocks until a connection is successfully accepted. If an error occurs that is not
diff --git a/network/impl/src/socket_common.c b/network/impl/src/socket_common.c
index 587318a2f..47db27f93 100644
--- a/network/impl/src/socket_common.c
+++ b/network/impl/src/socket_common.c
@@ -11,7 +11,7 @@
 #include <string.h> // strerror
 
 #include "util.h"
-#include "socket_common.h"
+#include "net_driver.h"
 
 #ifndef NUMBER_OF_FEDERATES
 #define NUMBER_OF_FEDERATES 1
@@ -133,6 +133,7 @@ static int set_socket_bind_option(int socket_descriptor, uint16_t specified_port
   return used_port;
 }
 
+//TODO: Fix on federate.
 int create_server(uint16_t port, int* final_socket, uint16_t* final_port, socket_type_t sock_type,
                   bool increment_port_on_retry) {
   int socket_descriptor;
@@ -170,6 +171,52 @@ int create_server(uint16_t port, int* final_socket, uint16_t* final_port, socket
   return 0;
 }
 
+int create_server_(netdrv_t* drv, server_type_t serv_type) {
+  socket_priv_t* priv = (socket_priv_t*)drv->priv;
+  int socket_descriptor;
+  struct timeval timeout_time;
+  // Create an IPv4 socket for TCP.
+  socket_descriptor = create_real_time_tcp_socket_errexit();
+  // Set the timeout time for the communications of the server
+  timeout_time = (struct timeval){.tv_sec = TCP_TIMEOUT_TIME / BILLION, .tv_usec = (TCP_TIMEOUT_TIME % BILLION) / 1000};
+  if (socket_descriptor < 0) {
+    lf_print_error("Failed to create TCP socket.");
+    return -1;
+  }
+  set_socket_timeout_option(socket_descriptor, &timeout_time);
+  bool increment_port_on_retry = (serv_type == RTI) ? true : false;
+
+  int used_port = set_socket_bind_option(socket_descriptor, priv->user_specified_port, increment_port_on_retry);
+  // Enable listening for socket connections.
+  // The second argument is the maximum number of queued socket requests,
+  // which according to the Mac man page is limited to 128.
+  if (listen(socket_descriptor, 128)) {
+    lf_print_error("Failed to listen on %d socket: %s.", socket_descriptor, strerror(errno));
+    return -1;
+  }
+  priv->socket_descriptor = socket_descriptor;
+  priv->port = used_port;
+  return 0;
+}
+
+int create_clock_server(uint16_t port, int* final_socket, uint16_t* final_port) {
+  int socket_descriptor;
+  struct timeval timeout_time;
+  // Create a UDP socket.
+  socket_descriptor = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
+  timeout_time = (struct timeval){.tv_sec = UDP_TIMEOUT_TIME / BILLION, .tv_usec = (UDP_TIMEOUT_TIME % BILLION) / 1000};
+
+  if (socket_descriptor < 0) {
+    lf_print_error("Failed to create UDP socket.");
+    return -1;
+  }
+  set_socket_timeout_option(socket_descriptor, &timeout_time);
+  int used_port = set_socket_bind_option(socket_descriptor, port, true);
+  *final_socket = socket_descriptor;
+  *final_port = used_port;
+  return 0;
+}
+
 /**
  * Return true if either the socket to the RTI is broken or the socket is
  * alive and the first unread byte on the socket's queue is MSG_TYPE_FAILED.

From fdf009cb0c2cf8717aff0272c24265574c1d44c2 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Wed, 15 Jan 2025 14:23:21 -0700
Subject: [PATCH 20/84] Add initialize_netdrv()

---
 core/federated/RTI/rti_remote.c      |  4 +++-
 network/api/net_driver.h             | 15 ++++++++++++-
 network/impl/src/lf_socket_support.c | 33 ++++++++++++++++++++++++++++
 3 files changed, 50 insertions(+), 2 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index 4eee736b2..c900fcb90 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -1479,7 +1479,9 @@ void initialize_federate(federate_info_t* fed, uint16_t id) {
 
 int32_t start_rti_server() {
   _lf_initialize_clock();
-  // Create the TCP socket server
+  // Initialize RTI's network driver.
+  rti_remote->rti_netdrv = initialize_netdrv();
+  // Create the server
   if (create_server_(rti_remote->rti_netdrv, RTI)) {
     lf_print_error_system_failure("RTI failed to create TCP server: %s.", strerror(errno));
   };
diff --git a/network/api/net_driver.h b/network/api/net_driver.h
index ce4d0b49d..85880173d 100644
--- a/network/api/net_driver.h
+++ b/network/api/net_driver.h
@@ -3,7 +3,7 @@
 
 #include "socket_common.h"
 
-typedef enum server_type_t { RTI, FED} server_type_t;
+typedef enum server_type_t { RTI, FED } server_type_t;
 typedef struct netdrv_t {
   void* priv;
   //   unsigned int read_remaining_bytes;
@@ -11,6 +11,19 @@ typedef struct netdrv_t {
   //   const char* federation_id;
 } netdrv_t;
 
+/**
+ * Allocate memory for the netdriver.
+ * @return netdrv_t*
+ */
+netdrv_t* initialize_netdrv();
+
+/**
+ * Create a netdriver server. This is such as a server socket which accepts connections. However this is only the creation of the server netdriver.
+ * 
+ * @param drv Server's network driver.
+ * @param serv_type Type of server, RTI or FED.
+ * @return int 0 for success, -1 for failure.
+ */
 int create_server_(netdrv_t* drv, server_type_t serv_type);
 
 #endif /* NET_DRIVER_H */
diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
index e69de29bb..144a2524b 100644
--- a/network/impl/src/lf_socket_support.c
+++ b/network/impl/src/lf_socket_support.c
@@ -0,0 +1,33 @@
+#include <stdlib.h>
+#include <string.h>
+
+#include "net_driver.h"
+#include "socket_common.h"
+#include "util.h"
+// #include "lf_socket_support.h"
+
+netdrv_t* initialize_netdrv() {
+  netdrv_t* drv = malloc(sizeof(netdrv_t));
+  if (drv == NULL) {
+    lf_print_error_and_exit("Falied to malloc netdrv_t.");
+  }
+  // Initialize priv.
+  socket_priv_t* priv = malloc(sizeof(socket_priv_t));
+  if (priv == NULL) {
+    lf_print_error_and_exit("Falied to malloc socket_priv_t.");
+  }
+  // Initialize to zero.
+  //   memset(priv, 0, sizeof(socket_priv_t));
+
+  // Server initialization.
+  priv->port = 0;
+  priv->socket_descriptor = -1;
+
+  // Federate initialization
+  strncpy(priv->server_hostname, "localhost", INET_ADDRSTRLEN);
+  priv->server_port = -1;
+
+  // Set drv->priv pointer to point the malloc'd priv.
+  drv->priv = (void*)priv;
+  return drv;
+}

From 83d92bd1411b593f85595c02371edb399c72f9b5 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Wed, 15 Jan 2025 16:49:21 -0700
Subject: [PATCH 21/84] Move create_server to lf_socket_support.c

---
 network/impl/src/lf_socket_support.c | 28 +++++++++++++
 network/impl/src/socket_common.c     | 59 +++++++++++++++-------------
 2 files changed, 60 insertions(+), 27 deletions(-)

diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
index 144a2524b..ed00d51b9 100644
--- a/network/impl/src/lf_socket_support.c
+++ b/network/impl/src/lf_socket_support.c
@@ -31,3 +31,31 @@ netdrv_t* initialize_netdrv() {
   drv->priv = (void*)priv;
   return drv;
 }
+
+int create_server_(netdrv_t* drv, server_type_t serv_type) {
+  socket_priv_t* priv = (socket_priv_t*)drv->priv;
+  int socket_descriptor;
+  struct timeval timeout_time;
+  // Create an IPv4 socket for TCP.
+  socket_descriptor = create_real_time_tcp_socket_errexit();
+  // Set the timeout time for the communications of the server
+  timeout_time = (struct timeval){.tv_sec = TCP_TIMEOUT_TIME / BILLION, .tv_usec = (TCP_TIMEOUT_TIME % BILLION) / 1000};
+  if (socket_descriptor < 0) {
+    lf_print_error("Failed to create TCP socket.");
+    return -1;
+  }
+  set_socket_timeout_option(socket_descriptor, &timeout_time);
+  bool increment_port_on_retry = (serv_type == RTI) ? true : false;
+
+  int used_port = set_socket_bind_option(socket_descriptor, priv->user_specified_port, increment_port_on_retry);
+  // Enable listening for socket connections.
+  // The second argument is the maximum number of queued socket requests,
+  // which according to the Mac man page is limited to 128.
+  if (listen(socket_descriptor, 128)) {
+    lf_print_error("Failed to listen on %d socket: %s.", socket_descriptor, strerror(errno));
+    return -1;
+  }
+  priv->socket_descriptor = socket_descriptor;
+  priv->port = used_port;
+  return 0;
+}
\ No newline at end of file
diff --git a/network/impl/src/socket_common.c b/network/impl/src/socket_common.c
index 47db27f93..f9b6311c9 100644
--- a/network/impl/src/socket_common.c
+++ b/network/impl/src/socket_common.c
@@ -171,33 +171,6 @@ int create_server(uint16_t port, int* final_socket, uint16_t* final_port, socket
   return 0;
 }
 
-int create_server_(netdrv_t* drv, server_type_t serv_type) {
-  socket_priv_t* priv = (socket_priv_t*)drv->priv;
-  int socket_descriptor;
-  struct timeval timeout_time;
-  // Create an IPv4 socket for TCP.
-  socket_descriptor = create_real_time_tcp_socket_errexit();
-  // Set the timeout time for the communications of the server
-  timeout_time = (struct timeval){.tv_sec = TCP_TIMEOUT_TIME / BILLION, .tv_usec = (TCP_TIMEOUT_TIME % BILLION) / 1000};
-  if (socket_descriptor < 0) {
-    lf_print_error("Failed to create TCP socket.");
-    return -1;
-  }
-  set_socket_timeout_option(socket_descriptor, &timeout_time);
-  bool increment_port_on_retry = (serv_type == RTI) ? true : false;
-
-  int used_port = set_socket_bind_option(socket_descriptor, priv->user_specified_port, increment_port_on_retry);
-  // Enable listening for socket connections.
-  // The second argument is the maximum number of queued socket requests,
-  // which according to the Mac man page is limited to 128.
-  if (listen(socket_descriptor, 128)) {
-    lf_print_error("Failed to listen on %d socket: %s.", socket_descriptor, strerror(errno));
-    return -1;
-  }
-  priv->socket_descriptor = socket_descriptor;
-  priv->port = used_port;
-  return 0;
-}
 
 int create_clock_server(uint16_t port, int* final_socket, uint16_t* final_port) {
   int socket_descriptor;
@@ -263,6 +236,38 @@ int accept_socket(int socket, int rti_socket) {
   return socket_id;
 }
 
+int accept_netdrv(netdrv_t server_drv, int rti_socket) {
+  struct sockaddr client_fd;
+  // Wait for an incoming connection request.
+  uint32_t client_length = sizeof(client_fd);
+  // The following blocks until a federate connects.
+  int socket_id = -1;
+  while (true) {
+    // When close(socket) is called, the accept() will return -1.
+    socket_id = accept(socket, &client_fd, &client_length);
+    if (socket_id >= 0) {
+      // Got a socket
+      break;
+    } else if (socket_id < 0 && (errno != EAGAIN || errno != EWOULDBLOCK || errno != EINTR)) {
+      lf_print_warning("Failed to accept the socket. %s.", strerror(errno));
+      break;
+    } else if (errno == EPERM) {
+      lf_print_error_system_failure("Firewall permissions prohibit connection.");
+    } else {
+      // For the federates, it should check if the rti_socket is still open, before retrying accept().
+      if (rti_socket == -1) {
+        if (check_socket_closed(rti_socket)) {
+          break;
+        }
+      }
+      // Try again
+      lf_print_warning("Failed to accept the socket. %s. Trying again.", strerror(errno));
+      continue;
+    }
+  }
+  return socket_id;
+}
+
 int connect_to_socket(int sock, const char* hostname, int port) {
   struct addrinfo hints;
   struct addrinfo* result;

From ce9ee880d28d3b5dafad9a65025275f4b5273700 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Wed, 15 Jan 2025 16:51:17 -0700
Subject: [PATCH 22/84] Fix start_rti_server() and wait_for_federates()
 parameters.

---
 core/federated/RTI/main.c       | 5 ++---
 core/federated/RTI/rti_remote.c | 8 +++++---
 core/federated/RTI/rti_remote.h | 9 ++++-----
 3 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/core/federated/RTI/main.c b/core/federated/RTI/main.c
index 7a5d418cb..c2be579c9 100644
--- a/core/federated/RTI/main.c
+++ b/core/federated/RTI/main.c
@@ -328,9 +328,8 @@ int main(int argc, const char* argv[]) {
     rti.base.scheduling_nodes[i] = (scheduling_node_t*)fed_info;
   }
 
-  int socket_descriptor = start_rti_server();
-  if (socket_descriptor >= 0) {
-    wait_for_federates(socket_descriptor);
+  if (!start_rti_server()) {
+    wait_for_federates();
     normal_termination = true;
     if (rti.base.tracing_enabled) {
       // No need for a mutex lock because all threads have exited.
diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index c900fcb90..9964cc3a1 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -1477,13 +1477,14 @@ void initialize_federate(federate_info_t* fed, uint16_t id) {
   fed->server_port = -1;
 }
 
-int32_t start_rti_server() {
+int start_rti_server() {
   _lf_initialize_clock();
   // Initialize RTI's network driver.
   rti_remote->rti_netdrv = initialize_netdrv();
   // Create the server
   if (create_server_(rti_remote->rti_netdrv, RTI)) {
     lf_print_error_system_failure("RTI failed to create TCP server: %s.", strerror(errno));
+    return -1;
   };
   lf_print("RTI: Listening for federates.");
   // Create the UDP socket server
@@ -1492,12 +1493,13 @@ int32_t start_rti_server() {
     if (create_clock_server(rti_remote->final_port_TCP + 1, &rti_remote->socket_descriptor_UDP,
                             &rti_remote->final_port_UDP)) {
       lf_print_error_system_failure("RTI failed to create UDP server: %s.", strerror(errno));
+      return -1;
     }
   }
-  return rti_remote->socket_descriptor_TCP;
+  return 0;
 }
 
-void wait_for_federates(int socket_descriptor) {
+void wait_for_federates() {
   // Wait for connections from federates and create a thread for each.
   lf_connect_to_federates(socket_descriptor);
 
diff --git a/core/federated/RTI/rti_remote.h b/core/federated/RTI/rti_remote.h
index bb3000cfe..2ec9b042a 100644
--- a/core/federated/RTI/rti_remote.h
+++ b/core/federated/RTI/rti_remote.h
@@ -375,17 +375,16 @@ void* respond_to_erroneous_connections(void* nothing);
 void initialize_federate(federate_info_t* fed, uint16_t id);
 
 /**
- * Start the socket server for the runtime infrastructure (RTI) and
- * return the socket descriptor.
+ * Start the socket server for the runtime infrastructure (RTI).
+ * @return 0 for success, -1 for failure.
  */
-int32_t start_rti_server();
+int start_rti_server();
 
 /**
  * Start the runtime infrastructure (RTI) interaction with the federates
  * and wait for the federates to exit.
- * @param socket_descriptor The socket descriptor returned by start_rti_server().
  */
-void wait_for_federates(int socket_descriptor);
+void wait_for_federates();
 
 /**
  * Print a usage message.

From f02fa2a415b887101b5fad6f7c1631038540364c Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Wed, 15 Jan 2025 17:19:58 -0700
Subject: [PATCH 23/84] Start fixing lf_connect_to_federates() && add
 accept_netdrv()

---
 core/federated/RTI/rti_remote.c      |  8 ++--
 core/federated/RTI/rti_remote.h      |  4 +-
 network/api/net_driver.h             |  6 ++-
 network/api/socket_common.h          | 27 +++++++++++++-
 network/impl/src/lf_socket_support.c | 43 ++++++++++++++++++++-
 network/impl/src/socket_common.c     | 56 ++--------------------------
 6 files changed, 79 insertions(+), 65 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index 9964cc3a1..05fb95cea 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -1384,9 +1384,9 @@ static bool authenticate_federate(int* socket) {
 }
 #endif
 
-void lf_connect_to_federates(int socket_descriptor) {
+void lf_connect_to_federates(netdrv_t* rti_netdrv) {
   for (int i = 0; i < rti_remote->base.number_of_scheduling_nodes; i++) {
-    int socket_id = accept_socket(rti_remote->socket_descriptor_TCP, -1);
+    int socket_id = accept_netdrv(rti_remote->socket_descriptor_TCP, -1);
 // Wait for the first message from the federate when RTI -a option is on.
 #ifdef __RTI_AUTH__
     if (rti_remote->authentication_enabled) {
@@ -1444,7 +1444,7 @@ void* respond_to_erroneous_connections(void* nothing) {
     // Wait for an incoming connection request.
     // The following will block until either a federate attempts to connect
     // or shutdown_socket(rti->socket_descriptor_TCP) is called.
-    int socket_id = accept_socket(rti_remote->socket_descriptor_TCP, -1);
+    int socket_id = accept_netdrv(rti_remote->socket_descriptor_TCP, -1);
     if (socket_id < 0) {
       return NULL;
     }
@@ -1501,7 +1501,7 @@ int start_rti_server() {
 
 void wait_for_federates() {
   // Wait for connections from federates and create a thread for each.
-  lf_connect_to_federates(socket_descriptor);
+  lf_connect_to_federates(rti_remote->rti_netdrv);
 
   // All federates have connected.
   lf_print("RTI: All expected federates have connected. Starting execution.");
diff --git a/core/federated/RTI/rti_remote.h b/core/federated/RTI/rti_remote.h
index 2ec9b042a..359ce2a91 100644
--- a/core/federated/RTI/rti_remote.h
+++ b/core/federated/RTI/rti_remote.h
@@ -357,9 +357,9 @@ void send_reject(int* socket_id, unsigned char error_code);
  * Wait for one incoming connection request from each federate,
  * and upon receiving it, create a thread to communicate with
  * that federate. Return when all federates have connected.
- * @param socket_descriptor The socket on which to accept connections.
+ * @param rti_netdrv The rti's network driver on which to accept connections.
  */
-void lf_connect_to_federates(int socket_descriptor);
+void lf_connect_to_federates(netdrv_t* rti_netdrv);
 
 /**
  * Thread to respond to new connections, which could be federates of other
diff --git a/network/api/net_driver.h b/network/api/net_driver.h
index 85880173d..833819785 100644
--- a/network/api/net_driver.h
+++ b/network/api/net_driver.h
@@ -12,8 +12,8 @@ typedef struct netdrv_t {
 } netdrv_t;
 
 /**
- * Allocate memory for the netdriver.
- * @return netdrv_t*
+ * Allocate memory for the network driver.
+ * @return netdrv_t* Initialized network driver.
  */
 netdrv_t* initialize_netdrv();
 
@@ -26,4 +26,6 @@ netdrv_t* initialize_netdrv();
  */
 int create_server_(netdrv_t* drv, server_type_t serv_type);
 
+netdrv_t* accept_netdrv(netdrv_t* server_drv, netdrv_t* rti_drv);
+
 #endif /* NET_DRIVER_H */
diff --git a/network/api/socket_common.h b/network/api/socket_common.h
index 7fd6c0f06..08c4770de 100644
--- a/network/api/socket_common.h
+++ b/network/api/socket_common.h
@@ -101,6 +101,24 @@ typedef struct socket_priv_t {
  */
 int create_real_time_tcp_socket_errexit();
 
+/**
+ * Set the socket timeout options.
+ * @param socket_descriptor The file descriptor of the socket on which to set options.
+ * @param timeout_time A pointer to a `struct timeval` that specifies the timeout duration
+ *                     for socket operations (receive and send).
+ */
+void set_socket_timeout_option(int socket_descriptor, struct timeval* timeout_time);
+
+/**
+ * Assign a port to the socket, and bind the socket.
+ *
+ * @param socket_descriptor The file descriptor of the socket to be bound to an address and port.
+ * @param specified_port The port number to bind the socket to.
+ * @param increment_port_on_retry Boolean to retry port increment.
+ * @return The final port number used.
+ */
+int set_socket_bind_option(int socket_descriptor, uint16_t specified_port, bool increment_port_on_retry);
+
 /**
  * @brief Create a TCP server that listens for socket connections.
  *
@@ -124,10 +142,15 @@ int create_real_time_tcp_socket_errexit();
 int create_server(uint16_t port, int* final_socket, uint16_t* final_port, socket_type_t sock_type,
                   bool increment_port_on_retry);
 
-
-
 int create_clock_server(uint16_t port, int* final_socket, uint16_t* final_port);
 
+/**
+ * Return true if either the socket to the RTI is broken or the socket is
+ * alive and the first unread byte on the socket's queue is MSG_TYPE_FAILED.
+ * @param socket Socket to check.
+ */
+bool check_socket_closed(int socket);
+
 /**
  * Wait for an incoming connection request on the specified server socket.
  * This blocks until a connection is successfully accepted. If an error occurs that is not
diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
index ed00d51b9..3f1cea0e4 100644
--- a/network/impl/src/lf_socket_support.c
+++ b/network/impl/src/lf_socket_support.c
@@ -1,5 +1,6 @@
-#include <stdlib.h>
-#include <string.h>
+#include <stdlib.h> // malloc
+#include <string.h> // strerror
+#include <errno.h>  // errno
 
 #include "net_driver.h"
 #include "socket_common.h"
@@ -58,4 +59,42 @@ int create_server_(netdrv_t* drv, server_type_t serv_type) {
   priv->socket_descriptor = socket_descriptor;
   priv->port = used_port;
   return 0;
+}
+
+netdrv_t* accept_netdrv(netdrv_t* server_drv, netdrv_t* rti_drv) {
+  socket_priv_t* serv_priv = (socket_priv_t*)server_drv->priv;
+  netdrv_t* fed_netdrv = initialize_netdrv();
+  socket_priv_t* fed_priv = (socket_priv_t*)fed_netdrv->priv;
+
+  struct sockaddr client_fd;
+  // Wait for an incoming connection request.
+  uint32_t client_length = sizeof(client_fd);
+  // The following blocks until a federate connects.
+  int socket_id = -1;
+  while (true) {
+    // When close(socket) is called, the accept() will return -1.
+    socket_id = accept(serv_priv->socket_descriptor, &client_fd, &client_length);
+    if (socket_id >= 0) {
+      // Got a socket
+      break;
+    } else if (socket_id < 0 && (errno != EAGAIN || errno != EWOULDBLOCK || errno != EINTR)) {
+      lf_print_warning("Failed to accept the socket. %s.", strerror(errno));
+      break;
+    } else if (errno == EPERM) {
+      lf_print_error_system_failure("Firewall permissions prohibit connection.");
+    } else {
+      // For the federates, it should check if the rti_socket is still open, before retrying accept().
+      socket_priv_t* rti_priv = (socket_priv_t*)rti_drv->priv;
+      if (rti_priv->socket_descriptor != -1) {
+        if (check_socket_closed(rti_priv->socket_descriptor)) {
+          break;
+        }
+      }
+      // Try again
+      lf_print_warning("Failed to accept the socket. %s. Trying again.", strerror(errno));
+      continue;
+    }
+  }
+  fed_priv->socket_descriptor = socket_id;
+  return fed_netdrv;
 }
\ No newline at end of file
diff --git a/network/impl/src/socket_common.c b/network/impl/src/socket_common.c
index f9b6311c9..e43181738 100644
--- a/network/impl/src/socket_common.c
+++ b/network/impl/src/socket_common.c
@@ -52,13 +52,7 @@ int create_real_time_tcp_socket_errexit() {
   return sock;
 }
 
-/**
- * Set the socket timeout options.
- * @param socket_descriptor The file descriptor of the socket on which to set options.
- * @param timeout_time A pointer to a `struct timeval` that specifies the timeout duration
- *                     for socket operations (receive and send).
- */
-static void set_socket_timeout_option(int socket_descriptor, struct timeval* timeout_time) {
+void set_socket_timeout_option(int socket_descriptor, struct timeval* timeout_time) {
   // Set the option for this socket to reuse the same address
   int true_variable = 1; // setsockopt() requires a reference to the value assigned to an option
   if (setsockopt(socket_descriptor, SOL_SOCKET, SO_REUSEADDR, &true_variable, sizeof(int32_t)) < 0) {
@@ -73,15 +67,7 @@ static void set_socket_timeout_option(int socket_descriptor, struct timeval* tim
   }
 }
 
-/**
- * Assign a port to the socket, and bind the socket.
- *
- * @param socket_descriptor The file descriptor of the socket to be bound to an address and port.
- * @param specified_port The port number to bind the socket to.
- * @param increment_port_on_retry Boolean to retry port increment.
- * @return The final port number used.
- */
-static int set_socket_bind_option(int socket_descriptor, uint16_t specified_port, bool increment_port_on_retry) {
+int set_socket_bind_option(int socket_descriptor, uint16_t specified_port, bool increment_port_on_retry) {
   // Server file descriptor.
   struct sockaddr_in server_fd;
   // Zero out the server address structure.
@@ -190,11 +176,7 @@ int create_clock_server(uint16_t port, int* final_socket, uint16_t* final_port)
   return 0;
 }
 
-/**
- * Return true if either the socket to the RTI is broken or the socket is
- * alive and the first unread byte on the socket's queue is MSG_TYPE_FAILED.
- */
-static bool check_socket_closed(int socket) {
+bool check_socket_closed(int socket) {
   unsigned char first_byte;
   ssize_t bytes = peek_from_socket(socket, &first_byte);
   if (bytes < 0 || (bytes == 1 && first_byte == MSG_TYPE_FAILED)) {
@@ -236,38 +218,6 @@ int accept_socket(int socket, int rti_socket) {
   return socket_id;
 }
 
-int accept_netdrv(netdrv_t server_drv, int rti_socket) {
-  struct sockaddr client_fd;
-  // Wait for an incoming connection request.
-  uint32_t client_length = sizeof(client_fd);
-  // The following blocks until a federate connects.
-  int socket_id = -1;
-  while (true) {
-    // When close(socket) is called, the accept() will return -1.
-    socket_id = accept(socket, &client_fd, &client_length);
-    if (socket_id >= 0) {
-      // Got a socket
-      break;
-    } else if (socket_id < 0 && (errno != EAGAIN || errno != EWOULDBLOCK || errno != EINTR)) {
-      lf_print_warning("Failed to accept the socket. %s.", strerror(errno));
-      break;
-    } else if (errno == EPERM) {
-      lf_print_error_system_failure("Firewall permissions prohibit connection.");
-    } else {
-      // For the federates, it should check if the rti_socket is still open, before retrying accept().
-      if (rti_socket == -1) {
-        if (check_socket_closed(rti_socket)) {
-          break;
-        }
-      }
-      // Try again
-      lf_print_warning("Failed to accept the socket. %s. Trying again.", strerror(errno));
-      continue;
-    }
-  }
-  return socket_id;
-}
-
 int connect_to_socket(int sock, const char* hostname, int port) {
   struct addrinfo hints;
   struct addrinfo* result;

From 839f58bb5b032f20aedc7243cd20ee3d32060aa5 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Wed, 15 Jan 2025 17:35:05 -0700
Subject: [PATCH 24/84] Fix lf_connect_to_federates() to use network drivers.

---
 core/federated/RTI/rti_remote.c | 33 +++++++++++++++++----------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index 05fb95cea..172a931bd 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -1016,11 +1016,12 @@ void send_reject(int* socket_id, unsigned char error_code) {
  * a federate ID and a federation ID. If the federation ID
  * matches this federation, send an MSG_TYPE_ACK and otherwise send
  * a MSG_TYPE_REJECT message.
- * @param socket_id Pointer to the socket on which to listen.
+ * @param fed_netdrv Pointer to the network driver on which to listen.
  * @param client_fd The socket address.
  * @return The federate ID for success or -1 for failure.
  */
-static int32_t receive_and_check_fed_id_message(int* socket_id) {
+// TODO: Check.
+static int32_t receive_and_check_fed_id_message(netdrv_t* fed_netdrv) {
   // Buffer for message ID, federate ID, and federation ID length.
   size_t length = 1 + sizeof(uint16_t) + 1; // Message ID, federate ID, length of fedration ID.
   unsigned char buffer[length];
@@ -1159,7 +1160,8 @@ static int32_t receive_and_check_fed_id_message(int* socket_id) {
  * out the relevant information in the federate's struct.
  * @return 1 on success and 0 on failure.
  */
-static int receive_connection_information(int* socket_id, uint16_t fed_id) {
+//TODO: Check.
+static int receive_connection_information(netdrv_t* fed_netdrv, uint16_t fed_id) {
   LF_PRINT_DEBUG("RTI waiting for MSG_TYPE_NEIGHBOR_STRUCTURE from federate %d.", fed_id);
   unsigned char connection_info_header[MSG_TYPE_NEIGHBOR_STRUCTURE_HEADER_SIZE];
   read_from_socket_fail_on_error(socket_id, MSG_TYPE_NEIGHBOR_STRUCTURE_HEADER_SIZE, connection_info_header, NULL,
@@ -1238,11 +1240,11 @@ static int receive_connection_information(int* socket_id, uint16_t fed_id) {
  * up to perform runtime clock synchronization using the UDP port number
  * specified in the payload to communicate with the federate's clock
  * synchronization logic.
- * @param socket_id The socket on which to listen.
+ * @param fed_netdrv The network driver on which to listen.
  * @param fed_id The federate ID.
  * @return 1 for success, 0 for failure.
  */
-static int receive_udp_message_and_set_up_clock_sync(int* socket_id, uint16_t fed_id) {
+static int receive_udp_message_and_set_up_clock_sync(netdrv_t* fed_netdrv, uint16_t fed_id) {
   // Read the MSG_TYPE_UDP_PORT message from the federate regardless of the status of
   // clock synchronization. This message will tell the RTI whether the federate
   // is doing clock synchronization, and if it is, what port to use for UDP.
@@ -1318,10 +1320,11 @@ static int receive_udp_message_and_set_up_clock_sync(int* socket_id, uint16_t fe
 /**
  * Authenticate incoming federate by performing HMAC-based authentication.
  *
- * @param socket Socket for the incoming federate tryting to authenticate.
+ * @param fed_netdrv Network driver for the incoming federate tryting to authenticate.
  * @return True if authentication is successful and false otherwise.
  */
-static bool authenticate_federate(int* socket) {
+//TODO: Fix.
+static bool authenticate_federate(netdrv_t* fed_netdrv) {
   // Wait for MSG_TYPE_FED_NONCE from federate.
   size_t fed_id_length = sizeof(uint16_t);
   unsigned char buffer[1 + fed_id_length + NONCE_LENGTH];
@@ -1386,13 +1389,14 @@ static bool authenticate_federate(int* socket) {
 
 void lf_connect_to_federates(netdrv_t* rti_netdrv) {
   for (int i = 0; i < rti_remote->base.number_of_scheduling_nodes; i++) {
-    int socket_id = accept_netdrv(rti_remote->socket_descriptor_TCP, -1);
+    netdrv_t* fed_netdrv = accept_netdrv(rti_netdrv, NULL);
 // Wait for the first message from the federate when RTI -a option is on.
 #ifdef __RTI_AUTH__
     if (rti_remote->authentication_enabled) {
-      if (!authenticate_federate(&socket_id)) {
+      if (!authenticate_federate(fed_netdrv)) {
         lf_print_warning("RTI failed to authenticate the incoming federate.");
         // Close the socket without reading until EOF.
+        //TODO: Check.
         shutdown_socket(&socket_id, false);
         // Ignore the federate that failed authentication.
         i--;
@@ -1402,9 +1406,9 @@ void lf_connect_to_federates(netdrv_t* rti_netdrv) {
 #endif
 
     // The first message from the federate should contain its ID and the federation ID.
-    int32_t fed_id = receive_and_check_fed_id_message(&socket_id);
-    if (fed_id >= 0 && socket_id >= 0 && receive_connection_information(&socket_id, (uint16_t)fed_id) &&
-        receive_udp_message_and_set_up_clock_sync(&socket_id, (uint16_t)fed_id)) {
+    int32_t fed_id = receive_and_check_fed_id_message(fed_netdrv);
+    if (fed_id >= 0 && receive_connection_information(fed_netdrv, (uint16_t)fed_id) &&
+        receive_udp_message_and_set_up_clock_sync(fed_netdrv, (uint16_t)fed_id)) {
 
       // Create a thread to communicate with the federate.
       // This has to be done after clock synchronization is finished
@@ -1444,10 +1448,7 @@ void* respond_to_erroneous_connections(void* nothing) {
     // Wait for an incoming connection request.
     // The following will block until either a federate attempts to connect
     // or shutdown_socket(rti->socket_descriptor_TCP) is called.
-    int socket_id = accept_netdrv(rti_remote->socket_descriptor_TCP, -1);
-    if (socket_id < 0) {
-      return NULL;
-    }
+    netdrv_t* fed_netdrv = accept_netdrv(rti_remote->rti_netdrv, NULL);
     if (rti_remote->all_federates_exited) {
       return NULL;
     }

From 6d03173dffec508202d570dc1fcb9500dfa5c406 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Wed, 15 Jan 2025 17:58:43 -0700
Subject: [PATCH 25/84] Add read() write() shutdown() functions using netdrver

---
 network/api/net_driver.h             | 107 ++++++++++++++++-
 network/impl/src/lf_socket_support.c | 164 ++++++++++++++++++++++++++-
 2 files changed, 266 insertions(+), 5 deletions(-)

diff --git a/network/api/net_driver.h b/network/api/net_driver.h
index 833819785..fe04dbefd 100644
--- a/network/api/net_driver.h
+++ b/network/api/net_driver.h
@@ -18,8 +18,9 @@ typedef struct netdrv_t {
 netdrv_t* initialize_netdrv();
 
 /**
- * Create a netdriver server. This is such as a server socket which accepts connections. However this is only the creation of the server netdriver.
- * 
+ * Create a netdriver server. This is such as a server socket which accepts connections. However this is only the
+ * creation of the server netdriver.
+ *
  * @param drv Server's network driver.
  * @param serv_type Type of server, RTI or FED.
  * @return int 0 for success, -1 for failure.
@@ -28,4 +29,106 @@ int create_server_(netdrv_t* drv, server_type_t serv_type);
 
 netdrv_t* accept_netdrv(netdrv_t* server_drv, netdrv_t* rti_drv);
 
+/**
+ * Read the specified number of bytes from the specified socket into the specified buffer.
+ * If an error occurs during this reading, return -1 and set errno to indicate
+ * the cause of the error. If the read succeeds in reading the specified number of bytes,
+ * return 0. If an EOF occurs before reading the specified number of bytes, return 1.
+ * This function repeats the read attempt until the specified number of bytes
+ * have been read, an EOF is read, or an error occurs. Specifically, errors EAGAIN,
+ * EWOULDBLOCK, and EINTR are not considered errors and instead trigger
+ * another attempt. A delay between attempts is given by DELAY_BETWEEN_SOCKET_RETRIES.
+ * @param drv The socket ID.
+ * @param num_bytes The number of bytes to read.
+ * @param buffer The buffer into which to put the bytes.
+ * @return 0 for success, 1 for EOF, and -1 for an error.
+ */
+int read_from_netdrv(netdrv_t* drv, size_t num_bytes, unsigned char* buffer);
+
+/**
+ * Read the specified number of bytes to the specified socket using read_from_socket
+ * and close the socket if an error occurs. If an error occurs, this will change the
+ * socket ID pointed to by the first argument to -1 and will return -1.
+ * @param socket Pointer to the socket ID.
+ * @param num_bytes The number of bytes to write.
+ * @param buffer The buffer from which to get the bytes.
+ * @return 0 for success, -1 for failure.
+ */
+int read_from_netdrv_close_on_error(netdrv_t* drv, size_t num_bytes, unsigned char* buffer);
+
+/**
+ * Read the specified number of bytes from the specified socket into the
+ * specified buffer. If a disconnect or an EOF occurs during this
+ * reading, then if format is non-null, report an error and exit.
+ * If the mutex argument is non-NULL, release the mutex before exiting.
+ * If format is null, then report the error, but do not exit.
+ * This function takes a formatted string and additional optional arguments
+ * similar to printf(format, ...) that is appended to the error messages.
+ * @param drv The socket ID.
+ * @param num_bytes The number of bytes to read.
+ * @param buffer The buffer into which to put the bytes.
+ * @param format A printf-style format string, followed by arguments to
+ *  fill the string, or NULL to not exit with an error message.
+ * @return The number of bytes read, or 0 if an EOF is received, or
+ *  a negative number for an error.
+ */
+void read_from_netdrv_fail_on_error(netdrv_t* drv, size_t num_bytes, unsigned char* buffer, lf_mutex_t* mutex,
+                                    char* format, ...);
+
+/**
+ * Write the specified number of bytes to the specified socket from the
+ * specified buffer. If an error occurs, return -1 and set errno to indicate
+ * the cause of the error. If the write succeeds, return 0.
+ * This function repeats the attempt until the specified number of bytes
+ * have been written or an error occurs. Specifically, errors EAGAIN,
+ * EWOULDBLOCK, and EINTR are not considered errors and instead trigger
+ * another attempt. A delay between attempts is given by
+ * DELAY_BETWEEN_SOCKET_RETRIES.
+ * @param drv The socket ID.
+ * @param num_bytes The number of bytes to write.
+ * @param buffer The buffer from which to get the bytes.
+ * @return 0 for success, -1 for failure.
+ */
+int write_to_netdrv(netdrv_t* drv, size_t num_bytes, unsigned char* buffer);
+
+/**
+ * Write the specified number of bytes to the specified socket using write_to_socket
+ * and close the socket if an error occurs. If an error occurs, this will change the
+ * socket ID pointed to by the first argument to -1 and will return -1.
+ * @param drv Pointer to the socket ID.
+ * @param num_bytes The number of bytes to write.
+ * @param buffer The buffer from which to get the bytes.
+ * @return 0 for success, -1 for failure.
+ */
+int write_to_netdrv_close_on_error(netdrv_t* drv, size_t num_bytes, unsigned char* buffer);
+
+/**
+ * Write the specified number of bytes to the specified socket using
+ * write_to_socket_close_on_error and exit with an error code if an error occurs.
+ * If the mutex argument is non-NULL, release the mutex before exiting.  If the
+ * format argument is non-null, then use it an any additional arguments to form
+ * the error message using printf conventions. Otherwise, print a generic error
+ * message.
+ * @param drv Pointer to the socket ID.
+ * @param num_bytes The number of bytes to write.
+ * @param buffer The buffer from which to get the bytes.
+ * @param mutex If non-NULL, the mutex to unlock before exiting.
+ * @param format A format string for error messages, followed by any number of
+ *  fields that will be used to fill the format string as in printf, or NULL
+ *  to print a generic error message.
+ */
+void write_to_netdrv_fail_on_error(netdrv_t* drv, size_t num_bytes, unsigned char* buffer, lf_mutex_t* mutex,
+                                   char* format, ...);
+
+/**
+ * @brief Gracefully shuts down and closes a socket, optionally reading until EOF.
+ * Shutdown and close the socket. If read_before_closing is false, it just immediately calls shutdown() with SHUT_RDWR
+ * and close(). If read_before_closing is true, it calls shutdown with SHUT_WR, only disallowing further writing. Then,
+ * it calls read() until EOF is received, and discards all received bytes.
+ * @param drv Pointer to the socket descriptor to shutdown and close.
+ * @param read_before_closing If true, read until EOF before closing the socket.
+ * @return int Returns 0 on success, -1 on failure (errno will indicate the error).
+ */
+int shutdown_netdrv(netdrv_t* drv, bool read_before_closing);
+
 #endif /* NET_DRIVER_H */
diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
index 3f1cea0e4..10ec1ba59 100644
--- a/network/impl/src/lf_socket_support.c
+++ b/network/impl/src/lf_socket_support.c
@@ -1,6 +1,7 @@
-#include <stdlib.h> // malloc
-#include <string.h> // strerror
+#include <stdlib.h> // malloc()
+#include <string.h> // strerror()
 #include <errno.h>  // errno
+#include <unistd.h> // read() write()
 
 #include "net_driver.h"
 #include "socket_common.h"
@@ -97,4 +98,161 @@ netdrv_t* accept_netdrv(netdrv_t* server_drv, netdrv_t* rti_drv) {
   }
   fed_priv->socket_descriptor = socket_id;
   return fed_netdrv;
-}
\ No newline at end of file
+}
+
+int read_from_netdrv(netdrv_t* drv, size_t num_bytes, unsigned char* buffer) {
+  socket_priv_t* priv = (socket_priv_t*)drv->priv;
+  int socket = priv->socket_descriptor;
+  if (socket < 0) {
+    // Socket is not open.
+    errno = EBADF;
+    return -1;
+  }
+  ssize_t bytes_read = 0;
+  while (bytes_read < (ssize_t)num_bytes) {
+    ssize_t more = read(socket, buffer + bytes_read, num_bytes - (size_t)bytes_read);
+    if (more < 0 && (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)) {
+      // Those error codes set by the socket indicates
+      // that we should try again (@see man errno).
+      LF_PRINT_DEBUG("Reading from socket %d failed with error: `%s`. Will try again.", socket, strerror(errno));
+      lf_sleep(DELAY_BETWEEN_SOCKET_RETRIES);
+      continue;
+    } else if (more < 0) {
+      // A more serious error occurred.
+      lf_print_error("Reading from socket %d failed. With error: `%s`", socket, strerror(errno));
+      return -1;
+    } else if (more == 0) {
+      // EOF received.
+      return 1;
+    }
+    bytes_read += more;
+  }
+  return 0;
+}
+
+int read_from_netdrv_close_on_error(netdrv_t* drv, size_t num_bytes, unsigned char* buffer) {
+  int read_failed = read_from_netdrv(drv, num_bytes, buffer);
+  if (read_failed) {
+    // Read failed.
+    // Socket has probably been closed from the other side.
+    // Shut down and close the socket from this side.
+    shutdown_netdrv(drv, false);
+    return -1;
+  }
+  return 0;
+}
+
+void read_from_netdrv_fail_on_error(netdrv_t* drv, size_t num_bytes, unsigned char* buffer, lf_mutex_t* mutex,
+                                    char* format, ...) {
+  va_list args;
+  int read_failed = read_from_netdrv_close_on_error(drv, num_bytes, buffer);
+  if (read_failed) {
+    // Read failed.
+    if (mutex != NULL) {
+      LF_MUTEX_UNLOCK(mutex);
+    }
+    if (format != NULL) {
+      va_start(args, format);
+      lf_print_error_system_failure(format, args);
+      va_end(args);
+    } else {
+      lf_print_error_system_failure("Failed to read from socket.");
+    }
+  }
+}
+
+int write_to_netdrv(netdrv_t* drv, size_t num_bytes, unsigned char* buffer) {
+  socket_priv_t* priv = (socket_priv_t*)drv->priv;
+  int socket = priv->socket_descriptor;
+  if (socket < 0) {
+    // Socket is not open.
+    errno = EBADF;
+    return -1;
+  }
+  ssize_t bytes_written = 0;
+  while (bytes_written < (ssize_t)num_bytes) {
+    ssize_t more = write(socket, buffer + bytes_written, num_bytes - (size_t)bytes_written);
+    if (more <= 0 && (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)) {
+      // The error codes EAGAIN or EWOULDBLOCK indicate
+      // that we should try again (@see man errno).
+      // The error code EINTR means the system call was interrupted before completing.
+      LF_PRINT_DEBUG("Writing to socket %d was blocked. Will try again.", socket);
+      lf_sleep(DELAY_BETWEEN_SOCKET_RETRIES);
+      continue;
+    } else if (more < 0) {
+      // A more serious error occurred.
+      lf_print_error("Writing to socket %d failed. With error: `%s`", socket, strerror(errno));
+      return -1;
+    }
+    bytes_written += more;
+  }
+  return 0;
+}
+
+int write_to_netdrv_close_on_error(netdrv_t* drv, size_t num_bytes, unsigned char* buffer) {
+  int result = write_to_netdrv(drv, num_bytes, buffer);
+  if (result) {
+    // Write failed.
+    // Socket has probably been closed from the other side.
+    // Shut down and close the socket from this side.
+    shutdown_netdrv(drv, false);
+  }
+  return result;
+}
+
+void write_to_netdrv_fail_on_error(netdrv_t* drv, size_t num_bytes, unsigned char* buffer, lf_mutex_t* mutex,
+                                   char* format, ...) {
+  va_list args;
+  int result = write_to_netdrv_close_on_error(drv, num_bytes, buffer);
+  if (result) {
+    // Write failed.
+    if (mutex != NULL) {
+      LF_MUTEX_UNLOCK(mutex);
+    }
+    if (format != NULL) {
+      va_start(args, format);
+      lf_print_error_system_failure(format, args);
+      va_end(args);
+    } else {
+      lf_print_error("Failed to write to socket. Closing it.");
+    }
+  }
+}
+
+int shutdown_netdrv(netdrv_t* drv, bool read_before_closing) {
+  socket_priv_t* priv = (socket_priv_t*)drv->priv;
+  if (!read_before_closing) {
+    if (shutdown(priv->socket_descriptor, SHUT_RDWR)) {
+      lf_print_log("On shutdown socket, received reply: %s", strerror(errno));
+      return -1;
+    }
+  } else {
+    // Signal the other side that no further writes are expected by sending a FIN packet.
+    // This indicates the write direction is closed. For more details, refer to:
+    // https://stackoverflow.com/questions/4160347/close-vs-shutdown-socket
+    if (shutdown(priv->socket_descriptor, SHUT_WR)) {
+      lf_print_log("Failed to shutdown socket: %s", strerror(errno));
+      return -1;
+    }
+
+    // Wait for the other side to send an EOF or encounter a socket error.
+    // Discard any incoming bytes. Normally, this read should return 0, indicating the peer has also closed the
+    // connection.
+    // This compensates for delayed ACKs and scenarios where Nagle's algorithm is disabled, ensuring the shutdown
+    // completes gracefully.
+    unsigned char buffer[10];
+    while (read(priv->socket_descriptor, buffer, 10) > 0)
+      ;
+  }
+  // NOTE: In all common TCP/IP stacks, there is a time period,
+  // typically between 30 and 120 seconds, called the TIME_WAIT period,
+  // before the port is released after this close. This is because
+  // the OS is preventing another program from accidentally receiving
+  // duplicated packets intended for this program.
+  if (close(priv->socket_descriptor)) {
+    lf_print_log("Error while closing socket: %s\n", strerror(errno));
+    return -1;
+  }
+  priv->socket_descriptor = -1;
+  return 0;
+}

From 9726b8602c9b4204d48486d765b87847ea4a8489 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Wed, 15 Jan 2025 21:09:58 -0700
Subject: [PATCH 26/84] Fix send_reject() read() write() shutdown() functions
 on remote.c && Formatting.

---
 core/federated/RTI/rti_remote.c | 73 +++++++++++++--------------------
 core/federated/RTI/rti_remote.h | 22 ++++------
 network/api/socket_common.h     |  6 +--
 3 files changed, 41 insertions(+), 60 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index 172a931bd..c9736e7eb 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -996,18 +996,18 @@ void* federate_info_thread_TCP(void* fed) {
   return NULL;
 }
 
-void send_reject(int* socket_id, unsigned char error_code) {
+void send_reject(netdrv_t* drv, unsigned char error_code) {
   LF_PRINT_DEBUG("RTI sending MSG_TYPE_REJECT.");
   unsigned char response[2];
   response[0] = MSG_TYPE_REJECT;
   response[1] = error_code;
   LF_MUTEX_LOCK(&rti_mutex);
   // NOTE: Ignore errors on this response.
-  if (write_to_socket(*socket_id, 2, response)) {
+  if (write_to_netdrv(drv, 2, response)) {
     lf_print_warning("RTI failed to write MSG_TYPE_REJECT message on the socket.");
   }
   // Close the socket without reading until EOF.
-  shutdown_socket(socket_id, false);
+  shutdown_netdrv(drv, false);
   LF_MUTEX_UNLOCK(&rti_mutex);
 }
 
@@ -1020,14 +1020,13 @@ void send_reject(int* socket_id, unsigned char error_code) {
  * @param client_fd The socket address.
  * @return The federate ID for success or -1 for failure.
  */
-// TODO: Check.
 static int32_t receive_and_check_fed_id_message(netdrv_t* fed_netdrv) {
   // Buffer for message ID, federate ID, and federation ID length.
   size_t length = 1 + sizeof(uint16_t) + 1; // Message ID, federate ID, length of fedration ID.
   unsigned char buffer[length];
 
   // Read bytes from the socket. We need 4 bytes.
-  if (read_from_socket_close_on_error(socket_id, length, buffer)) {
+  if (read_from_netdrv_close_on_error(fed_netdrv, length, buffer)) {
     lf_print_error("RTI failed to read from accepted socket.");
     return -1;
   }
@@ -1047,12 +1046,12 @@ static int32_t receive_and_check_fed_id_message(netdrv_t* fed_netdrv) {
       // of the peer they want to connect to from the RTI.
       // If the connection is a peer-to-peer connection between two
       // federates, reject the connection with the WRONG_SERVER error.
-      send_reject(socket_id, WRONG_SERVER);
+      send_reject(fed_netdrv, WRONG_SERVER);
     } else if (buffer[0] == MSG_TYPE_FED_NONCE) {
-      send_reject(socket_id, RTI_NOT_EXECUTED_WITH_AUTH);
+      send_reject(fed_netdrv, RTI_NOT_EXECUTED_WITH_AUTH);
       lf_print_error("RTI not executed with HMAC authentication option using -a or --auth.");
     } else {
-      send_reject(socket_id, UNEXPECTED_MESSAGE);
+      send_reject(fed_netdrv, UNEXPECTED_MESSAGE);
     }
     lf_print_error("RTI expected a MSG_TYPE_FED_IDS message. Got %u (see net_common.h).", buffer[0]);
     return -1;
@@ -1065,7 +1064,7 @@ static int32_t receive_and_check_fed_id_message(netdrv_t* fed_netdrv) {
     size_t federation_id_length = (size_t)buffer[sizeof(uint16_t) + 1];
     char federation_id_received[federation_id_length + 1]; // One extra for null terminator.
     // Next read the actual federation ID.
-    if (read_from_socket_close_on_error(socket_id, federation_id_length, (unsigned char*)federation_id_received)) {
+    if (read_from_netdrv_close_on_error(fed_netdrv, federation_id_length, (unsigned char*)federation_id_received)) {
       lf_print_error("RTI failed to read federation id from federate %d.", fed_id);
       return -1;
     }
@@ -1086,7 +1085,7 @@ static int32_t receive_and_check_fed_id_message(netdrv_t* fed_netdrv) {
       if (rti_remote->base.tracing_enabled) {
         tracepoint_rti_to_federate(send_REJECT, fed_id, NULL);
       }
-      send_reject(socket_id, FEDERATION_ID_DOES_NOT_MATCH);
+      send_reject(fed_netdrv, FEDERATION_ID_DOES_NOT_MATCH);
       return -1;
     } else {
       if (fed_id >= rti_remote->base.number_of_scheduling_nodes) {
@@ -1095,7 +1094,7 @@ static int32_t receive_and_check_fed_id_message(netdrv_t* fed_netdrv) {
         if (rti_remote->base.tracing_enabled) {
           tracepoint_rti_to_federate(send_REJECT, fed_id, NULL);
         }
-        send_reject(socket_id, FEDERATE_ID_OUT_OF_RANGE);
+        send_reject(fed_netdrv, FEDERATE_ID_OUT_OF_RANGE);
         return -1;
       } else {
         if ((rti_remote->base.scheduling_nodes[fed_id])->state != NOT_CONNECTED) {
@@ -1103,7 +1102,7 @@ static int32_t receive_and_check_fed_id_message(netdrv_t* fed_netdrv) {
           if (rti_remote->base.tracing_enabled) {
             tracepoint_rti_to_federate(send_REJECT, fed_id, NULL);
           }
-          send_reject(socket_id, FEDERATE_ID_IN_USE);
+          send_reject(fed_netdrv, FEDERATE_ID_IN_USE);
           return -1;
         }
       }
@@ -1113,23 +1112,10 @@ static int32_t receive_and_check_fed_id_message(netdrv_t* fed_netdrv) {
   // The MSG_TYPE_FED_IDS message has the right federation ID.
 
   // Get the peer address from the connected socket_id. Then assign it as the federate's socket server.
-  struct sockaddr_in peer_addr;
-  socklen_t addr_len = sizeof(peer_addr);
-  if (getpeername(*socket_id, (struct sockaddr*)&peer_addr, &addr_len) != 0) {
+  if(get_peer_address(fed_netdrv) != 0) {
     lf_print_error("RTI failed to get peer address.");
-  }
-  fed->server_ip_addr = peer_addr.sin_addr;
-
-#if LOG_LEVEL >= LOG_LEVEL_DEBUG
-  // Create the human readable format and copy that into
-  // the .server_hostname field of the federate.
-  char str[INET_ADDRSTRLEN + 1];
-  inet_ntop(AF_INET, &fed->server_ip_addr, str, INET_ADDRSTRLEN);
-  strncpy(fed->server_hostname, str, INET_ADDRSTRLEN);
-
-  LF_PRINT_DEBUG("RTI got address %s from federate %d.", fed->server_hostname, fed_id);
-#endif
-  fed->socket = *socket_id;
+  };
+  fed->fed_netdrv = fed_netdrv;
 
   // Set the federate's state as pending
   // because it is waiting for the start time to be
@@ -1164,7 +1150,7 @@ static int32_t receive_and_check_fed_id_message(netdrv_t* fed_netdrv) {
 static int receive_connection_information(netdrv_t* fed_netdrv, uint16_t fed_id) {
   LF_PRINT_DEBUG("RTI waiting for MSG_TYPE_NEIGHBOR_STRUCTURE from federate %d.", fed_id);
   unsigned char connection_info_header[MSG_TYPE_NEIGHBOR_STRUCTURE_HEADER_SIZE];
-  read_from_socket_fail_on_error(socket_id, MSG_TYPE_NEIGHBOR_STRUCTURE_HEADER_SIZE, connection_info_header, NULL,
+  read_from_netdrv_fail_on_error(fed_netdrv, MSG_TYPE_NEIGHBOR_STRUCTURE_HEADER_SIZE, connection_info_header, NULL,
                                  "RTI failed to read MSG_TYPE_NEIGHBOR_STRUCTURE message header from federate %d.",
                                  fed_id);
 
@@ -1172,7 +1158,7 @@ static int receive_connection_information(netdrv_t* fed_netdrv, uint16_t fed_id)
     lf_print_error("RTI was expecting a MSG_TYPE_UDP_PORT message from federate %d. Got %u instead. "
                    "Rejecting federate.",
                    fed_id, connection_info_header[0]);
-    send_reject(socket_id, UNEXPECTED_MESSAGE);
+    send_reject(fed_netdrv, UNEXPECTED_MESSAGE);
     return 0;
   } else {
     federate_info_t* fed = GET_FED_INFO(fed_id);
@@ -1206,7 +1192,7 @@ static int receive_connection_information(netdrv_t* fed_netdrv, uint16_t fed_id)
     if (connections_info_body_size > 0) {
       connections_info_body = (unsigned char*)malloc(connections_info_body_size);
       LF_ASSERT_NON_NULL(connections_info_body);
-      read_from_socket_fail_on_error(socket_id, connections_info_body_size, connections_info_body, NULL,
+      read_from_netdrv_fail_on_error(fed_netdrv, connections_info_body_size, connections_info_body, NULL,
                                      "RTI failed to read MSG_TYPE_NEIGHBOR_STRUCTURE message body from federate %d.",
                                      fed_id);
       // Keep track of where we are in the buffer
@@ -1250,13 +1236,13 @@ static int receive_udp_message_and_set_up_clock_sync(netdrv_t* fed_netdrv, uint1
   // is doing clock synchronization, and if it is, what port to use for UDP.
   LF_PRINT_DEBUG("RTI waiting for MSG_TYPE_UDP_PORT from federate %d.", fed_id);
   unsigned char response[1 + sizeof(uint16_t)];
-  read_from_socket_fail_on_error(socket_id, 1 + sizeof(uint16_t), response, NULL,
+  read_from_netdrv_fail_on_error(fed_netdrv, 1 + sizeof(uint16_t), response, NULL,
                                  "RTI failed to read MSG_TYPE_UDP_PORT message from federate %d.", fed_id);
   if (response[0] != MSG_TYPE_UDP_PORT) {
     lf_print_error("RTI was expecting a MSG_TYPE_UDP_PORT message from federate %d. Got %u instead. "
                    "Rejecting federate.",
                    fed_id, response[0]);
-    send_reject(socket_id, UNEXPECTED_MESSAGE);
+    send_reject(fed_netdrv, UNEXPECTED_MESSAGE);
     return 0;
   } else {
     federate_info_t* fed = GET_FED_INFO(fed_id);
@@ -1277,7 +1263,7 @@ static int receive_udp_message_and_set_up_clock_sync(netdrv_t* fed_netdrv, uint1
           // Listen for reply message, which should be T3.
           size_t message_size = 1 + sizeof(uint16_t);
           unsigned char buffer[message_size];
-          read_from_socket_fail_on_error(socket_id, message_size, buffer, NULL,
+          read_from_netdrv_fail_on_error(fed_netdrv, message_size, buffer, NULL,
                                          "Socket to federate %d unexpectedly closed.", fed_id);
           if (buffer[0] == MSG_TYPE_CLOCK_SYNC_T3) {
             uint16_t fed_id = extract_uint16(&(buffer[1]));
@@ -1285,7 +1271,7 @@ static int receive_udp_message_and_set_up_clock_sync(netdrv_t* fed_netdrv, uint1
             handle_physical_clock_sync_message(fed, TCP);
           } else {
             lf_print_error("Unexpected message %u from federate %d.", buffer[0], fed_id);
-            send_reject(socket_id, UNEXPECTED_MESSAGE);
+            send_reject(fed_netdrv, UNEXPECTED_MESSAGE);
             return 0;
           }
         }
@@ -1328,7 +1314,7 @@ static bool authenticate_federate(netdrv_t* fed_netdrv) {
   // Wait for MSG_TYPE_FED_NONCE from federate.
   size_t fed_id_length = sizeof(uint16_t);
   unsigned char buffer[1 + fed_id_length + NONCE_LENGTH];
-  read_from_socket_fail_on_error(socket, 1 + fed_id_length + NONCE_LENGTH, buffer, NULL,
+  read_from_netdrv_fail_on_error(fed_netdrv, 1 + fed_id_length + NONCE_LENGTH, buffer, NULL,
                                  "Failed to read MSG_TYPE_FED_NONCE");
   if (buffer[0] != MSG_TYPE_FED_NONCE) {
     lf_print_error_and_exit("Received unexpected response %u from the FED (see net_common.h).", buffer[0]);
@@ -1353,13 +1339,13 @@ static bool authenticate_federate(netdrv_t* fed_netdrv) {
   RAND_bytes(rti_nonce, NONCE_LENGTH);
   memcpy(&sender[1], rti_nonce, NONCE_LENGTH);
   memcpy(&sender[1 + NONCE_LENGTH], hmac_tag, hmac_length);
-  if (write_to_socket(*socket, 1 + NONCE_LENGTH + hmac_length, sender)) {
+  if (write_to_netdrv(fed_netdrv, 1 + NONCE_LENGTH + hmac_length, sender)) {
     lf_print_error("Failed to send nonce to federate.");
   }
 
   // Wait for MSG_TYPE_FED_RESPONSE
   unsigned char received[1 + hmac_length];
-  read_from_socket_fail_on_error(socket, 1 + hmac_length, received, NULL, "Failed to read federate response.");
+  read_from_netdrv_fail_on_error(fed_netdrv, 1 + hmac_length, received, NULL, "Failed to read federate response.");
   if (received[0] != MSG_TYPE_FED_RESPONSE) {
     lf_print_error_and_exit("Received unexpected response %u from the federate (see net_common.h).", received[0]);
     return false;
@@ -1378,7 +1364,7 @@ static bool authenticate_federate(netdrv_t* fed_netdrv) {
   if (memcmp(&received[1], rti_tag, hmac_length) != 0) {
     // Federation IDs do not match. Send back a HMAC_DOES_NOT_MATCH message.
     lf_print_warning("HMAC authentication failed. Rejecting the federate.");
-    send_reject(socket, HMAC_DOES_NOT_MATCH);
+    send_reject(fed_netdrv, HMAC_DOES_NOT_MATCH);
     return false;
   } else {
     LF_PRINT_LOG("Federate's HMAC verified.");
@@ -1396,8 +1382,7 @@ void lf_connect_to_federates(netdrv_t* rti_netdrv) {
       if (!authenticate_federate(fed_netdrv)) {
         lf_print_warning("RTI failed to authenticate the incoming federate.");
         // Close the socket without reading until EOF.
-        //TODO: Check.
-        shutdown_socket(&socket_id, false);
+        shutdown_netdrv(fed_netdrv, false);
         // Ignore the federate that failed authentication.
         i--;
         continue;
@@ -1458,11 +1443,11 @@ void* respond_to_erroneous_connections(void* nothing) {
     response[0] = MSG_TYPE_REJECT;
     response[1] = FEDERATION_ID_DOES_NOT_MATCH;
     // Ignore errors on this response.
-    if (write_to_socket(socket_id, 2, response)) {
+    if (write_to_netdrv(fed_netdrv, 2, response)) {
       lf_print_warning("RTI failed to write FEDERATION_ID_DOES_NOT_MATCH to erroneous incoming connection.");
     }
     // Close the socket without reading until EOF.
-    shutdown_socket(&socket_id, false);
+    shutdown_netdrv(fed_netdrv, false);
   }
   return NULL;
 }
@@ -1529,7 +1514,7 @@ void wait_for_federates() {
   // Shutdown and close the socket that is listening for incoming connections
   // so that the accept() call in respond_to_erroneous_connections returns.
   // That thread should then check rti->all_federates_exited and it should exit.
-  shutdown_socket(&socket_descriptor, false);
+  shutdown_netdrv(rti_remote->rti_netdrv, false);
 
   if (rti_remote->socket_descriptor_UDP > 0) {
     shutdown_socket(&rti_remote->socket_descriptor_UDP, false);
diff --git a/core/federated/RTI/rti_remote.h b/core/federated/RTI/rti_remote.h
index 359ce2a91..c29d0d53b 100644
--- a/core/federated/RTI/rti_remote.h
+++ b/core/federated/RTI/rti_remote.h
@@ -49,10 +49,13 @@
  */
 typedef struct federate_info_t {
   scheduling_node_t enclave;
-  bool requested_stop;                   // Indicates that the federate has requested stop or has replied
-                                         // to a request for stop from the RTI. Used to prevent double-counting
-                                         // a federate when handling lf_request_stop().
-  lf_thread_t thread_id;                 // The ID of the thread handling communication with this federate.
+  bool requested_stop;   // Indicates that the federate has requested stop or has replied
+                         // to a request for stop from the RTI. Used to prevent double-counting
+                         // a federate when handling lf_request_stop().
+  lf_thread_t thread_id; // The ID of the thread handling communication with this federate.
+
+  netdrv_t* fed_netdrv; // The netdriver that the RTI handling each federate.
+
   int socket;                            // The TCP socket descriptor for communicating with this federate.
   struct sockaddr_in UDP_addr;           // The UDP address for the federate.
   bool clock_synchronization_enabled;    // Indicates the status of clock synchronization
@@ -60,13 +63,6 @@ typedef struct federate_info_t {
   pqueue_tag_t* in_transit_message_tags; // Record of in-transit messages to this federate that are not
                                          // yet processed. This record is ordered based on the time
                                          // value of each message for a more efficient access.
-  char server_hostname[INET_ADDRSTRLEN]; // Human-readable IP address and
-  int32_t server_port;                   // port number of the socket server of the federate
-                                         // if it has any incoming direct connections from other federates.
-                                         // The port number will be -1 if there is no server or if the
-                                         // RTI has not been informed of the port number.
-  struct in_addr server_ip_addr;         // Information about the IP address of the socket
-                                         // server of the federate.
 } federate_info_t;
 
 /**
@@ -348,10 +344,10 @@ void* federate_info_thread_TCP(void* fed);
 
 /**
  * Send a MSG_TYPE_REJECT message to the specified socket and close the socket.
- * @param socket_id Pointer to the socket ID.
+ * @param drv Pointer to the network driver.
  * @param error_code An error code.
  */
-void send_reject(int* socket_id, unsigned char error_code);
+void send_reject(netdrv_t* drv, unsigned char error_code);
 
 /**
  * Wait for one incoming connection request from each federate,
diff --git a/network/api/socket_common.h b/network/api/socket_common.h
index 08c4770de..7e52c06a1 100644
--- a/network/api/socket_common.h
+++ b/network/api/socket_common.h
@@ -77,10 +77,10 @@ extern lf_mutex_t socket_mutex;
 
 typedef struct socket_priv_t {
   int socket_descriptor;
-  uint16_t port; // my port number // TODO: Only used in federate.c to send federate's port.
-  uint16_t user_specified_port;
+  uint16_t port;                // The port number. // TODO: Only used in federate.c to send federate's port.
+  uint16_t user_specified_port; // Default as 0 for both RTI and federate.
 
-  // The connected other side's info.
+  // The connected other side's info. The
   char server_hostname[INET_ADDRSTRLEN]; // Human-readable IP address and
   int32_t server_port;                   // port number of the socket server of the federate
                                          // if it has any incoming direct connections from other federates.

From 418d2c9e5e08994236aca1746d43a1d077fb966e Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Wed, 15 Jan 2025 21:21:28 -0700
Subject: [PATCH 27/84] Add get_peer_address()

---
 core/federated/RTI/rti_remote.c      |  6 +++---
 network/impl/src/lf_socket_support.c | 20 ++++++++++++++++++++
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index c9736e7eb..d9e6e7840 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -1112,7 +1112,7 @@ static int32_t receive_and_check_fed_id_message(netdrv_t* fed_netdrv) {
   // The MSG_TYPE_FED_IDS message has the right federation ID.
 
   // Get the peer address from the connected socket_id. Then assign it as the federate's socket server.
-  if(get_peer_address(fed_netdrv) != 0) {
+  if (get_peer_address(fed_netdrv) != 0) {
     lf_print_error("RTI failed to get peer address.");
   };
   fed->fed_netdrv = fed_netdrv;
@@ -1146,7 +1146,7 @@ static int32_t receive_and_check_fed_id_message(netdrv_t* fed_netdrv) {
  * out the relevant information in the federate's struct.
  * @return 1 on success and 0 on failure.
  */
-//TODO: Check.
+// TODO: Check.
 static int receive_connection_information(netdrv_t* fed_netdrv, uint16_t fed_id) {
   LF_PRINT_DEBUG("RTI waiting for MSG_TYPE_NEIGHBOR_STRUCTURE from federate %d.", fed_id);
   unsigned char connection_info_header[MSG_TYPE_NEIGHBOR_STRUCTURE_HEADER_SIZE];
@@ -1309,7 +1309,7 @@ static int receive_udp_message_and_set_up_clock_sync(netdrv_t* fed_netdrv, uint1
  * @param fed_netdrv Network driver for the incoming federate tryting to authenticate.
  * @return True if authentication is successful and false otherwise.
  */
-//TODO: Fix.
+// TODO: Fix.
 static bool authenticate_federate(netdrv_t* fed_netdrv) {
   // Wait for MSG_TYPE_FED_NONCE from federate.
   size_t fed_id_length = sizeof(uint16_t);
diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
index 10ec1ba59..31c95859b 100644
--- a/network/impl/src/lf_socket_support.c
+++ b/network/impl/src/lf_socket_support.c
@@ -100,6 +100,26 @@ netdrv_t* accept_netdrv(netdrv_t* server_drv, netdrv_t* rti_drv) {
   return fed_netdrv;
 }
 
+int get_peer_address(netdrv_t* drv) {
+  socket_priv_t* priv = (socket_priv_t*)drv->priv;
+  struct sockaddr_in peer_addr;
+  socklen_t addr_len = sizeof(peer_addr);
+  if (getpeername(priv->socket_descriptor, (struct sockaddr*)&peer_addr, &addr_len) != 0) {
+    lf_print_error("RTI failed to get peer address.");
+  }
+  priv->server_ip_addr = peer_addr.sin_addr;
+
+#if LOG_LEVEL >= LOG_LEVEL_DEBUG
+  // Create the human readable format and copy that into
+  // the .server_hostname field of the federate.
+  char str[INET_ADDRSTRLEN + 1];
+  inet_ntop(AF_INET, priv->server_ip_addr, str, INET_ADDRSTRLEN);
+  strncpy(priv->server_hostname, str, INET_ADDRSTRLEN);
+
+  LF_PRINT_DEBUG("RTI got address %s", priv->server_hostname);
+#endif
+}
+
 int read_from_netdrv(netdrv_t* drv, size_t num_bytes, unsigned char* buffer) {
   socket_priv_t* priv = (socket_priv_t*)drv->priv;
   int socket = priv->socket_descriptor;

From 32975a635c78bfba796c9c714cd4dba26989e684 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Wed, 15 Jan 2025 21:33:21 -0700
Subject: [PATCH 28/84] Minor fix on not using server_ip_addr

---
 network/impl/src/lf_socket_support.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
index 31c95859b..4d286df26 100644
--- a/network/impl/src/lf_socket_support.c
+++ b/network/impl/src/lf_socket_support.c
@@ -27,6 +27,7 @@ netdrv_t* initialize_netdrv() {
 
   // Federate initialization
   strncpy(priv->server_hostname, "localhost", INET_ADDRSTRLEN);
+  priv->server_ip_addr.s_addr = 0;
   priv->server_port = -1;
 
   // Set drv->priv pointer to point the malloc'd priv.

From 7922ae128d85ef26ab62fade48eb83565baef30b Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Wed, 15 Jan 2025 21:33:21 -0700
Subject: [PATCH 29/84] Minor fix on not using server_ip_addr

---
 core/federated/RTI/rti_remote.c      | 12 +++++-------
 network/api/net_driver.h             |  2 ++
 network/impl/src/lf_socket_support.c |  1 +
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index d9e6e7840..bd222c9c6 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -1146,7 +1146,6 @@ static int32_t receive_and_check_fed_id_message(netdrv_t* fed_netdrv) {
  * out the relevant information in the federate's struct.
  * @return 1 on success and 0 on failure.
  */
-// TODO: Check.
 static int receive_connection_information(netdrv_t* fed_netdrv, uint16_t fed_id) {
   LF_PRINT_DEBUG("RTI waiting for MSG_TYPE_NEIGHBOR_STRUCTURE from federate %d.", fed_id);
   unsigned char connection_info_header[MSG_TYPE_NEIGHBOR_STRUCTURE_HEADER_SIZE];
@@ -1283,7 +1282,11 @@ static int receive_udp_message_and_set_up_clock_sync(netdrv_t* fed_netdrv, uint1
           // Initialize the UDP_addr field of the federate struct
           fed->UDP_addr.sin_family = AF_INET;
           fed->UDP_addr.sin_port = htons(federate_UDP_port_number);
-          fed->UDP_addr.sin_addr = fed->server_ip_addr;
+          #ifdef COMM_TYPE_TCP
+          fed->UDP_addr.sin_addr = ((socket_priv_t*)fed_netdrv->priv)->server_ip_addr;
+          // fed->UDP_addr.sin_addr = fed->server_ip_addr;
+          #elif
+          #endif
         }
       } else {
         // Disable clock sync after initial round.
@@ -1309,7 +1312,6 @@ static int receive_udp_message_and_set_up_clock_sync(netdrv_t* fed_netdrv, uint1
  * @param fed_netdrv Network driver for the incoming federate tryting to authenticate.
  * @return True if authentication is successful and false otherwise.
  */
-// TODO: Fix.
 static bool authenticate_federate(netdrv_t* fed_netdrv) {
   // Wait for MSG_TYPE_FED_NONCE from federate.
   size_t fed_id_length = sizeof(uint16_t);
@@ -1455,12 +1457,8 @@ void* respond_to_erroneous_connections(void* nothing) {
 void initialize_federate(federate_info_t* fed, uint16_t id) {
   initialize_scheduling_node(&(fed->enclave), id);
   fed->requested_stop = false;
-  fed->socket = -1; // No socket.
   fed->clock_synchronization_enabled = true;
   fed->in_transit_message_tags = pqueue_tag_init(10);
-  strncpy(fed->server_hostname, "localhost", INET_ADDRSTRLEN);
-  fed->server_ip_addr.s_addr = 0;
-  fed->server_port = -1;
 }
 
 int start_rti_server() {
diff --git a/network/api/net_driver.h b/network/api/net_driver.h
index fe04dbefd..362efd3f0 100644
--- a/network/api/net_driver.h
+++ b/network/api/net_driver.h
@@ -29,6 +29,8 @@ int create_server_(netdrv_t* drv, server_type_t serv_type);
 
 netdrv_t* accept_netdrv(netdrv_t* server_drv, netdrv_t* rti_drv);
 
+int get_peer_address(netdrv_t* drv);
+
 /**
  * Read the specified number of bytes from the specified socket into the specified buffer.
  * If an error occurs during this reading, return -1 and set errno to indicate
diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
index 31c95859b..4d286df26 100644
--- a/network/impl/src/lf_socket_support.c
+++ b/network/impl/src/lf_socket_support.c
@@ -27,6 +27,7 @@ netdrv_t* initialize_netdrv() {
 
   // Federate initialization
   strncpy(priv->server_hostname, "localhost", INET_ADDRSTRLEN);
+  priv->server_ip_addr.s_addr = 0;
   priv->server_port = -1;
 
   // Set drv->priv pointer to point the malloc'd priv.

From 170f923809108363fe0d4d5034746bdeeeb72751 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Wed, 15 Jan 2025 21:36:19 -0700
Subject: [PATCH 30/84] Formatting on ifdefs.

---
 core/federated/RTI/rti_remote.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index bd222c9c6..87630141c 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -1282,11 +1282,11 @@ static int receive_udp_message_and_set_up_clock_sync(netdrv_t* fed_netdrv, uint1
           // Initialize the UDP_addr field of the federate struct
           fed->UDP_addr.sin_family = AF_INET;
           fed->UDP_addr.sin_port = htons(federate_UDP_port_number);
-          #ifdef COMM_TYPE_TCP
+#ifdef COMM_TYPE_TCP
           fed->UDP_addr.sin_addr = ((socket_priv_t*)fed_netdrv->priv)->server_ip_addr;
-          // fed->UDP_addr.sin_addr = fed->server_ip_addr;
-          #elif
-          #endif
+// fed->UDP_addr.sin_addr = fed->server_ip_addr;
+#elif
+#endif
         }
       } else {
         // Disable clock sync after initial round.

From 5f3d831ee20e438325945bfdfff345028e4c2ca4 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Wed, 15 Jan 2025 22:51:17 -0700
Subject: [PATCH 31/84] Fix handle_address_query to combine the message and
 write() it in once. && Change all read() write(), and shutdown() to use
 netdrv

---
 core/federated/RTI/rti_remote.c | 87 ++++++++++++++++++---------------
 1 file changed, 47 insertions(+), 40 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index 87630141c..5d2693a96 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -73,9 +73,10 @@ void notify_tag_advance_grant(scheduling_node_t* e, tag_t tag) {
     tracepoint_rti_to_federate(send_TAG, e->id, &tag);
   }
   // This function is called in notify_advance_grant_if_safe(), which is a long
-  // function. During this call, the socket might close, causing the following write_to_socket
+  // function. During this call, the socket might close, causing the following write_to_netdrv
   // to fail. Consider a failure here a soft failure and update the federate's status.
-  if (write_to_socket(((federate_info_t*)e)->socket, message_length, buffer)) {
+  // TODO: Check if works well.
+  if (write_to_netdrv(((federate_info_t*)e)->fed_netdrv, message_length, buffer)) {
     lf_print_error("RTI failed to send tag advance grant to federate %d.", e->id);
     e->state = NOT_CONNECTED;
   } else {
@@ -106,9 +107,9 @@ void notify_provisional_tag_advance_grant(scheduling_node_t* e, tag_t tag) {
     tracepoint_rti_to_federate(send_PTAG, e->id, &tag);
   }
   // This function is called in notify_advance_grant_if_safe(), which is a long
-  // function. During this call, the socket might close, causing the following write_to_socket
+  // function. During this call, the socket might close, causing the following write_to_netdrv
   // to fail. Consider a failure here a soft failure and update the federate's status.
-  if (write_to_socket(((federate_info_t*)e)->socket, message_length, buffer)) {
+  if (write_to_netdrv(((federate_info_t*)e)->fed_netdrv, message_length, buffer)) {
     lf_print_error("RTI failed to send tag advance grant to federate %d.", e->id);
     e->state = NOT_CONNECTED;
   } else {
@@ -158,7 +159,7 @@ void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_even
 void handle_port_absent_message(federate_info_t* sending_federate, unsigned char* buffer) {
   size_t message_size = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(int64_t) + sizeof(uint32_t);
 
-  read_from_socket_fail_on_error(&sending_federate->socket, message_size, &(buffer[1]), NULL,
+  read_from_netdrv_fail_on_error(sending_federate->fed_netdrv, message_size, &(buffer[1]), NULL,
                                  " RTI failed to read port absent message from federate %u.",
                                  sending_federate->enclave.id);
 
@@ -207,7 +208,7 @@ void handle_port_absent_message(federate_info_t* sending_federate, unsigned char
   }
 
   // Forward the message.
-  write_to_socket_fail_on_error(&fed->socket, message_size + 1, buffer, &rti_mutex,
+  write_to_netdrv_fail_on_error(fed->fed_netdrv, message_size + 1, buffer, &rti_mutex,
                                 "RTI failed to forward message to federate %d.", federate_id);
 
   LF_MUTEX_UNLOCK(&rti_mutex);
@@ -216,7 +217,7 @@ void handle_port_absent_message(federate_info_t* sending_federate, unsigned char
 void handle_timed_message(federate_info_t* sending_federate, unsigned char* buffer) {
   size_t header_size = 1 + sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint32_t) + sizeof(int64_t) + sizeof(uint32_t);
   // Read the header, minus the first byte which has already been read.
-  read_from_socket_fail_on_error(&sending_federate->socket, header_size - 1, &(buffer[1]), NULL,
+  read_from_netdrv_fail_on_error(sending_federate->fed_netdrv, header_size - 1, &(buffer[1]), NULL,
                                  "RTI failed to read the timed message header from remote federate.");
   // Extract the header information. of the sender
   uint16_t reactor_port_id;
@@ -245,7 +246,7 @@ void handle_timed_message(federate_info_t* sending_federate, unsigned char* buff
                sending_federate->enclave.id, federate_id, reactor_port_id, intended_tag.time - lf_time_start(),
                intended_tag.microstep);
 
-  read_from_socket_fail_on_error(&sending_federate->socket, bytes_to_read, &(buffer[header_size]), NULL,
+  read_from_netdrv_fail_on_error(sending_federate->fed_netdrv, bytes_to_read, &(buffer[header_size]), NULL,
                                  "RTI failed to read timed message from federate %d.", federate_id);
   size_t bytes_read = bytes_to_read + header_size;
   // Following only works for string messages.
@@ -281,7 +282,7 @@ void handle_timed_message(federate_info_t* sending_federate, unsigned char* buff
       if (bytes_to_read > FED_COM_BUFFER_SIZE) {
         bytes_to_read = FED_COM_BUFFER_SIZE;
       }
-      read_from_socket_fail_on_error(&sending_federate->socket, bytes_to_read, buffer, NULL,
+      read_from_netdrv_fail_on_error(sending_federate->fed_netdrv, bytes_to_read, buffer, NULL,
                                      "RTI failed to clear message chunks.");
       total_bytes_read += bytes_to_read;
     }
@@ -303,7 +304,7 @@ void handle_timed_message(federate_info_t* sending_federate, unsigned char* buff
     tracepoint_rti_to_federate(send_TAGGED_MSG, federate_id, &intended_tag);
   }
 
-  write_to_socket_fail_on_error(&fed->socket, bytes_read, buffer, &rti_mutex,
+  write_to_netdrv_fail_on_error(fed->fed_netdrv, bytes_read, buffer, &rti_mutex,
                                 "RTI failed to forward message to federate %d.", federate_id);
 
   // The message length may be longer than the buffer,
@@ -315,7 +316,7 @@ void handle_timed_message(federate_info_t* sending_federate, unsigned char* buff
     if (bytes_to_read > FED_COM_BUFFER_SIZE) {
       bytes_to_read = FED_COM_BUFFER_SIZE;
     }
-    read_from_socket_fail_on_error(&sending_federate->socket, bytes_to_read, buffer, NULL,
+    read_from_netdrv_fail_on_error(sending_federate->fed_netdrv, bytes_to_read, buffer, NULL,
                                    "RTI failed to read message chunks.");
     total_bytes_read += bytes_to_read;
 
@@ -323,7 +324,7 @@ void handle_timed_message(federate_info_t* sending_federate, unsigned char* buff
     // do not write to destination_socket and cause interleaving. However,
     // holding the rti_mutex might be very expensive. Instead, each outgoing
     // socket should probably have its own mutex.
-    write_to_socket_fail_on_error(&fed->socket, bytes_to_read, buffer, &rti_mutex,
+    write_to_netdrv_fail_on_error(fed->fed_netdrv, bytes_to_read, buffer, &rti_mutex,
                                   "RTI failed to send message chunks.");
   }
 
@@ -353,7 +354,7 @@ void handle_timed_message(federate_info_t* sending_federate, unsigned char* buff
 
 void handle_latest_tag_confirmed(federate_info_t* fed) {
   unsigned char buffer[sizeof(int64_t) + sizeof(uint32_t)];
-  read_from_socket_fail_on_error(&fed->socket, sizeof(int64_t) + sizeof(uint32_t), buffer, NULL,
+  read_from_netdrv_fail_on_error(fed->fed_netdrv, sizeof(int64_t) + sizeof(uint32_t), buffer, NULL,
                                  "RTI failed to read the content of the logical tag complete from federate %d.",
                                  fed->enclave.id);
   tag_t completed = extract_tag(buffer);
@@ -371,7 +372,7 @@ void handle_latest_tag_confirmed(federate_info_t* fed) {
 
 void handle_next_event_tag(federate_info_t* fed) {
   unsigned char buffer[sizeof(int64_t) + sizeof(uint32_t)];
-  read_from_socket_fail_on_error(&fed->socket, sizeof(int64_t) + sizeof(uint32_t), buffer, NULL,
+  read_from_netdrv_fail_on_error(fed->fed_netdrv, sizeof(int64_t) + sizeof(uint32_t), buffer, NULL,
                                  "RTI failed to read the content of the next event tag from federate %d.",
                                  fed->enclave.id);
 
@@ -429,7 +430,7 @@ static void broadcast_stop_time_to_federates_locked() {
     if (rti_remote->base.tracing_enabled) {
       tracepoint_rti_to_federate(send_STOP_GRN, fed->enclave.id, &rti_remote->base.max_stop_tag);
     }
-    write_to_socket_fail_on_error(&fed->socket, MSG_TYPE_STOP_GRANTED_LENGTH, outgoing_buffer, &rti_mutex,
+    write_to_netdrv_fail_on_error(fed->fed_netdrv, MSG_TYPE_STOP_GRANTED_LENGTH, outgoing_buffer, &rti_mutex,
                                   "RTI failed to send MSG_TYPE_STOP_GRANTED message to federate %d.", fed->enclave.id);
   }
 
@@ -482,7 +483,7 @@ void handle_stop_request_message(federate_info_t* fed) {
 
   size_t bytes_to_read = MSG_TYPE_STOP_REQUEST_LENGTH - 1;
   unsigned char buffer[bytes_to_read];
-  read_from_socket_fail_on_error(&fed->socket, bytes_to_read, buffer, NULL,
+  read_from_netdrv_fail_on_error(fed->fed_netdrv, bytes_to_read, buffer, NULL,
                                  "RTI failed to read the MSG_TYPE_STOP_REQUEST payload from federate %d.",
                                  fed->enclave.id);
 
@@ -549,7 +550,7 @@ void handle_stop_request_message(federate_info_t* fed) {
       if (rti_remote->base.tracing_enabled) {
         tracepoint_rti_to_federate(send_STOP_REQ, f->enclave.id, &rti_remote->base.max_stop_tag);
       }
-      write_to_socket_fail_on_error(&f->socket, MSG_TYPE_STOP_REQUEST_LENGTH, stop_request_buffer, &rti_mutex,
+      write_to_netdrv_fail_on_error(f->fed_netdrv, MSG_TYPE_STOP_REQUEST_LENGTH, stop_request_buffer, &rti_mutex,
                                     "RTI failed to forward MSG_TYPE_STOP_REQUEST message to federate %d.",
                                     f->enclave.id);
     }
@@ -562,7 +563,7 @@ void handle_stop_request_message(federate_info_t* fed) {
 void handle_stop_request_reply(federate_info_t* fed) {
   size_t bytes_to_read = MSG_TYPE_STOP_REQUEST_REPLY_LENGTH - 1;
   unsigned char buffer_stop_time[bytes_to_read];
-  read_from_socket_fail_on_error(&fed->socket, bytes_to_read, buffer_stop_time, NULL,
+  read_from_netdrv_fail_on_error(fed->fed_netdrv, bytes_to_read, buffer_stop_time, NULL,
                                  "RTI failed to read the reply to MSG_TYPE_STOP_REQUEST message from federate %d.",
                                  fed->enclave.id);
 
@@ -591,8 +592,8 @@ void handle_address_query(uint16_t fed_id) {
   federate_info_t* fed = GET_FED_INFO(fed_id);
   // Use buffer both for reading and constructing the reply.
   // The length is what is needed for the reply.
-  unsigned char buffer[1 + sizeof(int32_t)];
-  read_from_socket_fail_on_error(&fed->socket, sizeof(uint16_t), (unsigned char*)buffer, NULL,
+  unsigned char buffer[1 + sizeof(int32_t) + sizeof(uint32_t)];
+  read_from_netdrv_fail_on_error(fed->fed_netdrv, sizeof(uint16_t), (unsigned char*)buffer, NULL,
                                  "Failed to read address query.");
   uint16_t remote_fed_id = extract_uint16(buffer);
 
@@ -614,18 +615,23 @@ void handle_address_query(uint16_t fed_id) {
 
   // Send the port number (which could be -1).
   LF_MUTEX_LOCK(&rti_mutex);
-  encode_int32(remote_fed->server_port, (unsigned char*)&buffer[1]);
-  write_to_socket_fail_on_error(&fed->socket, sizeof(int32_t) + 1, (unsigned char*)buffer, &rti_mutex,
-                                "Failed to write port number to socket of federate %d.", fed_id);
-
-  // Send the server IP address to federate.
-  write_to_socket_fail_on_error(&fed->socket, sizeof(remote_fed->server_ip_addr),
-                                (unsigned char*)&remote_fed->server_ip_addr, &rti_mutex,
-                                "Failed to write ip address to socket of federate %d.", fed_id);
+  int32_t server_port = get_server_port(remote_fed->fed_netdrv);
+
+  encode_int32(server_port, (unsigned char*)&buffer[1]);
+  encode_uint32(get_server_ip_addr(remote_fed->fed_netdrv), buffer + 1 + sizeof(int32_t));
+
+  // Send the port number (which could be -1) and the server IP address to federate.
+  write_to_netdrv_fail_on_error(fed->fed_netdrv, 1 + sizeof(int32_t) + sizeof(uint32_t), (unsigned char*)buffer,
+                                &rti_mutex, "Failed to write port number and ip address to socket of federate %d.",
+                                fed_id);
+
+  // write_to_netdrv_fail_on_error(fed->fed_netdrv, sizeof(remote_fed->server_ip_addr),
+  //                               (unsigned char*)&remote_fed->server_ip_addr, &rti_mutex,
+  //                               "Failed to write ip address to socket of federate %d.", fed_id);
   LF_MUTEX_UNLOCK(&rti_mutex);
 
-  LF_PRINT_DEBUG("Replied to address query from federate %d with address %s:%d.", fed_id, remote_fed->server_hostname,
-                 remote_fed->server_port);
+  LF_PRINT_DEBUG("Replied to address query from federate %d with address %s:%d.", fed_id,
+                 get_server_hostname(remote_fed->fed_netdrv), server_port);
 }
 
 void handle_address_ad(uint16_t federate_id) {
@@ -634,7 +640,7 @@ void handle_address_ad(uint16_t federate_id) {
   // connections to other federates
   int32_t server_port = -1;
   unsigned char buffer[sizeof(int32_t)];
-  read_from_socket_fail_on_error(&fed->socket, sizeof(int32_t), (unsigned char*)buffer, NULL,
+  read_from_netdrv_fail_on_error(fed->fed_netdrv, sizeof(int32_t), (unsigned char*)buffer, NULL,
                                  "Error reading port data from federate %d.", federate_id);
 
   server_port = extract_int32(buffer);
@@ -642,7 +648,7 @@ void handle_address_ad(uint16_t federate_id) {
   assert(server_port < 65536);
 
   LF_MUTEX_LOCK(&rti_mutex);
-  fed->server_port = server_port;
+  set_port(fed->fed_netdrv, server_port);
   LF_MUTEX_UNLOCK(&rti_mutex);
 
   LF_PRINT_LOG("Received address advertisement with port %d from federate %d.", server_port, federate_id);
@@ -654,7 +660,7 @@ void handle_address_ad(uint16_t federate_id) {
 void handle_timestamp(federate_info_t* my_fed) {
   unsigned char buffer[sizeof(int64_t)];
   // Read bytes from the socket. We need 8 bytes.
-  read_from_socket_fail_on_error(&my_fed->socket, sizeof(int64_t), (unsigned char*)&buffer, NULL,
+  read_from_netdrv_fail_on_error(my_fed->fed_netdrv, sizeof(int64_t), (unsigned char*)&buffer, NULL,
                                  "ERROR reading timestamp from federate %d.\n", my_fed->enclave.id);
 
   int64_t timestamp = swap_bytes_if_big_endian_int64(*((int64_t*)(&buffer)));
@@ -696,7 +702,7 @@ void handle_timestamp(federate_info_t* my_fed) {
     tag_t tag = {.time = start_time, .microstep = 0};
     tracepoint_rti_to_federate(send_TIMESTAMP, my_fed->enclave.id, &tag);
   }
-  if (write_to_socket(my_fed->socket, MSG_TYPE_TIMESTAMP_LENGTH, start_time_buffer)) {
+  if (write_to_netdrv(my_fed->fed_netdrv, MSG_TYPE_TIMESTAMP_LENGTH, start_time_buffer)) {
     lf_print_error("Failed to send the starting time to federate %d.", my_fed->enclave.id);
   }
 
@@ -735,7 +741,7 @@ void send_physical_clock(unsigned char message_type, federate_info_t* fed, socke
   } else if (socket_type == TCP) {
     LF_PRINT_DEBUG("Clock sync:  RTI sending TCP message type %u.", buffer[0]);
     LF_MUTEX_LOCK(&rti_mutex);
-    write_to_socket_fail_on_error(&fed->socket, 1 + sizeof(int64_t), buffer, &rti_mutex,
+    write_to_netdrv_fail_on_error(fed->fed_netdrv, 1 + sizeof(int64_t), buffer, &rti_mutex,
                                   "Clock sync: RTI failed to send physical time to federate %d.", fed->enclave.id);
     LF_MUTEX_UNLOCK(&rti_mutex);
   }
@@ -871,7 +877,7 @@ static void handle_federate_failed(federate_info_t* my_fed) {
   // Indicate that there will no further events from this federate.
   my_fed->enclave.next_event = FOREVER_TAG;
 
-  shutdown_socket(&my_fed->socket, false);
+  shutdown_netdrv(my_fed->fed_netdrv, false);
 
   // Check downstream federates to see whether they should now be granted a TAG.
   // To handle cycles, need to create a boolean array to keep
@@ -910,7 +916,7 @@ static void handle_federate_resign(federate_info_t* my_fed) {
   // Indicate that there will no further events from this federate.
   my_fed->enclave.next_event = FOREVER_TAG;
 
-  shutdown_socket(&my_fed->socket, true);
+  shutdown_netdrv(my_fed->fed_netdrv, true);
 
   // Check downstream federates to see whether they should now be granted a TAG.
   // To handle cycles, need to create a boolean array to keep
@@ -934,7 +940,7 @@ void* federate_info_thread_TCP(void* fed) {
   // Listen for messages from the federate.
   while (my_fed->enclave.state != NOT_CONNECTED) {
     // Read no more than one byte to get the message type.
-    int read_failed = read_from_socket(my_fed->socket, 1, buffer);
+    int read_failed = read_from_netdrv(my_fed->fed_netdrv, 1, buffer);
     if (read_failed) {
       // Socket is closed
       lf_print_error("RTI: Socket to federate %d is closed. Exiting the thread.", my_fed->enclave.id);
@@ -942,7 +948,7 @@ void* federate_info_thread_TCP(void* fed) {
       // Nothing more to do. Close the socket and exit.
       // Prevent multiple threads from closing the same socket at the same time.
       LF_MUTEX_LOCK(&rti_mutex);
-      shutdown_socket(&my_fed->socket, false); //  from unistd.h
+      shutdown_netdrv(my_fed->fed_netdrv, false); //  from unistd.h
       LF_MUTEX_UNLOCK(&rti_mutex);
       // FIXME: We need better error handling here, but do not stop execution here.
       break;
@@ -1129,7 +1135,7 @@ static int32_t receive_and_check_fed_id_message(netdrv_t* fed_netdrv) {
     tracepoint_rti_to_federate(send_ACK, fed_id, NULL);
   }
   LF_MUTEX_LOCK(&rti_mutex);
-  if (write_to_socket_close_on_error(&fed->socket, 1, &ack_message)) {
+  if (write_to_netdrv_close_on_error(fed->fed_netdrv, 1, &ack_message)) {
     LF_MUTEX_UNLOCK(&rti_mutex);
     lf_print_error("RTI failed to write MSG_TYPE_ACK message to federate %d.", fed_id);
     return -1;
@@ -1515,6 +1521,7 @@ void wait_for_federates() {
   shutdown_netdrv(rti_remote->rti_netdrv, false);
 
   if (rti_remote->socket_descriptor_UDP > 0) {
+    // UDP only uses sockets.
     shutdown_socket(&rti_remote->socket_descriptor_UDP, false);
   }
 }

From 373a623d877839b146d58637f5ce8a3a036dba97 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Wed, 15 Jan 2025 22:51:41 -0700
Subject: [PATCH 32/84] Add get and set functions to be used in rti_remote.c

---
 network/api/net_driver.h             |  8 ++++
 network/impl/src/lf_socket_support.c | 66 ++++++++++++----------------
 2 files changed, 36 insertions(+), 38 deletions(-)

diff --git a/network/api/net_driver.h b/network/api/net_driver.h
index 362efd3f0..fec967843 100644
--- a/network/api/net_driver.h
+++ b/network/api/net_driver.h
@@ -133,4 +133,12 @@ void write_to_netdrv_fail_on_error(netdrv_t* drv, size_t num_bytes, unsigned cha
  */
 int shutdown_netdrv(netdrv_t* drv, bool read_before_closing);
 
+int32_t get_server_port(netdrv_t* drv);
+
+uint32_t get_server_ip_addr(netdrv_t* drv);
+
+char* get_server_hostname(netdrv_t* drv);
+
+void set_port(netdrv_t* drv, int32_t port);
+
 #endif /* NET_DRIVER_H */
diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
index 4d286df26..a6d8e0261 100644
--- a/network/impl/src/lf_socket_support.c
+++ b/network/impl/src/lf_socket_support.c
@@ -1,7 +1,8 @@
-#include <stdlib.h> // malloc()
-#include <string.h> // strerror()
-#include <errno.h>  // errno
-#include <unistd.h> // read() write()
+#include <stdlib.h>    // malloc()
+#include <string.h>    // strerror()
+#include <errno.h>     // errno
+#include <unistd.h>    // read() write()
+#include <arpa/inet.h> // inet_ntop
 
 #include "net_driver.h"
 #include "socket_common.h"
@@ -107,6 +108,7 @@ int get_peer_address(netdrv_t* drv) {
   socklen_t addr_len = sizeof(peer_addr);
   if (getpeername(priv->socket_descriptor, (struct sockaddr*)&peer_addr, &addr_len) != 0) {
     lf_print_error("RTI failed to get peer address.");
+    return -1;
   }
   priv->server_ip_addr = peer_addr.sin_addr;
 
@@ -114,11 +116,12 @@ int get_peer_address(netdrv_t* drv) {
   // Create the human readable format and copy that into
   // the .server_hostname field of the federate.
   char str[INET_ADDRSTRLEN + 1];
-  inet_ntop(AF_INET, priv->server_ip_addr, str, INET_ADDRSTRLEN);
+  inet_ntop(AF_INET, &priv->server_ip_addr, str, INET_ADDRSTRLEN);
   strncpy(priv->server_hostname, str, INET_ADDRSTRLEN);
 
   LF_PRINT_DEBUG("RTI got address %s", priv->server_hostname);
 #endif
+  return 0;
 }
 
 int read_from_netdrv(netdrv_t* drv, size_t num_bytes, unsigned char* buffer) {
@@ -242,38 +245,25 @@ void write_to_netdrv_fail_on_error(netdrv_t* drv, size_t num_bytes, unsigned cha
 
 int shutdown_netdrv(netdrv_t* drv, bool read_before_closing) {
   socket_priv_t* priv = (socket_priv_t*)drv->priv;
-  if (!read_before_closing) {
-    if (shutdown(priv->socket_descriptor, SHUT_RDWR)) {
-      lf_print_log("On shutdown socket, received reply: %s", strerror(errno));
-      return -1;
-    }
-  } else {
-    // Signal the other side that no further writes are expected by sending a FIN packet.
-    // This indicates the write direction is closed. For more details, refer to:
-    // https://stackoverflow.com/questions/4160347/close-vs-shutdown-socket
-    if (shutdown(priv->socket_descriptor, SHUT_WR)) {
-      lf_print_log("Failed to shutdown socket: %s", strerror(errno));
-      return -1;
-    }
+  return shutdown_socket(&priv->socket_descriptor, read_before_closing);
+}
 
-    // Wait for the other side to send an EOF or encounter a socket error.
-    // Discard any incoming bytes. Normally, this read should return 0, indicating the peer has also closed the
-    // connection.
-    // This compensates for delayed ACKs and scenarios where Nagle's algorithm is disabled, ensuring the shutdown
-    // completes gracefully.
-    unsigned char buffer[10];
-    while (read(priv->socket_descriptor, buffer, 10) > 0)
-      ;
-  }
-  // NOTE: In all common TCP/IP stacks, there is a time period,
-  // typically between 30 and 120 seconds, called the TIME_WAIT period,
-  // before the port is released after this close. This is because
-  // the OS is preventing another program from accidentally receiving
-  // duplicated packets intended for this program.
-  if (close(priv->socket_descriptor)) {
-    lf_print_log("Error while closing socket: %s\n", strerror(errno));
-    return -1;
-  }
-  priv->socket_descriptor = -1;
-  return 0;
+int32_t get_server_port(netdrv_t* drv) {
+  socket_priv_t* priv = (socket_priv_t*)drv->priv;
+  return priv->server_port;
+}
+
+uint32_t get_server_ip_addr(netdrv_t* drv) {
+  socket_priv_t* priv = (socket_priv_t*)drv->priv;
+  return priv->server_ip_addr.s_addr;
+}
+
+char* get_server_hostname(netdrv_t* drv) {
+  socket_priv_t* priv = (socket_priv_t*)drv->priv;
+  return priv->server_hostname;
+}
+
+void set_port(netdrv_t* drv, int32_t port) {
+  socket_priv_t* priv = (socket_priv_t*)drv->priv;
+  priv->server_port = port;
 }

From d2d982b9d6166b8ecf8e7e36b2b149493559f2c2 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Wed, 15 Jan 2025 22:51:58 -0700
Subject: [PATCH 33/84] Minor fix.

---
 network/api/socket_common.h      | 1 +
 network/impl/src/socket_common.c | 3 +--
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/network/api/socket_common.h b/network/api/socket_common.h
index 7e52c06a1..fd91640ac 100644
--- a/network/api/socket_common.h
+++ b/network/api/socket_common.h
@@ -4,6 +4,7 @@
 #include "low_level_platform.h"
 #include <sys/types.h>
 #include <netinet/in.h>
+#include <sys/time.h>
 
 /**
  * The amount of time to wait after a failed socket read or write before trying again. This defaults to 100 ms.
diff --git a/network/impl/src/socket_common.c b/network/impl/src/socket_common.c
index e43181738..202ad1125 100644
--- a/network/impl/src/socket_common.c
+++ b/network/impl/src/socket_common.c
@@ -119,7 +119,7 @@ int set_socket_bind_option(int socket_descriptor, uint16_t specified_port, bool
   return used_port;
 }
 
-//TODO: Fix on federate.
+// TODO: Fix on federate.
 int create_server(uint16_t port, int* final_socket, uint16_t* final_port, socket_type_t sock_type,
                   bool increment_port_on_retry) {
   int socket_descriptor;
@@ -157,7 +157,6 @@ int create_server(uint16_t port, int* final_socket, uint16_t* final_port, socket
   return 0;
 }
 
-
 int create_clock_server(uint16_t port, int* final_socket, uint16_t* final_port) {
   int socket_descriptor;
   struct timeval timeout_time;

From d6471208f3fc8c72504aa172994a4c0ab1d00274 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Wed, 15 Jan 2025 23:57:01 -0700
Subject: [PATCH 34/84] Revert to sending two write()s in
 handle_address_query()

---
 core/federated/RTI/rti_remote.c      | 19 +++++++++----------
 network/api/net_driver.h             |  2 +-
 network/impl/src/lf_socket_support.c |  7 +++++--
 3 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index 5d2693a96..35794371d 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -592,7 +592,7 @@ void handle_address_query(uint16_t fed_id) {
   federate_info_t* fed = GET_FED_INFO(fed_id);
   // Use buffer both for reading and constructing the reply.
   // The length is what is needed for the reply.
-  unsigned char buffer[1 + sizeof(int32_t) + sizeof(uint32_t)];
+  unsigned char buffer[1 + sizeof(int32_t)];
   read_from_netdrv_fail_on_error(fed->fed_netdrv, sizeof(uint16_t), (unsigned char*)buffer, NULL,
                                  "Failed to read address query.");
   uint16_t remote_fed_id = extract_uint16(buffer);
@@ -613,21 +613,21 @@ void handle_address_query(uint16_t fed_id) {
   // Encode the port number.
   federate_info_t* remote_fed = GET_FED_INFO(remote_fed_id);
 
+  // The fed_netdrv is NULL yet, which means, the RTI does not know the port number because it has not yet received an MSG_TYPE_ADDRESS_ADVERTISEMENT message from this federate. In that case, it will respond by sending -1.
+  if (remote_fed->fed_netdrv == NULL) {
+  }
   // Send the port number (which could be -1).
   LF_MUTEX_LOCK(&rti_mutex);
   int32_t server_port = get_server_port(remote_fed->fed_netdrv);
 
   encode_int32(server_port, (unsigned char*)&buffer[1]);
-  encode_uint32(get_server_ip_addr(remote_fed->fed_netdrv), buffer + 1 + sizeof(int32_t));
 
   // Send the port number (which could be -1) and the server IP address to federate.
-  write_to_netdrv_fail_on_error(fed->fed_netdrv, 1 + sizeof(int32_t) + sizeof(uint32_t), (unsigned char*)buffer,
-                                &rti_mutex, "Failed to write port number and ip address to socket of federate %d.",
-                                fed_id);
+  write_to_netdrv_fail_on_error(fed->fed_netdrv, 1 + sizeof(int32_t), (unsigned char*)buffer, &rti_mutex,
+                                "Failed to write port number to socket of federate %d.", fed_id);
 
-  // write_to_netdrv_fail_on_error(fed->fed_netdrv, sizeof(remote_fed->server_ip_addr),
-  //                               (unsigned char*)&remote_fed->server_ip_addr, &rti_mutex,
-  //                               "Failed to write ip address to socket of federate %d.", fed_id);
+  write_to_netdrv_fail_on_error(fed->fed_netdrv, sizeof(uint32_t), (unsigned char*)get_ip_addr(remote_fed->fed_netdrv),
+                                &rti_mutex, "Failed to write ip address to socket of federate %d.", fed_id);
   LF_MUTEX_UNLOCK(&rti_mutex);
 
   LF_PRINT_DEBUG("Replied to address query from federate %d with address %s:%d.", fed_id,
@@ -1289,8 +1289,7 @@ static int receive_udp_message_and_set_up_clock_sync(netdrv_t* fed_netdrv, uint1
           fed->UDP_addr.sin_family = AF_INET;
           fed->UDP_addr.sin_port = htons(federate_UDP_port_number);
 #ifdef COMM_TYPE_TCP
-          fed->UDP_addr.sin_addr = ((socket_priv_t*)fed_netdrv->priv)->server_ip_addr;
-// fed->UDP_addr.sin_addr = fed->server_ip_addr;
+          fed->UDP_addr.sin_addr = *get_ip_addr(fed_netdrv);
 #elif
 #endif
         }
diff --git a/network/api/net_driver.h b/network/api/net_driver.h
index fec967843..a70abca7e 100644
--- a/network/api/net_driver.h
+++ b/network/api/net_driver.h
@@ -135,7 +135,7 @@ int shutdown_netdrv(netdrv_t* drv, bool read_before_closing);
 
 int32_t get_server_port(netdrv_t* drv);
 
-uint32_t get_server_ip_addr(netdrv_t* drv);
+struct in_addr* get_ip_addr(netdrv_t* drv);
 
 char* get_server_hostname(netdrv_t* drv);
 
diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
index a6d8e0261..34844e8de 100644
--- a/network/impl/src/lf_socket_support.c
+++ b/network/impl/src/lf_socket_support.c
@@ -249,13 +249,16 @@ int shutdown_netdrv(netdrv_t* drv, bool read_before_closing) {
 }
 
 int32_t get_server_port(netdrv_t* drv) {
+  // if (drv == NULL) {
+  //   lf_print_warning("Netdriver is closed, returning -1.");
+  // }
   socket_priv_t* priv = (socket_priv_t*)drv->priv;
   return priv->server_port;
 }
 
-uint32_t get_server_ip_addr(netdrv_t* drv) {
+struct in_addr* get_ip_addr(netdrv_t* drv) {
   socket_priv_t* priv = (socket_priv_t*)drv->priv;
-  return priv->server_ip_addr.s_addr;
+  return &priv->server_ip_addr;
 }
 
 char* get_server_hostname(netdrv_t* drv) {

From 225e0b5ff0b3fda793a91b7465c105094ec8bff3 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Thu, 16 Jan 2025 10:21:08 -0700
Subject: [PATCH 35/84] Add logic in handle_address_query(), to check if the
 remote_fed's network driver is initialized, and send default values when not
 initialized.

---
 core/federated/RTI/rti_remote.c | 34 +++++++++++++++++++++++++--------
 1 file changed, 26 insertions(+), 8 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index 35794371d..d6d8f0c47 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -613,12 +613,26 @@ void handle_address_query(uint16_t fed_id) {
   // Encode the port number.
   federate_info_t* remote_fed = GET_FED_INFO(remote_fed_id);
 
-  // The fed_netdrv is NULL yet, which means, the RTI does not know the port number because it has not yet received an MSG_TYPE_ADDRESS_ADVERTISEMENT message from this federate. In that case, it will respond by sending -1.
+  int32_t server_port;
+  uint32_t* ip_address;
+  char* server_host_name;
+
+  LF_MUTEX_LOCK(&rti_mutex);
+  // Check if the RTI has initialized the remote federate's network driver.
   if (remote_fed->fed_netdrv == NULL) {
+    // RTI has not set up the remote federate. Respond with -1 to indicate an unknown port number.
+    server_port = -1;
+    uint32_t temp = 0;
+    ip_address = &temp;
+    server_host_name = "localhost";
+  } else {
+    // The network driver is initialized, but the RTI might still not know the port number. This can happen if the RTI
+    // has not yet received a MSG_TYPE_ADDRESS_ADVERTISEMENT message from the remote federate. In such cases, the
+    // returned port number might still be -1.
+    server_port = get_server_port(remote_fed->fed_netdrv);
+    ip_address = (uint32_t*)get_ip_addr(remote_fed->fed_netdrv);
+    server_host_name = get_server_hostname(remote_fed->fed_netdrv);
   }
-  // Send the port number (which could be -1).
-  LF_MUTEX_LOCK(&rti_mutex);
-  int32_t server_port = get_server_port(remote_fed->fed_netdrv);
 
   encode_int32(server_port, (unsigned char*)&buffer[1]);
 
@@ -626,12 +640,12 @@ void handle_address_query(uint16_t fed_id) {
   write_to_netdrv_fail_on_error(fed->fed_netdrv, 1 + sizeof(int32_t), (unsigned char*)buffer, &rti_mutex,
                                 "Failed to write port number to socket of federate %d.", fed_id);
 
-  write_to_netdrv_fail_on_error(fed->fed_netdrv, sizeof(uint32_t), (unsigned char*)get_ip_addr(remote_fed->fed_netdrv),
-                                &rti_mutex, "Failed to write ip address to socket of federate %d.", fed_id);
+  write_to_netdrv_fail_on_error(fed->fed_netdrv, sizeof(uint32_t), (unsigned char*)ip_address, &rti_mutex,
+                                "Failed to write ip address to socket of federate %d.", fed_id);
   LF_MUTEX_UNLOCK(&rti_mutex);
 
-  LF_PRINT_DEBUG("Replied to address query from federate %d with address %s:%d.", fed_id,
-                 get_server_hostname(remote_fed->fed_netdrv), server_port);
+  LF_PRINT_DEBUG("Replied to address query from federate %d with address %s:%d.", fed_id, server_host_name,
+                 server_port);
 }
 
 void handle_address_ad(uint16_t federate_id) {
@@ -1381,6 +1395,10 @@ static bool authenticate_federate(netdrv_t* fed_netdrv) {
 #endif
 
 void lf_connect_to_federates(netdrv_t* rti_netdrv) {
+  // netdrv_t* netdrv_array[rti_remote->base.number_of_scheduling_nodes];
+  // for (int i = 0; i < rti_remote->base.number_of_scheduling_nodes; i++) {
+  //   netdrv_array[i] = establish_communication_session(rti_netdrv);
+  // }
   for (int i = 0; i < rti_remote->base.number_of_scheduling_nodes; i++) {
     netdrv_t* fed_netdrv = accept_netdrv(rti_netdrv, NULL);
 // Wait for the first message from the federate when RTI -a option is on.

From 0de95842565a623b657dd2470a157e88d40f8597 Mon Sep 17 00:00:00 2001
From: Dongha Kim <74869052+Jakio815@users.noreply.github.com>
Date: Fri, 17 Jan 2025 09:55:04 -0700
Subject: [PATCH 36/84] Update CMakeLists.txt

---
 network/api/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/network/api/CMakeLists.txt b/network/api/CMakeLists.txt
index 6f724fe1a..75a3c2814 100644
--- a/network/api/CMakeLists.txt
+++ b/network/api/CMakeLists.txt
@@ -11,4 +11,4 @@ target_link_libraries(lf-network-api INTERFACE lf::low-level-platform-api)
 # target_include_directories(lf-network-api INTERFACE ${CMAKE_CURRENT_LIST_DIR}/type)
 target_include_directories(lf-network-api INTERFACE ${CMAKE_CURRENT_LIST_DIR}/../../include/core/utils)
 
-target_compile_definitions(lf-network-api INTERFACE COMM_TYPE_${COMM_TYPE})
\ No newline at end of file
+target_compile_definitions(lf-network-api INTERFACE COMM_TYPE_${COMM_TYPE})

From de834545aa310a82d15caf2e4d64a026bb58b639 Mon Sep 17 00:00:00 2001
From: Dongha Kim <74869052+Jakio815@users.noreply.github.com>
Date: Fri, 17 Jan 2025 09:56:05 -0700
Subject: [PATCH 37/84] Update CMakeLists.txt

---
 network/api/CMakeLists.txt | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/network/api/CMakeLists.txt b/network/api/CMakeLists.txt
index 75a3c2814..08aaf8105 100644
--- a/network/api/CMakeLists.txt
+++ b/network/api/CMakeLists.txt
@@ -6,9 +6,6 @@ add_library(lf::network-api ALIAS lf-network-api)
 target_link_libraries(lf-network-api INTERFACE lf::tag-api)
 target_link_libraries(lf-network-api INTERFACE lf::low-level-platform-api)
 
-
-# #TODO:CHECK.
-# target_include_directories(lf-network-api INTERFACE ${CMAKE_CURRENT_LIST_DIR}/type)
 target_include_directories(lf-network-api INTERFACE ${CMAKE_CURRENT_LIST_DIR}/../../include/core/utils)
 
 target_compile_definitions(lf-network-api INTERFACE COMM_TYPE_${COMM_TYPE})

From 5cf0a470e172ac0600741a01862a3ad30f4cb037 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sat, 18 Jan 2025 17:31:39 -0700
Subject: [PATCH 38/84] Fixing data types to netdriver.

---
 core/federated/federate.c         | 54 +++++++++++++++----------------
 include/core/federated/federate.h | 39 +++++++++++-----------
 2 files changed, 47 insertions(+), 46 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index a689ae090..52dac1281 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -78,7 +78,7 @@ int max_level_allowed_to_advance;
  */
 federate_instance_t _fed = {.socket_TCP_RTI = -1,
                             .number_of_inbound_p2p_connections = 0,
-                            .inbound_socket_listeners = NULL,
+                            .inbound_netdriv_listeners = NULL,
                             .number_of_outbound_p2p_connections = 0,
                             .inbound_p2p_handling_thread_id = 0,
                             .server_socket = -1,
@@ -411,8 +411,8 @@ static trigger_handle_t schedule_message_received_from_network_locked(environmen
  */
 static void close_inbound_socket(int fed_id) {
   LF_MUTEX_LOCK(&socket_mutex);
-  if (_fed.sockets_for_inbound_p2p_connections[fed_id] >= 0) {
-    shutdown_socket(&_fed.sockets_for_inbound_p2p_connections[fed_id], false);
+  if (_fed.netdrvs_for_inbound_p2p_connections[fed_id] >= 0) {
+    shutdown_socket(&_fed.netdrvs_for_inbound_p2p_connections[fed_id], false);
   }
   LF_MUTEX_UNLOCK(&socket_mutex);
 }
@@ -726,7 +726,7 @@ static int handle_port_absent_message(int* socket, int fed_id) {
  * peer federate and calls the appropriate handling function for
  * each message type. If an error occurs or an EOF is received
  * from the peer, then this procedure sets the corresponding
- * socket in _fed.sockets_for_inbound_p2p_connections
+ * socket in _fed.netdrvs_for_inbound_p2p_connections
  * to -1 and returns, terminating the thread.
  * @param _args The remote federate ID (cast to void*).
  * @param fed_id_ptr A pointer to a uint16_t containing federate ID being listened to.
@@ -738,7 +738,7 @@ static void* listen_to_federates(void* _args) {
 
   LF_PRINT_LOG("Listening to federate %d.", fed_id);
 
-  int* socket_id = &_fed.sockets_for_inbound_p2p_connections[fed_id];
+  int* socket_id = &_fed.netdrvs_for_inbound_p2p_connections[fed_id];
 
   // Buffer for incoming messages.
   // This does not constrain the message size
@@ -823,14 +823,14 @@ static void close_outbound_socket(int fed_id) {
   // abnormal termination, in which case it will just close the socket.
   if (_lf_normal_termination) {
     LF_MUTEX_LOCK(&lf_outbound_socket_mutex);
-    if (_fed.sockets_for_outbound_p2p_connections[fed_id] >= 0) {
+    if (_fed.netdrvs_for_outbound_p2p_connections[fed_id] >= 0) {
       // Close the socket by sending a FIN packet indicating that no further writes
       // are expected.  Then read until we get an EOF indication.
-      shutdown_socket(&_fed.sockets_for_outbound_p2p_connections[fed_id], true);
+      shutdown_socket(&_fed.netdrvs_for_outbound_p2p_connections[fed_id], true);
     }
     LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex);
   } else {
-    shutdown_socket(&_fed.sockets_for_outbound_p2p_connections[fed_id], false);
+    shutdown_socket(&_fed.netdrvs_for_outbound_p2p_connections[fed_id], false);
   }
 }
 
@@ -1632,11 +1632,11 @@ void lf_terminate_execution(environment_t* env) {
   for (int i = 0; i < NUMBER_OF_FEDERATES; i++) {
     close_inbound_socket(i);
     // Ignore errors. Mark the socket closed.
-    _fed.sockets_for_inbound_p2p_connections[i] = -1;
+    _fed.netdrvs_for_inbound_p2p_connections[i] = -1;
   }
 
   // Check for all outgoing physical connections in
-  // _fed.sockets_for_outbound_p2p_connections and
+  // _fed.netdrvs_for_outbound_p2p_connections and
   // if the socket ID is not -1, the connection is still open.
   // Send an EOF by closing the socket here.
   for (int i = 0; i < NUMBER_OF_FEDERATES; i++) {
@@ -1649,23 +1649,23 @@ void lf_terminate_execution(environment_t* env) {
 
   LF_PRINT_DEBUG("Waiting for inbound p2p socket listener threads.");
   // Wait for each inbound socket listener thread to close.
-  if (_fed.number_of_inbound_p2p_connections > 0 && _fed.inbound_socket_listeners != NULL) {
+  if (_fed.number_of_inbound_p2p_connections > 0 && _fed.inbound_netdriv_listeners != NULL) {
     LF_PRINT_LOG("Waiting for %zu threads listening for incoming messages to exit.",
                  _fed.number_of_inbound_p2p_connections);
     for (size_t i = 0; i < _fed.number_of_inbound_p2p_connections; i++) {
       // Ignoring errors here.
-      lf_thread_join(_fed.inbound_socket_listeners[i], NULL);
+      lf_thread_join(_fed.inbound_netdriv_listeners[i], NULL);
     }
   }
 
-  LF_PRINT_DEBUG("Waiting for RTI's socket listener threads.");
+  LF_PRINT_DEBUG("Waiting for RTI's network driver listener threads.");
   // Wait for the thread listening for messages from the RTI to close.
-  lf_thread_join(_fed.RTI_socket_listener, NULL);
+  lf_thread_join(_fed.RTI_netdrv_listener, NULL);
 
   // For abnormal termination, there is no need to free memory.
   if (_lf_normal_termination) {
     LF_PRINT_DEBUG("Freeing memory occupied by the federate.");
-    free(_fed.inbound_socket_listeners);
+    free(_fed.inbound_netdriv_listeners);
     free(federation_metadata.rti_host);
     free(federation_metadata.rti_user);
   }
@@ -1801,7 +1801,7 @@ void lf_connect_to_federate(uint16_t remote_federate_id) {
   }
   // Once we set this variable, then all future calls to close() on this
   // socket ID should reset it to -1 within a critical section.
-  _fed.sockets_for_outbound_p2p_connections[remote_federate_id] = socket_id;
+  _fed.netdrvs_for_outbound_p2p_connections[remote_federate_id] = socket_id;
 }
 
 void lf_connect_to_rti(const char* hostname, int port) {
@@ -1967,7 +1967,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
   LF_ASSERT_NON_NULL(env_arg);
   size_t received_federates = 0;
   // Allocate memory to store thread IDs.
-  _fed.inbound_socket_listeners = (lf_thread_t*)calloc(_fed.number_of_inbound_p2p_connections, sizeof(lf_thread_t));
+  _fed.inbound_netdriv_listeners = (lf_thread_t*)calloc(_fed.number_of_inbound_p2p_connections, sizeof(lf_thread_t));
   while (received_federates < _fed.number_of_inbound_p2p_connections && !_lf_termination_executed) {
     // Wait for an incoming connection request.
     int socket_id = accept_socket(_fed.server_socket, _fed.socket_TCP_RTI);
@@ -2028,7 +2028,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
     // element should be reset to -1 during that critical section.
     // Otherwise, there can be race condition where, during termination,
     // two threads attempt to simultaneously close the socket.
-    _fed.sockets_for_inbound_p2p_connections[remote_fed_id] = socket_id;
+    _fed.netdrvs_for_inbound_p2p_connections[remote_fed_id] = socket_id;
 
     // Send an MSG_TYPE_ACK message.
     unsigned char response = MSG_TYPE_ACK;
@@ -2037,7 +2037,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
     tracepoint_federate_to_federate(send_ACK, _lf_my_fed_id, remote_fed_id, NULL);
 
     LF_MUTEX_LOCK(&lf_outbound_socket_mutex);
-    write_to_socket_fail_on_error(&_fed.sockets_for_inbound_p2p_connections[remote_fed_id], 1,
+    write_to_socket_fail_on_error(&_fed.netdrvs_for_inbound_p2p_connections[remote_fed_id], 1,
                                   (unsigned char*)&response, &lf_outbound_socket_mutex,
                                   "Failed to write MSG_TYPE_ACK in response to federate %d.", remote_fed_id);
     LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex);
@@ -2045,13 +2045,13 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
     // Start a thread to listen for incoming messages from other federates.
     // The fed_id is a uint16_t, which we assume can be safely cast to and from void*.
     void* fed_id_arg = (void*)(uintptr_t)remote_fed_id;
-    int result = lf_thread_create(&_fed.inbound_socket_listeners[received_federates], listen_to_federates, fed_id_arg);
+    int result = lf_thread_create(&_fed.inbound_netdriv_listeners[received_federates], listen_to_federates, fed_id_arg);
     if (result != 0) {
       // Failed to create a listening thread.
       LF_MUTEX_LOCK(&socket_mutex);
-      if (_fed.sockets_for_inbound_p2p_connections[remote_fed_id] != -1) {
+      if (_fed.netdrvs_for_inbound_p2p_connections[remote_fed_id] != -1) {
         shutdown_socket(&socket_id, false);
-        _fed.sockets_for_inbound_p2p_connections[remote_fed_id] = -1;
+        _fed.netdrvs_for_inbound_p2p_connections[remote_fed_id] = -1;
       }
       LF_MUTEX_UNLOCK(&socket_mutex);
       lf_print_error_and_exit("Failed to create a thread to listen for incoming physical connection. Error code: %d.",
@@ -2160,7 +2160,7 @@ int lf_send_message(int message_type, unsigned short port, unsigned short federa
   // Use a mutex lock to prevent multiple threads from simultaneously sending.
   LF_MUTEX_LOCK(&lf_outbound_socket_mutex);
 
-  int* socket = &_fed.sockets_for_outbound_p2p_connections[federate];
+  int* socket = &_fed.netdrvs_for_outbound_p2p_connections[federate];
 
   // Trace the event when tracing is enabled
   tracepoint_federate_to_federate(send_P2P_MSG, _lf_my_fed_id, federate, NULL);
@@ -2340,7 +2340,7 @@ void lf_send_port_absent_to_federate(environment_t* env, interval_t additional_d
   int* socket = &_fed.socket_TCP_RTI;
 #else
   // Send the absent message directly to the federate
-  int* socket = &_fed.sockets_for_outbound_p2p_connections[fed_ID];
+  int* socket = &_fed.netdrvs_for_outbound_p2p_connections[fed_ID];
 #endif
 
   if (socket == &_fed.socket_TCP_RTI) {
@@ -2453,7 +2453,7 @@ int lf_send_tagged_message(environment_t* env, interval_t additional_delay, int
 
   int* socket;
   if (message_type == MSG_TYPE_P2P_TAGGED_MESSAGE) {
-    socket = &_fed.sockets_for_outbound_p2p_connections[federate];
+    socket = &_fed.netdrvs_for_outbound_p2p_connections[federate];
     tracepoint_federate_to_federate(send_P2P_TAGGED_MSG, _lf_my_fed_id, federate, &current_message_intended_tag);
   } else {
     socket = &_fed.socket_TCP_RTI;
@@ -2508,11 +2508,11 @@ void lf_synchronize_with_other_federates(void) {
   start_time = get_start_time_from_rti(lf_time_physical());
   lf_tracing_set_start_time(start_time);
 
-  // Start a thread to listen for incoming TCP messages from the RTI.
+  // Start a thread to listen for incoming messages from the RTI.
   // @note Up until this point, the federate has been listening for messages
   //  from the RTI in a sequential manner in the main thread. From now on, a
   //  separate thread is created to allow for asynchronous communication.
-  lf_thread_create(&_fed.RTI_socket_listener, listen_to_rti_TCP, NULL);
+  lf_thread_create(&_fed.RTI_netdrv_listener, listen_to_rti_TCP, NULL);
   lf_thread_t thread_id;
   if (create_clock_sync_thread(&thread_id)) {
     lf_print_warning("Failed to create thread to handle clock synchronization.");
diff --git a/include/core/federated/federate.h b/include/core/federated/federate.h
index 50c59daa1..a8cb325d0 100644
--- a/include/core/federated/federate.h
+++ b/include/core/federated/federate.h
@@ -18,6 +18,7 @@
 #include "lf_types.h"
 #include "environment.h"
 #include "low_level_platform.h"
+#include "net_driver.h"
 
 #ifndef ADVANCE_MESSAGE_INTERVAL
 #define ADVANCE_MESSAGE_INTERVAL MSEC(10)
@@ -31,16 +32,16 @@
  */
 typedef struct federate_instance_t {
   /**
-   * The TCP socket descriptor for this federate to communicate with the RTI.
+   * The network driver for this federate to communicate with the RTI.
    * This is set by lf_connect_to_rti(), which must be called before other
    * functions that communicate with the rti are called.
    */
-  int socket_TCP_RTI;
+  netdrv_t* netdrv_to_RTI;
 
   /**
-   * Thread listening for incoming TCP messages from the RTI.
+   * Thread listening for incoming messages from the RTI.
    */
-  lf_thread_t RTI_socket_listener;
+  lf_thread_t RTI_netdrv_listener;
 
   /**
    * Number of inbound physical connections to the federate.
@@ -54,7 +55,7 @@ typedef struct federate_instance_t {
    * This is NULL if there are none and otherwise has size given by
    * number_of_inbound_p2p_connections.
    */
-  lf_thread_t* inbound_socket_listeners;
+  lf_thread_t* inbound_netdriv_listeners;
 
   /**
    * Number of outbound peer-to-peer connections from the federate.
@@ -64,34 +65,34 @@ typedef struct federate_instance_t {
   size_t number_of_outbound_p2p_connections;
 
   /**
-   * An array that holds the socket descriptors for inbound
+   * An array that holds the network drivers for inbound
    * connections from each federate. The index will be the federate
    * ID of the remote sending federate. This is initialized at startup
-   * to -1 and is set to a socket ID by lf_handle_p2p_connections_from_federates()
-   * when the socket is opened.
+   * to -1 and is set to a socket ID by lf_handle_p2p_connections_from_federates() //TODO: Check here.
+   * when the network drivers is opened.
    *
-   * @note There will not be an inbound socket unless a physical connection
+   * @note There will not be an inbound network driver unless a physical connection
    * or a p2p logical connection (by setting the coordination target property
    * to "distributed") is specified in the Lingua Franca program where this
    * federate is the destination. Multiple incoming p2p connections from the
-   * same remote federate will use the same socket.
+   * same remote federate will use the same network driver.
    */
-  int sockets_for_inbound_p2p_connections[NUMBER_OF_FEDERATES];
+  netdrv_t* netdrvs_for_inbound_p2p_connections[NUMBER_OF_FEDERATES];
 
   /**
-   * An array that holds the socket descriptors for outbound direct
+   * An array that holds the network drivers for outbound direct
    * connections to each remote federate. The index will be the federate
    * ID of the remote receiving federate. This is initialized at startup
-   * to -1 and is set to a socket ID by lf_connect_to_federate()
-   * when the socket is opened.
+   * to -1 and is set to a socket ID by lf_connect_to_federate() //TODO: Check here.
+   * when the network drivers is opened.
    *
-   * @note This federate will not open an outbound socket unless a physical
+   * @note This federate will not open an outbound network drivers unless a physical
    * connection or a p2p logical connection (by setting the coordination target
    * property to "distributed") is specified in the Lingua Franca
    * program where this federate acts as the source. Multiple outgoing p2p
-   * connections to the same remote federate will use the same socket.
+   * connections to the same remote federate will use the same network drivers.
    */
-  int sockets_for_outbound_p2p_connections[NUMBER_OF_FEDERATES];
+  netdrv_t* netdrvs_for_outbound_p2p_connections[NUMBER_OF_FEDERATES];
 
   /**
    * Thread ID for a thread that accepts sockets and then supervises
@@ -105,7 +106,7 @@ typedef struct federate_instance_t {
    * This socket is used to listen to incoming physical connections from
    * remote federates. Once an incoming connection is accepted, the
    * opened socket will be stored in
-   * federate_sockets_for_inbound_p2p_connections.
+   * federate_netdrvs_for_inbound_p2p_connections.
    */
   int server_socket;
 
@@ -227,7 +228,7 @@ extern lf_cond_t lf_port_status_changed;
  * the IP address and port number of the specified federate. It then attempts
  * to establish a socket connection to the specified federate.
  * If this fails, the program exits. If it succeeds, it sets element [id] of
- * the _fed.sockets_for_outbound_p2p_connections global array to
+ * the _fed.netdrvs_for_outbound_p2p_connections global array to
  * refer to the socket for communicating directly with the federate.
  * @param remote_federate_id The ID of the remote federate.
  */

From 9c7c4ab429fe3d368c57e61382b51a96b73cb3f0 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sat, 18 Jan 2025 18:07:44 -0700
Subject: [PATCH 39/84] Fix names to netdriver.

---
 core/federated/federate.c         | 76 +++++++++++++++----------------
 include/core/federated/federate.h | 51 +++++++++------------
 2 files changed, 60 insertions(+), 67 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index 52dac1281..e7f3143cc 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -50,7 +50,7 @@ extern instant_t start_time;
 extern bool _lf_termination_executed;
 
 // Global variables references in federate.h
-lf_mutex_t lf_outbound_socket_mutex;
+lf_mutex_t lf_outbound_netdrv_mutex;
 lf_cond_t lf_port_status_changed;
 
 /**
@@ -100,7 +100,7 @@ federation_metadata_t federation_metadata = {
 // Static functions (used only internally)
 
 /**
- * Send a time to the RTI. This acquires the lf_outbound_socket_mutex.
+ * Send a time to the RTI. This acquires the lf_outbound_netdrv_mutex.
  * @param type The message type (MSG_TYPE_TIMESTAMP).
  * @param time The time.
  */
@@ -115,15 +115,15 @@ static void send_time(unsigned char type, instant_t time) {
   tag_t tag = {.time = time, .microstep = 0};
   tracepoint_federate_to_rti(send_TIMESTAMP, _lf_my_fed_id, &tag);
 
-  LF_MUTEX_LOCK(&lf_outbound_socket_mutex);
-  write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_write, buffer, &lf_outbound_socket_mutex,
+  LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
+  write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_write, buffer, &lf_outbound_netdrv_mutex,
                                 "Failed to send time " PRINTF_TIME " to the RTI.", time - start_time);
-  LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex);
+  LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
 }
 
 /**
  * Send a tag to the RTI.
- * This function acquires the lf_outbound_socket_mutex.
+ * This function acquires the lf_outbound_netdrv_mutex.
  * @param type The message type (MSG_TYPE_NEXT_EVENT_TAG or MSG_TYPE_LATEST_TAG_CONFIRMED).
  * @param tag The tag.
  */
@@ -134,18 +134,18 @@ static void send_tag(unsigned char type, tag_t tag) {
   buffer[0] = type;
   encode_tag(&(buffer[1]), tag);
 
-  LF_MUTEX_LOCK(&lf_outbound_socket_mutex);
+  LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
   if (_fed.socket_TCP_RTI < 0) {
     lf_print_warning("Socket is no longer connected. Dropping message.");
-    LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex);
+    LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
     return;
   }
   trace_event_t event_type = (type == MSG_TYPE_NEXT_EVENT_TAG) ? send_NET : send_LTC;
   // Trace the event when tracing is enabled
   tracepoint_federate_to_rti(event_type, _lf_my_fed_id, &tag);
-  write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_write, buffer, &lf_outbound_socket_mutex,
+  write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_write, buffer, &lf_outbound_netdrv_mutex,
                                 "Failed to send tag " PRINTF_TAG " to the RTI.", tag.time - start_time, tag.microstep);
-  LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex);
+  LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
 }
 
 /**
@@ -811,7 +811,7 @@ static void* listen_to_federates(void* _args) {
 
 /**
  * Close the socket that sends outgoing messages to the
- * specified federate ID. This function acquires the lf_outbound_socket_mutex mutex lock
+ * specified federate ID. This function acquires the lf_outbound_netdrv_mutex mutex lock
  * if _lf_normal_termination is true and otherwise proceeds without the lock.
  * @param fed_id The ID of the peer federate receiving messages from this
  *  federate, or -1 if the RTI (centralized coordination).
@@ -822,13 +822,13 @@ static void close_outbound_socket(int fed_id) {
   // This will result in EOF being sent to the remote federate, except for
   // abnormal termination, in which case it will just close the socket.
   if (_lf_normal_termination) {
-    LF_MUTEX_LOCK(&lf_outbound_socket_mutex);
+    LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
     if (_fed.netdrvs_for_outbound_p2p_connections[fed_id] >= 0) {
       // Close the socket by sending a FIN packet indicating that no further writes
       // are expected.  Then read until we get an EOF indication.
       shutdown_socket(&_fed.netdrvs_for_outbound_p2p_connections[fed_id], true);
     }
-    LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex);
+    LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
   } else {
     shutdown_socket(&_fed.netdrvs_for_outbound_p2p_connections[fed_id], false);
   }
@@ -1379,7 +1379,7 @@ static void handle_stop_request_message() {
   // is guarded by the outbound socket mutex.
   // The second is guarded by the global mutex.
   // Note that the RTI should not send stop requests more than once to federates.
-  LF_MUTEX_LOCK(&lf_outbound_socket_mutex);
+  LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
   if (_fed.received_stop_request_from_rti) {
     LF_PRINT_LOG("Redundant MSG_TYPE_STOP_REQUEST from RTI. Ignoring it.");
     already_blocked = true;
@@ -1388,7 +1388,7 @@ static void handle_stop_request_message() {
     // prevent lf_request_stop from sending.
     _fed.received_stop_request_from_rti = true;
   }
-  LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex);
+  LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
 
   if (already_blocked) {
     // Either we have sent a stop request to the RTI ourselves,
@@ -1422,11 +1422,11 @@ static void handle_stop_request_message() {
   tracepoint_federate_to_rti(send_STOP_REQ_REP, _lf_my_fed_id, &tag_to_stop);
 
   // Send the current logical time to the RTI.
-  LF_MUTEX_LOCK(&lf_outbound_socket_mutex);
+  LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
   write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, MSG_TYPE_STOP_REQUEST_REPLY_LENGTH, outgoing_buffer,
-                                &lf_outbound_socket_mutex,
+                                &lf_outbound_netdrv_mutex,
                                 "Failed to send the answer to MSG_TYPE_STOP_REQUEST to RTI.");
-  LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex);
+  LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
 
   LF_PRINT_DEBUG("Sent MSG_TYPE_STOP_REQUEST_REPLY to RTI with tag " PRINTF_TAG, tag_to_stop.time,
                  tag_to_stop.microstep);
@@ -1439,10 +1439,10 @@ static void send_resign_signal() {
   size_t bytes_to_write = 1;
   unsigned char buffer[bytes_to_write];
   buffer[0] = MSG_TYPE_RESIGN;
-  LF_MUTEX_LOCK(&lf_outbound_socket_mutex);
-  write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_write, &(buffer[0]), &lf_outbound_socket_mutex,
+  LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
+  write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_write, &(buffer[0]), &lf_outbound_netdrv_mutex,
                                 "Failed to send MSG_TYPE_RESIGN.");
-  LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex);
+  LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
   LF_PRINT_LOG("Resigned.");
 }
 
@@ -1694,10 +1694,10 @@ void lf_connect_to_federate(uint16_t remote_federate_id) {
     // Trace the event when tracing is enabled
     tracepoint_federate_to_rti(send_ADR_QR, _lf_my_fed_id, NULL);
 
-    LF_MUTEX_LOCK(&lf_outbound_socket_mutex);
-    write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, sizeof(uint16_t) + 1, buffer, &lf_outbound_socket_mutex,
+    LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
+    write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, sizeof(uint16_t) + 1, buffer, &lf_outbound_netdrv_mutex,
                                   "Failed to send address query for federate %d to RTI.", remote_federate_id);
-    LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex);
+    LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
 
     // Read RTI's response.
     read_from_socket_fail_on_error(&_fed.socket_TCP_RTI, sizeof(int32_t) + 1, buffer, NULL,
@@ -2036,11 +2036,11 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
     // Trace the event when tracing is enabled
     tracepoint_federate_to_federate(send_ACK, _lf_my_fed_id, remote_fed_id, NULL);
 
-    LF_MUTEX_LOCK(&lf_outbound_socket_mutex);
+    LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
     write_to_socket_fail_on_error(&_fed.netdrvs_for_inbound_p2p_connections[remote_fed_id], 1,
-                                  (unsigned char*)&response, &lf_outbound_socket_mutex,
+                                  (unsigned char*)&response, &lf_outbound_netdrv_mutex,
                                   "Failed to write MSG_TYPE_ACK in response to federate %d.", remote_fed_id);
-    LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex);
+    LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
 
     // Start a thread to listen for incoming messages from other federates.
     // The fed_id is a uint16_t, which we assume can be safely cast to and from void*.
@@ -2158,7 +2158,7 @@ int lf_send_message(int message_type, unsigned short port, unsigned short federa
   const int header_length = 1 + sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint32_t);
 
   // Use a mutex lock to prevent multiple threads from simultaneously sending.
-  LF_MUTEX_LOCK(&lf_outbound_socket_mutex);
+  LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
 
   int* socket = &_fed.netdrvs_for_outbound_p2p_connections[federate];
 
@@ -2174,7 +2174,7 @@ int lf_send_message(int message_type, unsigned short port, unsigned short federa
     // Message did not send. Since this is used for physical connections, this is not critical.
     lf_print_warning("Failed to send message to %s. Dropping the message.", next_destination_str);
   }
-  LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex);
+  LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
   return result;
 }
 
@@ -2349,9 +2349,9 @@ void lf_send_port_absent_to_federate(environment_t* env, interval_t additional_d
     tracepoint_federate_to_federate(send_PORT_ABS, _lf_my_fed_id, fed_ID, &current_message_intended_tag);
   }
 
-  LF_MUTEX_LOCK(&lf_outbound_socket_mutex);
+  LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
   int result = write_to_socket_close_on_error(socket, message_length, buffer);
-  LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex);
+  LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
 
   if (result != 0) {
     // Write failed. Response depends on whether coordination is centralized.
@@ -2374,7 +2374,7 @@ int lf_send_stop_request_to_rti(tag_t stop_tag) {
   stop_tag.microstep++;
   ENCODE_STOP_REQUEST(buffer, stop_tag.time, stop_tag.microstep);
 
-  LF_MUTEX_LOCK(&lf_outbound_socket_mutex);
+  LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
   // Do not send a stop request if a stop request has been previously received from the RTI.
   if (!_fed.received_stop_request_from_rti) {
     LF_PRINT_LOG("Sending to RTI a MSG_TYPE_STOP_REQUEST message with tag " PRINTF_TAG ".", stop_tag.time - start_time,
@@ -2382,21 +2382,21 @@ int lf_send_stop_request_to_rti(tag_t stop_tag) {
 
     if (_fed.socket_TCP_RTI < 0) {
       lf_print_warning("Socket is no longer connected. Dropping message.");
-      LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex);
+      LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
       return -1;
     }
     // Trace the event when tracing is enabled
     tracepoint_federate_to_rti(send_STOP_REQ, _lf_my_fed_id, &stop_tag);
 
-    write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, MSG_TYPE_STOP_REQUEST_LENGTH, buffer, &lf_outbound_socket_mutex,
+    write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, MSG_TYPE_STOP_REQUEST_LENGTH, buffer, &lf_outbound_netdrv_mutex,
                                   "Failed to send stop time " PRINTF_TIME " to the RTI.", stop_tag.time - start_time);
 
     // Treat this sending  as equivalent to having received a stop request from the RTI.
     _fed.received_stop_request_from_rti = true;
-    LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex);
+    LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
     return 0;
   } else {
-    LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex);
+    LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
     return 1;
   }
 }
@@ -2449,7 +2449,7 @@ int lf_send_tagged_message(environment_t* env, interval_t additional_delay, int
                current_message_intended_tag.microstep, next_destination_str);
 
   // Use a mutex lock to prevent multiple threads from simultaneously sending.
-  LF_MUTEX_LOCK(&lf_outbound_socket_mutex);
+  LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
 
   int* socket;
   if (message_type == MSG_TYPE_P2P_TAGGED_MESSAGE) {
@@ -2474,7 +2474,7 @@ int lf_send_tagged_message(environment_t* env, interval_t additional_delay, int
                                     next_destination_str, errno, strerror(errno));
     }
   }
-  LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex);
+  LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
   return result;
 }
 
diff --git a/include/core/federated/federate.h b/include/core/federated/federate.h
index a8cb325d0..0a62dba43 100644
--- a/include/core/federated/federate.h
+++ b/include/core/federated/federate.h
@@ -95,27 +95,20 @@ typedef struct federate_instance_t {
   netdrv_t* netdrvs_for_outbound_p2p_connections[NUMBER_OF_FEDERATES];
 
   /**
-   * Thread ID for a thread that accepts sockets and then supervises
-   * listening to those sockets for incoming P2P (physical) connections.
+   * Thread ID for a thread that accepts network drivers and then supervises
+   * listening to those network drivers for incoming P2P (physical) connections.
    */
   lf_thread_t inbound_p2p_handling_thread_id;
 
   /**
-   * A socket descriptor for the socket server of the federate.
+   * A network driver for the server of the federate.
    * This is assigned in lf_create_server().
-   * This socket is used to listen to incoming physical connections from
+   * This network driver is used to listen to incoming physical connections from
    * remote federates. Once an incoming connection is accepted, the
-   * opened socket will be stored in
+   * opened network driver will be stored in
    * federate_netdrvs_for_inbound_p2p_connections.
    */
-  int server_socket;
-
-  /**
-   * The port used for the server socket to listen for messages from other federates.
-   * The federate informs the RTI of this port once it has created its socket server by
-   * sending an ADDRESS_AD message (@see rti.h).
-   */
-  int server_port;
+  netdrv_t* server_netdrv;
 
   /**
    * Most recent tag advance grant (TAG) received from the RTI, or NEVER if none
@@ -207,9 +200,9 @@ typedef enum parse_rti_code_t { SUCCESS, INVALID_PORT, INVALID_HOST, INVALID_USE
 // Global variables
 
 /**
- * Mutex lock held while performing socket write and close operations.
+ * Mutex lock held while performing network driver write and close operations.
  */
-extern lf_mutex_t lf_outbound_socket_mutex;
+extern lf_mutex_t lf_outbound_netdrv_mutex;
 
 /**
  * Condition variable for blocking on unkonwn federate input ports.
@@ -226,10 +219,10 @@ extern lf_cond_t lf_port_status_changed;
  * to send messages directly to the specified federate.
  * This function first sends an MSG_TYPE_ADDRESS_QUERY message to the RTI to obtain
  * the IP address and port number of the specified federate. It then attempts
- * to establish a socket connection to the specified federate.
+ * to establish a network driver connection to the specified federate.
  * If this fails, the program exits. If it succeeds, it sets element [id] of
  * the _fed.netdrvs_for_outbound_p2p_connections global array to
- * refer to the socket for communicating directly with the federate.
+ * refer to the network driver for communicating directly with the federate.
  * @param remote_federate_id The ID of the remote federate.
  */
 void lf_connect_to_federate(uint16_t);
@@ -237,12 +230,12 @@ void lf_connect_to_federate(uint16_t);
 /**
  * @brief Connect to the RTI at the specified host and port.
  *
- * This will return the socket descriptor for the connection.
+ * This will return the network driver for the connection.
  * If port_number is 0, then start at DEFAULT_PORT and increment
  * the port number on each attempt. If an attempt fails, wait CONNECT_RETRY_INTERVAL
  * and try again.  If it fails after CONNECT_TIMEOUT, the program exits.
- * If it succeeds, it sets the _fed.socket_TCP_RTI global variable to refer to
- * the socket for communicating with the RTI.
+ * If it succeeds, it sets the _fed.netdrv_to_RTI global variable to refer to
+ * the network driver for communicating with the RTI.
  * @param hostname A hostname, such as "localhost".
  * @param port_number A port number or 0 to start with the default.
  */
@@ -252,8 +245,8 @@ void lf_connect_to_rti(const char* hostname, int port_number);
  * @brief Create a server to listen to incoming P2P connections.
  *
  * Such connections are used for physical connections or any connection if using
- * decentralized coordination. This function only handles the creation of the server socket.
- * The bound port for the server socket is then sent to the RTI by sending an
+ * decentralized coordination. This function only handles the creation of the server network driver.
+ * The bound port for the server network driver is then sent to the RTI by sending an
  * MSG_TYPE_ADDRESS_ADVERTISEMENT message (@see net_common.h).
  * This function expects no response from the RTI.
  *
@@ -280,8 +273,8 @@ void lf_enqueue_port_absent_reactions(environment_t* env);
  *
  * This thread accepts connections from federates that send messages directly
  * to this one (not through the RTI). This thread starts a thread for
- * each accepted socket connection to read messages and, once it has opened all expected
- * sockets, exits.
+ * each accepted network driver connection to read messages and, once it has opened all expected
+ * network drivers, exits.
  * @param ignored No argument needed for this thread.
  */
 void* lf_handle_p2p_connections_from_federates(void*);
@@ -323,7 +316,7 @@ void lf_reset_status_fields_on_input_port_triggers();
  * between federates. If the socket connection to the remote federate or the RTI has been broken,
  * then this returns -1 without sending. Otherwise, it returns 0.
  *
- * This method assumes that the caller does not hold the lf_outbound_socket_mutex lock,
+ * This method assumes that the caller does not hold the lf_outbound_netdrv_mutex lock,
  * which it acquires to perform the send.
  *
  * @param message_type The type of the message being sent (currently only MSG_TYPE_P2P_MESSAGE).
@@ -425,7 +418,7 @@ void lf_send_port_absent_to_federate(environment_t* env, interval_t additional_d
  *
  * The payload is the specified tag plus one microstep. If this federate has previously
  * received a stop request from the RTI, then do not send the message and
- * return 1. Return -1 if the socket is disconnected. Otherwise, return 0.
+ * return 1. Return -1 if the network driver is disconnected. Otherwise, return 0.
  * @return 0 if the message is sent.
  */
 int lf_send_stop_request_to_rti(tag_t stop_tag);
@@ -437,7 +430,7 @@ int lf_send_stop_request_to_rti(tag_t stop_tag);
  * If the delayed tag falls after the timeout time, then the message is not sent and -1 is returned.
  * The caller can reuse or free the memory storing the message after this returns.
  *
- * If the message fails to send (e.g. the socket connection is broken), then the
+ * If the message fails to send (e.g. the network driver connection is broken), then the
  * response depends on the message_type.  For MSG_TYPE_TAGGED_MESSAGE, the message is
  * supposed to go via the RTI, and failure to communicate with the RTI is a critical failure.
  * In this case, the program will exit with an error message. If the message type is
@@ -446,7 +439,7 @@ int lf_send_stop_request_to_rti(tag_t stop_tag);
  * to believe that there were no messages forthcoming.  In this case, on failure to send
  * the message, this function returns -11.
  *
- * This method assumes that the caller does not hold the lf_outbound_socket_mutex lock,
+ * This method assumes that the caller does not hold the lf_outbound_netdrv_mutex lock,
  * which it acquires to perform the send.
  *
  * @param env The environment from which to get the current tag.
@@ -502,7 +495,7 @@ void lf_stall_advance_level_federation_locked(size_t level);
  * @brief Synchronize the start with other federates via the RTI.
  *
  * This assumes that a connection to the RTI is already made
- * and _lf_rti_socket_TCP is valid. It then sends the current logical
+ * and _lf_rti_socket_TCP is valid. It then sends the current logical //TODO: Check.
  * time to the RTI and waits for the RTI to respond with a specified
  * time. It starts a thread to listen for messages from the RTI.
  */

From aa3237e3858d84f1d9acc2de96b4cc84625be28e Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sat, 18 Jan 2025 18:44:08 -0700
Subject: [PATCH 40/84] Remove old TCP related parameters.

---
 core/federated/RTI/rti_remote.c | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index d6d8f0c47..634a99b06 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -1560,10 +1560,6 @@ void initialize_RTI(rti_remote_t* rti) {
   rti_remote->all_federates_exited = false;
   rti_remote->federation_id = "Unidentified Federation";
   rti_remote->user_specified_port = 0;
-  // TODO: Erase
-  rti_remote->final_port_TCP = 0;
-  rti_remote->socket_descriptor_TCP = -1;
-
   rti_remote->final_port_UDP = UINT16_MAX;
   rti_remote->socket_descriptor_UDP = -1;
   rti_remote->clock_sync_global_status = clock_sync_init;

From bfea77a2cfe2157692a718305178499703d6631a Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sat, 18 Jan 2025 18:44:08 -0700
Subject: [PATCH 41/84] Remove old TCP related parameters.

---
 core/federated/RTI/rti_remote.c | 4 ----
 core/federated/RTI/rti_remote.h | 6 ------
 2 files changed, 10 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index d6d8f0c47..634a99b06 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -1560,10 +1560,6 @@ void initialize_RTI(rti_remote_t* rti) {
   rti_remote->all_federates_exited = false;
   rti_remote->federation_id = "Unidentified Federation";
   rti_remote->user_specified_port = 0;
-  // TODO: Erase
-  rti_remote->final_port_TCP = 0;
-  rti_remote->socket_descriptor_TCP = -1;
-
   rti_remote->final_port_UDP = UINT16_MAX;
   rti_remote->socket_descriptor_UDP = -1;
   rti_remote->clock_sync_global_status = clock_sync_init;
diff --git a/core/federated/RTI/rti_remote.h b/core/federated/RTI/rti_remote.h
index c29d0d53b..abf7cb20e 100644
--- a/core/federated/RTI/rti_remote.h
+++ b/core/federated/RTI/rti_remote.h
@@ -114,12 +114,6 @@ typedef struct rti_remote_t {
    */
   uint16_t user_specified_port;
 
-  /** The final port number that the TCP socket server ends up using. */
-  uint16_t final_port_TCP;
-
-  /** The TCP socket descriptor for the socket server. */
-  int socket_descriptor_TCP;
-
   /************* UDP server information *************/
   /** The final port number that the UDP socket server ends up using. */
   uint16_t final_port_UDP;

From f8f45e2857d84f690a56e4b7f4e6508a5c1f3405 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sat, 18 Jan 2025 18:48:47 -0700
Subject: [PATCH 42/84] Minor fix.

---
 network/impl/src/lf_socket_support.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
index 34844e8de..90ab3d84c 100644
--- a/network/impl/src/lf_socket_support.c
+++ b/network/impl/src/lf_socket_support.c
@@ -19,8 +19,6 @@ netdrv_t* initialize_netdrv() {
   if (priv == NULL) {
     lf_print_error_and_exit("Falied to malloc socket_priv_t.");
   }
-  // Initialize to zero.
-  //   memset(priv, 0, sizeof(socket_priv_t));
 
   // Server initialization.
   priv->port = 0;

From dc79adf739a36aa76f4ec8968481a13cf4b15bf9 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sat, 18 Jan 2025 22:28:09 -0700
Subject: [PATCH 43/84] Set hostname and port for federate. Also move
 get_peer_address to bottom of code.

---
 core/federated/federate.c            |  8 ++++-
 network/impl/src/lf_socket_support.c | 49 +++++++++++++++-------------
 2 files changed, 34 insertions(+), 23 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index e7f3143cc..cd188c5fe 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -1811,7 +1811,13 @@ void lf_connect_to_rti(const char* hostname, int port) {
   hostname = federation_metadata.rti_host ? federation_metadata.rti_host : hostname;
   port = federation_metadata.rti_port >= 0 ? federation_metadata.rti_port : port;
 
-  // Create a socket
+
+
+  // Create a network driver.
+  _fed.netdrv_to_RTI = initialize_netdrv();
+  set_host_name(_fed.netdrv_to_RTI, hostname);
+  set_port(_fed.netdrv_to_RTI, port);
+  
   _fed.socket_TCP_RTI = create_real_time_tcp_socket_errexit();
   if (connect_to_socket(_fed.socket_TCP_RTI, hostname, port) < 0) {
     lf_print_error_and_exit("Failed to connect() to RTI.");
diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
index 90ab3d84c..e6a753d77 100644
--- a/network/impl/src/lf_socket_support.c
+++ b/network/impl/src/lf_socket_support.c
@@ -100,28 +100,6 @@ netdrv_t* accept_netdrv(netdrv_t* server_drv, netdrv_t* rti_drv) {
   return fed_netdrv;
 }
 
-int get_peer_address(netdrv_t* drv) {
-  socket_priv_t* priv = (socket_priv_t*)drv->priv;
-  struct sockaddr_in peer_addr;
-  socklen_t addr_len = sizeof(peer_addr);
-  if (getpeername(priv->socket_descriptor, (struct sockaddr*)&peer_addr, &addr_len) != 0) {
-    lf_print_error("RTI failed to get peer address.");
-    return -1;
-  }
-  priv->server_ip_addr = peer_addr.sin_addr;
-
-#if LOG_LEVEL >= LOG_LEVEL_DEBUG
-  // Create the human readable format and copy that into
-  // the .server_hostname field of the federate.
-  char str[INET_ADDRSTRLEN + 1];
-  inet_ntop(AF_INET, &priv->server_ip_addr, str, INET_ADDRSTRLEN);
-  strncpy(priv->server_hostname, str, INET_ADDRSTRLEN);
-
-  LF_PRINT_DEBUG("RTI got address %s", priv->server_hostname);
-#endif
-  return 0;
-}
-
 int read_from_netdrv(netdrv_t* drv, size_t num_bytes, unsigned char* buffer) {
   socket_priv_t* priv = (socket_priv_t*)drv->priv;
   int socket = priv->socket_descriptor;
@@ -246,6 +224,28 @@ int shutdown_netdrv(netdrv_t* drv, bool read_before_closing) {
   return shutdown_socket(&priv->socket_descriptor, read_before_closing);
 }
 
+int get_peer_address(netdrv_t* drv) {
+  socket_priv_t* priv = (socket_priv_t*)drv->priv;
+  struct sockaddr_in peer_addr;
+  socklen_t addr_len = sizeof(peer_addr);
+  if (getpeername(priv->socket_descriptor, (struct sockaddr*)&peer_addr, &addr_len) != 0) {
+    lf_print_error("RTI failed to get peer address.");
+    return -1;
+  }
+  priv->server_ip_addr = peer_addr.sin_addr;
+
+#if LOG_LEVEL >= LOG_LEVEL_DEBUG
+  // Create the human readable format and copy that into
+  // the .server_hostname field of the federate.
+  char str[INET_ADDRSTRLEN + 1];
+  inet_ntop(AF_INET, &priv->server_ip_addr, str, INET_ADDRSTRLEN);
+  strncpy(priv->server_hostname, str, INET_ADDRSTRLEN);
+
+  LF_PRINT_DEBUG("RTI got address %s", priv->server_hostname);
+#endif
+  return 0;
+}
+
 int32_t get_server_port(netdrv_t* drv) {
   // if (drv == NULL) {
   //   lf_print_warning("Netdriver is closed, returning -1.");
@@ -268,3 +268,8 @@ void set_port(netdrv_t* drv, int32_t port) {
   socket_priv_t* priv = (socket_priv_t*)drv->priv;
   priv->server_port = port;
 }
+
+void set_host_name(netdrv_t* drv, const char* hostname) {
+  socket_priv_t* priv = (socket_priv_t*)drv->priv;
+  memcpy(priv->server_hostname, hostname, INET_ADDRSTRLEN);
+}
\ No newline at end of file

From 56ff3be892c019c8b2f83e342354ec9f04f5d8c6 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sat, 18 Jan 2025 22:31:14 -0700
Subject: [PATCH 44/84] Add create client.

---
 core/federated/federate.c            | 4 +++-
 network/impl/src/lf_socket_support.c | 4 ++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index cd188c5fe..ad0bf3b2d 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -1817,7 +1817,9 @@ void lf_connect_to_rti(const char* hostname, int port) {
   _fed.netdrv_to_RTI = initialize_netdrv();
   set_host_name(_fed.netdrv_to_RTI, hostname);
   set_port(_fed.netdrv_to_RTI, port);
-  
+
+  create_client();
+
   _fed.socket_TCP_RTI = create_real_time_tcp_socket_errexit();
   if (connect_to_socket(_fed.socket_TCP_RTI, hostname, port) < 0) {
     lf_print_error_and_exit("Failed to connect() to RTI.");
diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
index e6a753d77..03cd18223 100644
--- a/network/impl/src/lf_socket_support.c
+++ b/network/impl/src/lf_socket_support.c
@@ -246,6 +246,10 @@ int get_peer_address(netdrv_t* drv) {
   return 0;
 }
 
+void create_client(netdrv_t* drv) {
+  socket_priv_t* priv = (socket_priv_t*)drv->priv;
+  priv->socket_descriptor = create_real_time_tcp_socket_errexit();
+}
 int32_t get_server_port(netdrv_t* drv) {
   // if (drv == NULL) {
   //   lf_print_warning("Netdriver is closed, returning -1.");

From 7001175a310ca4e3ffa2b309d9aba38a124f42b9 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sat, 18 Jan 2025 22:47:48 -0700
Subject: [PATCH 45/84] Minor fix setting server port and host name.

---
 core/federated/RTI/rti_remote.c      |  2 +-
 core/federated/federate.c            |  8 ++++----
 network/api/net_driver.h             |  4 +++-
 network/impl/src/lf_socket_support.c | 19 +++++++++++++------
 4 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index 634a99b06..39847ce31 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -662,7 +662,7 @@ void handle_address_ad(uint16_t federate_id) {
   assert(server_port < 65536);
 
   LF_MUTEX_LOCK(&rti_mutex);
-  set_port(fed->fed_netdrv, server_port);
+  set_server_port(fed->fed_netdrv, server_port);
   LF_MUTEX_UNLOCK(&rti_mutex);
 
   LF_PRINT_LOG("Received address advertisement with port %d from federate %d.", server_port, federate_id);
diff --git a/core/federated/federate.c b/core/federated/federate.c
index ad0bf3b2d..f2efe4edf 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -1815,12 +1815,12 @@ void lf_connect_to_rti(const char* hostname, int port) {
 
   // Create a network driver.
   _fed.netdrv_to_RTI = initialize_netdrv();
-  set_host_name(_fed.netdrv_to_RTI, hostname);
-  set_port(_fed.netdrv_to_RTI, port);
+  // Set the user specified host name and port to the network driver.
+  set_server_port(_fed.netdrv_to_RTI, hostname);
+  set_server_host_name(_fed.netdrv_to_RTI, port);
 
+  // 
   create_client();
-
-  _fed.socket_TCP_RTI = create_real_time_tcp_socket_errexit();
   if (connect_to_socket(_fed.socket_TCP_RTI, hostname, port) < 0) {
     lf_print_error_and_exit("Failed to connect() to RTI.");
   }
diff --git a/network/api/net_driver.h b/network/api/net_driver.h
index a70abca7e..c6410a0c1 100644
--- a/network/api/net_driver.h
+++ b/network/api/net_driver.h
@@ -139,6 +139,8 @@ struct in_addr* get_ip_addr(netdrv_t* drv);
 
 char* get_server_hostname(netdrv_t* drv);
 
-void set_port(netdrv_t* drv, int32_t port);
+void set_server_port(netdrv_t* drv, int32_t port);
+
+void set_server_host_name(netdrv_t* drv, const char* hostname);
 
 #endif /* NET_DRIVER_H */
diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
index 03cd18223..5b5375f15 100644
--- a/network/impl/src/lf_socket_support.c
+++ b/network/impl/src/lf_socket_support.c
@@ -100,6 +100,17 @@ netdrv_t* accept_netdrv(netdrv_t* server_drv, netdrv_t* rti_drv) {
   return fed_netdrv;
 }
 
+void create_client(netdrv_t* drv) {
+  socket_priv_t* priv = (socket_priv_t*)drv->priv;
+  priv->socket_descriptor = create_real_time_tcp_socket_errexit();
+}
+
+int connect_to_netdrv(netdrv_t* drv) {
+  socket_priv_t* priv = (socket_priv_t*)drv->priv;
+
+  return connect_to_socket(priv->socket_descriptor, priv->server_hostname, priv->server_port);
+}
+
 int read_from_netdrv(netdrv_t* drv, size_t num_bytes, unsigned char* buffer) {
   socket_priv_t* priv = (socket_priv_t*)drv->priv;
   int socket = priv->socket_descriptor;
@@ -246,10 +257,6 @@ int get_peer_address(netdrv_t* drv) {
   return 0;
 }
 
-void create_client(netdrv_t* drv) {
-  socket_priv_t* priv = (socket_priv_t*)drv->priv;
-  priv->socket_descriptor = create_real_time_tcp_socket_errexit();
-}
 int32_t get_server_port(netdrv_t* drv) {
   // if (drv == NULL) {
   //   lf_print_warning("Netdriver is closed, returning -1.");
@@ -268,12 +275,12 @@ char* get_server_hostname(netdrv_t* drv) {
   return priv->server_hostname;
 }
 
-void set_port(netdrv_t* drv, int32_t port) {
+void set_server_port(netdrv_t* drv, int32_t port) {
   socket_priv_t* priv = (socket_priv_t*)drv->priv;
   priv->server_port = port;
 }
 
-void set_host_name(netdrv_t* drv, const char* hostname) {
+void set_server_host_name(netdrv_t* drv, const char* hostname) {
   socket_priv_t* priv = (socket_priv_t*)drv->priv;
   memcpy(priv->server_hostname, hostname, INET_ADDRSTRLEN);
 }
\ No newline at end of file

From caebf9b1e7aa8236e010502f16a555cffd59eeb6 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sat, 18 Jan 2025 22:48:46 -0700
Subject: [PATCH 46/84] Add connect to network driver.

---
 core/federated/federate.c            | 2 +-
 network/impl/src/lf_socket_support.c | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index f2efe4edf..f4a07218c 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -1819,7 +1819,7 @@ void lf_connect_to_rti(const char* hostname, int port) {
   set_server_port(_fed.netdrv_to_RTI, hostname);
   set_server_host_name(_fed.netdrv_to_RTI, port);
 
-  // 
+  // Create the client network driver.
   create_client();
   if (connect_to_socket(_fed.socket_TCP_RTI, hostname, port) < 0) {
     lf_print_error_and_exit("Failed to connect() to RTI.");
diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
index 5b5375f15..1001f7a27 100644
--- a/network/impl/src/lf_socket_support.c
+++ b/network/impl/src/lf_socket_support.c
@@ -107,7 +107,6 @@ void create_client(netdrv_t* drv) {
 
 int connect_to_netdrv(netdrv_t* drv) {
   socket_priv_t* priv = (socket_priv_t*)drv->priv;
-
   return connect_to_socket(priv->socket_descriptor, priv->server_hostname, priv->server_port);
 }
 

From 12a88f0f71ccf323d7574212d517b8915153698c Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sat, 18 Jan 2025 22:49:09 -0700
Subject: [PATCH 47/84] Minor fix.

---
 core/federated/federate.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index f4a07218c..6063b6db2 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -1821,7 +1821,7 @@ void lf_connect_to_rti(const char* hostname, int port) {
 
   // Create the client network driver.
   create_client();
-  if (connect_to_socket(_fed.socket_TCP_RTI, hostname, port) < 0) {
+  if (connect_to_netdrv(__fed.netdrv_to_RTI) < 0) {
     lf_print_error_and_exit("Failed to connect() to RTI.");
   }
 

From 83cd10e2e45755e679c1fad55f815fa61a82f50e Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sat, 18 Jan 2025 22:55:28 -0700
Subject: [PATCH 48/84] Fix all read and write namings.

---
 core/federated/federate.c | 92 +++++++++++++++++++--------------------
 1 file changed, 46 insertions(+), 46 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index 6063b6db2..13bc08f67 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -116,7 +116,7 @@ static void send_time(unsigned char type, instant_t time) {
   tracepoint_federate_to_rti(send_TIMESTAMP, _lf_my_fed_id, &tag);
 
   LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
-  write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_write, buffer, &lf_outbound_netdrv_mutex,
+  write_to_netdrv_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_write, buffer, &lf_outbound_netdrv_mutex,
                                 "Failed to send time " PRINTF_TIME " to the RTI.", time - start_time);
   LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
 }
@@ -143,7 +143,7 @@ static void send_tag(unsigned char type, tag_t tag) {
   trace_event_t event_type = (type == MSG_TYPE_NEXT_EVENT_TAG) ? send_NET : send_LTC;
   // Trace the event when tracing is enabled
   tracepoint_federate_to_rti(event_type, _lf_my_fed_id, &tag);
-  write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_write, buffer, &lf_outbound_netdrv_mutex,
+  write_to_netdrv_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_write, buffer, &lf_outbound_netdrv_mutex,
                                 "Failed to send tag " PRINTF_TAG " to the RTI.", tag.time - start_time, tag.microstep);
   LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
 }
@@ -473,7 +473,7 @@ static int handle_message(int* socket, int fed_id) {
   // Read the header.
   size_t bytes_to_read = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint32_t);
   unsigned char buffer[bytes_to_read];
-  if (read_from_socket_close_on_error(socket, bytes_to_read, buffer)) {
+  if (read_from_netdrv_close_on_error(socket, bytes_to_read, buffer)) {
     // Read failed, which means the socket has been closed between reading the
     // message ID byte and here.
     return -1;
@@ -494,7 +494,7 @@ static int handle_message(int* socket, int fed_id) {
   // Read the payload.
   // Allocate memory for the message contents.
   unsigned char* message_contents = (unsigned char*)malloc(length);
-  if (read_from_socket_close_on_error(socket, length, message_contents)) {
+  if (read_from_netdrv_close_on_error(socket, length, message_contents)) {
     return -1;
   }
   // Trace the event when tracing is enabled
@@ -531,7 +531,7 @@ static int handle_tagged_message(int* socket, int fed_id) {
   size_t bytes_to_read =
       sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint32_t) + sizeof(instant_t) + sizeof(microstep_t);
   unsigned char buffer[bytes_to_read];
-  if (read_from_socket_close_on_error(socket, bytes_to_read, buffer)) {
+  if (read_from_netdrv_close_on_error(socket, bytes_to_read, buffer)) {
     return -1; // Read failed.
   }
 
@@ -580,7 +580,7 @@ static int handle_tagged_message(int* socket, int fed_id) {
   // Read the payload.
   // Allocate memory for the message contents.
   unsigned char* message_contents = (unsigned char*)malloc(length);
-  if (read_from_socket_close_on_error(socket, length, message_contents)) {
+  if (read_from_netdrv_close_on_error(socket, length, message_contents)) {
 #ifdef FEDERATED_DECENTRALIZED
     _lf_decrement_tag_barrier_locked(env);
 #endif
@@ -689,7 +689,7 @@ static int handle_tagged_message(int* socket, int fed_id) {
 static int handle_port_absent_message(int* socket, int fed_id) {
   size_t bytes_to_read = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(instant_t) + sizeof(microstep_t);
   unsigned char buffer[bytes_to_read];
-  if (read_from_socket_close_on_error(socket, bytes_to_read, buffer)) {
+  if (read_from_netdrv_close_on_error(socket, bytes_to_read, buffer)) {
     return -1;
   }
 
@@ -751,7 +751,7 @@ static void* listen_to_federates(void* _args) {
     // Read one byte to get the message type.
     LF_PRINT_DEBUG("Waiting for a P2P message on socket %d.", *socket_id);
     bool bad_message = false;
-    if (read_from_socket_close_on_error(socket_id, 1, buffer)) {
+    if (read_from_netdrv_close_on_error(socket_id, 1, buffer)) {
       // Socket has been closed.
       lf_print("Socket from federate %d is closed.", fed_id);
       // Stop listening to this federate.
@@ -852,14 +852,14 @@ static int perform_hmac_authentication() {
   RAND_bytes(fed_nonce, NONCE_LENGTH);
   memcpy(&fed_hello_buf[1 + fed_id_length], fed_nonce, NONCE_LENGTH);
 
-  write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, message_length, fed_hello_buf, NULL, "Failed to write nonce.");
+  write_to_netdrv_fail_on_error(&_fed.socket_TCP_RTI, message_length, fed_hello_buf, NULL, "Failed to write nonce.");
 
   // Check HMAC of received FED_RESPONSE message.
   unsigned int hmac_length = SHA256_HMAC_LENGTH;
   size_t federation_id_length = strnlen(federation_metadata.federation_id, 255);
 
   unsigned char received[1 + NONCE_LENGTH + hmac_length];
-  if (read_from_socket_close_on_error(&_fed.socket_TCP_RTI, 1 + NONCE_LENGTH + hmac_length, received)) {
+  if (read_from_netdrv_close_on_error(&_fed.socket_TCP_RTI, 1 + NONCE_LENGTH + hmac_length, received)) {
     lf_print_warning("Failed to read RTI response.");
     return -1;
   }
@@ -893,7 +893,7 @@ static int perform_hmac_authentication() {
     response[1] = HMAC_DOES_NOT_MATCH;
 
     // Ignore errors on writing back.
-    write_to_socket(_fed.socket_TCP_RTI, 2, response);
+    write_to_netdrv(_fed.socket_TCP_RTI, 2, response);
     return -1;
   } else {
     LF_PRINT_LOG("HMAC verified.");
@@ -907,7 +907,7 @@ static int perform_hmac_authentication() {
     HMAC(EVP_sha256(), federation_metadata.federation_id, federation_id_length, mac_buf, 1 + NONCE_LENGTH, &sender[1],
          &hmac_length);
 
-    write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, 1 + hmac_length, sender, NULL, "Failed to write fed response.");
+    write_to_netdrv_fail_on_error(&_fed.socket_TCP_RTI, 1 + hmac_length, sender, NULL, "Failed to write fed response.");
   }
   return 0;
 }
@@ -931,7 +931,7 @@ static instant_t get_start_time_from_rti(instant_t my_physical_time) {
   size_t buffer_length = 1 + sizeof(instant_t);
   unsigned char buffer[buffer_length];
 
-  read_from_socket_fail_on_error(&_fed.socket_TCP_RTI, buffer_length, buffer, NULL,
+  read_from_netdrv_fail_on_error(&_fed.socket_TCP_RTI, buffer_length, buffer, NULL,
                                  "Failed to read MSG_TYPE_TIMESTAMP message from RTI.");
   LF_PRINT_DEBUG("Read 9 bytes.");
 
@@ -975,7 +975,7 @@ static void handle_tag_advance_grant(void) {
 
   size_t bytes_to_read = sizeof(instant_t) + sizeof(microstep_t);
   unsigned char buffer[bytes_to_read];
-  read_from_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_read, buffer, NULL,
+  read_from_netdrv_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_read, buffer, NULL,
                                  "Failed to read tag advance grant from RTI.");
   tag_t TAG = extract_tag(buffer);
 
@@ -1216,7 +1216,7 @@ static void handle_provisional_tag_advance_grant() {
 
   size_t bytes_to_read = sizeof(instant_t) + sizeof(microstep_t);
   unsigned char buffer[bytes_to_read];
-  read_from_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_read, buffer, NULL,
+  read_from_netdrv_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_read, buffer, NULL,
                                  "Failed to read provisional tag advance grant from RTI.");
   tag_t PTAG = extract_tag(buffer);
 
@@ -1306,7 +1306,7 @@ static void handle_stop_granted_message() {
 
   size_t bytes_to_read = MSG_TYPE_STOP_GRANTED_LENGTH - 1;
   unsigned char buffer[bytes_to_read];
-  read_from_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_read, buffer, NULL,
+  read_from_netdrv_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_read, buffer, NULL,
                                  "Failed to read stop granted from RTI.");
 
   tag_t received_stop_tag = extract_tag(buffer);
@@ -1350,7 +1350,7 @@ static void handle_stop_granted_message() {
 static void handle_stop_request_message() {
   size_t bytes_to_read = MSG_TYPE_STOP_REQUEST_LENGTH - 1;
   unsigned char buffer[bytes_to_read];
-  read_from_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_read, buffer, NULL,
+  read_from_netdrv_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_read, buffer, NULL,
                                  "Failed to read stop request from RTI.");
   tag_t tag_to_stop = extract_tag(buffer);
 
@@ -1423,7 +1423,7 @@ static void handle_stop_request_message() {
 
   // Send the current logical time to the RTI.
   LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
-  write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, MSG_TYPE_STOP_REQUEST_REPLY_LENGTH, outgoing_buffer,
+  write_to_netdrv_fail_on_error(&_fed.socket_TCP_RTI, MSG_TYPE_STOP_REQUEST_REPLY_LENGTH, outgoing_buffer,
                                 &lf_outbound_netdrv_mutex,
                                 "Failed to send the answer to MSG_TYPE_STOP_REQUEST to RTI.");
   LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
@@ -1440,7 +1440,7 @@ static void send_resign_signal() {
   unsigned char buffer[bytes_to_write];
   buffer[0] = MSG_TYPE_RESIGN;
   LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
-  write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_write, &(buffer[0]), &lf_outbound_netdrv_mutex,
+  write_to_netdrv_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_write, &(buffer[0]), &lf_outbound_netdrv_mutex,
                                 "Failed to send MSG_TYPE_RESIGN.");
   LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
   LF_PRINT_LOG("Resigned.");
@@ -1453,7 +1453,7 @@ static void send_failed_signal() {
   size_t bytes_to_write = 1;
   unsigned char buffer[bytes_to_write];
   buffer[0] = MSG_TYPE_FAILED;
-  write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_write, &(buffer[0]), NULL,
+  write_to_netdrv_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_write, &(buffer[0]), NULL,
                                 "Failed to send MSG_TYPE_FAILED.");
   LF_PRINT_LOG("Failed.");
 }
@@ -1488,7 +1488,7 @@ static void* listen_to_rti_TCP(void* args) {
     }
     // Read one byte to get the message type.
     // This will exit if the read fails.
-    int read_failed = read_from_socket(_fed.socket_TCP_RTI, 1, buffer);
+    int read_failed = read_from_netdrv(_fed.socket_TCP_RTI, 1, buffer);
     if (read_failed < 0) {
       if (errno == ECONNRESET) {
         lf_print_error("Socket connection to the RTI was closed by the RTI without"
@@ -1695,12 +1695,12 @@ void lf_connect_to_federate(uint16_t remote_federate_id) {
     tracepoint_federate_to_rti(send_ADR_QR, _lf_my_fed_id, NULL);
 
     LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
-    write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, sizeof(uint16_t) + 1, buffer, &lf_outbound_netdrv_mutex,
+    write_to_netdrv_fail_on_error(&_fed.socket_TCP_RTI, sizeof(uint16_t) + 1, buffer, &lf_outbound_netdrv_mutex,
                                   "Failed to send address query for federate %d to RTI.", remote_federate_id);
     LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
 
     // Read RTI's response.
-    read_from_socket_fail_on_error(&_fed.socket_TCP_RTI, sizeof(int32_t) + 1, buffer, NULL,
+    read_from_netdrv_fail_on_error(&_fed.socket_TCP_RTI, sizeof(int32_t) + 1, buffer, NULL,
                                    "Failed to read the requested port number for federate %d from RTI.",
                                    remote_federate_id);
 
@@ -1714,7 +1714,7 @@ void lf_connect_to_federate(uint16_t remote_federate_id) {
     }
     port = extract_int32(&buffer[1]);
 
-    read_from_socket_fail_on_error(&_fed.socket_TCP_RTI, sizeof(host_ip_addr), (unsigned char*)&host_ip_addr, NULL,
+    read_from_netdrv_fail_on_error(&_fed.socket_TCP_RTI, sizeof(host_ip_addr), (unsigned char*)&host_ip_addr, NULL,
                                    "Failed to read the IP address for federate %d from RTI.", remote_federate_id);
 
     // A reply of -1 for the port means that the RTI does not know
@@ -1772,17 +1772,17 @@ void lf_connect_to_federate(uint16_t remote_federate_id) {
     tracepoint_federate_to_federate(send_FED_ID, _lf_my_fed_id, remote_federate_id, NULL);
 
     // No need for a mutex because we have the only handle on the socket.
-    write_to_socket_fail_on_error(&socket_id, buffer_length, buffer, NULL, "Failed to send fed_id to federate %d.",
+    write_to_netdrv_fail_on_error(&socket_id, buffer_length, buffer, NULL, "Failed to send fed_id to federate %d.",
                                   remote_federate_id);
-    write_to_socket_fail_on_error(&socket_id, federation_id_length, (unsigned char*)federation_metadata.federation_id,
+    write_to_netdrv_fail_on_error(&socket_id, federation_id_length, (unsigned char*)federation_metadata.federation_id,
                                   NULL, "Failed to send federation id to federate %d.", remote_federate_id);
 
-    read_from_socket_fail_on_error(&socket_id, 1, (unsigned char*)buffer, NULL,
+    read_from_netdrv_fail_on_error(&socket_id, 1, (unsigned char*)buffer, NULL,
                                    "Failed to read MSG_TYPE_ACK from federate %d in response to sending fed_id.",
                                    remote_federate_id);
     if (buffer[0] != MSG_TYPE_ACK) {
       // Get the error code.
-      read_from_socket_fail_on_error(&socket_id, 1, (unsigned char*)buffer, NULL,
+      read_from_netdrv_fail_on_error(&socket_id, 1, (unsigned char*)buffer, NULL,
                                      "Failed to read error code from federate %d in response to sending fed_id.",
                                      remote_federate_id);
       lf_print_error("Received MSG_TYPE_REJECT message from remote federate (%d).", buffer[0]);
@@ -1863,12 +1863,12 @@ void lf_connect_to_rti(const char* hostname, int port) {
     tracepoint_federate_to_rti(send_FED_ID, _lf_my_fed_id, NULL);
 
     // No need for a mutex here because no other threads are writing to this socket.
-    if (write_to_socket(_fed.socket_TCP_RTI, 2 + sizeof(uint16_t), buffer)) {
+    if (write_to_netdrv(_fed.netdrv_to_RTI, 2 + sizeof(uint16_t), buffer)) {
       continue; // Try again, possibly on a new port.
     }
 
     // Next send the federation ID itself.
-    if (write_to_socket(_fed.socket_TCP_RTI, federation_id_length, (unsigned char*)federation_metadata.federation_id)) {
+    if (write_to_netdrv(_fed.netdrv_to_RTI, federation_id_length, (unsigned char*)federation_metadata.federation_id)) {
       continue; // Try again.
     }
 
@@ -1880,7 +1880,7 @@ void lf_connect_to_rti(const char* hostname, int port) {
 
     LF_PRINT_DEBUG("Waiting for response to federation ID from the RTI.");
 
-    if (read_from_socket(_fed.socket_TCP_RTI, 1, &response)) {
+    if (read_from_netdrv(_fed.netdrv_to_RTI, 1, &response)) {
       continue; // Try again.
     }
     if (response == MSG_TYPE_REJECT) {
@@ -1888,7 +1888,7 @@ void lf_connect_to_rti(const char* hostname, int port) {
       tracepoint_federate_from_rti(receive_REJECT, _lf_my_fed_id, NULL);
       // Read one more byte to determine the cause of rejection.
       unsigned char cause;
-      read_from_socket_fail_on_error(&_fed.socket_TCP_RTI, 1, &cause, NULL,
+      read_from_netdrv_fail_on_error(&_fed.netdrv_to_RTI, 1, &cause, NULL,
                                      "Failed to read the cause of rejection by the RTI.");
       if (cause == FEDERATION_ID_DOES_NOT_MATCH || cause == WRONG_SERVER) {
         lf_print_warning("Connected to the wrong RTI. Will try again");
@@ -1912,7 +1912,7 @@ void lf_connect_to_rti(const char* hostname, int port) {
   // about connections between this federate and other federates
   // where messages are routed through the RTI.
   // @see MSG_TYPE_NEIGHBOR_STRUCTURE in net_common.h
-  lf_send_neighbor_structure_to_RTI(_fed.socket_TCP_RTI);
+  lf_send_neighbor_structure_to_RTI(_fed.netdrv_to_RTI);
 
   uint16_t udp_port = setup_clock_synchronization_with_rti();
 
@@ -1920,7 +1920,7 @@ void lf_connect_to_rti(const char* hostname, int port) {
   unsigned char UDP_port_number[1 + sizeof(uint16_t)];
   UDP_port_number[0] = MSG_TYPE_UDP_PORT;
   encode_uint16(udp_port, &(UDP_port_number[1]));
-  write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, 1 + sizeof(uint16_t), UDP_port_number, NULL,
+  write_to_netdrv_fail_on_error(&_fed.netdrv_to_RTI, 1 + sizeof(uint16_t), UDP_port_number, NULL,
                                 "Failed to send the UDP port number to the RTI.");
 }
 
@@ -1941,7 +1941,7 @@ void lf_create_server(int specified_port) {
   tracepoint_federate_to_rti(send_ADR_AD, _lf_my_fed_id, NULL);
 
   // No need for a mutex because we have the only handle on this socket.
-  write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, sizeof(int32_t) + 1, (unsigned char*)buffer, NULL,
+  write_to_netdrv_fail_on_error(&_fed.socket_TCP_RTI, sizeof(int32_t) + 1, (unsigned char*)buffer, NULL,
                                 "Failed to send address advertisement.");
 
   LF_PRINT_DEBUG("Sent port %d to the RTI.", _fed.server_port);
@@ -1987,7 +1987,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
 
     size_t header_length = 1 + sizeof(uint16_t) + 1;
     unsigned char buffer[header_length];
-    int read_failed = read_from_socket(socket_id, header_length, (unsigned char*)&buffer);
+    int read_failed = read_from_netdrv(socket_id, header_length, (unsigned char*)&buffer);
     if (read_failed || buffer[0] != MSG_TYPE_P2P_SENDING_FED_ID) {
       lf_print_warning("Federate received invalid first message on P2P socket. Closing socket.");
       if (read_failed == 0) {
@@ -1998,7 +1998,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
         // Trace the event when tracing is enabled
         tracepoint_federate_to_federate(send_REJECT, _lf_my_fed_id, -3, NULL);
         // Ignore errors on this response.
-        write_to_socket(socket_id, 2, response);
+        write_to_netdrv(socket_id, 2, response);
       }
       shutdown_socket(&socket_id, false);
       continue;
@@ -2007,7 +2007,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
     // Get the federation ID and check it.
     unsigned char federation_id_length = buffer[header_length - 1];
     char remote_federation_id[federation_id_length];
-    read_failed = read_from_socket(socket_id, federation_id_length, (unsigned char*)remote_federation_id);
+    read_failed = read_from_netdrv(socket_id, federation_id_length, (unsigned char*)remote_federation_id);
     if (read_failed || (strncmp(federation_metadata.federation_id, remote_federation_id,
                                 strnlen(federation_metadata.federation_id, 255)) != 0)) {
       lf_print_warning("Received invalid federation ID. Closing socket.");
@@ -2018,7 +2018,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
         // Trace the event when tracing is enabled
         tracepoint_federate_to_federate(send_REJECT, _lf_my_fed_id, -3, NULL);
         // Ignore errors on this response.
-        write_to_socket(socket_id, 2, response);
+        write_to_netdrv(socket_id, 2, response);
       }
       shutdown_socket(&socket_id, false);
       continue;
@@ -2045,7 +2045,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
     tracepoint_federate_to_federate(send_ACK, _lf_my_fed_id, remote_fed_id, NULL);
 
     LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
-    write_to_socket_fail_on_error(&_fed.netdrvs_for_inbound_p2p_connections[remote_fed_id], 1,
+    write_to_netdrv_fail_on_error(&_fed.netdrvs_for_inbound_p2p_connections[remote_fed_id], 1,
                                   (unsigned char*)&response, &lf_outbound_netdrv_mutex,
                                   "Failed to write MSG_TYPE_ACK in response to federate %d.", remote_fed_id);
     LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
@@ -2173,10 +2173,10 @@ int lf_send_message(int message_type, unsigned short port, unsigned short federa
   // Trace the event when tracing is enabled
   tracepoint_federate_to_federate(send_P2P_MSG, _lf_my_fed_id, federate, NULL);
 
-  int result = write_to_socket_close_on_error(socket, header_length, header_buffer);
+  int result = write_to_netdrv_close_on_error(socket, header_length, header_buffer);
   if (result == 0) {
     // Header sent successfully. Send the body.
-    result = write_to_socket_close_on_error(socket, length, message);
+    result = write_to_netdrv_close_on_error(socket, length, message);
   }
   if (result != 0) {
     // Message did not send. Since this is used for physical connections, this is not critical.
@@ -2358,7 +2358,7 @@ void lf_send_port_absent_to_federate(environment_t* env, interval_t additional_d
   }
 
   LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
-  int result = write_to_socket_close_on_error(socket, message_length, buffer);
+  int result = write_to_netdrv_close_on_error(socket, message_length, buffer);
   LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
 
   if (result != 0) {
@@ -2396,7 +2396,7 @@ int lf_send_stop_request_to_rti(tag_t stop_tag) {
     // Trace the event when tracing is enabled
     tracepoint_federate_to_rti(send_STOP_REQ, _lf_my_fed_id, &stop_tag);
 
-    write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, MSG_TYPE_STOP_REQUEST_LENGTH, buffer, &lf_outbound_netdrv_mutex,
+    write_to_netdrv_fail_on_error(&_fed.socket_TCP_RTI, MSG_TYPE_STOP_REQUEST_LENGTH, buffer, &lf_outbound_netdrv_mutex,
                                   "Failed to send stop time " PRINTF_TIME " to the RTI.", stop_tag.time - start_time);
 
     // Treat this sending  as equivalent to having received a stop request from the RTI.
@@ -2468,10 +2468,10 @@ int lf_send_tagged_message(environment_t* env, interval_t additional_delay, int
     tracepoint_federate_to_rti(send_TAGGED_MSG, _lf_my_fed_id, &current_message_intended_tag);
   }
 
-  int result = write_to_socket_close_on_error(socket, header_length, header_buffer);
+  int result = write_to_netdrv_close_on_error(socket, header_length, header_buffer);
   if (result == 0) {
     // Header sent successfully. Send the body.
-    result = write_to_socket_close_on_error(socket, length, message);
+    result = write_to_netdrv_close_on_error(socket, length, message);
   }
   if (result != 0) {
     // Message did not send. Handling depends on message type.

From 9d47c70cbefeb924994bb7295defba3be7dbac82 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sat, 18 Jan 2025 22:55:43 -0700
Subject: [PATCH 49/84] Fix parameter on lf_send_neighbor_structure_to_RTI

---
 include/core/federated/federate.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/core/federated/federate.h b/include/core/federated/federate.h
index 0a62dba43..b2432e6f2 100644
--- a/include/core/federated/federate.h
+++ b/include/core/federated/federate.h
@@ -339,7 +339,7 @@ int lf_send_message(int message_type, unsigned short port, unsigned short federa
  * information is needed for the RTI to perform the centralized coordination.
  * @see MSG_TYPE_NEIGHBOR_STRUCTURE in net_common.h
  */
-void lf_send_neighbor_structure_to_RTI(int);
+void lf_send_neighbor_structure_to_RTI(netdrv_t*);
 
 /**
  * @brief Send a next event tag (NET) signal.

From 4b172d15e044ddeab49fc5a0ea8ede483e088bfd Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sat, 18 Jan 2025 23:02:04 -0700
Subject: [PATCH 50/84] Change socket_mutex to netdrv_mutex

---
 core/federated/clock-sync.c         |  2 +-
 core/federated/federate.c           | 10 +++++-----
 include/core/federated/clock-sync.h |  2 +-
 network/api/socket_common.h         |  2 +-
 network/impl/src/socket_common.c    |  6 +++---
 5 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/core/federated/clock-sync.c b/core/federated/clock-sync.c
index 60a7bd16f..f4b578cf5 100644
--- a/core/federated/clock-sync.c
+++ b/core/federated/clock-sync.c
@@ -304,7 +304,7 @@ int handle_T1_clock_sync_message(unsigned char* buffer, int socket, instant_t t2
  * the T4 and the coded probe message is not as expected, then reject
  * this clock synchronization round. If it is not rejected, then make
  * an adjustment to the clock offset based on the estimated error.
- * This function does not acquire the socket_mutex lock.
+ * This function does not acquire the netdrv_mutex lock.
  * The caller should acquire it unless it is sure there is only one thread running.
  * @param buffer The buffer containing the message, including the message type.
  * @param socket The socket (either _lf_rti_socket_TCP or _lf_rti_socket_UDP).
diff --git a/core/federated/federate.c b/core/federated/federate.c
index 13bc08f67..a8d35eb2e 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -410,11 +410,11 @@ static trigger_handle_t schedule_message_received_from_network_locked(environmen
  *  federate.
  */
 static void close_inbound_socket(int fed_id) {
-  LF_MUTEX_LOCK(&socket_mutex);
+  LF_MUTEX_LOCK(&netdrv_mutex);
   if (_fed.netdrvs_for_inbound_p2p_connections[fed_id] >= 0) {
     shutdown_socket(&_fed.netdrvs_for_inbound_p2p_connections[fed_id], false);
   }
-  LF_MUTEX_UNLOCK(&socket_mutex);
+  LF_MUTEX_UNLOCK(&netdrv_mutex);
 }
 
 /**
@@ -2032,7 +2032,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
     tracepoint_federate_to_federate(receive_FED_ID, _lf_my_fed_id, remote_fed_id, NULL);
 
     // Once we record the socket_id here, all future calls to close() on
-    // the socket should be done while holding the socket_mutex, and this array
+    // the socket should be done while holding the netdrv_mutex, and this array
     // element should be reset to -1 during that critical section.
     // Otherwise, there can be race condition where, during termination,
     // two threads attempt to simultaneously close the socket.
@@ -2056,12 +2056,12 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
     int result = lf_thread_create(&_fed.inbound_netdriv_listeners[received_federates], listen_to_federates, fed_id_arg);
     if (result != 0) {
       // Failed to create a listening thread.
-      LF_MUTEX_LOCK(&socket_mutex);
+      LF_MUTEX_LOCK(&netdrv_mutex);
       if (_fed.netdrvs_for_inbound_p2p_connections[remote_fed_id] != -1) {
         shutdown_socket(&socket_id, false);
         _fed.netdrvs_for_inbound_p2p_connections[remote_fed_id] = -1;
       }
-      LF_MUTEX_UNLOCK(&socket_mutex);
+      LF_MUTEX_UNLOCK(&netdrv_mutex);
       lf_print_error_and_exit("Failed to create a thread to listen for incoming physical connection. Error code: %d.",
                               result);
     }
diff --git a/include/core/federated/clock-sync.h b/include/core/federated/clock-sync.h
index b003a3150..1b0cacd3e 100644
--- a/include/core/federated/clock-sync.h
+++ b/include/core/federated/clock-sync.h
@@ -190,7 +190,7 @@ int handle_T1_clock_sync_message(unsigned char* buffer, int socket, instant_t t2
  * the T4 and the coded probe message is not as expected, then reject
  * this clock synchronization round. If it is not rejected, then make
  * an adjustment to the clock offset based on the estimated error.
- * This function does not acquire the socket_mutex lock.
+ * This function does not acquire the netdrv_mutex lock.
  * The caller should acquire it unless it is sure there is only one thread running.
  * @param buffer The buffer containing the message, including the message type.
  * @param socket The socket (either _lf_rti_socket_TCP or _lf_rti_socket_UDP).
diff --git a/network/api/socket_common.h b/network/api/socket_common.h
index fd91640ac..63665942b 100644
--- a/network/api/socket_common.h
+++ b/network/api/socket_common.h
@@ -74,7 +74,7 @@ typedef enum socket_type_t { TCP, UDP } socket_type_t;
 /**
  * Mutex protecting socket close operations.
  */
-extern lf_mutex_t socket_mutex;
+extern lf_mutex_t netdrv_mutex;
 
 typedef struct socket_priv_t {
   int socket_descriptor;
diff --git a/network/impl/src/socket_common.c b/network/impl/src/socket_common.c
index 202ad1125..279cb51ef 100644
--- a/network/impl/src/socket_common.c
+++ b/network/impl/src/socket_common.c
@@ -20,9 +20,9 @@
 /** Number of nanoseconds to sleep before retrying a socket read. */
 #define SOCKET_READ_RETRY_INTERVAL 1000000
 
-// Mutex lock held while performing socket close operations.
-// A deadlock can occur if two threads simulataneously attempt to close the same socket.
-lf_mutex_t socket_mutex;
+// Mutex lock held while performing networ driver close operations.
+// A deadlock can occur if two threads simulataneously attempt to close the same network driver.
+lf_mutex_t netdrv_mutex;
 
 int create_real_time_tcp_socket_errexit() {
   int sock = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);

From 9dbf5be93511129af5cce2fefd89cc20cbf34272 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sat, 18 Jan 2025 23:11:14 -0700
Subject: [PATCH 51/84] Remove server_type. just leave it as true false for
 increment on retry.

---
 core/federated/RTI/rti_remote.c      | 2 +-
 core/federated/clock-sync.c          | 1 +
 core/federated/federate.c            | 7 ++++---
 network/api/net_driver.h             | 3 +--
 network/impl/src/lf_socket_support.c | 3 +--
 5 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index 39847ce31..38de11ef5 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -1489,7 +1489,7 @@ int start_rti_server() {
   // Initialize RTI's network driver.
   rti_remote->rti_netdrv = initialize_netdrv();
   // Create the server
-  if (create_server_(rti_remote->rti_netdrv, RTI)) {
+  if (create_server_(rti_remote->rti_netdrv, true)) {
     lf_print_error_system_failure("RTI failed to create TCP server: %s.", strerror(errno));
     return -1;
   };
diff --git a/core/federated/clock-sync.c b/core/federated/clock-sync.c
index f4b578cf5..cd95aa0b2 100644
--- a/core/federated/clock-sync.c
+++ b/core/federated/clock-sync.c
@@ -208,6 +208,7 @@ uint16_t setup_clock_synchronization_with_rti() {
   return port_to_return;
 }
 
+//TODO: Fix clocks.
 void synchronize_initial_physical_clock_with_rti(int* rti_socket_TCP) {
   LF_PRINT_DEBUG("Waiting for initial clock synchronization messages from the RTI.");
 
diff --git a/core/federated/federate.c b/core/federated/federate.c
index a8d35eb2e..74443c0d5 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -1811,8 +1811,6 @@ void lf_connect_to_rti(const char* hostname, int port) {
   hostname = federation_metadata.rti_host ? federation_metadata.rti_host : hostname;
   port = federation_metadata.rti_port >= 0 ? federation_metadata.rti_port : port;
 
-
-
   // Create a network driver.
   _fed.netdrv_to_RTI = initialize_netdrv();
   // Set the user specified host name and port to the network driver.
@@ -1926,7 +1924,10 @@ void lf_connect_to_rti(const char* hostname, int port) {
 
 void lf_create_server(int specified_port) {
   assert(specified_port <= UINT16_MAX && specified_port >= 0);
-  if (create_server(specified_port, &_fed.server_socket, (uint16_t*)&_fed.server_port, TCP, false)) {
+
+  netdrv_t* server_netdrv = initialize_netdrv();
+
+  if (create_server_(server_netdrv, false)) {
     lf_print_error_system_failure("RTI failed to create TCP server: %s.", strerror(errno));
   };
   LF_PRINT_LOG("Server for communicating with other federates started using port %d.", _fed.server_port);
diff --git a/network/api/net_driver.h b/network/api/net_driver.h
index c6410a0c1..e2a0a9606 100644
--- a/network/api/net_driver.h
+++ b/network/api/net_driver.h
@@ -3,7 +3,6 @@
 
 #include "socket_common.h"
 
-typedef enum server_type_t { RTI, FED } server_type_t;
 typedef struct netdrv_t {
   void* priv;
   //   unsigned int read_remaining_bytes;
@@ -25,7 +24,7 @@ netdrv_t* initialize_netdrv();
  * @param serv_type Type of server, RTI or FED.
  * @return int 0 for success, -1 for failure.
  */
-int create_server_(netdrv_t* drv, server_type_t serv_type);
+int create_server_(netdrv_t* drv, bool increment_port_on_retry);
 
 netdrv_t* accept_netdrv(netdrv_t* server_drv, netdrv_t* rti_drv);
 
diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
index 1001f7a27..c71f5763e 100644
--- a/network/impl/src/lf_socket_support.c
+++ b/network/impl/src/lf_socket_support.c
@@ -34,7 +34,7 @@ netdrv_t* initialize_netdrv() {
   return drv;
 }
 
-int create_server_(netdrv_t* drv, server_type_t serv_type) {
+int create_server_(netdrv_t* drv, bool increment_port_on_retry) {
   socket_priv_t* priv = (socket_priv_t*)drv->priv;
   int socket_descriptor;
   struct timeval timeout_time;
@@ -47,7 +47,6 @@ int create_server_(netdrv_t* drv, server_type_t serv_type) {
     return -1;
   }
   set_socket_timeout_option(socket_descriptor, &timeout_time);
-  bool increment_port_on_retry = (serv_type == RTI) ? true : false;
 
   int used_port = set_socket_bind_option(socket_descriptor, priv->user_specified_port, increment_port_on_retry);
   // Enable listening for socket connections.

From df0c3ef920032a9bace21617f6f06b2ec9e71f48 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sat, 18 Jan 2025 23:30:58 -0700
Subject: [PATCH 52/84] Fix lf_create_server.

---
 core/federated/federate.c | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index 74443c0d5..f316ee41c 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -1926,26 +1926,30 @@ void lf_create_server(int specified_port) {
   assert(specified_port <= UINT16_MAX && specified_port >= 0);
 
   netdrv_t* server_netdrv = initialize_netdrv();
+  set_server_port(server_netdrv, specified_port);
 
   if (create_server_(server_netdrv, false)) {
-    lf_print_error_system_failure("RTI failed to create TCP server: %s.", strerror(errno));
+    lf_print_error_system_failure("RTI failed to create server: %s.", strerror(errno));
   };
-  LF_PRINT_LOG("Server for communicating with other federates started using port %d.", _fed.server_port);
+  // Get the final server port set.
+  int32_t server_port = get_server_port(server_netdrv);
+
+  LF_PRINT_LOG("Server for communicating with other federates started using port %d.", server_port);
 
   // Send the server port number to the RTI
   // on an MSG_TYPE_ADDRESS_ADVERTISEMENT message (@see net_common.h).
   unsigned char buffer[sizeof(int32_t) + 1];
   buffer[0] = MSG_TYPE_ADDRESS_ADVERTISEMENT;
-  encode_int32(_fed.server_port, &(buffer[1]));
+  encode_int32(server_port, &(buffer[1]));
 
   // Trace the event when tracing is enabled
   tracepoint_federate_to_rti(send_ADR_AD, _lf_my_fed_id, NULL);
 
   // No need for a mutex because we have the only handle on this socket.
-  write_to_netdrv_fail_on_error(&_fed.socket_TCP_RTI, sizeof(int32_t) + 1, (unsigned char*)buffer, NULL,
+  write_to_netdrv_fail_on_error(server_netdrv, sizeof(int32_t) + 1, (unsigned char*)buffer, NULL,
                                 "Failed to send address advertisement.");
 
-  LF_PRINT_DEBUG("Sent port %d to the RTI.", _fed.server_port);
+  LF_PRINT_DEBUG("Sent port %d to the RTI.", server_port);
 }
 
 void lf_enqueue_port_absent_reactions(environment_t* env) {

From df2bcd587c40c5fa644c8a294fadd8d28c836623 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sat, 18 Jan 2025 23:33:21 -0700
Subject: [PATCH 53/84] Add setting server netdriver to federate.

---
 core/federated/federate.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index f316ee41c..67617af0f 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -1931,6 +1931,7 @@ void lf_create_server(int specified_port) {
   if (create_server_(server_netdrv, false)) {
     lf_print_error_system_failure("RTI failed to create server: %s.", strerror(errno));
   };
+  _fed.server_netdrv = server_netdrv;
   // Get the final server port set.
   int32_t server_port = get_server_port(server_netdrv);
 

From 6cc28d55893b70598e254e0b44aa4a1763b2c212 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sat, 18 Jan 2025 23:43:37 -0700
Subject: [PATCH 54/84] Fix on accept netdriver, to return NULL, when failed to
 accept.

---
 core/federated/federate.c            | 25 +++++++++++++------------
 network/impl/src/lf_socket_support.c |  6 ++++--
 2 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index 67617af0f..4510b5548 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -1984,8 +1984,9 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
   _fed.inbound_netdriv_listeners = (lf_thread_t*)calloc(_fed.number_of_inbound_p2p_connections, sizeof(lf_thread_t));
   while (received_federates < _fed.number_of_inbound_p2p_connections && !_lf_termination_executed) {
     // Wait for an incoming connection request.
-    int socket_id = accept_socket(_fed.server_socket, _fed.socket_TCP_RTI);
-    if (socket_id < 0) {
+    netdrv_t* netdrv = accept_netdrv(_fed.server_netdrv, _fed.netdrv_to_RTI);
+    // int socket_id = accept_socket(_fed.server_socket, _fed.socket_TCP_RTI);
+    if (netdrv == NULL) {
       lf_print_warning("Federate failed to accept the socket.");
       return NULL;
     }
@@ -1993,7 +1994,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
 
     size_t header_length = 1 + sizeof(uint16_t) + 1;
     unsigned char buffer[header_length];
-    int read_failed = read_from_netdrv(socket_id, header_length, (unsigned char*)&buffer);
+    int read_failed = read_from_netdrv(netdrv, header_length, (unsigned char*)&buffer);
     if (read_failed || buffer[0] != MSG_TYPE_P2P_SENDING_FED_ID) {
       lf_print_warning("Federate received invalid first message on P2P socket. Closing socket.");
       if (read_failed == 0) {
@@ -2004,7 +2005,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
         // Trace the event when tracing is enabled
         tracepoint_federate_to_federate(send_REJECT, _lf_my_fed_id, -3, NULL);
         // Ignore errors on this response.
-        write_to_netdrv(socket_id, 2, response);
+        write_to_netdrv(netdrv, 2, response);
       }
       shutdown_socket(&socket_id, false);
       continue;
@@ -2013,7 +2014,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
     // Get the federation ID and check it.
     unsigned char federation_id_length = buffer[header_length - 1];
     char remote_federation_id[federation_id_length];
-    read_failed = read_from_netdrv(socket_id, federation_id_length, (unsigned char*)remote_federation_id);
+    read_failed = read_from_netdrv(netdrv, federation_id_length, (unsigned char*)remote_federation_id);
     if (read_failed || (strncmp(federation_metadata.federation_id, remote_federation_id,
                                 strnlen(federation_metadata.federation_id, 255)) != 0)) {
       lf_print_warning("Received invalid federation ID. Closing socket.");
@@ -2024,7 +2025,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
         // Trace the event when tracing is enabled
         tracepoint_federate_to_federate(send_REJECT, _lf_my_fed_id, -3, NULL);
         // Ignore errors on this response.
-        write_to_netdrv(socket_id, 2, response);
+        write_to_netdrv(netdrv, 2, response);
       }
       shutdown_socket(&socket_id, false);
       continue;
@@ -2037,12 +2038,12 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
     // Trace the event when tracing is enabled
     tracepoint_federate_to_federate(receive_FED_ID, _lf_my_fed_id, remote_fed_id, NULL);
 
-    // Once we record the socket_id here, all future calls to close() on
-    // the socket should be done while holding the netdrv_mutex, and this array
+    // Once we record the network driver here, all future calls to close() on
+    // the network driver should be done while holding the netdrv_mutex, and this array
     // element should be reset to -1 during that critical section.
     // Otherwise, there can be race condition where, during termination,
-    // two threads attempt to simultaneously close the socket.
-    _fed.netdrvs_for_inbound_p2p_connections[remote_fed_id] = socket_id;
+    // two threads attempt to simultaneously close the network driver.
+    _fed.netdrvs_for_inbound_p2p_connections[remote_fed_id] = netdrv;
 
     // Send an MSG_TYPE_ACK message.
     unsigned char response = MSG_TYPE_ACK;
@@ -2063,9 +2064,9 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
     if (result != 0) {
       // Failed to create a listening thread.
       LF_MUTEX_LOCK(&netdrv_mutex);
-      if (_fed.netdrvs_for_inbound_p2p_connections[remote_fed_id] != -1) {
+      if (_fed.netdrvs_for_inbound_p2p_connections[remote_fed_id] != NULL) {
         shutdown_socket(&socket_id, false);
-        _fed.netdrvs_for_inbound_p2p_connections[remote_fed_id] = -1;
+        _fed.netdrvs_for_inbound_p2p_connections[remote_fed_id] = NULL;
       }
       LF_MUTEX_UNLOCK(&netdrv_mutex);
       lf_print_error_and_exit("Failed to create a thread to listen for incoming physical connection. Error code: %d.",
diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
index c71f5763e..185cdead6 100644
--- a/network/impl/src/lf_socket_support.c
+++ b/network/impl/src/lf_socket_support.c
@@ -79,15 +79,17 @@ netdrv_t* accept_netdrv(netdrv_t* server_drv, netdrv_t* rti_drv) {
       break;
     } else if (socket_id < 0 && (errno != EAGAIN || errno != EWOULDBLOCK || errno != EINTR)) {
       lf_print_warning("Failed to accept the socket. %s.", strerror(errno));
-      break;
+      //TODO: Must free memory.
+      return NULL;
     } else if (errno == EPERM) {
       lf_print_error_system_failure("Firewall permissions prohibit connection.");
+      return NULL;
     } else {
       // For the federates, it should check if the rti_socket is still open, before retrying accept().
       socket_priv_t* rti_priv = (socket_priv_t*)rti_drv->priv;
       if (rti_priv->socket_descriptor != -1) {
         if (check_socket_closed(rti_priv->socket_descriptor)) {
-          break;
+          return NULL;
         }
       }
       // Try again

From bde87c4494e5695a35a6cb2a8b18c052e645164a Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sat, 18 Jan 2025 23:55:33 -0700
Subject: [PATCH 55/84] Add free_netdrv()

---
 core/federated/federate.c            | 18 +++++++++---------
 network/impl/src/lf_socket_support.c | 19 ++++++++++++++++---
 2 files changed, 25 insertions(+), 12 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index 4510b5548..35f1bea80 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -412,7 +412,7 @@ static trigger_handle_t schedule_message_received_from_network_locked(environmen
 static void close_inbound_socket(int fed_id) {
   LF_MUTEX_LOCK(&netdrv_mutex);
   if (_fed.netdrvs_for_inbound_p2p_connections[fed_id] >= 0) {
-    shutdown_socket(&_fed.netdrvs_for_inbound_p2p_connections[fed_id], false);
+    shutdown_netdrv(&_fed.netdrvs_for_inbound_p2p_connections[fed_id], false);
   }
   LF_MUTEX_UNLOCK(&netdrv_mutex);
 }
@@ -826,11 +826,11 @@ static void close_outbound_socket(int fed_id) {
     if (_fed.netdrvs_for_outbound_p2p_connections[fed_id] >= 0) {
       // Close the socket by sending a FIN packet indicating that no further writes
       // are expected.  Then read until we get an EOF indication.
-      shutdown_socket(&_fed.netdrvs_for_outbound_p2p_connections[fed_id], true);
+      shutdown_netdrv(&_fed.netdrvs_for_outbound_p2p_connections[fed_id], true);
     }
     LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
   } else {
-    shutdown_socket(&_fed.netdrvs_for_outbound_p2p_connections[fed_id], false);
+    shutdown_netdrv(&_fed.netdrvs_for_outbound_p2p_connections[fed_id], false);
   }
 }
 
@@ -1494,7 +1494,7 @@ static void* listen_to_rti_TCP(void* args) {
         lf_print_error("Socket connection to the RTI was closed by the RTI without"
                        " properly sending an EOF first. Considering this a soft error.");
         // FIXME: If this happens, possibly a new RTI must be elected.
-        shutdown_socket(&_fed.socket_TCP_RTI, false);
+        shutdown_netdrv(&_fed.socket_TCP_RTI, false);
         return NULL;
       } else {
         lf_print_error("Socket connection to the RTI has been broken with error %d: %s."
@@ -1502,13 +1502,13 @@ static void* listen_to_rti_TCP(void* args) {
                        " Considering this a soft error.",
                        errno, strerror(errno));
         // FIXME: If this happens, possibly a new RTI must be elected.
-        shutdown_socket(&_fed.socket_TCP_RTI, false);
+        shutdown_netdrv(&_fed.socket_TCP_RTI, false);
         return NULL;
       }
     } else if (read_failed > 0) {
       // EOF received.
       lf_print("Connection to the RTI closed with an EOF.");
-      shutdown_socket(&_fed.socket_TCP_RTI, false);
+      shutdown_netdrv(&_fed.socket_TCP_RTI, false);
       return NULL;
     }
     switch (buffer[0]) {
@@ -2007,7 +2007,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
         // Ignore errors on this response.
         write_to_netdrv(netdrv, 2, response);
       }
-      shutdown_socket(&socket_id, false);
+      shutdown_netdrv(netdrv, false);
       continue;
     }
 
@@ -2027,7 +2027,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
         // Ignore errors on this response.
         write_to_netdrv(netdrv, 2, response);
       }
-      shutdown_socket(&socket_id, false);
+      shutdown_netdrv(netdrv, false);
       continue;
     }
 
@@ -2065,7 +2065,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
       // Failed to create a listening thread.
       LF_MUTEX_LOCK(&netdrv_mutex);
       if (_fed.netdrvs_for_inbound_p2p_connections[remote_fed_id] != NULL) {
-        shutdown_socket(&socket_id, false);
+        shutdown_netdrv(&socket_id, false);
         _fed.netdrvs_for_inbound_p2p_connections[remote_fed_id] = NULL;
       }
       LF_MUTEX_UNLOCK(&netdrv_mutex);
diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
index 185cdead6..97fbfd9d5 100644
--- a/network/impl/src/lf_socket_support.c
+++ b/network/impl/src/lf_socket_support.c
@@ -34,6 +34,12 @@ netdrv_t* initialize_netdrv() {
   return drv;
 }
 
+void free_netdrv(netdrv_t* drv) {
+  socket_priv_t* priv = (socket_priv_t*)drv->priv;
+  free(priv);
+  free(drv);
+}
+
 int create_server_(netdrv_t* drv, bool increment_port_on_retry) {
   socket_priv_t* priv = (socket_priv_t*)drv->priv;
   int socket_descriptor;
@@ -79,16 +85,18 @@ netdrv_t* accept_netdrv(netdrv_t* server_drv, netdrv_t* rti_drv) {
       break;
     } else if (socket_id < 0 && (errno != EAGAIN || errno != EWOULDBLOCK || errno != EINTR)) {
       lf_print_warning("Failed to accept the socket. %s.", strerror(errno));
-      //TODO: Must free memory.
+      free_netdrv(fed_netdrv);
       return NULL;
     } else if (errno == EPERM) {
       lf_print_error_system_failure("Firewall permissions prohibit connection.");
+      free_netdrv(fed_netdrv);
       return NULL;
     } else {
       // For the federates, it should check if the rti_socket is still open, before retrying accept().
       socket_priv_t* rti_priv = (socket_priv_t*)rti_drv->priv;
       if (rti_priv->socket_descriptor != -1) {
         if (check_socket_closed(rti_priv->socket_descriptor)) {
+          free_netdrv(fed_netdrv);
           return NULL;
         }
       }
@@ -232,7 +240,12 @@ void write_to_netdrv_fail_on_error(netdrv_t* drv, size_t num_bytes, unsigned cha
 
 int shutdown_netdrv(netdrv_t* drv, bool read_before_closing) {
   socket_priv_t* priv = (socket_priv_t*)drv->priv;
-  return shutdown_socket(&priv->socket_descriptor, read_before_closing);
+  int ret = shutdown_socket(&priv->socket_descriptor, read_before_closing);
+  if (ret != 0) {
+     lf_print_error("Failed to shutdown socket.");
+  }
+  free_netdrv(drv);
+  return ret;
 }
 
 int get_peer_address(netdrv_t* drv) {
@@ -283,4 +296,4 @@ void set_server_port(netdrv_t* drv, int32_t port) {
 void set_server_host_name(netdrv_t* drv, const char* hostname) {
   socket_priv_t* priv = (socket_priv_t*)drv->priv;
   memcpy(priv->server_hostname, hostname, INET_ADDRSTRLEN);
-}
\ No newline at end of file
+}

From 96567d6261177076f5b320b05ca6a9b7344b7bd0 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sun, 19 Jan 2025 00:37:39 -0700
Subject: [PATCH 56/84] Add get_socket_id()

---
 network/api/net_driver.h             | 2 ++
 network/impl/src/lf_socket_support.c | 5 +++++
 2 files changed, 7 insertions(+)

diff --git a/network/api/net_driver.h b/network/api/net_driver.h
index e2a0a9606..44a95cf25 100644
--- a/network/api/net_driver.h
+++ b/network/api/net_driver.h
@@ -138,6 +138,8 @@ struct in_addr* get_ip_addr(netdrv_t* drv);
 
 char* get_server_hostname(netdrv_t* drv);
 
+int get_socket_id(netdrv_t* drv);
+
 void set_server_port(netdrv_t* drv, int32_t port);
 
 void set_server_host_name(netdrv_t* drv, const char* hostname);
diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
index 97fbfd9d5..9b29a64ef 100644
--- a/network/impl/src/lf_socket_support.c
+++ b/network/impl/src/lf_socket_support.c
@@ -288,6 +288,11 @@ char* get_server_hostname(netdrv_t* drv) {
   return priv->server_hostname;
 }
 
+int get_socket_id(netdrv_t* drv) {
+  socket_priv_t* priv = (socket_priv_t*)drv->priv;
+  return priv->socket_descriptor;
+}
+
 void set_server_port(netdrv_t* drv, int32_t port) {
   socket_priv_t* priv = (socket_priv_t*)drv->priv;
   priv->server_port = port;

From 21d2f2334440857f55afd3e1f425c8a962dd1049 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sun, 19 Jan 2025 00:38:45 -0700
Subject: [PATCH 57/84] Fix on listen_to_federates and lf_connect_to_federate

---
 core/federated/federate.c | 118 +++++++++++++++++++++-----------------
 1 file changed, 64 insertions(+), 54 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index 35f1bea80..c8e553339 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -409,10 +409,10 @@ static trigger_handle_t schedule_message_received_from_network_locked(environmen
  * @param fed_id The ID of the peer federate sending messages to this
  *  federate.
  */
-static void close_inbound_socket(int fed_id) {
+static void close_inbound_netdrv(int fed_id) {
   LF_MUTEX_LOCK(&netdrv_mutex);
-  if (_fed.netdrvs_for_inbound_p2p_connections[fed_id] >= 0) {
-    shutdown_netdrv(&_fed.netdrvs_for_inbound_p2p_connections[fed_id], false);
+  if (_fed.netdrvs_for_inbound_p2p_connections[fed_id] != NULL) {
+    shutdown_netdrv(_fed.netdrvs_for_inbound_p2p_connections[fed_id], false);
   }
   LF_MUTEX_UNLOCK(&netdrv_mutex);
 }
@@ -464,17 +464,17 @@ static bool handle_message_now(environment_t* env, trigger_t* trigger, tag_t int
  * Handle a message being received from a remote federate.
  *
  * This function assumes the caller does not hold the mutex lock.
- * @param socket Pointer to the socket to read the message from.
+ * @param netdrv Pointer to the network driver to read the message from.
  * @param fed_id The sending federate ID or -1 if the centralized coordination.
  * @return 0 for success, -1 for failure.
  */
-static int handle_message(int* socket, int fed_id) {
+static int handle_message(netdrv_t* netdrv, int fed_id) {
   (void)fed_id;
   // Read the header.
   size_t bytes_to_read = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint32_t);
   unsigned char buffer[bytes_to_read];
-  if (read_from_netdrv_close_on_error(socket, bytes_to_read, buffer)) {
-    // Read failed, which means the socket has been closed between reading the
+  if (read_from_netdrv_close_on_error(netdrv, bytes_to_read, buffer)) {
+    // Read failed, which means the network driver has been closed between reading the
     // message ID byte and here.
     return -1;
   }
@@ -494,7 +494,7 @@ static int handle_message(int* socket, int fed_id) {
   // Read the payload.
   // Allocate memory for the message contents.
   unsigned char* message_contents = (unsigned char*)malloc(length);
-  if (read_from_netdrv_close_on_error(socket, length, message_contents)) {
+  if (read_from_netdrv_close_on_error(netdrv, length, message_contents)) {
     return -1;
   }
   // Trace the event when tracing is enabled
@@ -518,11 +518,11 @@ static int handle_message(int* socket, int fed_id) {
  * will not advance to the tag of the message if it is in the future, or
  * the tag will not advance at all if the tag of the message is
  * now or in the past.
- * @param socket Pointer to the socket to read the message from.
+ * @param netdrv Pointer to the network driver to read the message from.
  * @param fed_id The sending federate ID or -1 if the centralized coordination.
- * @return 0 on successfully reading the message, -1 on failure (e.g. due to socket closed).
+ * @return 0 on successfully reading the message, -1 on failure (e.g. due to network driver closed).
  */
-static int handle_tagged_message(int* socket, int fed_id) {
+static int handle_tagged_message(netdrv_t* netdrv, int fed_id) {
   // Environment is always the one corresponding to the top-level scheduling enclave.
   environment_t* env;
   _lf_get_environments(&env);
@@ -531,7 +531,7 @@ static int handle_tagged_message(int* socket, int fed_id) {
   size_t bytes_to_read =
       sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint32_t) + sizeof(instant_t) + sizeof(microstep_t);
   unsigned char buffer[bytes_to_read];
-  if (read_from_netdrv_close_on_error(socket, bytes_to_read, buffer)) {
+  if (read_from_netdrv_close_on_error(netdrv, bytes_to_read, buffer)) {
     return -1; // Read failed.
   }
 
@@ -580,7 +580,7 @@ static int handle_tagged_message(int* socket, int fed_id) {
   // Read the payload.
   // Allocate memory for the message contents.
   unsigned char* message_contents = (unsigned char*)malloc(length);
-  if (read_from_netdrv_close_on_error(socket, length, message_contents)) {
+  if (read_from_netdrv_close_on_error(netdrv, length, message_contents)) {
 #ifdef FEDERATED_DECENTRALIZED
     _lf_decrement_tag_barrier_locked(env);
 #endif
@@ -649,11 +649,11 @@ static int handle_tagged_message(int* socket, int fed_id) {
     if (lf_tag_compare(env->current_tag, env->stop_tag) >= 0 && env->execution_started) {
       lf_print_error("Received message too late. Already at stop tag.\n"
                      "    Current tag is " PRINTF_TAG " and intended tag is " PRINTF_TAG ".\n"
-                     "    Discarding message and closing the socket.",
+                     "    Discarding message and closing the network driver.",
                      env->current_tag.time - start_time, env->current_tag.microstep, intended_tag.time - start_time,
                      intended_tag.microstep);
-      // Close socket, reading any incoming data and discarding it.
-      close_inbound_socket(fed_id);
+      // Close network driver, reading any incoming data and discarding it.
+      close_inbound_netdrv(fed_id);
     } else {
       // Need to use intended_tag here, not actual_tag, so that STP violations are detected.
       // It will become actual_tag (that is when the reactions will be invoked).
@@ -682,14 +682,14 @@ static int handle_tagged_message(int* socket, int fed_id) {
  * This just sets the last known status tag of the port specified
  * in the message.
  *
- * @param socket Pointer to the socket to read the message from
+ * @param netdrv Pointer to the network driver to read the message from
  * @param fed_id The sending federate ID or -1 if the centralized coordination.
  * @return 0 for success, -1 for failure to complete the read.
  */
-static int handle_port_absent_message(int* socket, int fed_id) {
+static int handle_port_absent_message(netdrv_t* netdrv, int fed_id) {
   size_t bytes_to_read = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(instant_t) + sizeof(microstep_t);
   unsigned char buffer[bytes_to_read];
-  if (read_from_netdrv_close_on_error(socket, bytes_to_read, buffer)) {
+  if (read_from_netdrv_close_on_error(netdrv, bytes_to_read, buffer)) {
     return -1;
   }
 
@@ -726,7 +726,7 @@ static int handle_port_absent_message(int* socket, int fed_id) {
  * peer federate and calls the appropriate handling function for
  * each message type. If an error occurs or an EOF is received
  * from the peer, then this procedure sets the corresponding
- * socket in _fed.netdrvs_for_inbound_p2p_connections
+ * network driver in _fed.netdrvs_for_inbound_p2p_connections
  * to -1 and returns, terminating the thread.
  * @param _args The remote federate ID (cast to void*).
  * @param fed_id_ptr A pointer to a uint16_t containing federate ID being listened to.
@@ -738,53 +738,55 @@ static void* listen_to_federates(void* _args) {
 
   LF_PRINT_LOG("Listening to federate %d.", fed_id);
 
-  int* socket_id = &_fed.netdrvs_for_inbound_p2p_connections[fed_id];
+  netdrv_t* netdrv = _fed.netdrvs_for_inbound_p2p_connections[fed_id];
 
   // Buffer for incoming messages.
   // This does not constrain the message size
   // because the message will be put into malloc'd memory.
   unsigned char buffer[FED_COM_BUFFER_SIZE];
 
+  int socket_id = get_socket_id(netdrv);
+
   // Listen for messages from the federate.
   while (1) {
-    bool socket_closed = false;
+    bool netdrv_closed = false;
     // Read one byte to get the message type.
-    LF_PRINT_DEBUG("Waiting for a P2P message on socket %d.", *socket_id);
+    LF_PRINT_DEBUG("Waiting for a P2P message on socket %d.", socket_id);
     bool bad_message = false;
-    if (read_from_netdrv_close_on_error(socket_id, 1, buffer)) {
-      // Socket has been closed.
-      lf_print("Socket from federate %d is closed.", fed_id);
+    if (read_from_netdrv_close_on_error(netdrv, 1, buffer)) {
+      // Network driver has been closed.
+      lf_print("Network driver from federate %d is closed.", fed_id);
       // Stop listening to this federate.
-      socket_closed = true;
+      netdrv_closed = true;
     } else {
-      LF_PRINT_DEBUG("Received a P2P message on socket %d of type %d.", *socket_id, buffer[0]);
+      LF_PRINT_DEBUG("Received a P2P message on socket %d of type %d.", socket_id, buffer[0]);
       switch (buffer[0]) {
       case MSG_TYPE_P2P_MESSAGE:
         LF_PRINT_LOG("Received untimed message from federate %d.", fed_id);
-        if (handle_message(socket_id, fed_id)) {
+        if (handle_message(netdrv, fed_id)) {
           // Failed to complete the reading of a message on a physical connection.
           lf_print_warning("Failed to complete reading of message on physical connection.");
-          socket_closed = true;
+          netdrv_closed = true;
         }
         break;
       case MSG_TYPE_P2P_TAGGED_MESSAGE:
         LF_PRINT_LOG("Received tagged message from federate %d.", fed_id);
-        if (handle_tagged_message(socket_id, fed_id)) {
+        if (handle_tagged_message(netdrv, fed_id)) {
           // P2P tagged messages are only used in decentralized coordination, and
           // it is not a fatal error if the socket is closed before the whole message is read.
           // But this thread should exit.
           lf_print_warning("Failed to complete reading of tagged message.");
-          socket_closed = true;
+          netdrv_closed = true;
         }
         break;
       case MSG_TYPE_PORT_ABSENT:
         LF_PRINT_LOG("Received port absent message from federate %d.", fed_id);
-        if (handle_port_absent_message(socket_id, fed_id)) {
+        if (handle_port_absent_message(netdrv, fed_id)) {
           // P2P tagged messages are only used in decentralized coordination, and
           // it is not a fatal error if the socket is closed before the whole message is read.
           // But this thread should exit.
           lf_print_warning("Failed to complete reading of tagged message.");
-          socket_closed = true;
+          netdrv_closed = true;
         }
         break;
       default:
@@ -797,7 +799,7 @@ static void* listen_to_federates(void* _args) {
       tracepoint_federate_from_federate(receive_UNIDENTIFIED, _lf_my_fed_id, fed_id, NULL);
       break; // while loop
     }
-    if (socket_closed) {
+    if (netdrv_closed) {
       // For decentralized execution, once this socket is closed, we
       // update last known tags of all ports connected to the specified federate to FOREVER_TAG,
       // which would eliminate the need to wait for STAA to assume an input is absent.
@@ -1695,12 +1697,12 @@ void lf_connect_to_federate(uint16_t remote_federate_id) {
     tracepoint_federate_to_rti(send_ADR_QR, _lf_my_fed_id, NULL);
 
     LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
-    write_to_netdrv_fail_on_error(&_fed.socket_TCP_RTI, sizeof(uint16_t) + 1, buffer, &lf_outbound_netdrv_mutex,
+    write_to_netdrv_fail_on_error(_fed.netdrv_to_RTI, sizeof(uint16_t) + 1, buffer, &lf_outbound_netdrv_mutex,
                                   "Failed to send address query for federate %d to RTI.", remote_federate_id);
     LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
 
     // Read RTI's response.
-    read_from_netdrv_fail_on_error(&_fed.socket_TCP_RTI, sizeof(int32_t) + 1, buffer, NULL,
+    read_from_netdrv_fail_on_error(_fed.netdrv_to_RTI, sizeof(int32_t) + 1, buffer, NULL,
                                    "Failed to read the requested port number for federate %d from RTI.",
                                    remote_federate_id);
 
@@ -1714,7 +1716,7 @@ void lf_connect_to_federate(uint16_t remote_federate_id) {
     }
     port = extract_int32(&buffer[1]);
 
-    read_from_netdrv_fail_on_error(&_fed.socket_TCP_RTI, sizeof(host_ip_addr), (unsigned char*)&host_ip_addr, NULL,
+    read_from_netdrv_fail_on_error(_fed.netdrv_to_RTI, sizeof(host_ip_addr), (unsigned char*)&host_ip_addr, NULL,
                                    "Failed to read the IP address for federate %d from RTI.", remote_federate_id);
 
     // A reply of -1 for the port means that the RTI does not know
@@ -1735,10 +1737,18 @@ void lf_connect_to_federate(uint16_t remote_federate_id) {
 
   char hostname[INET_ADDRSTRLEN];
   inet_ntop(AF_INET, &host_ip_addr, hostname, INET_ADDRSTRLEN);
-  int socket_id = create_real_time_tcp_socket_errexit();
-  if (connect_to_socket(socket_id, (const char*)hostname, uport) < 0) {
+
+  // Create a network driver.
+  netdrv_t* netdrv = initialize_netdrv();
+  // Set the received host name and port to the network driver.
+  set_server_port(netdrv, uport);
+  set_server_host_name(netdrv, hostname);
+  // Create the client network driver.
+  create_client(netdrv);
+  if (connect_to_netdrv(netdrv) < 0) {
     lf_print_error_and_exit("Failed to connect() to RTI.");
   }
+
   // Iterate until we either successfully connect or we exceed the CONNECT_TIMEOUT
   start_connect = lf_time_physical();
   while (result < 0 && !_lf_termination_executed) {
@@ -1755,6 +1765,7 @@ void lf_connect_to_federate(uint16_t remote_federate_id) {
     }
 
     // Check whether the RTI is still there.
+    // TODO: Fix.
     if (rti_failed()) {
       break;
     }
@@ -1772,17 +1783,17 @@ void lf_connect_to_federate(uint16_t remote_federate_id) {
     tracepoint_federate_to_federate(send_FED_ID, _lf_my_fed_id, remote_federate_id, NULL);
 
     // No need for a mutex because we have the only handle on the socket.
-    write_to_netdrv_fail_on_error(&socket_id, buffer_length, buffer, NULL, "Failed to send fed_id to federate %d.",
+    write_to_netdrv_fail_on_error(netdrv, buffer_length, buffer, NULL, "Failed to send fed_id to federate %d.",
                                   remote_federate_id);
-    write_to_netdrv_fail_on_error(&socket_id, federation_id_length, (unsigned char*)federation_metadata.federation_id,
-                                  NULL, "Failed to send federation id to federate %d.", remote_federate_id);
+    write_to_netdrv_fail_on_error(netdrv, federation_id_length, (unsigned char*)federation_metadata.federation_id, NULL,
+                                  "Failed to send federation id to federate %d.", remote_federate_id);
 
-    read_from_netdrv_fail_on_error(&socket_id, 1, (unsigned char*)buffer, NULL,
+    read_from_netdrv_fail_on_error(netdrv, 1, (unsigned char*)buffer, NULL,
                                    "Failed to read MSG_TYPE_ACK from federate %d in response to sending fed_id.",
                                    remote_federate_id);
     if (buffer[0] != MSG_TYPE_ACK) {
       // Get the error code.
-      read_from_netdrv_fail_on_error(&socket_id, 1, (unsigned char*)buffer, NULL,
+      read_from_netdrv_fail_on_error(netdrv, 1, (unsigned char*)buffer, NULL,
                                      "Failed to read error code from federate %d in response to sending fed_id.",
                                      remote_federate_id);
       lf_print_error("Received MSG_TYPE_REJECT message from remote federate (%d).", buffer[0]);
@@ -1800,8 +1811,8 @@ void lf_connect_to_federate(uint16_t remote_federate_id) {
     }
   }
   // Once we set this variable, then all future calls to close() on this
-  // socket ID should reset it to -1 within a critical section.
-  _fed.netdrvs_for_outbound_p2p_connections[remote_federate_id] = socket_id;
+  // network driver should reset it to NULL within a critical section.
+  _fed.netdrvs_for_outbound_p2p_connections[remote_federate_id] = netdrv;
 }
 
 void lf_connect_to_rti(const char* hostname, int port) {
@@ -1814,11 +1825,11 @@ void lf_connect_to_rti(const char* hostname, int port) {
   // Create a network driver.
   _fed.netdrv_to_RTI = initialize_netdrv();
   // Set the user specified host name and port to the network driver.
-  set_server_port(_fed.netdrv_to_RTI, hostname);
-  set_server_host_name(_fed.netdrv_to_RTI, port);
+  set_server_port(_fed.netdrv_to_RTI, port);
+  set_server_host_name(_fed.netdrv_to_RTI, hostname);
 
   // Create the client network driver.
-  create_client();
+  create_client(_fed.netdrv_to_RTI);
   if (connect_to_netdrv(__fed.netdrv_to_RTI) < 0) {
     lf_print_error_and_exit("Failed to connect() to RTI.");
   }
@@ -1860,7 +1871,7 @@ void lf_connect_to_rti(const char* hostname, int port) {
     // Trace the event when tracing is enabled
     tracepoint_federate_to_rti(send_FED_ID, _lf_my_fed_id, NULL);
 
-    // No need for a mutex here because no other threads are writing to this socket.
+    // No need for a mutex here because no other threads are writing to this network driver.
     if (write_to_netdrv(_fed.netdrv_to_RTI, 2 + sizeof(uint16_t), buffer)) {
       continue; // Try again, possibly on a new port.
     }
@@ -1985,7 +1996,6 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
   while (received_federates < _fed.number_of_inbound_p2p_connections && !_lf_termination_executed) {
     // Wait for an incoming connection request.
     netdrv_t* netdrv = accept_netdrv(_fed.server_netdrv, _fed.netdrv_to_RTI);
-    // int socket_id = accept_socket(_fed.server_socket, _fed.socket_TCP_RTI);
     if (netdrv == NULL) {
       lf_print_warning("Federate failed to accept the socket.");
       return NULL;
@@ -2040,7 +2050,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
 
     // Once we record the network driver here, all future calls to close() on
     // the network driver should be done while holding the netdrv_mutex, and this array
-    // element should be reset to -1 during that critical section.
+    // element should be reset to NULL during that critical section.
     // Otherwise, there can be race condition where, during termination,
     // two threads attempt to simultaneously close the network driver.
     _fed.netdrvs_for_inbound_p2p_connections[remote_fed_id] = netdrv;
@@ -2065,7 +2075,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
       // Failed to create a listening thread.
       LF_MUTEX_LOCK(&netdrv_mutex);
       if (_fed.netdrvs_for_inbound_p2p_connections[remote_fed_id] != NULL) {
-        shutdown_netdrv(&socket_id, false);
+        shutdown_netdrv(netdrv, false);
         _fed.netdrvs_for_inbound_p2p_connections[remote_fed_id] = NULL;
       }
       LF_MUTEX_UNLOCK(&netdrv_mutex);

From 255ddb396da8d3fabdebf0816198b57c54cb9265 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sun, 19 Jan 2025 00:56:48 -0700
Subject: [PATCH 58/84] Fix listen_to_rti_TCP to use netdriver.

---
 core/federated/federate.c | 30 +++++++++---------------------
 1 file changed, 9 insertions(+), 21 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index c8e553339..cfc7fdb4d 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -1484,38 +1484,26 @@ static void* listen_to_rti_TCP(void* args) {
   // Listen for messages from the federate.
   while (1) {
     // Check whether the RTI socket is still valid
-    if (_fed.socket_TCP_RTI < 0) {
-      lf_print_warning("Socket to the RTI unexpectedly closed.");
+    if (_fed.netdrv_to_RTI == NULL) {
+      lf_print_warning("Network driver to the RTI unexpectedly closed.");
       return NULL;
     }
     // Read one byte to get the message type.
     // This will exit if the read fails.
-    int read_failed = read_from_netdrv(_fed.socket_TCP_RTI, 1, buffer);
+    int read_failed = read_from_netdrv(_fed.netdrv_to_RTI, 1, buffer);
     if (read_failed < 0) {
-      if (errno == ECONNRESET) {
-        lf_print_error("Socket connection to the RTI was closed by the RTI without"
-                       " properly sending an EOF first. Considering this a soft error.");
-        // FIXME: If this happens, possibly a new RTI must be elected.
-        shutdown_netdrv(&_fed.socket_TCP_RTI, false);
-        return NULL;
-      } else {
-        lf_print_error("Socket connection to the RTI has been broken with error %d: %s."
-                       " The RTI should close connections with an EOF first."
-                       " Considering this a soft error.",
-                       errno, strerror(errno));
-        // FIXME: If this happens, possibly a new RTI must be elected.
-        shutdown_netdrv(&_fed.socket_TCP_RTI, false);
-        return NULL;
-      }
+      lf_print_error("Connection to the RTI was closed by the RTI with an error. Considering this a soft error.");
+      shutdown_netdrv(_fed.netdrv_to_RTI, false);
+      return NULL;
     } else if (read_failed > 0) {
       // EOF received.
       lf_print("Connection to the RTI closed with an EOF.");
-      shutdown_netdrv(&_fed.socket_TCP_RTI, false);
+      shutdown_netdrv(_fed.netdrv_to_RTI, false);
       return NULL;
     }
     switch (buffer[0]) {
     case MSG_TYPE_TAGGED_MESSAGE:
-      if (handle_tagged_message(&_fed.socket_TCP_RTI, -1)) {
+      if (handle_tagged_message(_fed.netdrv_to_RTI, -1)) {
         // Failures to complete the read of messages from the RTI are fatal.
         lf_print_error_and_exit("Failed to complete the reading of a message from the RTI.");
       }
@@ -1533,7 +1521,7 @@ static void* listen_to_rti_TCP(void* args) {
       handle_stop_granted_message();
       break;
     case MSG_TYPE_PORT_ABSENT:
-      if (handle_port_absent_message(&_fed.socket_TCP_RTI, -1)) {
+      if (handle_port_absent_message(_fed.netdrv_to_RTI, -1)) {
         // Failures to complete the read of absent messages from the RTI are fatal.
         lf_print_error_and_exit("Failed to complete the reading of an absent message from the RTI.");
       }

From a64adfcae0a610be553dbb5df386d3ca246a855b Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sun, 19 Jan 2025 00:57:20 -0700
Subject: [PATCH 59/84] Fix read to handle ECONNRESET errors.

---
 network/impl/src/lf_socket_support.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
index 9b29a64ef..561bce69b 100644
--- a/network/impl/src/lf_socket_support.c
+++ b/network/impl/src/lf_socket_support.c
@@ -136,6 +136,10 @@ int read_from_netdrv(netdrv_t* drv, size_t num_bytes, unsigned char* buffer) {
       LF_PRINT_DEBUG("Reading from socket %d failed with error: `%s`. Will try again.", socket, strerror(errno));
       lf_sleep(DELAY_BETWEEN_SOCKET_RETRIES);
       continue;
+    } else if (more < 0 && errno == ECONNRESET) {
+      lf_print_error(
+          "Connection was closed by the peer without properly sending an EOF first. Considering this a soft error.");
+      return -1;
     } else if (more < 0) {
       // A more serious error occurred.
       lf_print_error("Reading from socket %d failed. With error: `%s`", socket, strerror(errno));
@@ -242,7 +246,7 @@ int shutdown_netdrv(netdrv_t* drv, bool read_before_closing) {
   socket_priv_t* priv = (socket_priv_t*)drv->priv;
   int ret = shutdown_socket(&priv->socket_descriptor, read_before_closing);
   if (ret != 0) {
-     lf_print_error("Failed to shutdown socket.");
+    lf_print_error("Failed to shutdown socket.");
   }
   free_netdrv(drv);
   return ret;

From 89c8b077586e0df700e6b0c829ecabcd3e0db17c Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sun, 19 Jan 2025 00:58:55 -0700
Subject: [PATCH 60/84] Fix perform_hmac_authentication() to use network
 driver.

---
 core/federated/federate.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index cfc7fdb4d..7d87ae481 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -854,14 +854,14 @@ static int perform_hmac_authentication() {
   RAND_bytes(fed_nonce, NONCE_LENGTH);
   memcpy(&fed_hello_buf[1 + fed_id_length], fed_nonce, NONCE_LENGTH);
 
-  write_to_netdrv_fail_on_error(&_fed.socket_TCP_RTI, message_length, fed_hello_buf, NULL, "Failed to write nonce.");
+  write_to_netdrv_fail_on_error(_fed.netdrv_to_RTI, message_length, fed_hello_buf, NULL, "Failed to write nonce.");
 
   // Check HMAC of received FED_RESPONSE message.
   unsigned int hmac_length = SHA256_HMAC_LENGTH;
   size_t federation_id_length = strnlen(federation_metadata.federation_id, 255);
 
   unsigned char received[1 + NONCE_LENGTH + hmac_length];
-  if (read_from_netdrv_close_on_error(&_fed.socket_TCP_RTI, 1 + NONCE_LENGTH + hmac_length, received)) {
+  if (read_from_netdrv_close_on_error(_fed.netdrv_to_RTI, 1 + NONCE_LENGTH + hmac_length, received)) {
     lf_print_warning("Failed to read RTI response.");
     return -1;
   }
@@ -895,7 +895,7 @@ static int perform_hmac_authentication() {
     response[1] = HMAC_DOES_NOT_MATCH;
 
     // Ignore errors on writing back.
-    write_to_netdrv(_fed.socket_TCP_RTI, 2, response);
+    write_to_netdrv(_fed.netdrv_to_RTI, 2, response);
     return -1;
   } else {
     LF_PRINT_LOG("HMAC verified.");
@@ -909,7 +909,7 @@ static int perform_hmac_authentication() {
     HMAC(EVP_sha256(), federation_metadata.federation_id, federation_id_length, mac_buf, 1 + NONCE_LENGTH, &sender[1],
          &hmac_length);
 
-    write_to_netdrv_fail_on_error(&_fed.socket_TCP_RTI, 1 + hmac_length, sender, NULL, "Failed to write fed response.");
+    write_to_netdrv_fail_on_error(_fed.netdrv_to_RTI, 1 + hmac_length, sender, NULL, "Failed to write fed response.");
   }
   return 0;
 }

From 0e73e792bc3e2de955e174f2d35b92ff2076a583 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sun, 19 Jan 2025 01:03:38 -0700
Subject: [PATCH 61/84] Fix all RTI sockets to network drivers.

---
 core/federated/federate.c | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index 7d87ae481..9c6cfa12b 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -116,7 +116,7 @@ static void send_time(unsigned char type, instant_t time) {
   tracepoint_federate_to_rti(send_TIMESTAMP, _lf_my_fed_id, &tag);
 
   LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
-  write_to_netdrv_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_write, buffer, &lf_outbound_netdrv_mutex,
+  write_to_netdrv_fail_on_error(_fed.netdrv_to_RTI, bytes_to_write, buffer, &lf_outbound_netdrv_mutex,
                                 "Failed to send time " PRINTF_TIME " to the RTI.", time - start_time);
   LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
 }
@@ -135,7 +135,7 @@ static void send_tag(unsigned char type, tag_t tag) {
   encode_tag(&(buffer[1]), tag);
 
   LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
-  if (_fed.socket_TCP_RTI < 0) {
+  if (_fed.netdrv_to_RTI == NULL) {
     lf_print_warning("Socket is no longer connected. Dropping message.");
     LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
     return;
@@ -143,7 +143,7 @@ static void send_tag(unsigned char type, tag_t tag) {
   trace_event_t event_type = (type == MSG_TYPE_NEXT_EVENT_TAG) ? send_NET : send_LTC;
   // Trace the event when tracing is enabled
   tracepoint_federate_to_rti(event_type, _lf_my_fed_id, &tag);
-  write_to_netdrv_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_write, buffer, &lf_outbound_netdrv_mutex,
+  write_to_netdrv_fail_on_error(_fed.netdrv_to_RTI, bytes_to_write, buffer, &lf_outbound_netdrv_mutex,
                                 "Failed to send tag " PRINTF_TAG " to the RTI.", tag.time - start_time, tag.microstep);
   LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
 }
@@ -933,7 +933,7 @@ static instant_t get_start_time_from_rti(instant_t my_physical_time) {
   size_t buffer_length = 1 + sizeof(instant_t);
   unsigned char buffer[buffer_length];
 
-  read_from_netdrv_fail_on_error(&_fed.socket_TCP_RTI, buffer_length, buffer, NULL,
+  read_from_netdrv_fail_on_error(_fed.netdrv_to_RTI, buffer_length, buffer, NULL,
                                  "Failed to read MSG_TYPE_TIMESTAMP message from RTI.");
   LF_PRINT_DEBUG("Read 9 bytes.");
 
@@ -977,7 +977,7 @@ static void handle_tag_advance_grant(void) {
 
   size_t bytes_to_read = sizeof(instant_t) + sizeof(microstep_t);
   unsigned char buffer[bytes_to_read];
-  read_from_netdrv_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_read, buffer, NULL,
+  read_from_netdrv_fail_on_error(_fed.netdrv_to_RTI, bytes_to_read, buffer, NULL,
                                  "Failed to read tag advance grant from RTI.");
   tag_t TAG = extract_tag(buffer);
 
@@ -1218,7 +1218,7 @@ static void handle_provisional_tag_advance_grant() {
 
   size_t bytes_to_read = sizeof(instant_t) + sizeof(microstep_t);
   unsigned char buffer[bytes_to_read];
-  read_from_netdrv_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_read, buffer, NULL,
+  read_from_netdrv_fail_on_error(_fed.netdrv_to_RTI, bytes_to_read, buffer, NULL,
                                  "Failed to read provisional tag advance grant from RTI.");
   tag_t PTAG = extract_tag(buffer);
 
@@ -1308,7 +1308,7 @@ static void handle_stop_granted_message() {
 
   size_t bytes_to_read = MSG_TYPE_STOP_GRANTED_LENGTH - 1;
   unsigned char buffer[bytes_to_read];
-  read_from_netdrv_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_read, buffer, NULL,
+  read_from_netdrv_fail_on_error(_fed.netdrv_to_RTI, bytes_to_read, buffer, NULL,
                                  "Failed to read stop granted from RTI.");
 
   tag_t received_stop_tag = extract_tag(buffer);
@@ -1352,7 +1352,7 @@ static void handle_stop_granted_message() {
 static void handle_stop_request_message() {
   size_t bytes_to_read = MSG_TYPE_STOP_REQUEST_LENGTH - 1;
   unsigned char buffer[bytes_to_read];
-  read_from_netdrv_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_read, buffer, NULL,
+  read_from_netdrv_fail_on_error(_fed.netdrv_to_RTI, bytes_to_read, buffer, NULL,
                                  "Failed to read stop request from RTI.");
   tag_t tag_to_stop = extract_tag(buffer);
 
@@ -1425,7 +1425,7 @@ static void handle_stop_request_message() {
 
   // Send the current logical time to the RTI.
   LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
-  write_to_netdrv_fail_on_error(&_fed.socket_TCP_RTI, MSG_TYPE_STOP_REQUEST_REPLY_LENGTH, outgoing_buffer,
+  write_to_netdrv_fail_on_error(_fed.netdrv_to_RTI, MSG_TYPE_STOP_REQUEST_REPLY_LENGTH, outgoing_buffer,
                                 &lf_outbound_netdrv_mutex,
                                 "Failed to send the answer to MSG_TYPE_STOP_REQUEST to RTI.");
   LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
@@ -1442,7 +1442,7 @@ static void send_resign_signal() {
   unsigned char buffer[bytes_to_write];
   buffer[0] = MSG_TYPE_RESIGN;
   LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
-  write_to_netdrv_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_write, &(buffer[0]), &lf_outbound_netdrv_mutex,
+  write_to_netdrv_fail_on_error(_fed.netdrv_to_RTI, bytes_to_write, &(buffer[0]), &lf_outbound_netdrv_mutex,
                                 "Failed to send MSG_TYPE_RESIGN.");
   LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
   LF_PRINT_LOG("Resigned.");
@@ -1455,7 +1455,7 @@ static void send_failed_signal() {
   size_t bytes_to_write = 1;
   unsigned char buffer[bytes_to_write];
   buffer[0] = MSG_TYPE_FAILED;
-  write_to_netdrv_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_write, &(buffer[0]), NULL,
+  write_to_netdrv_fail_on_error(_fed.netdrv_to_RTI, bytes_to_write, &(buffer[0]), NULL,
                                 "Failed to send MSG_TYPE_FAILED.");
   LF_PRINT_LOG("Failed.");
 }
@@ -1607,7 +1607,7 @@ void lf_terminate_execution(environment_t* env) {
 
   // For an abnormal termination (e.g. a SIGINT), we need to send a
   // MSG_TYPE_FAILED message to the RTI, but we should not acquire a mutex.
-  if (_fed.socket_TCP_RTI >= 0) {
+  if (_fed.netdrv_to_RTI != NULL) {
     if (_lf_normal_termination) {
       tracepoint_federate_to_rti(send_RESIGN, _lf_my_fed_id, &env->current_tag);
       send_resign_signal();
@@ -2350,13 +2350,13 @@ void lf_send_port_absent_to_federate(environment_t* env, interval_t additional_d
 
 #ifdef FEDERATED_CENTRALIZED
   // Send the absent message through the RTI
-  int* socket = &_fed.socket_TCP_RTI;
+  int* socket = _fed.netdrv_to_RTI;
 #else
   // Send the absent message directly to the federate
   int* socket = &_fed.netdrvs_for_outbound_p2p_connections[fed_ID];
 #endif
 
-  if (socket == &_fed.socket_TCP_RTI) {
+  if (socket == _fed.netdrv_to_RTI) {
     tracepoint_federate_to_rti(send_PORT_ABS, _lf_my_fed_id, &current_message_intended_tag);
   } else {
     tracepoint_federate_to_federate(send_PORT_ABS, _lf_my_fed_id, fed_ID, &current_message_intended_tag);
@@ -2368,7 +2368,7 @@ void lf_send_port_absent_to_federate(environment_t* env, interval_t additional_d
 
   if (result != 0) {
     // Write failed. Response depends on whether coordination is centralized.
-    if (socket == &_fed.socket_TCP_RTI) {
+    if (socket == _fed.netdrv_to_RTI) {
       // Centralized coordination. This is a critical error.
       lf_print_error_system_failure("Failed to send port absent message for port %hu to federate %hu.", port_ID,
                                     fed_ID);
@@ -2393,7 +2393,7 @@ int lf_send_stop_request_to_rti(tag_t stop_tag) {
     LF_PRINT_LOG("Sending to RTI a MSG_TYPE_STOP_REQUEST message with tag " PRINTF_TAG ".", stop_tag.time - start_time,
                  stop_tag.microstep);
 
-    if (_fed.socket_TCP_RTI < 0) {
+    if (_fed.netdrv_to_RTI == NULL) {
       lf_print_warning("Socket is no longer connected. Dropping message.");
       LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
       return -1;
@@ -2401,7 +2401,7 @@ int lf_send_stop_request_to_rti(tag_t stop_tag) {
     // Trace the event when tracing is enabled
     tracepoint_federate_to_rti(send_STOP_REQ, _lf_my_fed_id, &stop_tag);
 
-    write_to_netdrv_fail_on_error(&_fed.socket_TCP_RTI, MSG_TYPE_STOP_REQUEST_LENGTH, buffer, &lf_outbound_netdrv_mutex,
+    write_to_netdrv_fail_on_error(_fed.netdrv_to_RTI, MSG_TYPE_STOP_REQUEST_LENGTH, buffer, &lf_outbound_netdrv_mutex,
                                   "Failed to send stop time " PRINTF_TIME " to the RTI.", stop_tag.time - start_time);
 
     // Treat this sending  as equivalent to having received a stop request from the RTI.
@@ -2469,7 +2469,7 @@ int lf_send_tagged_message(environment_t* env, interval_t additional_delay, int
     socket = &_fed.netdrvs_for_outbound_p2p_connections[federate];
     tracepoint_federate_to_federate(send_P2P_TAGGED_MSG, _lf_my_fed_id, federate, &current_message_intended_tag);
   } else {
-    socket = &_fed.socket_TCP_RTI;
+    socket = _fed.netdrv_to_RTI;
     tracepoint_federate_to_rti(send_TAGGED_MSG, _lf_my_fed_id, &current_message_intended_tag);
   }
 

From 04e71e3b55720addaf27676b28e44fa4ce8267a7 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sun, 19 Jan 2025 01:05:05 -0700
Subject: [PATCH 62/84] Remove server socket and port from federate instance.

---
 core/federated/federate.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index 9c6cfa12b..94dac8648 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -81,8 +81,6 @@ federate_instance_t _fed = {.socket_TCP_RTI = -1,
                             .inbound_netdriv_listeners = NULL,
                             .number_of_outbound_p2p_connections = 0,
                             .inbound_p2p_handling_thread_id = 0,
-                            .server_socket = -1,
-                            .server_port = -1,
                             .last_TAG = {.time = NEVER, .microstep = 0u},
                             .is_last_TAG_provisional = false,
                             .has_upstream = false,

From 6a851e3cd5594b16a8983d1d4b4639bfc7476da9 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sun, 19 Jan 2025 01:10:05 -0700
Subject: [PATCH 63/84] Fix peek_from_netdrv().

---
 core/federated/federate.c            | 2 +-
 network/api/net_driver.h             | 2 ++
 network/impl/src/lf_socket_support.c | 5 +++++
 3 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index 94dac8648..d014137a5 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -152,7 +152,7 @@ static void send_tag(unsigned char type, tag_t tag) {
  */
 static bool rti_failed() {
   unsigned char first_byte;
-  ssize_t bytes = peek_from_socket(_fed.socket_TCP_RTI, &first_byte);
+  ssize_t bytes = peek_from_netdrv(_fed.netdrv_to_RTI, &first_byte);
   if (bytes < 0 || (bytes == 1 && first_byte == MSG_TYPE_FAILED))
     return true;
   else
diff --git a/network/api/net_driver.h b/network/api/net_driver.h
index 44a95cf25..221e6fb3f 100644
--- a/network/api/net_driver.h
+++ b/network/api/net_driver.h
@@ -121,6 +121,8 @@ int write_to_netdrv_close_on_error(netdrv_t* drv, size_t num_bytes, unsigned cha
 void write_to_netdrv_fail_on_error(netdrv_t* drv, size_t num_bytes, unsigned char* buffer, lf_mutex_t* mutex,
                                    char* format, ...);
 
+ssize_t peek_from_netdrv(netdrv_t* drv, unsigned char* result);
+
 /**
  * @brief Gracefully shuts down and closes a socket, optionally reading until EOF.
  * Shutdown and close the socket. If read_before_closing is false, it just immediately calls shutdown() with SHUT_RDWR
diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
index 561bce69b..81a2bb302 100644
--- a/network/impl/src/lf_socket_support.c
+++ b/network/impl/src/lf_socket_support.c
@@ -242,6 +242,11 @@ void write_to_netdrv_fail_on_error(netdrv_t* drv, size_t num_bytes, unsigned cha
   }
 }
 
+ssize_t peek_from_netdrv(netdrv_t* drv, unsigned char* result) {
+  socket_priv_t* priv = (socket_priv_t*)drv->priv;
+  return peek_from_socket(priv->socket_descriptor, result);
+}
+
 int shutdown_netdrv(netdrv_t* drv, bool read_before_closing) {
   socket_priv_t* priv = (socket_priv_t*)drv->priv;
   int ret = shutdown_socket(&priv->socket_descriptor, read_before_closing);

From 31496157e1fb8041d84d2a60bd7bbad55b2f4f39 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sun, 19 Jan 2025 01:10:39 -0700
Subject: [PATCH 64/84] Remove socket_TCP_RTI

---
 core/federated/federate.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index d014137a5..20e7d9b19 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -76,8 +76,7 @@ int max_level_allowed_to_advance;
  * The state of this federate instance. Each executable has exactly one federate instance,
  * and the _fed global variable refers to that instance.
  */
-federate_instance_t _fed = {.socket_TCP_RTI = -1,
-                            .number_of_inbound_p2p_connections = 0,
+federate_instance_t _fed = {.number_of_inbound_p2p_connections = 0,
                             .inbound_netdriv_listeners = NULL,
                             .number_of_outbound_p2p_connections = 0,
                             .inbound_p2p_handling_thread_id = 0,

From 18caa30bb149af6c368ca9b2b64f607d5b091bd0 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sun, 19 Jan 2025 01:20:06 -0700
Subject: [PATCH 65/84] Fix all socket to netdrivers.

---
 core/federated/federate.c | 99 +++++++++++++++++++--------------------
 1 file changed, 49 insertions(+), 50 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index 20e7d9b19..e06dfc50a 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -133,7 +133,7 @@ static void send_tag(unsigned char type, tag_t tag) {
 
   LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
   if (_fed.netdrv_to_RTI == NULL) {
-    lf_print_warning("Socket is no longer connected. Dropping message.");
+    lf_print_warning("RTI is no longer connected. Dropping message.");
     LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
     return;
   }
@@ -146,8 +146,8 @@ static void send_tag(unsigned char type, tag_t tag) {
 }
 
 /**
- * Return true if either the socket to the RTI is broken or the socket is
- * alive and the first unread byte on the socket's queue is MSG_TYPE_FAILED.
+ * Return true if either the network driver to the RTI is broken or the network driver is
+ * alive and the first unread byte on the network driver's queue is MSG_TYPE_FAILED.
  */
 static bool rti_failed() {
   unsigned char first_byte;
@@ -400,7 +400,7 @@ static trigger_handle_t schedule_message_received_from_network_locked(environmen
 }
 
 /**
- * Close the socket that receives incoming messages from the
+ * Close the network driver that receives incoming messages from the
  * specified federate ID.
  *
  * @param fed_id The ID of the peer federate sending messages to this
@@ -770,7 +770,7 @@ static void* listen_to_federates(void* _args) {
         LF_PRINT_LOG("Received tagged message from federate %d.", fed_id);
         if (handle_tagged_message(netdrv, fed_id)) {
           // P2P tagged messages are only used in decentralized coordination, and
-          // it is not a fatal error if the socket is closed before the whole message is read.
+          // it is not a fatal error if the network driver is closed before the whole message is read.
           // But this thread should exit.
           lf_print_warning("Failed to complete reading of tagged message.");
           netdrv_closed = true;
@@ -780,7 +780,7 @@ static void* listen_to_federates(void* _args) {
         LF_PRINT_LOG("Received port absent message from federate %d.", fed_id);
         if (handle_port_absent_message(netdrv, fed_id)) {
           // P2P tagged messages are only used in decentralized coordination, and
-          // it is not a fatal error if the socket is closed before the whole message is read.
+          // it is not a fatal error if the network driver is closed before the whole message is read.
           // But this thread should exit.
           lf_print_warning("Failed to complete reading of tagged message.");
           netdrv_closed = true;
@@ -791,13 +791,13 @@ static void* listen_to_federates(void* _args) {
       }
     }
     if (bad_message) {
-      lf_print_error("Received erroneous message type: %d. Closing the socket.", buffer[0]);
+      lf_print_error("Received erroneous message type: %d. Closing the network driver.", buffer[0]);
       // Trace the event when tracing is enabled
       tracepoint_federate_from_federate(receive_UNIDENTIFIED, _lf_my_fed_id, fed_id, NULL);
       break; // while loop
     }
     if (netdrv_closed) {
-      // For decentralized execution, once this socket is closed, we
+      // For decentralized execution, once this network driver is closed, we
       // update last known tags of all ports connected to the specified federate to FOREVER_TAG,
       // which would eliminate the need to wait for STAA to assume an input is absent.
       mark_inputs_known_absent(fed_id);
@@ -809,21 +809,21 @@ static void* listen_to_federates(void* _args) {
 }
 
 /**
- * Close the socket that sends outgoing messages to the
+ * Close the network driver that sends outgoing messages to the
  * specified federate ID. This function acquires the lf_outbound_netdrv_mutex mutex lock
  * if _lf_normal_termination is true and otherwise proceeds without the lock.
  * @param fed_id The ID of the peer federate receiving messages from this
  *  federate, or -1 if the RTI (centralized coordination).
  */
-static void close_outbound_socket(int fed_id) {
+static void close_outbound_netdrv(int fed_id) {
   assert(fed_id >= 0 && fed_id < NUMBER_OF_FEDERATES);
   // Close outbound connections, in case they have not closed themselves.
   // This will result in EOF being sent to the remote federate, except for
-  // abnormal termination, in which case it will just close the socket.
+  // abnormal termination, in which case it will just close the network driver.
   if (_lf_normal_termination) {
     LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
     if (_fed.netdrvs_for_outbound_p2p_connections[fed_id] >= 0) {
-      // Close the socket by sending a FIN packet indicating that no further writes
+      // Close the network driver by sending a FIN packet indicating that no further writes
       // are expected.  Then read until we get an EOF indication.
       shutdown_netdrv(&_fed.netdrvs_for_outbound_p2p_connections[fed_id], true);
     }
@@ -925,7 +925,7 @@ static instant_t get_start_time_from_rti(instant_t my_physical_time) {
   // Send the timestamp marker first.
   send_time(MSG_TYPE_TIMESTAMP, my_physical_time);
 
-  // Read bytes from the socket. We need 9 bytes.
+  // Read bytes from the network driver. We need 9 bytes.
   // Buffer for message ID plus timestamp.
   size_t buffer_length = 1 + sizeof(instant_t);
   unsigned char buffer[buffer_length];
@@ -1375,7 +1375,7 @@ static void handle_stop_request_message() {
   // or we have previously sent a stop request to the RTI,
   // then we have already blocked tag advance in enclaves.
   // Do not do this twice. The record of whether the first has occurred
-  // is guarded by the outbound socket mutex.
+  // is guarded by the outbound network driver mutex.
   // The second is guarded by the global mutex.
   // Note that the RTI should not send stop requests more than once to federates.
   LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
@@ -1480,7 +1480,7 @@ static void* listen_to_rti_TCP(void* args) {
 
   // Listen for messages from the federate.
   while (1) {
-    // Check whether the RTI socket is still valid
+    // Check whether the RTI network driver is still valid
     if (_fed.netdrv_to_RTI == NULL) {
       lf_print_warning("Network driver to the RTI unexpectedly closed.");
       return NULL;
@@ -1528,7 +1528,7 @@ static void* listen_to_rti_TCP(void* args) {
       break;
     case MSG_TYPE_CLOCK_SYNC_T1:
     case MSG_TYPE_CLOCK_SYNC_T4:
-      lf_print_error("Federate %d received unexpected clock sync message from RTI on TCP socket.", _lf_my_fed_id);
+      lf_print_error("Federate %d received unexpected clock sync message from RTI.", _lf_my_fed_id);
       break;
     default:
       lf_print_error_and_exit("Received from RTI an unrecognized TCP message type: %hhx.", buffer[0]);
@@ -1593,7 +1593,7 @@ static bool bounded_NET(tag_t* tag) {
 // An empty version of this function is code generated for unfederated execution.
 
 /**
- * Close sockets used to communicate with other federates, if they are open,
+ * Close network drivers used to communicate with other federates, if they are open,
  * and send a MSG_TYPE_RESIGN message to the RTI. This implements the function
  * defined in reactor.h. For unfederated execution, the code generator
  * generates an empty implementation.
@@ -1614,28 +1614,28 @@ void lf_terminate_execution(environment_t* env) {
     }
   }
 
-  LF_PRINT_DEBUG("Closing incoming P2P sockets.");
-  // Close any incoming P2P sockets that are still open.
+  LF_PRINT_DEBUG("Closing incoming P2P network drivers.");
+  // Close any incoming P2P network drivers that are still open.
   for (int i = 0; i < NUMBER_OF_FEDERATES; i++) {
-    close_inbound_socket(i);
-    // Ignore errors. Mark the socket closed.
-    _fed.netdrvs_for_inbound_p2p_connections[i] = -1;
+    close_inbound_netdrv(i);
+    // Ignore errors. Mark the network driver closed.
+    _fed.netdrvs_for_inbound_p2p_connections[i] = NULL;
   }
 
   // Check for all outgoing physical connections in
   // _fed.netdrvs_for_outbound_p2p_connections and
-  // if the socket ID is not -1, the connection is still open.
-  // Send an EOF by closing the socket here.
+  // if the network driver ID is not NULL, the connection is still open.
+  // Send an EOF by closing the network driver here.
   for (int i = 0; i < NUMBER_OF_FEDERATES; i++) {
 
     // Close outbound connections, in case they have not closed themselves.
     // This will result in EOF being sent to the remote federate, except for
-    // abnormal termination, in which case it will just close the socket.
-    close_outbound_socket(i);
+    // abnormal termination, in which case it will just close the network driver.
+    close_outbound_netdrv(i);
   }
 
-  LF_PRINT_DEBUG("Waiting for inbound p2p socket listener threads.");
-  // Wait for each inbound socket listener thread to close.
+  LF_PRINT_DEBUG("Waiting for inbound p2p network driver listener threads.");
+  // Wait for each inbound network driver listener thread to close.
   if (_fed.number_of_inbound_p2p_connections > 0 && _fed.inbound_netdriv_listeners != NULL) {
     LF_PRINT_LOG("Waiting for %zu threads listening for incoming messages to exit.",
                  _fed.number_of_inbound_p2p_connections);
@@ -1739,8 +1739,8 @@ void lf_connect_to_federate(uint16_t remote_federate_id) {
   while (result < 0 && !_lf_termination_executed) {
     // Try again after some time if the connection failed.
     // Note that this should not really happen since the remote federate should be
-    // accepting socket connections. But possibly it will be busy (in process of accepting
-    // another socket connection?). Hence, we retry.
+    // accepting  connections. But possibly it will be busy (in process of accepting
+    // another connection?). Hence, we retry.
     if (CHECK_TIMEOUT(start_connect, CONNECT_TIMEOUT)) {
       // If the remote federate is not accepting the connection after CONNECT_TIMEOUT
       // treat it as a soft error condition and return.
@@ -1750,7 +1750,6 @@ void lf_connect_to_federate(uint16_t remote_federate_id) {
     }
 
     // Check whether the RTI is still there.
-    // TODO: Fix.
     if (rti_failed()) {
       break;
     }
@@ -1767,7 +1766,7 @@ void lf_connect_to_federate(uint16_t remote_federate_id) {
     // Trace the event when tracing is enabled
     tracepoint_federate_to_federate(send_FED_ID, _lf_my_fed_id, remote_federate_id, NULL);
 
-    // No need for a mutex because we have the only handle on the socket.
+    // No need for a mutex because we have the only handle on the network driver.
     write_to_netdrv_fail_on_error(netdrv, buffer_length, buffer, NULL, "Failed to send fed_id to federate %d.",
                                   remote_federate_id);
     write_to_netdrv_fail_on_error(netdrv, federation_id_length, (unsigned char*)federation_metadata.federation_id, NULL,
@@ -1942,7 +1941,7 @@ void lf_create_server(int specified_port) {
   // Trace the event when tracing is enabled
   tracepoint_federate_to_rti(send_ADR_AD, _lf_my_fed_id, NULL);
 
-  // No need for a mutex because we have the only handle on this socket.
+  // No need for a mutex because we have the only handle on this network driver.
   write_to_netdrv_fail_on_error(server_netdrv, sizeof(int32_t) + 1, (unsigned char*)buffer, NULL,
                                 "Failed to send address advertisement.");
 
@@ -1982,7 +1981,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
     // Wait for an incoming connection request.
     netdrv_t* netdrv = accept_netdrv(_fed.server_netdrv, _fed.netdrv_to_RTI);
     if (netdrv == NULL) {
-      lf_print_warning("Federate failed to accept the socket.");
+      lf_print_warning("Federate failed to accept the network driver.");
       return NULL;
     }
     LF_PRINT_LOG("Accepted new connection from remote federate.");
@@ -1991,7 +1990,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
     unsigned char buffer[header_length];
     int read_failed = read_from_netdrv(netdrv, header_length, (unsigned char*)&buffer);
     if (read_failed || buffer[0] != MSG_TYPE_P2P_SENDING_FED_ID) {
-      lf_print_warning("Federate received invalid first message on P2P socket. Closing socket.");
+      lf_print_warning("Federate received invalid first message on P2P network driver. Closing network driver.");
       if (read_failed == 0) {
         // Wrong message received.
         unsigned char response[2];
@@ -2012,7 +2011,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
     read_failed = read_from_netdrv(netdrv, federation_id_length, (unsigned char*)remote_federation_id);
     if (read_failed || (strncmp(federation_metadata.federation_id, remote_federation_id,
                                 strnlen(federation_metadata.federation_id, 255)) != 0)) {
-      lf_print_warning("Received invalid federation ID. Closing socket.");
+      lf_print_warning("Received invalid federation ID. Closing network driver.");
       if (read_failed == 0) {
         unsigned char response[2];
         response[0] = MSG_TYPE_REJECT;
@@ -2170,15 +2169,15 @@ int lf_send_message(int message_type, unsigned short port, unsigned short federa
   // Use a mutex lock to prevent multiple threads from simultaneously sending.
   LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
 
-  int* socket = &_fed.netdrvs_for_outbound_p2p_connections[federate];
+  netdrv_t* netdrv = _fed.netdrvs_for_outbound_p2p_connections[federate];
 
   // Trace the event when tracing is enabled
   tracepoint_federate_to_federate(send_P2P_MSG, _lf_my_fed_id, federate, NULL);
 
-  int result = write_to_netdrv_close_on_error(socket, header_length, header_buffer);
+  int result = write_to_netdrv_close_on_error(netdrv, header_length, header_buffer);
   if (result == 0) {
     // Header sent successfully. Send the body.
-    result = write_to_netdrv_close_on_error(socket, length, message);
+    result = write_to_netdrv_close_on_error(netdrv, length, message);
   }
   if (result != 0) {
     // Message did not send. Since this is used for physical connections, this is not critical.
@@ -2347,25 +2346,25 @@ void lf_send_port_absent_to_federate(environment_t* env, interval_t additional_d
 
 #ifdef FEDERATED_CENTRALIZED
   // Send the absent message through the RTI
-  int* socket = _fed.netdrv_to_RTI;
+  netdrv_t* netdrv = _fed.netdrv_to_RTI;
 #else
   // Send the absent message directly to the federate
-  int* socket = &_fed.netdrvs_for_outbound_p2p_connections[fed_ID];
+  netdrv_t* netdrv  = &_fed.netdrvs_for_outbound_p2p_connections[fed_ID];
 #endif
 
-  if (socket == _fed.netdrv_to_RTI) {
+  if (netdrv == _fed.netdrv_to_RTI) {
     tracepoint_federate_to_rti(send_PORT_ABS, _lf_my_fed_id, &current_message_intended_tag);
   } else {
     tracepoint_federate_to_federate(send_PORT_ABS, _lf_my_fed_id, fed_ID, &current_message_intended_tag);
   }
 
   LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
-  int result = write_to_netdrv_close_on_error(socket, message_length, buffer);
+  int result = write_to_netdrv_close_on_error(netdrv, message_length, buffer);
   LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
 
   if (result != 0) {
     // Write failed. Response depends on whether coordination is centralized.
-    if (socket == _fed.netdrv_to_RTI) {
+    if (netdrv == _fed.netdrv_to_RTI) {
       // Centralized coordination. This is a critical error.
       lf_print_error_system_failure("Failed to send port absent message for port %hu to federate %hu.", port_ID,
                                     fed_ID);
@@ -2391,7 +2390,7 @@ int lf_send_stop_request_to_rti(tag_t stop_tag) {
                  stop_tag.microstep);
 
     if (_fed.netdrv_to_RTI == NULL) {
-      lf_print_warning("Socket is no longer connected. Dropping message.");
+      lf_print_warning("RTI is no longer connected. Dropping message.");
       LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
       return -1;
     }
@@ -2461,19 +2460,19 @@ int lf_send_tagged_message(environment_t* env, interval_t additional_delay, int
   // Use a mutex lock to prevent multiple threads from simultaneously sending.
   LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
 
-  int* socket;
+  netdrv_t* netdrv ;
   if (message_type == MSG_TYPE_P2P_TAGGED_MESSAGE) {
-    socket = &_fed.netdrvs_for_outbound_p2p_connections[federate];
+    netdrv = _fed.netdrvs_for_outbound_p2p_connections[federate];
     tracepoint_federate_to_federate(send_P2P_TAGGED_MSG, _lf_my_fed_id, federate, &current_message_intended_tag);
   } else {
-    socket = _fed.netdrv_to_RTI;
+    netdrv = _fed.netdrv_to_RTI;
     tracepoint_federate_to_rti(send_TAGGED_MSG, _lf_my_fed_id, &current_message_intended_tag);
   }
 
-  int result = write_to_netdrv_close_on_error(socket, header_length, header_buffer);
+  int result = write_to_netdrv_close_on_error(netdrv, header_length, header_buffer);
   if (result == 0) {
     // Header sent successfully. Send the body.
-    result = write_to_netdrv_close_on_error(socket, length, message);
+    result = write_to_netdrv_close_on_error(netdrv, length, message);
   }
   if (result != 0) {
     // Message did not send. Handling depends on message type.

From ce8be1482bdffe47adb873d4ee59b02675ea2fc3 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sun, 19 Jan 2025 01:34:09 -0700
Subject: [PATCH 66/84] Remove socket.h header from federate.c

---
 core/federated/federate.c | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index e06dfc50a..f56877018 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -17,7 +17,6 @@
 #include <arpa/inet.h>  // inet_ntop & inet_pton
 #include <netdb.h>      // Defines getaddrinfo(), freeaddrinfo() and struct addrinfo.
 #include <netinet/in.h> // Defines struct sockaddr_in
-#include <sys/socket.h>
 #include <unistd.h> // Defines read(), write(), and close()
 #include <string.h> // Defines memset(), strnlen(), strncmp(), strncpy()
 #include <stdio.h>  // Defines strerror()
@@ -825,11 +824,11 @@ static void close_outbound_netdrv(int fed_id) {
     if (_fed.netdrvs_for_outbound_p2p_connections[fed_id] >= 0) {
       // Close the network driver by sending a FIN packet indicating that no further writes
       // are expected.  Then read until we get an EOF indication.
-      shutdown_netdrv(&_fed.netdrvs_for_outbound_p2p_connections[fed_id], true);
+      shutdown_netdrv(_fed.netdrvs_for_outbound_p2p_connections[fed_id], true);
     }
     LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
   } else {
-    shutdown_netdrv(&_fed.netdrvs_for_outbound_p2p_connections[fed_id], false);
+    shutdown_netdrv(_fed.netdrvs_for_outbound_p2p_connections[fed_id], false);
   }
 }
 
@@ -1814,7 +1813,7 @@ void lf_connect_to_rti(const char* hostname, int port) {
 
   // Create the client network driver.
   create_client(_fed.netdrv_to_RTI);
-  if (connect_to_netdrv(__fed.netdrv_to_RTI) < 0) {
+  if (connect_to_netdrv(_fed.netdrv_to_RTI) < 0) {
     lf_print_error_and_exit("Failed to connect() to RTI.");
   }
 
@@ -1881,7 +1880,7 @@ void lf_connect_to_rti(const char* hostname, int port) {
       tracepoint_federate_from_rti(receive_REJECT, _lf_my_fed_id, NULL);
       // Read one more byte to determine the cause of rejection.
       unsigned char cause;
-      read_from_netdrv_fail_on_error(&_fed.netdrv_to_RTI, 1, &cause, NULL,
+      read_from_netdrv_fail_on_error(_fed.netdrv_to_RTI, 1, &cause, NULL,
                                      "Failed to read the cause of rejection by the RTI.");
       if (cause == FEDERATION_ID_DOES_NOT_MATCH || cause == WRONG_SERVER) {
         lf_print_warning("Connected to the wrong RTI. Will try again");
@@ -1913,7 +1912,7 @@ void lf_connect_to_rti(const char* hostname, int port) {
   unsigned char UDP_port_number[1 + sizeof(uint16_t)];
   UDP_port_number[0] = MSG_TYPE_UDP_PORT;
   encode_uint16(udp_port, &(UDP_port_number[1]));
-  write_to_netdrv_fail_on_error(&_fed.netdrv_to_RTI, 1 + sizeof(uint16_t), UDP_port_number, NULL,
+  write_to_netdrv_fail_on_error(_fed.netdrv_to_RTI, 1 + sizeof(uint16_t), UDP_port_number, NULL,
                                 "Failed to send the UDP port number to the RTI.");
 }
 
@@ -2046,7 +2045,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
     tracepoint_federate_to_federate(send_ACK, _lf_my_fed_id, remote_fed_id, NULL);
 
     LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
-    write_to_netdrv_fail_on_error(&_fed.netdrvs_for_inbound_p2p_connections[remote_fed_id], 1,
+    write_to_netdrv_fail_on_error(_fed.netdrvs_for_inbound_p2p_connections[remote_fed_id], 1,
                                   (unsigned char*)&response, &lf_outbound_netdrv_mutex,
                                   "Failed to write MSG_TYPE_ACK in response to federate %d.", remote_fed_id);
     LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
@@ -2349,7 +2348,7 @@ void lf_send_port_absent_to_federate(environment_t* env, interval_t additional_d
   netdrv_t* netdrv = _fed.netdrv_to_RTI;
 #else
   // Send the absent message directly to the federate
-  netdrv_t* netdrv  = &_fed.netdrvs_for_outbound_p2p_connections[fed_ID];
+  netdrv_t* netdrv  = _fed.netdrvs_for_outbound_p2p_connections[fed_ID];
 #endif
 
   if (netdrv == _fed.netdrv_to_RTI) {

From 2e53546ea298dd4010bf6bc2e52230e41bc1295d Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sun, 19 Jan 2025 01:34:32 -0700
Subject: [PATCH 67/84] Add create_client() and connect_to_netdrv() in
 net_driver.h header file.

---
 network/api/net_driver.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/network/api/net_driver.h b/network/api/net_driver.h
index 221e6fb3f..8feb706cd 100644
--- a/network/api/net_driver.h
+++ b/network/api/net_driver.h
@@ -28,6 +28,10 @@ int create_server_(netdrv_t* drv, bool increment_port_on_retry);
 
 netdrv_t* accept_netdrv(netdrv_t* server_drv, netdrv_t* rti_drv);
 
+void create_client(netdrv_t* drv);
+
+int connect_to_netdrv(netdrv_t* drv);
+
 int get_peer_address(netdrv_t* drv);
 
 /**

From 1c44f2e6c3f7e7910384067280e6aa141a194186 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sun, 19 Jan 2025 01:35:07 -0700
Subject: [PATCH 68/84] Fix accept_netdrv to use accept_socket.

---
 network/impl/src/lf_socket_support.c | 40 +++++-----------------------
 network/impl/src/socket_common.c     |  5 ++--
 2 files changed, 10 insertions(+), 35 deletions(-)

diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
index 81a2bb302..1793b0db0 100644
--- a/network/impl/src/lf_socket_support.c
+++ b/network/impl/src/lf_socket_support.c
@@ -69,43 +69,17 @@ int create_server_(netdrv_t* drv, bool increment_port_on_retry) {
 
 netdrv_t* accept_netdrv(netdrv_t* server_drv, netdrv_t* rti_drv) {
   socket_priv_t* serv_priv = (socket_priv_t*)server_drv->priv;
+  socket_priv_t* rti_priv = (socket_priv_t*)rti_drv->priv;
+  
   netdrv_t* fed_netdrv = initialize_netdrv();
   socket_priv_t* fed_priv = (socket_priv_t*)fed_netdrv->priv;
 
-  struct sockaddr client_fd;
-  // Wait for an incoming connection request.
-  uint32_t client_length = sizeof(client_fd);
-  // The following blocks until a federate connects.
-  int socket_id = -1;
-  while (true) {
-    // When close(socket) is called, the accept() will return -1.
-    socket_id = accept(serv_priv->socket_descriptor, &client_fd, &client_length);
-    if (socket_id >= 0) {
-      // Got a socket
-      break;
-    } else if (socket_id < 0 && (errno != EAGAIN || errno != EWOULDBLOCK || errno != EINTR)) {
-      lf_print_warning("Failed to accept the socket. %s.", strerror(errno));
-      free_netdrv(fed_netdrv);
-      return NULL;
-    } else if (errno == EPERM) {
-      lf_print_error_system_failure("Firewall permissions prohibit connection.");
-      free_netdrv(fed_netdrv);
-      return NULL;
-    } else {
-      // For the federates, it should check if the rti_socket is still open, before retrying accept().
-      socket_priv_t* rti_priv = (socket_priv_t*)rti_drv->priv;
-      if (rti_priv->socket_descriptor != -1) {
-        if (check_socket_closed(rti_priv->socket_descriptor)) {
-          free_netdrv(fed_netdrv);
-          return NULL;
-        }
-      }
-      // Try again
-      lf_print_warning("Failed to accept the socket. %s. Trying again.", strerror(errno));
-      continue;
-    }
+  int sock = accept_socket(serv_priv->socket_descriptor, rti_priv->socket_descriptor);
+  if (sock == -1) {
+    free_netdrv(fed_netdrv);
+    return NULL;
   }
-  fed_priv->socket_descriptor = socket_id;
+  fed_priv->socket_descriptor = sock;
   return fed_netdrv;
 }
 
diff --git a/network/impl/src/socket_common.c b/network/impl/src/socket_common.c
index 279cb51ef..d6e393e12 100644
--- a/network/impl/src/socket_common.c
+++ b/network/impl/src/socket_common.c
@@ -199,14 +199,15 @@ int accept_socket(int socket, int rti_socket) {
       break;
     } else if (socket_id < 0 && (errno != EAGAIN || errno != EWOULDBLOCK || errno != EINTR)) {
       lf_print_warning("Failed to accept the socket. %s.", strerror(errno));
-      break;
+      return -1;
     } else if (errno == EPERM) {
       lf_print_error_system_failure("Firewall permissions prohibit connection.");
+      return -1;
     } else {
       // For the federates, it should check if the rti_socket is still open, before retrying accept().
       if (rti_socket == -1) {
         if (check_socket_closed(rti_socket)) {
-          break;
+          return -1;
         }
       }
       // Try again

From 9e45794665e0122c845cf2aa8bca1f2afe0e54df Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sun, 19 Jan 2025 01:42:39 -0700
Subject: [PATCH 69/84] Fix create_server() to call create_socket_server()

---
 core/federated/RTI/rti_remote.c      |  2 +-
 core/federated/federate.c            |  2 +-
 network/api/net_driver.h             |  2 +-
 network/api/socket_common.h          |  4 +--
 network/impl/src/lf_socket_support.c | 29 +++------------------
 network/impl/src/socket_common.c     | 39 ++++++++++------------------
 6 files changed, 22 insertions(+), 56 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index 38de11ef5..14d1da2f1 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -1489,7 +1489,7 @@ int start_rti_server() {
   // Initialize RTI's network driver.
   rti_remote->rti_netdrv = initialize_netdrv();
   // Create the server
-  if (create_server_(rti_remote->rti_netdrv, true)) {
+  if (create_server(rti_remote->rti_netdrv, true)) {
     lf_print_error_system_failure("RTI failed to create TCP server: %s.", strerror(errno));
     return -1;
   };
diff --git a/core/federated/federate.c b/core/federated/federate.c
index f56877018..95465219e 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -1922,7 +1922,7 @@ void lf_create_server(int specified_port) {
   netdrv_t* server_netdrv = initialize_netdrv();
   set_server_port(server_netdrv, specified_port);
 
-  if (create_server_(server_netdrv, false)) {
+  if (create_server(server_netdrv, false)) {
     lf_print_error_system_failure("RTI failed to create server: %s.", strerror(errno));
   };
   _fed.server_netdrv = server_netdrv;
diff --git a/network/api/net_driver.h b/network/api/net_driver.h
index 8feb706cd..8faf031c0 100644
--- a/network/api/net_driver.h
+++ b/network/api/net_driver.h
@@ -24,7 +24,7 @@ netdrv_t* initialize_netdrv();
  * @param serv_type Type of server, RTI or FED.
  * @return int 0 for success, -1 for failure.
  */
-int create_server_(netdrv_t* drv, bool increment_port_on_retry);
+int create_server(netdrv_t* drv, bool increment_port_on_retry);
 
 netdrv_t* accept_netdrv(netdrv_t* server_drv, netdrv_t* rti_drv);
 
diff --git a/network/api/socket_common.h b/network/api/socket_common.h
index 63665942b..b46e759a3 100644
--- a/network/api/socket_common.h
+++ b/network/api/socket_common.h
@@ -136,12 +136,10 @@ int set_socket_bind_option(int socket_descriptor, uint16_t specified_port, bool
  * @param port The port number to use or 0 to let the OS pick or 1 to start trying at DEFAULT_PORT.
  * @param final_socket Pointer to the returned socket descriptor on which accepting connections will occur.
  * @param final_port Pointer to the final port the server will use.
- * @param sock_type Type of the socket, TCP or UDP.
  * @param increment_port_on_retry Boolean to retry port increment.
  * @return 0 for success, -1 for failure.
  */
-int create_server(uint16_t port, int* final_socket, uint16_t* final_port, socket_type_t sock_type,
-                  bool increment_port_on_retry);
+int create_socket_server(uint16_t port, int* final_socket, uint16_t* final_port, bool increment_port_on_retry);
 
 int create_clock_server(uint16_t port, int* final_socket, uint16_t* final_port);
 
diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
index 1793b0db0..8cb017f0b 100644
--- a/network/impl/src/lf_socket_support.c
+++ b/network/impl/src/lf_socket_support.c
@@ -40,37 +40,16 @@ void free_netdrv(netdrv_t* drv) {
   free(drv);
 }
 
-int create_server_(netdrv_t* drv, bool increment_port_on_retry) {
+int create_server(netdrv_t* drv, bool increment_port_on_retry) {
   socket_priv_t* priv = (socket_priv_t*)drv->priv;
-  int socket_descriptor;
-  struct timeval timeout_time;
-  // Create an IPv4 socket for TCP.
-  socket_descriptor = create_real_time_tcp_socket_errexit();
-  // Set the timeout time for the communications of the server
-  timeout_time = (struct timeval){.tv_sec = TCP_TIMEOUT_TIME / BILLION, .tv_usec = (TCP_TIMEOUT_TIME % BILLION) / 1000};
-  if (socket_descriptor < 0) {
-    lf_print_error("Failed to create TCP socket.");
-    return -1;
-  }
-  set_socket_timeout_option(socket_descriptor, &timeout_time);
-
-  int used_port = set_socket_bind_option(socket_descriptor, priv->user_specified_port, increment_port_on_retry);
-  // Enable listening for socket connections.
-  // The second argument is the maximum number of queued socket requests,
-  // which according to the Mac man page is limited to 128.
-  if (listen(socket_descriptor, 128)) {
-    lf_print_error("Failed to listen on %d socket: %s.", socket_descriptor, strerror(errno));
-    return -1;
-  }
-  priv->socket_descriptor = socket_descriptor;
-  priv->port = used_port;
-  return 0;
+  return create_socket_server(priv->user_specified_port, &priv->socket_descriptor, &priv->port,
+                              increment_port_on_retry);
 }
 
 netdrv_t* accept_netdrv(netdrv_t* server_drv, netdrv_t* rti_drv) {
   socket_priv_t* serv_priv = (socket_priv_t*)server_drv->priv;
   socket_priv_t* rti_priv = (socket_priv_t*)rti_drv->priv;
-  
+
   netdrv_t* fed_netdrv = initialize_netdrv();
   socket_priv_t* fed_priv = (socket_priv_t*)fed_netdrv->priv;
 
diff --git a/network/impl/src/socket_common.c b/network/impl/src/socket_common.c
index d6e393e12..9884d9f18 100644
--- a/network/impl/src/socket_common.c
+++ b/network/impl/src/socket_common.c
@@ -119,38 +119,27 @@ int set_socket_bind_option(int socket_descriptor, uint16_t specified_port, bool
   return used_port;
 }
 
-// TODO: Fix on federate.
-int create_server(uint16_t port, int* final_socket, uint16_t* final_port, socket_type_t sock_type,
-                  bool increment_port_on_retry) {
+int create_socket_server(uint16_t port, int* final_socket, uint16_t* final_port, bool increment_port_on_retry) {
   int socket_descriptor;
   struct timeval timeout_time;
-  if (sock_type == TCP) {
-    // Create an IPv4 socket for TCP.
-    socket_descriptor = create_real_time_tcp_socket_errexit();
-    // Set the timeout time for the communications of the server
-    timeout_time =
-        (struct timeval){.tv_sec = TCP_TIMEOUT_TIME / BILLION, .tv_usec = (TCP_TIMEOUT_TIME % BILLION) / 1000};
-  } else {
-    // Create a UDP socket.
-    socket_descriptor = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
-    timeout_time =
-        (struct timeval){.tv_sec = UDP_TIMEOUT_TIME / BILLION, .tv_usec = (UDP_TIMEOUT_TIME % BILLION) / 1000};
-  }
-  char* type = (sock_type == TCP) ? "TCP" : "UDP";
+
+  // Create an IPv4 socket for TCP.
+  socket_descriptor = create_real_time_tcp_socket_errexit();
+  // Set the timeout time for the communications of the server
+  timeout_time = (struct timeval){.tv_sec = TCP_TIMEOUT_TIME / BILLION, .tv_usec = (TCP_TIMEOUT_TIME % BILLION) / 1000};
+
   if (socket_descriptor < 0) {
-    lf_print_error("Failed to create %s socket.", type);
+    lf_print_error("Failed to create TCP socket.");
     return -1;
   }
   set_socket_timeout_option(socket_descriptor, &timeout_time);
   int used_port = set_socket_bind_option(socket_descriptor, port, increment_port_on_retry);
-  if (sock_type == TCP) {
-    // Enable listening for socket connections.
-    // The second argument is the maximum number of queued socket requests,
-    // which according to the Mac man page is limited to 128.
-    if (listen(socket_descriptor, 128)) {
-      lf_print_error("Failed to listen on %d socket: %s.", socket_descriptor, strerror(errno));
-      return -1;
-    }
+  // Enable listening for socket connections.
+  // The second argument is the maximum number of queued socket requests,
+  // which according to the Mac man page is limited to 128.
+  if (listen(socket_descriptor, 128)) {
+    lf_print_error("Failed to listen on %d socket: %s.", socket_descriptor, strerror(errno));
+    return -1;
   }
   *final_socket = socket_descriptor;
   *final_port = used_port;

From 02f041bf2b604b67ae3361210b66307f4c4312a8 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sun, 19 Jan 2025 01:45:51 -0700
Subject: [PATCH 70/84] Create UDP default port, because there can be 
 situations not using TCP for RTI.

---
 core/federated/RTI/rti_remote.c | 3 +--
 network/api/socket_common.h     | 2 ++
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index 14d1da2f1..0ab6880ea 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -1495,9 +1495,8 @@ int start_rti_server() {
   };
   lf_print("RTI: Listening for federates.");
   // Create the UDP socket server
-  // Try to get the rti_remote->final_port_TCP + 1 port
   if (rti_remote->clock_sync_global_status >= clock_sync_on) {
-    if (create_clock_server(rti_remote->final_port_TCP + 1, &rti_remote->socket_descriptor_UDP,
+    if (create_clock_server(DEFAULT_UDP_PORT, &rti_remote->socket_descriptor_UDP,
                             &rti_remote->final_port_UDP)) {
       lf_print_error_system_failure("RTI failed to create UDP server: %s.", strerror(errno));
       return -1;
diff --git a/network/api/socket_common.h b/network/api/socket_common.h
index b46e759a3..bae6a5b3f 100644
--- a/network/api/socket_common.h
+++ b/network/api/socket_common.h
@@ -64,6 +64,8 @@
  */
 #define DEFAULT_PORT 15045u
 
+#define DEFAULT_UDP_PORT 15061u
+
 /**
  * Byte identifying that the federate or the RTI has failed.
  */

From aacd379ca46c6ec9a097ddbae66d60ff8a0ed75c Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Sun, 19 Jan 2025 02:24:32 -0700
Subject: [PATCH 71/84] Add logic to check the netdriver is null, when the
 rti_drv is not available.

---
 core/federated/federate.c            |  6 +++---
 network/api/net_driver.h             |  2 ++
 network/impl/src/lf_socket_support.c | 23 +++++++++++++++++------
 3 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index 95465219e..66617c2b3 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -1783,7 +1783,7 @@ void lf_connect_to_federate(uint16_t remote_federate_id) {
       result = -1;
       // Wait ADDRESS_QUERY_RETRY_INTERVAL nanoseconds.
       lf_sleep(ADDRESS_QUERY_RETRY_INTERVAL);
-      lf_print_warning("Could not connect to federate %d. Will try again every" PRINTF_TIME "nanoseconds.\n",
+      lf_print_warning("Could not connect to federate %d. Will try again every " PRINTF_TIME "nanoseconds.\n",
                        remote_federate_id, ADDRESS_QUERY_RETRY_INTERVAL);
       continue;
     } else {
@@ -1927,7 +1927,7 @@ void lf_create_server(int specified_port) {
   };
   _fed.server_netdrv = server_netdrv;
   // Get the final server port set.
-  int32_t server_port = get_server_port(server_netdrv);
+  int32_t server_port = get_my_port(server_netdrv);
 
   LF_PRINT_LOG("Server for communicating with other federates started using port %d.", server_port);
 
@@ -1941,7 +1941,7 @@ void lf_create_server(int specified_port) {
   tracepoint_federate_to_rti(send_ADR_AD, _lf_my_fed_id, NULL);
 
   // No need for a mutex because we have the only handle on this network driver.
-  write_to_netdrv_fail_on_error(server_netdrv, sizeof(int32_t) + 1, (unsigned char*)buffer, NULL,
+  write_to_netdrv_fail_on_error(_fed.netdrv_to_RTI, sizeof(int32_t) + 1, (unsigned char*)buffer, NULL,
                                 "Failed to send address advertisement.");
 
   LF_PRINT_DEBUG("Sent port %d to the RTI.", server_port);
diff --git a/network/api/net_driver.h b/network/api/net_driver.h
index 8faf031c0..6006a9230 100644
--- a/network/api/net_driver.h
+++ b/network/api/net_driver.h
@@ -138,6 +138,8 @@ ssize_t peek_from_netdrv(netdrv_t* drv, unsigned char* result);
  */
 int shutdown_netdrv(netdrv_t* drv, bool read_before_closing);
 
+int32_t get_my_port(netdrv_t* drv);
+
 int32_t get_server_port(netdrv_t* drv);
 
 struct in_addr* get_ip_addr(netdrv_t* drv);
diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
index 8cb017f0b..cbd8f4035 100644
--- a/network/impl/src/lf_socket_support.c
+++ b/network/impl/src/lf_socket_support.c
@@ -48,12 +48,17 @@ int create_server(netdrv_t* drv, bool increment_port_on_retry) {
 
 netdrv_t* accept_netdrv(netdrv_t* server_drv, netdrv_t* rti_drv) {
   socket_priv_t* serv_priv = (socket_priv_t*)server_drv->priv;
-  socket_priv_t* rti_priv = (socket_priv_t*)rti_drv->priv;
-
+  int rti_socket;
+  if (rti_drv == NULL) {
+    rti_socket = -1;
+  } else {
+    socket_priv_t* rti_priv = (socket_priv_t*)rti_drv->priv;
+    rti_socket = rti_priv->socket_descriptor;
+  }
   netdrv_t* fed_netdrv = initialize_netdrv();
   socket_priv_t* fed_priv = (socket_priv_t*)fed_netdrv->priv;
 
-  int sock = accept_socket(serv_priv->socket_descriptor, rti_priv->socket_descriptor);
+  int sock = accept_socket(serv_priv->socket_descriptor, rti_socket);
   if (sock == -1) {
     free_netdrv(fed_netdrv);
     return NULL;
@@ -201,6 +206,10 @@ ssize_t peek_from_netdrv(netdrv_t* drv, unsigned char* result) {
 }
 
 int shutdown_netdrv(netdrv_t* drv, bool read_before_closing) {
+  if (drv == NULL) {
+    lf_print("Socket already closed.");
+    return 0;
+  }
   socket_priv_t* priv = (socket_priv_t*)drv->priv;
   int ret = shutdown_socket(&priv->socket_descriptor, read_before_closing);
   if (ret != 0) {
@@ -232,10 +241,12 @@ int get_peer_address(netdrv_t* drv) {
   return 0;
 }
 
+int32_t get_my_port(netdrv_t* drv) {
+  socket_priv_t* priv = (socket_priv_t*)drv->priv;
+  return priv->port;
+}
+
 int32_t get_server_port(netdrv_t* drv) {
-  // if (drv == NULL) {
-  //   lf_print_warning("Netdriver is closed, returning -1.");
-  // }
   socket_priv_t* priv = (socket_priv_t*)drv->priv;
   return priv->server_port;
 }

From 0406a0270aa0c02cfffe53ff0ac7ad922a0ef161 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Mon, 20 Jan 2025 10:21:01 -0700
Subject: [PATCH 72/84] Fis to not shutdown the netdrv, but the socket.

---
 network/impl/src/lf_socket_support.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
index cbd8f4035..10c003a68 100644
--- a/network/impl/src/lf_socket_support.c
+++ b/network/impl/src/lf_socket_support.c
@@ -112,12 +112,13 @@ int read_from_netdrv(netdrv_t* drv, size_t num_bytes, unsigned char* buffer) {
 }
 
 int read_from_netdrv_close_on_error(netdrv_t* drv, size_t num_bytes, unsigned char* buffer) {
+  socket_priv_t* priv = (socket_priv_t*)drv->priv;
   int read_failed = read_from_netdrv(drv, num_bytes, buffer);
   if (read_failed) {
     // Read failed.
     // Socket has probably been closed from the other side.
     // Shut down and close the socket from this side.
-    shutdown_netdrv(drv, false);
+    shutdown_socket(&priv->socket_descriptor, false);
     return -1;
   }
   return 0;
@@ -171,12 +172,13 @@ int write_to_netdrv(netdrv_t* drv, size_t num_bytes, unsigned char* buffer) {
 }
 
 int write_to_netdrv_close_on_error(netdrv_t* drv, size_t num_bytes, unsigned char* buffer) {
+  socket_priv_t* priv = (socket_priv_t*)drv->priv;
   int result = write_to_netdrv(drv, num_bytes, buffer);
   if (result) {
     // Write failed.
     // Socket has probably been closed from the other side.
     // Shut down and close the socket from this side.
-    shutdown_netdrv(drv, false);
+    shutdown_socket(&priv->socket_descriptor, false);
   }
   return result;
 }

From 8f589ea44298aa0a8f8b01db5025d2331005470f Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Mon, 20 Jan 2025 10:34:19 -0700
Subject: [PATCH 73/84] Remove comments.

---
 core/federated/RTI/rti_remote.c | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index 0ab6880ea..9fd6a3b8d 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -1395,10 +1395,6 @@ static bool authenticate_federate(netdrv_t* fed_netdrv) {
 #endif
 
 void lf_connect_to_federates(netdrv_t* rti_netdrv) {
-  // netdrv_t* netdrv_array[rti_remote->base.number_of_scheduling_nodes];
-  // for (int i = 0; i < rti_remote->base.number_of_scheduling_nodes; i++) {
-  //   netdrv_array[i] = establish_communication_session(rti_netdrv);
-  // }
   for (int i = 0; i < rti_remote->base.number_of_scheduling_nodes; i++) {
     netdrv_t* fed_netdrv = accept_netdrv(rti_netdrv, NULL);
 // Wait for the first message from the federate when RTI -a option is on.

From 36b29eea9de19ee6204d346b7fef263b4ddebc49 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Mon, 20 Jan 2025 14:49:15 -0700
Subject: [PATCH 74/84] Fix clock sync to split down network drivers with UDP.

---
 core/federated/RTI/rti_remote.c     | 62 +++++++++++++++-------
 core/federated/RTI/rti_remote.h     | 18 +++++--
 core/federated/clock-sync.c         | 80 +++++++++++++++++++----------
 include/core/federated/clock-sync.h | 47 +++++++++++------
 4 files changed, 142 insertions(+), 65 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index 9fd6a3b8d..a611c67f2 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -730,7 +730,14 @@ void handle_timestamp(federate_info_t* my_fed) {
   LF_MUTEX_UNLOCK(&rti_mutex);
 }
 
-void send_physical_clock(unsigned char message_type, federate_info_t* fed, socket_type_t socket_type) {
+/**
+ * Helper function to send the current physical clock time to a federate.
+ * This supports both TCP and UDP sockets.
+ * @param message_type The type of message being sent.
+ * @param fed Information about the federate receiving the message.
+ * @param is_udp Flag indicating whether to use UDP (true) or TCP (false).
+ */
+static void send_physical_clock_helper(unsigned char message_type, federate_info_t* fed, bool is_udp) {
   if (fed->enclave.state == NOT_CONNECTED) {
     lf_print_warning("Clock sync: RTI failed to send physical time to federate %d. Socket not connected.\n",
                      fed->enclave.id);
@@ -741,9 +748,8 @@ void send_physical_clock(unsigned char message_type, federate_info_t* fed, socke
   int64_t current_physical_time = lf_time_physical();
   encode_int64(current_physical_time, &(buffer[1]));
 
-  // Send the message
-  if (socket_type == UDP) {
-    // FIXME: UDP_addr is never initialized.
+  if (is_udp) {
+    // Send using UDP
     LF_PRINT_DEBUG("Clock sync: RTI sending UDP message type %u.", buffer[0]);
     ssize_t bytes_written = sendto(rti_remote->socket_descriptor_UDP, buffer, 1 + sizeof(int64_t), 0,
                                    (struct sockaddr*)&fed->UDP_addr, sizeof(fed->UDP_addr));
@@ -752,8 +758,9 @@ void send_physical_clock(unsigned char message_type, federate_info_t* fed, socke
                        strerror(errno));
       return;
     }
-  } else if (socket_type == TCP) {
-    LF_PRINT_DEBUG("Clock sync:  RTI sending TCP message type %u.", buffer[0]);
+  } else {
+    // Send using TCP
+    LF_PRINT_DEBUG("Clock sync: RTI sending TCP message type %u.", buffer[0]);
     LF_MUTEX_LOCK(&rti_mutex);
     write_to_netdrv_fail_on_error(fed->fed_netdrv, 1 + sizeof(int64_t), buffer, &rti_mutex,
                                   "Clock sync: RTI failed to send physical time to federate %d.", fed->enclave.id);
@@ -763,20 +770,40 @@ void send_physical_clock(unsigned char message_type, federate_info_t* fed, socke
                  current_physical_time, fed->enclave.id);
 }
 
-void handle_physical_clock_sync_message(federate_info_t* my_fed, socket_type_t socket_type) {
-  // Lock the mutex to prevent interference between sending the two
-  // coded probe messages.
+void send_physical_clock(unsigned char message_type, federate_info_t* fed) {
+  send_physical_clock_helper(message_type, fed, false);
+}
+
+void send_physical_clock_UDP(unsigned char message_type, federate_info_t* fed) {
+  send_physical_clock_helper(message_type, fed, true);
+}
+
+/**
+ * Handle a physical clock synchronization message, sending the required messages.
+ * UDP sends a coded probe message.
+ * @param my_fed The federate information.
+ * @param send_coded_probe Boolean to send a coded probe message (for UDP only).
+ */
+static void handle_physical_clock_sync_message_helper(federate_info_t* my_fed, bool send_coded_probe) {
   LF_MUTEX_LOCK(&rti_mutex);
   // Reply with a T4 type message
-  send_physical_clock(MSG_TYPE_CLOCK_SYNC_T4, my_fed, socket_type);
+  send_physical_clock(MSG_TYPE_CLOCK_SYNC_T4, my_fed);
   // Send the corresponding coded probe immediately after,
   // but only if this is a UDP channel.
-  if (socket_type == UDP) {
-    send_physical_clock(MSG_TYPE_CLOCK_SYNC_CODED_PROBE, my_fed, socket_type);
+  if (send_coded_probe) {
+    send_physical_clock_UDP(MSG_TYPE_CLOCK_SYNC_CODED_PROBE, my_fed);
   }
   LF_MUTEX_UNLOCK(&rti_mutex);
 }
 
+void handle_physical_clock_sync_message(federate_info_t* my_fed) {
+  handle_physical_clock_sync_message_helper(my_fed, false);
+}
+
+void handle_physical_clock_sync_message_UDP(federate_info_t* my_fed) {
+  handle_physical_clock_sync_message_helper(my_fed, true);
+}
+
 void* clock_synchronization_thread(void* noargs) {
   initialize_lf_thread_id();
   // Wait until all federates have been notified of the start time.
@@ -814,7 +841,7 @@ void* clock_synchronization_thread(void* noargs) {
       // Send the RTI's current physical time to the federate
       // Send on UDP.
       LF_PRINT_DEBUG("RTI sending T1 message to initiate clock sync round.");
-      send_physical_clock(MSG_TYPE_CLOCK_SYNC_T1, fed, UDP);
+      send_physical_clock_UDP(MSG_TYPE_CLOCK_SYNC_T1, fed);
 
       // Listen for reply message, which should be T3.
       size_t message_size = 1 + sizeof(uint16_t);
@@ -839,7 +866,7 @@ void* clock_synchronization_thread(void* noargs) {
               continue;
             }
             LF_PRINT_DEBUG("Clock sync: RTI received T3 message from federate %d.", fed_id_2);
-            handle_physical_clock_sync_message(GET_FED_INFO(fed_id_2), UDP);
+            handle_physical_clock_sync_message_UDP(GET_FED_INFO(fed_id_2));
             break;
           } else {
             // The message is not a T3 message. Discard the message and
@@ -1277,7 +1304,7 @@ static int receive_udp_message_and_set_up_clock_sync(netdrv_t* fed_netdrv, uint1
         // Send the required number of messages for clock synchronization
         for (int i = 0; i < rti_remote->clock_sync_exchanges_per_interval; i++) {
           // Send the RTI's current physical time T1 to the federate.
-          send_physical_clock(MSG_TYPE_CLOCK_SYNC_T1, fed, TCP);
+          send_physical_clock(MSG_TYPE_CLOCK_SYNC_T1, fed);
 
           // Listen for reply message, which should be T3.
           size_t message_size = 1 + sizeof(uint16_t);
@@ -1287,7 +1314,7 @@ static int receive_udp_message_and_set_up_clock_sync(netdrv_t* fed_netdrv, uint1
           if (buffer[0] == MSG_TYPE_CLOCK_SYNC_T3) {
             uint16_t fed_id = extract_uint16(&(buffer[1]));
             LF_PRINT_DEBUG("RTI received T3 clock sync message from federate %d.", fed_id);
-            handle_physical_clock_sync_message(fed, TCP);
+            handle_physical_clock_sync_message(fed);
           } else {
             lf_print_error("Unexpected message %u from federate %d.", buffer[0], fed_id);
             send_reject(fed_netdrv, UNEXPECTED_MESSAGE);
@@ -1492,8 +1519,7 @@ int start_rti_server() {
   lf_print("RTI: Listening for federates.");
   // Create the UDP socket server
   if (rti_remote->clock_sync_global_status >= clock_sync_on) {
-    if (create_clock_server(DEFAULT_UDP_PORT, &rti_remote->socket_descriptor_UDP,
-                            &rti_remote->final_port_UDP)) {
+    if (create_clock_server(DEFAULT_UDP_PORT, &rti_remote->socket_descriptor_UDP, &rti_remote->final_port_UDP)) {
       lf_print_error_system_failure("RTI failed to create UDP server: %s.", strerror(errno));
       return -1;
     }
diff --git a/core/federated/RTI/rti_remote.h b/core/federated/RTI/rti_remote.h
index abf7cb20e..900ee39b9 100644
--- a/core/federated/RTI/rti_remote.h
+++ b/core/federated/RTI/rti_remote.h
@@ -291,15 +291,22 @@ void handle_timestamp(federate_info_t* my_fed);
 
 /**
  * Take a snapshot of the physical clock time and send
- * it to federate fed_id.
+ * it to federate fed_id using the network driver.
  *
  * This version assumes the caller holds the mutex lock.
  *
  * @param message_type The type of the clock sync message (see net_common.h).
  * @param fed The federate to send the physical time to.
- * @param socket_type The socket type (TCP or UDP).
  */
-void send_physical_clock(unsigned char message_type, federate_info_t* fed, socket_type_t socket_type);
+void send_physical_clock(unsigned char message_type, federate_info_t* fed);
+
+/**
+ * This does the same function with send_physical_clock(), but uses UDP.
+ *
+ * @param message_type The type of the clock sync message (see net_common.h).
+ * @param fed The federate to send the physical time to.
+ */
+void send_physical_clock_UDP(unsigned char message_type, federate_info_t* fed);
 
 /**
  * Handle clock synchronization T3 messages from federates.
@@ -312,9 +319,10 @@ void send_physical_clock(unsigned char message_type, federate_info_t* fed, socke
  * clock synchronization round.
  *
  * @param my_fed The sending federate.
- * @param socket_type The RTI's socket type used for the communication (TCP or UDP)
  */
-void handle_physical_clock_sync_message(federate_info_t* my_fed, socket_type_t socket_type);
+void handle_physical_clock_sync_message(federate_info_t* my_fed);
+
+void handle_physical_clock_sync_message_UDP(federate_info_t* my_fed);
 
 /**
  * A (quasi-)periodic thread that performs clock synchronization with each
diff --git a/core/federated/clock-sync.c b/core/federated/clock-sync.c
index cd95aa0b2..d2b1e14e8 100644
--- a/core/federated/clock-sync.c
+++ b/core/federated/clock-sync.c
@@ -208,8 +208,7 @@ uint16_t setup_clock_synchronization_with_rti() {
   return port_to_return;
 }
 
-//TODO: Fix clocks.
-void synchronize_initial_physical_clock_with_rti(int* rti_socket_TCP) {
+void synchronize_initial_physical_clock_with_rti(netdrv_t* rti_netdrv) {
   LF_PRINT_DEBUG("Waiting for initial clock synchronization messages from the RTI.");
 
   size_t message_size = 1 + sizeof(instant_t);
@@ -217,7 +216,7 @@ void synchronize_initial_physical_clock_with_rti(int* rti_socket_TCP) {
 
   for (int i = 0; i < _LF_CLOCK_SYNC_EXCHANGES_PER_INTERVAL; i++) {
     // The first message expected from the RTI is MSG_TYPE_CLOCK_SYNC_T1
-    read_from_socket_fail_on_error(rti_socket_TCP, message_size, buffer, NULL,
+    read_from_netdrv_fail_on_error(rti_netdrv, message_size, buffer, NULL,
                                    "Federate %d did not get the initial clock synchronization message T1 from the RTI.",
                                    _lf_my_fed_id);
 
@@ -231,12 +230,12 @@ void synchronize_initial_physical_clock_with_rti(int* rti_socket_TCP) {
     // Handle the message and send a reply T3 message.
     // NOTE: No need to acquire the mutex lock during initialization because only
     // one thread is running.
-    if (handle_T1_clock_sync_message(buffer, *rti_socket_TCP, receive_time) != 0) {
+    if (handle_T1_clock_sync_message(buffer, rti_netdrv, receive_time) != 0) {
       lf_print_error_and_exit("Initial clock sync: Failed to send T3 reply to RTI.");
     }
 
     // Next message from the RTI is required to be MSG_TYPE_CLOCK_SYNC_T4
-    read_from_socket_fail_on_error(rti_socket_TCP, message_size, buffer, NULL,
+    read_from_netdrv_fail_on_error(rti_netdrv, message_size, buffer, NULL,
                                    "Federate %d did not get the clock synchronization message T4 from the RTI.",
                                    _lf_my_fed_id);
 
@@ -246,7 +245,7 @@ void synchronize_initial_physical_clock_with_rti(int* rti_socket_TCP) {
     }
 
     // Handle the message.
-    handle_T4_clock_sync_message(buffer, *rti_socket_TCP, receive_time);
+    handle_T4_clock_sync_message(buffer, rti_netdrv, receive_time);
   }
 
   LF_PRINT_LOG("Finished initial clock synchronization with the RTI.");
@@ -259,11 +258,12 @@ void synchronize_initial_physical_clock_with_rti(int* rti_socket_TCP) {
  * It also measures the time it takes between when the method is
  * called and the reply has been sent.
  * @param buffer The buffer containing the message, including the message type.
- * @param socket The socket (either _lf_rti_socket_TCP or _lf_rti_socket_UDP).
+ * @param socket_or_netdrv The pointer of either UDP socket or the network driver.
  * @param t2 The physical time at which the T1 message was received.
  * @return 0 if T3 reply is successfully sent, -1 otherwise.
  */
-int handle_T1_clock_sync_message(unsigned char* buffer, int socket, instant_t t2) {
+static int handle_T1_clock_sync_message_common(unsigned char* buffer, void* socket_or_netdrv, instant_t t2,
+                                               bool use_udp) {
   // Extract the payload
   instant_t t1 = extract_int64(&(buffer[1]));
 
@@ -283,7 +283,10 @@ int handle_T1_clock_sync_message(unsigned char* buffer, int socket, instant_t t2
 
   // Write the reply to the socket.
   LF_PRINT_DEBUG("Sending T3 message to RTI.");
-  if (write_to_socket(socket, 1 + sizeof(uint16_t), reply_buffer)) {
+  int result = use_udp ? write_to_socket(*(int*)socket_or_netdrv, sizeof(reply_buffer), reply_buffer)
+                       : write_to_netdrv((netdrv_t*)socket_or_netdrv, sizeof(reply_buffer), reply_buffer);
+
+  if (result) {
     lf_print_error("Clock sync: Failed to send T3 message to RTI.");
     return -1;
   }
@@ -295,12 +298,22 @@ int handle_T1_clock_sync_message(unsigned char* buffer, int socket, instant_t t2
   return 0;
 }
 
+// Wrapper for handling clock synchronization over netdrv (e.g., TCP)
+int handle_T1_clock_sync_message(unsigned char* buffer, netdrv_t* netdrv, instant_t t2) {
+  return handle_T1_clock_sync_message_common(buffer, (void*)netdrv, t2, false);
+}
+
+// Wrapper for handling clock synchronization over UDP
+int handle_T1_clock_sync_message_UDP(unsigned char* buffer, int* UDP_socket, instant_t t2) {
+  return handle_T1_clock_sync_message_common(buffer, (void*)UDP_socket, t2, true);
+}
+
 /**
  * Handle a clock synchronization message T4 coming from the RTI.
- * If the socket is _lf_rti_socket_TCP, then assume we are in the
+ * If using the network driver, then assume we are in the
  * initial clock synchronization phase and set the clock offset
  * based on the estimated clock synchronization error.
- * Otherwise, if the socket is _lf_rti_socket_UDP, then this looks also for a
+ * Otherwise, if using the UDP socket, then this looks also for a
  * subsequent "coded probe" message on the socket. If the delay between
  * the T4 and the coded probe message is not as expected, then reject
  * this clock synchronization round. If it is not rejected, then make
@@ -308,10 +321,11 @@ int handle_T1_clock_sync_message(unsigned char* buffer, int socket, instant_t t2
  * This function does not acquire the netdrv_mutex lock.
  * The caller should acquire it unless it is sure there is only one thread running.
  * @param buffer The buffer containing the message, including the message type.
- * @param socket The socket (either _lf_rti_socket_TCP or _lf_rti_socket_UDP).
+ * @param socket_or_netdrv The pointer of either UDP socket or the network driver.
  * @param r4 The physical time at which this T4 message was received.
  */
-void handle_T4_clock_sync_message(unsigned char* buffer, int socket, instant_t r4) {
+static void handle_T4_clock_sync_message_common(unsigned char* buffer, void* socket_or_netdrv, instant_t r4,
+                                                int use_udp) {
   // Increment the number of received T4 messages
   _lf_rti_socket_stat.received_T4_messages_in_current_sync_window++;
 
@@ -343,10 +357,10 @@ void handle_T4_clock_sync_message(unsigned char* buffer, int socket, instant_t r
   // If the socket is _lf_rti_socket_UDP, then
   // after sending T4, the RTI sends a "coded probe" message,
   // which can be used to filter out noise.
-  if (socket == _lf_rti_socket_UDP) {
+  if (use_udp) {
     // Read the coded probe message.
     // We can reuse the same buffer.
-    int read_failed = read_from_socket(socket, 1 + sizeof(instant_t), buffer);
+    int read_failed = read_from_socket(*(int*)socket_or_netdrv, 1 + sizeof(instant_t), buffer);
 
     instant_t r5 = lf_time_physical();
 
@@ -377,17 +391,15 @@ void handle_T4_clock_sync_message(unsigned char* buffer, int socket, instant_t r
       _lf_rti_socket_stat.received_T4_messages_in_current_sync_window--;
       return;
     }
-    // Apply a jitter attenuator to the estimated clock error to prevent
-    // large jumps in the underlying clock.
-    // Note that estimated_clock_error is calculated using lf_time_physical() which includes
-    // the clock sync adjustment.
-    adjustment = estimated_clock_error / _LF_CLOCK_SYNC_ATTENUATION;
-  } else {
-    // Use of TCP socket means we are in the startup phase, so
-    // rather than adjust the clock offset, we simply set it to the
-    // estimated error.
-    adjustment = estimated_clock_error;
   }
+  // If use UDP, apply a jitter attenuator to the estimated clock error to prevent
+  // large jumps in the underlying clock.
+  // Note that estimated_clock_error is calculated using lf_time_physical() which includes
+  // the clock sync adjustment.
+  // Use of TCP socket means we are in the startup phase, so
+  // rather than adjust the clock offset, we simply set it to the
+  // estimated error.
+  adjustment = use_udp ? estimated_clock_error / _LF_CLOCK_SYNC_ATTENUATION : estimated_clock_error;
 
 #ifdef _LF_CLOCK_SYNC_COLLECT_STATS // Enabled by default
   // Update RTI's socket stats
@@ -437,6 +449,20 @@ void handle_T4_clock_sync_message(unsigned char* buffer, int socket, instant_t r
   }
 }
 
+/**
+ * Wrapper for handling clock synchronization messages via TCP.
+ */
+void handle_T4_clock_sync_message(unsigned char* buffer, netdrv_t* netdrv, instant_t r4) {
+  handle_T4_clock_sync_message_common(buffer, (void*)netdrv, r4, 0);
+}
+
+/**
+ * Wrapper for handling clock synchronization messages via UDP.
+ */
+void handle_T4_clock_sync_message_UDP(unsigned char* buffer, int* UDP_socket, instant_t r4) {
+  handle_T4_clock_sync_message_common(buffer, (void*)UDP_socket, r4, 1);
+}
+
 /**
  * Thread that listens for UDP inputs from the RTI.
  */
@@ -502,7 +528,7 @@ void* listen_to_rti_UDP_thread(void* args) {
           break;
         }
         connected = true;
-        if (handle_T1_clock_sync_message(buffer, _lf_rti_socket_UDP, receive_time) != 0) {
+        if (handle_T1_clock_sync_message_UDP(buffer, &_lf_rti_socket_UDP, receive_time) != 0) {
           // Failed to send T3 reply. Wait for the next T1.
           waiting_for_T1 = true;
           continue;
@@ -515,7 +541,7 @@ void* listen_to_rti_UDP_thread(void* args) {
         continue;
       }
     } else if (buffer[0] == MSG_TYPE_CLOCK_SYNC_T4) {
-      handle_T4_clock_sync_message(buffer, _lf_rti_socket_UDP, receive_time);
+      handle_T4_clock_sync_message_UDP(buffer, &_lf_rti_socket_UDP, receive_time);
       waiting_for_T1 = true;
     } else {
       lf_print_warning("Clock sync: Received from RTI an unexpected UDP message type: %u. "
diff --git a/include/core/federated/clock-sync.h b/include/core/federated/clock-sync.h
index 1b0cacd3e..18fe33848 100644
--- a/include/core/federated/clock-sync.h
+++ b/include/core/federated/clock-sync.h
@@ -34,6 +34,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define CLOCK_SYNC_H
 
 #include "low_level_platform.h"
+#include "net_driver.h"
 
 // Clock synchronization defaults to performing clock synchronization only at initialization.
 #define LF_CLOCK_SYNC_OFF 1
@@ -157,15 +158,15 @@ uint16_t setup_clock_synchronization_with_rti(void);
  * is required.
  *
  * This is a blocking function that expects
- * to read a MSG_TYPE_CLOCK_SYNC_T1 from the RTI TCP socket.
+ * to read a MSG_TYPE_CLOCK_SYNC_T1 from the RTI network driver.
  * It will then follow the PTP protocol to synchronize the local
  * physical clock with the RTI.
  * Failing to complete this protocol is treated as a catastrophic
  * error that causes the federate to exit.
  *
- * @param rti_socket_TCP Pointer to the RTI's socket
+ * @param rti_netdrv Pointer to the RTI's network driver.
  */
-void synchronize_initial_physical_clock_with_rti(int* rti_socket_TCP);
+void synchronize_initial_physical_clock_with_rti(netdrv_t* rti_netdrv);
 
 /**
  * Handle a clock synchroninzation message T1 coming from the RTI.
@@ -174,29 +175,45 @@ void synchronize_initial_physical_clock_with_rti(int* rti_socket_TCP);
  * It also measures the time it takes between when the method is
  * called and the reply has been sent.
  * @param buffer The buffer containing the message, including the message type.
- * @param socket The socket (either _lf_rti_socket_TCP or _lf_rti_socket_UDP).
+ * @param netdrv_t The pointer to the network driver.
  * @param t2 The physical time at which the T1 message was received.
  * @return 0 if T3 reply is successfully sent, -1 otherwise.
  */
-int handle_T1_clock_sync_message(unsigned char* buffer, int socket, instant_t t2);
+int handle_T1_clock_sync_message(unsigned char* buffer, netdrv_t* netdrv, instant_t t2);
 
 /**
- * Handle a clock synchronization message T4 coming from the RTI.
- * If the socket is _lf_rti_socket_TCP, then assume we are in the
- * initial clock synchronization phase and set the clock offset
+ * This does the same function as handle_T1_clock_sync_message(), only using a UDP socket.
+ * @param buffer The buffer containing the message, including the message type.
+ * @param socket The pointer to the UDP socket.
+ * @param t2 The physical time at which the T1 message was received.
+ * @return 0 if T3 reply is successfully sent, -1 otherwise.
+ */
+int handle_T1_clock_sync_message_UDP(unsigned char* buffer, int* UDP_socket, instant_t t2);
+
+/**
+ * Handle a clock synchronization message T4 coming from the RTI using the network driver.
+ * Assume this is the initial clock synchronization phase and set the clock offset
  * based on the estimated clock synchronization error.
- * Otherwise, if the socket is _lf_rti_socket_UDP, then this looks also for a
- * subsequent "coded probe" message on the socket. If the delay between
- * the T4 and the coded probe message is not as expected, then reject
- * this clock synchronization round. If it is not rejected, then make
- * an adjustment to the clock offset based on the estimated error.
  * This function does not acquire the netdrv_mutex lock.
  * The caller should acquire it unless it is sure there is only one thread running.
  * @param buffer The buffer containing the message, including the message type.
- * @param socket The socket (either _lf_rti_socket_TCP or _lf_rti_socket_UDP).
+ * @param netdrv_t The pointer to the network driver.
+ * @param r4 The physical time at which this T4 message was received.
+ */
+void handle_T4_clock_sync_message(unsigned char* buffer, netdrv_t* netdrv, instant_t r4);
+
+/**
+ * Handle a clock synchronization message T4 coming from the RTI using a UDP socket.
+ * Look for a subsequent "coded probe" message on the socket.
+ * If the delay between the T4 and the coded probe message is not as expected, reject the clock synchronization round.
+ * If it is not rejected, then make an adjustment to the clock offset based on the estimated error.
+ * This function does not acquire the netdrv_mutex lock.
+ * The caller should acquire it unless it is sure there is only one thread running.
+ * @param buffer The buffer containing the message, including the message type.
+ * @param socket The pointer to the UDP socket.
  * @param r4 The physical time at which this T4 message was received.
  */
-void handle_T4_clock_sync_message(unsigned char* buffer, int socket, instant_t r4);
+void handle_T4_clock_sync_message_UDP(unsigned char* buffer, int* UDP_socket, instant_t r4);
 
 /**
  * Thread that listens for UDP inputs from the RTI.

From fb46c59e77eb51c2c1b2816540850d5b3369ca60 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Mon, 20 Jan 2025 15:22:41 -0700
Subject: [PATCH 75/84] Code cleanup.

---
 network/api/net_driver.h         |  2 +-
 network/api/socket_common.h      |  7 ----
 network/impl/src/socket_common.c | 72 +++-----------------------------
 3 files changed, 7 insertions(+), 74 deletions(-)

diff --git a/network/api/net_driver.h b/network/api/net_driver.h
index 6006a9230..a604bc481 100644
--- a/network/api/net_driver.h
+++ b/network/api/net_driver.h
@@ -109,7 +109,7 @@ int write_to_netdrv_close_on_error(netdrv_t* drv, size_t num_bytes, unsigned cha
 
 /**
  * Write the specified number of bytes to the specified socket using
- * write_to_socket_close_on_error and exit with an error code if an error occurs.
+ * write_to_netdrv_close_on_error and exit with an error code if an error occurs.
  * If the mutex argument is non-NULL, release the mutex before exiting.  If the
  * format argument is non-null, then use it an any additional arguments to form
  * the error message using printf conventions. Otherwise, print a generic error
diff --git a/network/api/socket_common.h b/network/api/socket_common.h
index bae6a5b3f..debcdfe8e 100644
--- a/network/api/socket_common.h
+++ b/network/api/socket_common.h
@@ -145,13 +145,6 @@ int create_socket_server(uint16_t port, int* final_socket, uint16_t* final_port,
 
 int create_clock_server(uint16_t port, int* final_socket, uint16_t* final_port);
 
-/**
- * Return true if either the socket to the RTI is broken or the socket is
- * alive and the first unread byte on the socket's queue is MSG_TYPE_FAILED.
- * @param socket Socket to check.
- */
-bool check_socket_closed(int socket);
-
 /**
  * Wait for an incoming connection request on the specified server socket.
  * This blocks until a connection is successfully accepted. If an error occurs that is not
diff --git a/network/impl/src/socket_common.c b/network/impl/src/socket_common.c
index 9884d9f18..2f5890815 100644
--- a/network/impl/src/socket_common.c
+++ b/network/impl/src/socket_common.c
@@ -164,7 +164,12 @@ int create_clock_server(uint16_t port, int* final_socket, uint16_t* final_port)
   return 0;
 }
 
-bool check_socket_closed(int socket) {
+/**
+ * Return true if either the socket to the RTI is broken or the socket is
+ * alive and the first unread byte on the socket's queue is MSG_TYPE_FAILED.
+ * @param socket Socket to check.
+ */
+static bool check_socket_closed(int socket) {
   unsigned char first_byte;
   ssize_t bytes = peek_from_socket(socket, &first_byte);
   if (bytes < 0 || (bytes == 1 && first_byte == MSG_TYPE_FAILED)) {
@@ -289,39 +294,6 @@ int read_from_socket(int socket, size_t num_bytes, unsigned char* buffer) {
   return 0;
 }
 
-int read_from_socket_close_on_error(int* socket, size_t num_bytes, unsigned char* buffer) {
-  assert(socket);
-  int read_failed = read_from_socket(*socket, num_bytes, buffer);
-  if (read_failed) {
-    // Read failed.
-    // Socket has probably been closed from the other side.
-    // Shut down and close the socket from this side.
-    shutdown_socket(socket, false);
-    return -1;
-  }
-  return 0;
-}
-
-void read_from_socket_fail_on_error(int* socket, size_t num_bytes, unsigned char* buffer, lf_mutex_t* mutex,
-                                    char* format, ...) {
-  va_list args;
-  assert(socket);
-  int read_failed = read_from_socket_close_on_error(socket, num_bytes, buffer);
-  if (read_failed) {
-    // Read failed.
-    if (mutex != NULL) {
-      LF_MUTEX_UNLOCK(mutex);
-    }
-    if (format != NULL) {
-      va_start(args, format);
-      lf_print_error_system_failure(format, args);
-      va_end(args);
-    } else {
-      lf_print_error_system_failure("Failed to read from socket.");
-    }
-  }
-}
-
 ssize_t peek_from_socket(int socket, unsigned char* result) {
   ssize_t bytes_read = recv(socket, result, 1, MSG_DONTWAIT | MSG_PEEK);
   if (bytes_read < 0 && (errno == EAGAIN || errno == EWOULDBLOCK))
@@ -356,38 +328,6 @@ int write_to_socket(int socket, size_t num_bytes, unsigned char* buffer) {
   return 0;
 }
 
-int write_to_socket_close_on_error(int* socket, size_t num_bytes, unsigned char* buffer) {
-  assert(socket);
-  int result = write_to_socket(*socket, num_bytes, buffer);
-  if (result) {
-    // Write failed.
-    // Socket has probably been closed from the other side.
-    // Shut down and close the socket from this side.
-    shutdown_socket(socket, false);
-  }
-  return result;
-}
-
-void write_to_socket_fail_on_error(int* socket, size_t num_bytes, unsigned char* buffer, lf_mutex_t* mutex,
-                                   char* format, ...) {
-  va_list args;
-  assert(socket);
-  int result = write_to_socket_close_on_error(socket, num_bytes, buffer);
-  if (result) {
-    // Write failed.
-    if (mutex != NULL) {
-      LF_MUTEX_UNLOCK(mutex);
-    }
-    if (format != NULL) {
-      va_start(args, format);
-      lf_print_error_system_failure(format, args);
-      va_end(args);
-    } else {
-      lf_print_error("Failed to write to socket. Closing it.");
-    }
-  }
-}
-
 int shutdown_socket(int* socket, bool read_before_closing) {
   if (!read_before_closing) {
     if (shutdown(*socket, SHUT_RDWR)) {

From 4464284ac64f3a775dafae54bbbeb8f20645ab2c Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Mon, 20 Jan 2025 15:25:36 -0700
Subject: [PATCH 76/84] Cleanup read and write functions.

---
 network/impl/src/lf_socket_support.c | 56 +---------------------------
 1 file changed, 2 insertions(+), 54 deletions(-)

diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
index 10c003a68..4c31efc2b 100644
--- a/network/impl/src/lf_socket_support.c
+++ b/network/impl/src/lf_socket_support.c
@@ -79,36 +79,7 @@ int connect_to_netdrv(netdrv_t* drv) {
 
 int read_from_netdrv(netdrv_t* drv, size_t num_bytes, unsigned char* buffer) {
   socket_priv_t* priv = (socket_priv_t*)drv->priv;
-  int socket = priv->socket_descriptor;
-  if (socket < 0) {
-    // Socket is not open.
-    errno = EBADF;
-    return -1;
-  }
-  ssize_t bytes_read = 0;
-  while (bytes_read < (ssize_t)num_bytes) {
-    ssize_t more = read(socket, buffer + bytes_read, num_bytes - (size_t)bytes_read);
-    if (more < 0 && (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)) {
-      // Those error codes set by the socket indicates
-      // that we should try again (@see man errno).
-      LF_PRINT_DEBUG("Reading from socket %d failed with error: `%s`. Will try again.", socket, strerror(errno));
-      lf_sleep(DELAY_BETWEEN_SOCKET_RETRIES);
-      continue;
-    } else if (more < 0 && errno == ECONNRESET) {
-      lf_print_error(
-          "Connection was closed by the peer without properly sending an EOF first. Considering this a soft error.");
-      return -1;
-    } else if (more < 0) {
-      // A more serious error occurred.
-      lf_print_error("Reading from socket %d failed. With error: `%s`", socket, strerror(errno));
-      return -1;
-    } else if (more == 0) {
-      // EOF received.
-      return 1;
-    }
-    bytes_read += more;
-  }
-  return 0;
+  return read_from_socket(priv->socket_descriptor, num_bytes, buffer);
 }
 
 int read_from_netdrv_close_on_error(netdrv_t* drv, size_t num_bytes, unsigned char* buffer) {
@@ -145,30 +116,7 @@ void read_from_netdrv_fail_on_error(netdrv_t* drv, size_t num_bytes, unsigned ch
 
 int write_to_netdrv(netdrv_t* drv, size_t num_bytes, unsigned char* buffer) {
   socket_priv_t* priv = (socket_priv_t*)drv->priv;
-  int socket = priv->socket_descriptor;
-  if (socket < 0) {
-    // Socket is not open.
-    errno = EBADF;
-    return -1;
-  }
-  ssize_t bytes_written = 0;
-  while (bytes_written < (ssize_t)num_bytes) {
-    ssize_t more = write(socket, buffer + bytes_written, num_bytes - (size_t)bytes_written);
-    if (more <= 0 && (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)) {
-      // The error codes EAGAIN or EWOULDBLOCK indicate
-      // that we should try again (@see man errno).
-      // The error code EINTR means the system call was interrupted before completing.
-      LF_PRINT_DEBUG("Writing to socket %d was blocked. Will try again.", socket);
-      lf_sleep(DELAY_BETWEEN_SOCKET_RETRIES);
-      continue;
-    } else if (more < 0) {
-      // A more serious error occurred.
-      lf_print_error("Writing to socket %d failed. With error: `%s`", socket, strerror(errno));
-      return -1;
-    }
-    bytes_written += more;
-  }
-  return 0;
+  return write_to_socket(priv->socket_descriptor, num_bytes, buffer);
 }
 
 int write_to_netdrv_close_on_error(netdrv_t* drv, size_t num_bytes, unsigned char* buffer) {

From d4d78a92acd6947080f9af869310de60b18b2aa9 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Mon, 20 Jan 2025 18:22:56 -0700
Subject: [PATCH 77/84] Rollback create socket server to one function.

---
 network/api/socket_common.h          |  3 +-
 network/impl/src/lf_socket_support.c |  2 +-
 network/impl/src/socket_common.c     | 54 +++++++++++++---------------
 3 files changed, 28 insertions(+), 31 deletions(-)

diff --git a/network/api/socket_common.h b/network/api/socket_common.h
index debcdfe8e..6653a813d 100644
--- a/network/api/socket_common.h
+++ b/network/api/socket_common.h
@@ -141,7 +141,8 @@ int set_socket_bind_option(int socket_descriptor, uint16_t specified_port, bool
  * @param increment_port_on_retry Boolean to retry port increment.
  * @return 0 for success, -1 for failure.
  */
-int create_socket_server(uint16_t port, int* final_socket, uint16_t* final_port, bool increment_port_on_retry);
+int create_socket_server(uint16_t port, int* final_socket, uint16_t* final_port, socket_type_t sock_type,
+                         bool increment_port_on_retry);
 
 int create_clock_server(uint16_t port, int* final_socket, uint16_t* final_port);
 
diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
index 4c31efc2b..f8f7504ee 100644
--- a/network/impl/src/lf_socket_support.c
+++ b/network/impl/src/lf_socket_support.c
@@ -42,7 +42,7 @@ void free_netdrv(netdrv_t* drv) {
 
 int create_server(netdrv_t* drv, bool increment_port_on_retry) {
   socket_priv_t* priv = (socket_priv_t*)drv->priv;
-  return create_socket_server(priv->user_specified_port, &priv->socket_descriptor, &priv->port,
+  return create_socket_server(priv->user_specified_port, &priv->socket_descriptor, &priv->port, TCP,
                               increment_port_on_retry);
 }
 
diff --git a/network/impl/src/socket_common.c b/network/impl/src/socket_common.c
index 2f5890815..b68152680 100644
--- a/network/impl/src/socket_common.c
+++ b/network/impl/src/socket_common.c
@@ -119,27 +119,37 @@ int set_socket_bind_option(int socket_descriptor, uint16_t specified_port, bool
   return used_port;
 }
 
-int create_socket_server(uint16_t port, int* final_socket, uint16_t* final_port, bool increment_port_on_retry) {
+int create_socket_server(uint16_t port, int* final_socket, uint16_t* final_port, socket_type_t sock_type,
+                  bool increment_port_on_retry) {
   int socket_descriptor;
   struct timeval timeout_time;
-
-  // Create an IPv4 socket for TCP.
-  socket_descriptor = create_real_time_tcp_socket_errexit();
-  // Set the timeout time for the communications of the server
-  timeout_time = (struct timeval){.tv_sec = TCP_TIMEOUT_TIME / BILLION, .tv_usec = (TCP_TIMEOUT_TIME % BILLION) / 1000};
-
+  if (sock_type == TCP) {
+    // Create an IPv4 socket for TCP.
+    socket_descriptor = create_real_time_tcp_socket_errexit();
+    // Set the timeout time for the communications of the server
+    timeout_time =
+        (struct timeval){.tv_sec = TCP_TIMEOUT_TIME / BILLION, .tv_usec = (TCP_TIMEOUT_TIME % BILLION) / 1000};
+  } else {
+    // Create a UDP socket.
+    socket_descriptor = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
+    timeout_time =
+        (struct timeval){.tv_sec = UDP_TIMEOUT_TIME / BILLION, .tv_usec = (UDP_TIMEOUT_TIME % BILLION) / 1000};
+  }
+  char* type = (sock_type == TCP) ? "TCP" : "UDP";
   if (socket_descriptor < 0) {
-    lf_print_error("Failed to create TCP socket.");
+    lf_print_error("Failed to create %s socket.", type);
     return -1;
   }
   set_socket_timeout_option(socket_descriptor, &timeout_time);
   int used_port = set_socket_bind_option(socket_descriptor, port, increment_port_on_retry);
-  // Enable listening for socket connections.
-  // The second argument is the maximum number of queued socket requests,
-  // which according to the Mac man page is limited to 128.
-  if (listen(socket_descriptor, 128)) {
-    lf_print_error("Failed to listen on %d socket: %s.", socket_descriptor, strerror(errno));
-    return -1;
+  if (sock_type == TCP) {
+    // Enable listening for socket connections.
+    // The second argument is the maximum number of queued socket requests,
+    // which according to the Mac man page is limited to 128.
+    if (listen(socket_descriptor, 128)) {
+      lf_print_error("Failed to listen on %d socket: %s.", socket_descriptor, strerror(errno));
+      return -1;
+    }
   }
   *final_socket = socket_descriptor;
   *final_port = used_port;
@@ -147,21 +157,7 @@ int create_socket_server(uint16_t port, int* final_socket, uint16_t* final_port,
 }
 
 int create_clock_server(uint16_t port, int* final_socket, uint16_t* final_port) {
-  int socket_descriptor;
-  struct timeval timeout_time;
-  // Create a UDP socket.
-  socket_descriptor = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
-  timeout_time = (struct timeval){.tv_sec = UDP_TIMEOUT_TIME / BILLION, .tv_usec = (UDP_TIMEOUT_TIME % BILLION) / 1000};
-
-  if (socket_descriptor < 0) {
-    lf_print_error("Failed to create UDP socket.");
-    return -1;
-  }
-  set_socket_timeout_option(socket_descriptor, &timeout_time);
-  int used_port = set_socket_bind_option(socket_descriptor, port, true);
-  *final_socket = socket_descriptor;
-  *final_port = used_port;
-  return 0;
+  return create_socket_server(port, final_socket, final_port, UDP, true);
 }
 
 /**

From 5d1bf720f17a3a92041c7386c0583f6184324e08 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Mon, 20 Jan 2025 19:16:11 -0700
Subject: [PATCH 78/84] Minor fix and formatting.

---
 core/federated/RTI/rti_remote.c |  2 +-
 core/federated/federate.c       | 18 +++++++++---------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index a611c67f2..b5c53d80e 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -989,7 +989,7 @@ void* federate_info_thread_TCP(void* fed) {
       // Nothing more to do. Close the socket and exit.
       // Prevent multiple threads from closing the same socket at the same time.
       LF_MUTEX_LOCK(&rti_mutex);
-      shutdown_netdrv(my_fed->fed_netdrv, false); //  from unistd.h
+      shutdown_netdrv(my_fed->fed_netdrv, false);
       LF_MUTEX_UNLOCK(&rti_mutex);
       // FIXME: We need better error handling here, but do not stop execution here.
       break;
diff --git a/core/federated/federate.c b/core/federated/federate.c
index 66617c2b3..7690d8cad 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -17,9 +17,9 @@
 #include <arpa/inet.h>  // inet_ntop & inet_pton
 #include <netdb.h>      // Defines getaddrinfo(), freeaddrinfo() and struct addrinfo.
 #include <netinet/in.h> // Defines struct sockaddr_in
-#include <unistd.h> // Defines read(), write(), and close()
-#include <string.h> // Defines memset(), strnlen(), strncmp(), strncpy()
-#include <stdio.h>  // Defines strerror()
+#include <unistd.h>     // Defines read(), write(), and close()
+#include <string.h>     // Defines memset(), strnlen(), strncmp(), strncpy()
+#include <stdio.h>      // Defines strerror()
 
 #include <assert.h>
 #include <errno.h>   // Defined perror(), errno
@@ -2045,9 +2045,9 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
     tracepoint_federate_to_federate(send_ACK, _lf_my_fed_id, remote_fed_id, NULL);
 
     LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
-    write_to_netdrv_fail_on_error(_fed.netdrvs_for_inbound_p2p_connections[remote_fed_id], 1,
-                                  (unsigned char*)&response, &lf_outbound_netdrv_mutex,
-                                  "Failed to write MSG_TYPE_ACK in response to federate %d.", remote_fed_id);
+    write_to_netdrv_fail_on_error(_fed.netdrvs_for_inbound_p2p_connections[remote_fed_id], 1, (unsigned char*)&response,
+                                  &lf_outbound_netdrv_mutex, "Failed to write MSG_TYPE_ACK in response to federate %d.",
+                                  remote_fed_id);
     LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex);
 
     // Start a thread to listen for incoming messages from other federates.
@@ -2058,7 +2058,7 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) {
       // Failed to create a listening thread.
       LF_MUTEX_LOCK(&netdrv_mutex);
       if (_fed.netdrvs_for_inbound_p2p_connections[remote_fed_id] != NULL) {
-        shutdown_netdrv(netdrv, false);
+        shutdown_netdrv(_fed.netdrvs_for_inbound_p2p_connections[remote_fed_id], false);
         _fed.netdrvs_for_inbound_p2p_connections[remote_fed_id] = NULL;
       }
       LF_MUTEX_UNLOCK(&netdrv_mutex);
@@ -2348,7 +2348,7 @@ void lf_send_port_absent_to_federate(environment_t* env, interval_t additional_d
   netdrv_t* netdrv = _fed.netdrv_to_RTI;
 #else
   // Send the absent message directly to the federate
-  netdrv_t* netdrv  = _fed.netdrvs_for_outbound_p2p_connections[fed_ID];
+  netdrv_t* netdrv = _fed.netdrvs_for_outbound_p2p_connections[fed_ID];
 #endif
 
   if (netdrv == _fed.netdrv_to_RTI) {
@@ -2459,7 +2459,7 @@ int lf_send_tagged_message(environment_t* env, interval_t additional_delay, int
   // Use a mutex lock to prevent multiple threads from simultaneously sending.
   LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
 
-  netdrv_t* netdrv ;
+  netdrv_t* netdrv;
   if (message_type == MSG_TYPE_P2P_TAGGED_MESSAGE) {
     netdrv = _fed.netdrvs_for_outbound_p2p_connections[federate];
     tracepoint_federate_to_federate(send_P2P_TAGGED_MSG, _lf_my_fed_id, federate, &current_message_intended_tag);

From 4c6e5e71525157bf0cd813e40565cb216d9508b5 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Mon, 20 Jan 2025 19:42:35 -0700
Subject: [PATCH 79/84] Minor fix on clock-sync.

---
 core/federated/clock-sync.c      | 4 ++--
 network/impl/src/socket_common.c | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/core/federated/clock-sync.c b/core/federated/clock-sync.c
index d2b1e14e8..273af494b 100644
--- a/core/federated/clock-sync.c
+++ b/core/federated/clock-sync.c
@@ -283,8 +283,8 @@ static int handle_T1_clock_sync_message_common(unsigned char* buffer, void* sock
 
   // Write the reply to the socket.
   LF_PRINT_DEBUG("Sending T3 message to RTI.");
-  int result = use_udp ? write_to_socket(*(int*)socket_or_netdrv, sizeof(reply_buffer), reply_buffer)
-                       : write_to_netdrv((netdrv_t*)socket_or_netdrv, sizeof(reply_buffer), reply_buffer);
+  int result = use_udp ? write_to_socket(*(int*)socket_or_netdrv, 1 + sizeof(uint16_t), reply_buffer)
+                       : write_to_netdrv((netdrv_t*)socket_or_netdrv, 1 + sizeof(uint16_t), reply_buffer);
 
   if (result) {
     lf_print_error("Clock sync: Failed to send T3 message to RTI.");
diff --git a/network/impl/src/socket_common.c b/network/impl/src/socket_common.c
index b68152680..963145338 100644
--- a/network/impl/src/socket_common.c
+++ b/network/impl/src/socket_common.c
@@ -157,7 +157,7 @@ int create_socket_server(uint16_t port, int* final_socket, uint16_t* final_port,
 }
 
 int create_clock_server(uint16_t port, int* final_socket, uint16_t* final_port) {
-  return create_socket_server(port, final_socket, final_port, UDP, true);
+  return create_socket_server(port, final_socket, final_port, UDP, false);
 }
 
 /**

From e0d7fd1ef25b654440f65a191fe03fb70dd0401e Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Mon, 20 Jan 2025 20:34:11 -0700
Subject: [PATCH 80/84] Fix sending clocks.

---
 core/federated/RTI/rti_remote.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index b5c53d80e..e54f45e33 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -786,16 +786,17 @@ void send_physical_clock_UDP(unsigned char message_type, federate_info_t* fed) {
  */
 static void handle_physical_clock_sync_message_helper(federate_info_t* my_fed, bool send_coded_probe) {
   LF_MUTEX_LOCK(&rti_mutex);
+  if (!send_coded_probe) {
   // Reply with a T4 type message
   send_physical_clock(MSG_TYPE_CLOCK_SYNC_T4, my_fed);
   // Send the corresponding coded probe immediately after,
   // but only if this is a UDP channel.
-  if (send_coded_probe) {
+  else {
+    send_physical_clock_UDP(MSG_TYPE_CLOCK_SYNC_T4, my_fed);
     send_physical_clock_UDP(MSG_TYPE_CLOCK_SYNC_CODED_PROBE, my_fed);
   }
   LF_MUTEX_UNLOCK(&rti_mutex);
 }
-
 void handle_physical_clock_sync_message(federate_info_t* my_fed) {
   handle_physical_clock_sync_message_helper(my_fed, false);
 }

From 30ecc8b757ad32901d8e361a4361c70c837e220d Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Mon, 20 Jan 2025 20:34:11 -0700
Subject: [PATCH 81/84] Fix sending clocks.

---
 core/federated/RTI/rti_remote.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index b5c53d80e..ec577de07 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -786,16 +786,17 @@ void send_physical_clock_UDP(unsigned char message_type, federate_info_t* fed) {
  */
 static void handle_physical_clock_sync_message_helper(federate_info_t* my_fed, bool send_coded_probe) {
   LF_MUTEX_LOCK(&rti_mutex);
+  if (!send_coded_probe) {
   // Reply with a T4 type message
   send_physical_clock(MSG_TYPE_CLOCK_SYNC_T4, my_fed);
   // Send the corresponding coded probe immediately after,
   // but only if this is a UDP channel.
-  if (send_coded_probe) {
+  } else {
+    send_physical_clock_UDP(MSG_TYPE_CLOCK_SYNC_T4, my_fed);
     send_physical_clock_UDP(MSG_TYPE_CLOCK_SYNC_CODED_PROBE, my_fed);
   }
   LF_MUTEX_UNLOCK(&rti_mutex);
 }
-
 void handle_physical_clock_sync_message(federate_info_t* my_fed) {
   handle_physical_clock_sync_message_helper(my_fed, false);
 }

From 087b9e2af2b7b1e709cf19570a3adb04e34a1e9b Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Tue, 21 Jan 2025 12:01:35 -0700
Subject: [PATCH 82/84] Fix netdrv pointer check.

---
 core/federated/federate.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index 7690d8cad..919732b4e 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -821,7 +821,7 @@ static void close_outbound_netdrv(int fed_id) {
   // abnormal termination, in which case it will just close the network driver.
   if (_lf_normal_termination) {
     LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex);
-    if (_fed.netdrvs_for_outbound_p2p_connections[fed_id] >= 0) {
+    if (_fed.netdrvs_for_outbound_p2p_connections[fed_id] != NULL) {
       // Close the network driver by sending a FIN packet indicating that no further writes
       // are expected.  Then read until we get an EOF indication.
       shutdown_netdrv(_fed.netdrvs_for_outbound_p2p_connections[fed_id], true);

From 6dfabf56c102c50281826231a3d68c79ef1b3e9b Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Tue, 21 Jan 2025 12:24:49 -0700
Subject: [PATCH 83/84] Fix memory uninitialized error.

---
 network/impl/src/lf_socket_support.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/network/impl/src/lf_socket_support.c b/network/impl/src/lf_socket_support.c
index f8f7504ee..a73def181 100644
--- a/network/impl/src/lf_socket_support.c
+++ b/network/impl/src/lf_socket_support.c
@@ -22,6 +22,7 @@ netdrv_t* initialize_netdrv() {
 
   // Server initialization.
   priv->port = 0;
+  priv->user_specified_port = 0;
   priv->socket_descriptor = -1;
 
   // Federate initialization

From 6147df62685fa0aaa07d3db6fea9fd14a89c0ee6 Mon Sep 17 00:00:00 2001
From: Dongha Kim <dongha@asu.edu>
Date: Tue, 21 Jan 2025 12:36:29 -0700
Subject: [PATCH 84/84] Fix get getaddrinfo memory error.

---
 network/impl/src/socket_common.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/network/impl/src/socket_common.c b/network/impl/src/socket_common.c
index 963145338..56670ddb8 100644
--- a/network/impl/src/socket_common.c
+++ b/network/impl/src/socket_common.c
@@ -120,7 +120,7 @@ int set_socket_bind_option(int socket_descriptor, uint16_t specified_port, bool
 }
 
 int create_socket_server(uint16_t port, int* final_socket, uint16_t* final_port, socket_type_t sock_type,
-                  bool increment_port_on_retry) {
+                         bool increment_port_on_retry) {
   int socket_descriptor;
   struct timeval timeout_time;
   if (sock_type == TCP) {
@@ -252,8 +252,10 @@ int connect_to_socket(int sock, const char* hostname, int port) {
       }
       lf_print_warning("Could not connect. Will try again every " PRINTF_TIME " nanoseconds. Connecting to port %d.\n",
                        CONNECT_RETRY_INTERVAL, used_port);
+      freeaddrinfo(result);
       continue;
     } else {
+      freeaddrinfo(result);
       break;
     }
     freeaddrinfo(result);