eliminate network_requests table, fix beta/canary appcast bug

api/Sources/Api/Configure/migrations.swift

@@ -21,5 +21,6 @@ extension Configure {
     app.migrations.add(DuringSuspensionActivity())
     app.migrations.add(ReworkPayments())
     app.migrations.add(AddUserShowSuspensionActivity())
+    app.migrations.add(EliminateNetworkDecisionsTable())
   }
 }

api/Sources/Api/Database/Migration.swift

@@ -12,6 +12,8 @@ enum MigrationDirection: String {
   case down
 }
 
+enum Deleted {}
+
 protocol TableNamingMigration {
   static var tableName: String { get }
 }

api/Sources/Api/Database/Migrations/001_AdminTables.swift

@@ -21,16 +21,16 @@ struct AdminTables: GertieMigration {
   // table: admins
 
   func upAdmins(_ sql: SQLDatabase) async throws {
-    try await sql.create(enum: Admin.M1.LegacySubscriptionStatus.self)
+    try await sql.create(enum: Admin.M1.Deleted.SubscriptionStatus.self)
     try await sql.create(table: Admin.M1.self) {
       Column(.id, .uuid, .primaryKey)
       Column(Admin.M1.email, .text, .unique)
       Column(Admin.M1.password, .text)
       Column(Admin.M1.subscriptionId, .text, .nullable)
       Column(
         Admin.M1.subscriptionStatus,
-        .enum(Admin.M1.LegacySubscriptionStatus.self),
-        default: .enumValue(Admin.M1.LegacySubscriptionStatus.pendingEmailVerification)
+        .enum(Admin.M1.Deleted.SubscriptionStatus.self),
+        default: .enumValue(Admin.M1.Deleted.SubscriptionStatus.pendingEmailVerification)
       )
       Column(.createdAt, .timestampWithTimezone)
       Column(.updatedAt, .timestampWithTimezone)
@@ -160,19 +160,21 @@ extension Admin {
     static let subscriptionStatus = FieldKey("subscription_status")
     static let subscriptionStatusTypeName = "enum_admin_subscription_status"
 
-    enum LegacySubscriptionStatus: String, Codable, CaseIterable, PostgresEnum {
-      var typeName: String { Admin.M1.subscriptionStatusTypeName }
-      case pendingEmailVerification
-      case emailVerified
-      case signupCanceled
-      case complimentary
-      case incomplete
-      case incompleteExpired
-      case trialing
-      case active
-      case pastDue
-      case canceled
-      case unpaid
+    enum Deleted {
+      enum SubscriptionStatus: String, Codable, CaseIterable, PostgresEnum {
+        var typeName: String { Admin.M1.subscriptionStatusTypeName }
+        case pendingEmailVerification
+        case emailVerified
+        case signupCanceled
+        case complimentary
+        case incomplete
+        case incompleteExpired
+        case trialing
+        case active
+        case pastDue
+        case canceled
+        case unpaid
+      }
     }
   }
 }

api/Sources/Api/Database/Migrations/005_RequestTables.swift

@@ -19,26 +19,26 @@ struct RequestTables: GertieMigration {
   // table: network_decisions
 
   let networkDecisionDeviceFk = Constraint.foreignKey(
-    from: NetworkDecision.M5.self,
+    from: Deleted.NetworkDecisionTable.M5.self,
     to: Device.M3.self,
-    thru: NetworkDecision.M5.deviceId,
+    thru: Deleted.NetworkDecisionTable.M5.deviceId,
     onDelete: .cascade
   )
 
   let networkDecisionKeyFk = Constraint.foreignKey(
-    from: NetworkDecision.M5.self,
+    from: Deleted.NetworkDecisionTable.M5.self,
     to: Key.M2.self,
-    thru: NetworkDecision.M5.responsibleKeyId,
+    thru: Deleted.NetworkDecisionTable.M5.responsibleKeyId,
     onDelete: .cascade
   )
 
   func upNetworkDecisions(_ sql: SQLDatabase) async throws {
-    typealias M = NetworkDecision.M5
+    typealias M = Deleted.NetworkDecisionTable.M5
 
     try await sql.create(enum: NetworkDecisionVerdict.self)
     try await sql.create(enum: NetworkDecisionReason.self)
 
-    try await sql.create(table: NetworkDecision.M5.self) {
+    try await sql.create(table: Deleted.NetworkDecisionTable.M5.self) {
       Column(.id, .uuid, .primaryKey)
       Column(M.deviceId, .uuid)
       Column(M.verdict, .enum(NetworkDecisionVerdict.self))
@@ -60,7 +60,7 @@ struct RequestTables: GertieMigration {
   func downNetworkDecisions(_ sql: SQLDatabase) async throws {
     try await sql.drop(constraint: networkDecisionDeviceFk)
     try await sql.drop(constraint: networkDecisionKeyFk)
-    try await sql.drop(table: NetworkDecision.M5.self)
+    try await sql.drop(table: Deleted.NetworkDecisionTable.M5.self)
     try await sql.drop(enum: NetworkDecisionVerdict.self)
     try await sql.drop(enum: NetworkDecisionReason.self)
   }
@@ -76,7 +76,7 @@ struct RequestTables: GertieMigration {
 
   let unlockRequestNetworkDecisionFk = Constraint.foreignKey(
     from: UnlockRequest.M5.self,
-    to: NetworkDecision.M5.self,
+    to: Deleted.NetworkDecisionTable.M5.self,
     thru: UnlockRequest.M5.networkDecisionId,
     onDelete: .cascade
   )
@@ -140,21 +140,23 @@ extension RequestTables {
   }
 }
 
-extension NetworkDecision {
-  enum M5: TableNamingMigration {
-    static let tableName = "network_decisions"
-    static let verdictTypeName = "enum_network_decision_verdict"
-    static let reasonTypeName = "enum_network_decision_reason"
-    static let deviceId = FieldKey("device_id")
-    static let verdict = FieldKey("verdict")
-    static let reason = FieldKey("reason")
-    static let ipProtocolNumber = FieldKey("ip_protocol_number")
-    static let hostname = FieldKey("hostname")
-    static let ipAddress = FieldKey("ip_address")
-    static let url = FieldKey("url")
-    static let appBundleId = FieldKey("app_bundle_id")
-    static let count = FieldKey("count")
-    static let responsibleKeyId = FieldKey("responsible_key_id")
+extension Deleted {
+  enum NetworkDecisionTable {
+    enum M5: TableNamingMigration {
+      static let tableName = "network_decisions"
+      static let verdictTypeName = "enum_network_decision_verdict"
+      static let reasonTypeName = "enum_network_decision_reason"
+      static let deviceId = FieldKey("device_id")
+      static let verdict = FieldKey("verdict")
+      static let reason = FieldKey("reason")
+      static let ipProtocolNumber = FieldKey("ip_protocol_number")
+      static let hostname = FieldKey("hostname")
+      static let ipAddress = FieldKey("ip_address")
+      static let url = FieldKey("url")
+      static let appBundleId = FieldKey("app_bundle_id")
+      static let count = FieldKey("count")
+      static let responsibleKeyId = FieldKey("responsible_key_id")
+    }
   }
 }
 

api/Sources/Api/Database/Migrations/011_DeviceRefactor.swift

@@ -258,9 +258,9 @@ struct DeviceRefactor: GertieMigration {
   }
 
   let networkDecisionFk = Constraint.foreignKey(
-    from: NetworkDecision.M5.self,
+    from: Deleted.NetworkDecisionTable.M5.self,
     to: UserDevice.M11.self,
-    thru: NetworkDecision.M11.userDeviceId,
+    thru: Deleted.NetworkDecisionTable.M11.userDeviceId,
     onDelete: .cascade
   )
 
@@ -349,9 +349,9 @@ struct DeviceRefactor: GertieMigration {
       to: Screenshot.M11.userDeviceId
     )
     try await sql.renameColumn(
-      on: NetworkDecision.M5.self,
-      from: NetworkDecision.M5.deviceId,
-      to: NetworkDecision.M11.userDeviceId
+      on: Deleted.NetworkDecisionTable.M5.self,
+      from: Deleted.NetworkDecisionTable.M5.deviceId,
+      to: Deleted.NetworkDecisionTable.M11.userDeviceId
     )
     try await sql.renameColumn(
       on: UnlockRequest.M5.self,
@@ -387,9 +387,9 @@ struct DeviceRefactor: GertieMigration {
       to: Screenshot.M4.deviceId
     )
     try await sql.renameColumn(
-      on: NetworkDecision.M5.self,
-      from: NetworkDecision.M11.userDeviceId,
-      to: NetworkDecision.M5.deviceId
+      on: Deleted.NetworkDecisionTable.M5.self,
+      from: Deleted.NetworkDecisionTable.M11.userDeviceId,
+      to: Deleted.NetworkDecisionTable.M5.deviceId
     )
     try await sql.renameColumn(
       on: UnlockRequest.M5.self,
@@ -509,7 +509,7 @@ extension Screenshot {
   }
 }
 
-extension NetworkDecision {
+extension Deleted.NetworkDecisionTable {
   enum M11 {
     static let userDeviceId = FieldKey("user_device_id")
   }

api/Sources/Api/Database/Migrations/016_ReworkPayments.swift

@@ -5,7 +5,7 @@ struct ReworkPayments: GertieMigration {
   func up(sql: SQLDatabase) async throws {
     let admins = try await getAdmins(sql)
     try await sql.dropColumn(Admin.M1.subscriptionStatus, on: Admin.M1.self)
-    try await sql.drop(enum: Admin.M1.LegacySubscriptionStatus.self)
+    try await sql.drop(enum: Admin.M1.Deleted.SubscriptionStatus.self)
     try await sql.create(enum: Admin.SubscriptionStatus.self)
     try await sql.dropColumn(.deletedAt, on: Admin.M1.self)
 
@@ -68,7 +68,7 @@ struct ReworkPayments: GertieMigration {
 
     try await sql.dropColumn(Admin.M1.subscriptionStatus, on: Admin.M1.self)
     try await sql.drop(enum: Admin.SubscriptionStatus.self)
-    try await sql.create(enum: Admin.M1.LegacySubscriptionStatus.self)
+    try await sql.create(enum: Admin.M1.Deleted.SubscriptionStatus.self)
 
     try await sql.addColumn(
       .deletedAt,
@@ -80,13 +80,13 @@ struct ReworkPayments: GertieMigration {
     try await sql.addColumn(
       Admin.M1.subscriptionStatus,
       on: Admin.M1.self,
-      type: .enum(Admin.M1.LegacySubscriptionStatus.self),
-      default: .enumValue(Admin.M1.LegacySubscriptionStatus.pendingEmailVerification) // <-- temp
+      type: .enum(Admin.M1.Deleted.SubscriptionStatus.self),
+      default: .enumValue(Admin.M1.Deleted.SubscriptionStatus.pendingEmailVerification) // <-- temp
     )
 
     try await sql.dropColumn(Admin.M16.subscriptionStatusExpiration, on: Admin.M1.self)
 
-    let updates: [(UUID, Admin.M1.LegacySubscriptionStatus)] = admins.map { admin in
+    let updates: [(UUID, Admin.M1.Deleted.SubscriptionStatus)] = admins.map { admin in
       switch admin.subscriptionStatus {
       case Admin.SubscriptionStatus.pendingEmailVerification.rawValue:
         return (admin.id, .pendingEmailVerification)

api/Sources/Api/Database/Migrations/018_EliminateNetworkDecisionsTable.swift

@@ -0,0 +1,152 @@
+import DuetSQL
+import FluentSQL
+import Gertie
+
+struct EliminateNetworkDecisionsTable: GertieMigration {
+  func up(sql: SQLDatabase) async throws {
+    // query legacy data for migration
+    let decisions = try await Current.db.customQuery(LegacyDecisions.self)
+    let unlockRequestIds = try await Current.db.customQuery(UnlockRequestIds.self)
+
+    // alter unlock_requests table
+    try await sql.drop(constraint: RequestTables().unlockRequestNetworkDecisionFk)
+    try await sql.dropColumn(
+      UnlockRequest.M5.networkDecisionId,
+      on: UnlockRequest.M5.self
+    )
+    try await sql.addColumn(
+      UnlockRequest.M18.appBundleId,
+      on: UnlockRequest.M5.self,
+      type: .text,
+      default: .text("--temp--") // temp, during migration
+    )
+    try await sql.addColumn(
+      UnlockRequest.M18.url,
+      on: UnlockRequest.M5.self,
+      type: .text,
+      nullable: true
+    )
+    try await sql.addColumn(
+      UnlockRequest.M18.hostname,
+      on: UnlockRequest.M5.self,
+      type: .text,
+      nullable: true
+    )
+    try await sql.addColumn(
+      UnlockRequest.M18.ipAddress,
+      on: UnlockRequest.M5.self,
+      type: .text,
+      nullable: true
+    )
+
+    // transfer data from network_decisions to unlock_requests
+    for unlock in unlockRequestIds {
+      guard let decision = decisions.first(where: { $0.id == unlock.networkDecisionId }) else {
+        fatalError("Decision not found for unlock request `\(unlock.id)`")
+      }
+      try await sql.execute("""
+        UPDATE \(table: UnlockRequest.M5.self)
+        SET
+          \(col: UnlockRequest.M18.appBundleId) = '\(raw: decision.appBundleId)',
+          \(col: UnlockRequest.M18.url) = \(nullable: decision.url),
+          \(col: UnlockRequest.M18.hostname) = \(nullable: decision.hostname),
+          \(col: UnlockRequest.M18.ipAddress) = \(nullable: decision.ipAddress)
+        WHERE \(col: .id) = '\(uuid: unlock.id)'
+      """)
+    }
+
+    // remove temporary default, all rows should have a real value now
+    // and all new rows will have a value
+    try await sql.dropDefault(from: UnlockRequest.M18.appBundleId, on: UnlockRequest.M5.self)
+
+    // drop network_decisions table
+    try await sql.drop(constraint: RequestTables().networkDecisionKeyFk)
+    try await sql.drop(constraint: DeviceRefactor().networkDecisionFk)
+    try await sql.drop(table: Deleted.NetworkDecisionTable.M5.self)
+    try await sql.drop(enum: NetworkDecisionVerdict.self)
+    try await sql.drop(enum: NetworkDecisionReason.self)
+  }
+
+  func down(sql: SQLDatabase) async throws {
+    // recreate network_decisions table
+    try await sql.create(enum: NetworkDecisionVerdict.self)
+    try await sql.create(enum: NetworkDecisionReason.self)
+    try await sql.create(table: Deleted.NetworkDecisionTable.M5.self) {
+      Column(.id, .uuid, .primaryKey)
+      Column(Deleted.NetworkDecisionTable.M11.userDeviceId, .uuid)
+      Column(Deleted.NetworkDecisionTable.M5.verdict, .enum(NetworkDecisionVerdict.self))
+      Column(Deleted.NetworkDecisionTable.M5.reason, .enum(NetworkDecisionReason.self))
+      Column(Deleted.NetworkDecisionTable.M5.ipProtocolNumber, .bigint, .nullable)
+      Column(Deleted.NetworkDecisionTable.M5.hostname, .text, .nullable)
+      Column(Deleted.NetworkDecisionTable.M5.ipAddress, .text, .nullable)
+      Column(Deleted.NetworkDecisionTable.M5.url, .text, .nullable)
+      Column(Deleted.NetworkDecisionTable.M5.appBundleId, .text, .nullable)
+      Column(Deleted.NetworkDecisionTable.M5.count, .bigint)
+      Column(Deleted.NetworkDecisionTable.M5.responsibleKeyId, .uuid, .nullable)
+      Column(.createdAt, .timestampWithTimezone)
+    }
+    try await sql.add(constraint: RequestTables().networkDecisionKeyFk)
+    try await sql.add(constraint: DeviceRefactor().networkDecisionFk)
+
+    // drop all unlock_requests, we don't have required FK to restore them
+    // and they are transient requests anyway, and this down should never run in prod
+    try await sql.execute("DELETE FROM \(table: UnlockRequest.M5.self)")
+
+    // restore unlock_requests table
+    try await sql.dropColumn(UnlockRequest.M18.appBundleId, on: UnlockRequest.M5.self)
+    try await sql.dropColumn(UnlockRequest.M18.url, on: UnlockRequest.M5.self)
+    try await sql.dropColumn(UnlockRequest.M18.hostname, on: UnlockRequest.M5.self)
+    try await sql.dropColumn(UnlockRequest.M18.ipAddress, on: UnlockRequest.M5.self)
+    try await sql.addColumn(
+      UnlockRequest.M5.networkDecisionId,
+      on: UnlockRequest.M5.self,
+      type: .uuid
+    )
+    try await sql.add(constraint: RequestTables().unlockRequestNetworkDecisionFk)
+  }
+}
+
+// extensions
+
+extension UnlockRequest {
+  enum M18 {
+    static let appBundleId = FieldKey("app_bundle_id")
+    static let url = FieldKey("url")
+    static let hostname = FieldKey("hostname")
+    static let ipAddress = FieldKey("ip_address")
+  }
+}
+
+// query
+
+private struct LegacyDecisions: CustomQueryable {
+  var id: UUID
+  var appBundleId: String
+  var url: String?
+  var hostname: String?
+  var ipAddress: String?
+
+  static func query(numBindings: Int) -> String {
+    """
+    SELECT
+      id,
+      \(Deleted.NetworkDecisionTable.M5.appBundleId),
+      \(Deleted.NetworkDecisionTable.M5.url),
+      \(Deleted.NetworkDecisionTable.M5.hostname),
+      \(Deleted.NetworkDecisionTable.M5.ipAddress)
+    FROM \(Deleted.NetworkDecisionTable.M5.tableName)
+    """
+  }
+}
+
+private struct UnlockRequestIds: CustomQueryable {
+  var id: UUID
+  var networkDecisionId: UUID
+
+  static func query(numBindings: Int) -> String {
+    """
+    SELECT id, \(UnlockRequest.M5.networkDecisionId)
+    FROM \(UnlockRequest.M5.tableName)
+    """
+  }
+}