From 93aeabdbd1bc0839558fe98b7b639f98ff4a7deb Mon Sep 17 00:00:00 2001 From: Jared Henderson <jaredh159@users.noreply.github.com> Date: Tue, 7 Jan 2025 16:46:50 -0500 Subject: [PATCH] api: dump/sync pub keychains and app data to staging --- api/Sources/Api/Configure/jobs.swift | 3 + api/Sources/Api/Configure/router.swift | 7 + api/Sources/Api/Environment/EnvVars.swift | 4 + api/Sources/Api/Extend/Request.swift | 6 + api/Sources/Api/Extend/XSlack.swift | 15 ++- api/Sources/Api/Routes/Reset/AdminBetsy.swift | 9 +- api/Sources/Api/Routes/Reset/Reset.swift | 49 ++----- api/Sources/Api/Routes/Reset/ResetRoute.swift | 21 ++- .../Api/Services/SyncStagingData.swift | 121 ++++++++++++++++++ justfile | 56 ++++---- x-kit/Sources/XCore/Error.swift | 20 +++ 11 files changed, 244 insertions(+), 67 deletions(-) create mode 100644 api/Sources/Api/Services/SyncStagingData.swift diff --git a/api/Sources/Api/Configure/jobs.swift b/api/Sources/Api/Configure/jobs.swift index dfff0d9f..933d62c3 100644 --- a/api/Sources/Api/Configure/jobs.swift +++ b/api/Sources/Api/Configure/jobs.swift @@ -11,5 +11,8 @@ public extension Configure { app.queues.schedule(DiskSpaceJob()).hourly().at(0) try app.queues.startScheduledJobs() + + app.asyncCommands.use(ResetCommand(), as: "reset") + app.asyncCommands.use(SyncStagingDataCommand(), as: "sync-staging-data") } } diff --git a/api/Sources/Api/Configure/router.swift b/api/Sources/Api/Configure/router.swift index 890f095e..2870e4ce 100644 --- a/api/Sources/Api/Configure/router.swift +++ b/api/Sources/Api/Configure/router.swift @@ -47,6 +47,13 @@ public extension Configure { } #endif + if app.env.mode != .staging { + app.get( + "dump-staging-data", ":token", + use: DumpStagingDataRoute.handler(_:) + ) + } + app.get( "releases", use: ReleasesRoute.handler(_:) diff --git a/api/Sources/Api/Environment/EnvVars.swift b/api/Sources/Api/Environment/EnvVars.swift index cadf933f..5c4fd2fa 100644 --- a/api/Sources/Api/Environment/EnvVars.swift +++ b/api/Sources/Api/Environment/EnvVars.swift @@ -162,4 +162,8 @@ extension Env: TestDependencyKey { public static var testValue: Env { .fromProcess(mode: .testing) } + + func getUUID(_ key: String) -> UUID? { + self.get(key).flatMap(UUID.init(uuidString:)) + } } diff --git a/api/Sources/Api/Extend/Request.swift b/api/Sources/Api/Extend/Request.swift index 01251ed2..abad46a4 100644 --- a/api/Sources/Api/Extend/Request.swift +++ b/api/Sources/Api/Extend/Request.swift @@ -92,6 +92,12 @@ extension Request { } } +extension Parameters { + func getUUID(_ key: String) -> UUID? { + self.get("token").flatMap(UUID.init(uuidString:)) + } +} + // helpers private extension UUID { diff --git a/api/Sources/Api/Extend/XSlack.swift b/api/Sources/Api/Extend/XSlack.swift index d5e07006..5607f065 100644 --- a/api/Sources/Api/Extend/XSlack.swift +++ b/api/Sources/Api/Extend/XSlack.swift @@ -19,6 +19,8 @@ extension XSlack.Slack.Client { func sysLog(to channel: String = "info", _ message: String) async { @Dependency(\.env) var env + @Dependency(\.logger) var logger + guard let token = env.get("SLACK_API_TOKEN"), env.mode != .staging else { return @@ -31,8 +33,17 @@ extension XSlack.Slack.Client { ) if let error = await send(slack, token) { - with(dependency: \.logger) - .error("Error sending slack: \(error)") + logger.error("Error sending slack: \(error)") + } + + if channel == "errors" { + logger.error("Slack sysLog to #errors: \(message)") + } else { + logger.info("Slack sysLog to #info: \(message)") } } + + func sysLogErr(_ message: String) async { + await self.sysLog(to: "errors", message) + } } diff --git a/api/Sources/Api/Routes/Reset/AdminBetsy.swift b/api/Sources/Api/Routes/Reset/AdminBetsy.swift index 335ef302..d775f469 100644 --- a/api/Sources/Api/Routes/Reset/AdminBetsy.swift +++ b/api/Sources/Api/Routes/Reset/AdminBetsy.swift @@ -1,4 +1,5 @@ import Dependencies +import DuetSQL import Gertie import Vapor import XCore @@ -58,12 +59,16 @@ enum AdminBetsy { let (jimmy, sally, _) = try await createUsers(betsy) let (musicTheory, misc) = try await createKeychains(betsy) + let firstPublicKeychain = try await Keychain.query() + .where(.isPublic == true) + .first(in: db) + try await db.create([ UserKeychain(userId: jimmy.id, keychainId: musicTheory.id), UserKeychain(userId: jimmy.id, keychainId: misc.id), - UserKeychain(userId: jimmy.id, keychainId: Reset.Ids.htcKeychain), + UserKeychain(userId: jimmy.id, keychainId: firstPublicKeychain.id), UserKeychain(userId: sally.id, keychainId: misc.id), - UserKeychain(userId: sally.id, keychainId: Reset.Ids.htcKeychain), + UserKeychain(userId: sally.id, keychainId: firstPublicKeychain.id), ]) try await self.createUserActivity() diff --git a/api/Sources/Api/Routes/Reset/Reset.swift b/api/Sources/Api/Routes/Reset/Reset.swift index 4e6751e8..73e15659 100644 --- a/api/Sources/Api/Routes/Reset/Reset.swift +++ b/api/Sources/Api/Routes/Reset/Reset.swift @@ -7,43 +7,22 @@ import XCore enum Reset { static func run() async throws { @Dependency(\.db) var db - try await db.delete(all: Admin.self) - try await self.createHtcPublicKeychain() + try await Admin.query() + .where(.id != Admin.Id.stagingPublicKeychainOwner) + .delete(in: db) try await AdminBetsy.create() } - static func createHtcPublicKeychain() async throws { + static func ensurePublicKeychainOwner() async throws { @Dependency(\.db) var db - let jared = try await db.create(Admin( - email: "jared-htc-author" |> self.testEmail, - password: try Bcrypt.hash("jared123") - )) - try await self.createKeychain( - id: Ids.htcKeychain, - adminId: jared.id, - name: "HTC", - isPublic: true, - description: "Keys for student's in Jared's How to Computer (HTC) class.", - keys: [ - .anySubdomain(domain: .init("howtocomputer.link")!, scope: .unrestricted), - .anySubdomain(domain: .init("tailwind.css")!, scope: .webBrowsers), - .anySubdomain(domain: .init("vsassets.io")!, scope: .single(.identifiedAppSlug("vscode"))), - .anySubdomain( - domain: .init("executeprogram.com")!, - scope: .single(.bundleId("com.apple.Safari")) - ), - .domain(domain: .init("friendslibrary.com")!, scope: .unrestricted), - .domain(domain: .init("developer.mozilla.org")!, scope: .webBrowsers), - .domain(domain: .init("nextjs.org")!, scope: .webBrowsers), - .domain(domain: .init("www.snowpack.dev")!, scope: .webBrowsers), - .domain(domain: .init("regexr.com")!, scope: .webBrowsers), - .domain(domain: .init("api.netlify.com")!, scope: .unrestricted), - .domain(domain: .init("registry.npmjs.org")!, scope: .single(.bundleId(".node"))), - .skeleton(scope: .identifiedAppSlug("slack")), - .skeleton(scope: .bundleId("Y48LQG59RS.com.sequelpro.SequelPro")), - .ipAddress(ipAddress: .init("76.88.114.31")!, scope: .unrestricted), - ] - ) + let existing = try? await db.find(Admin.Id.stagingPublicKeychainOwner) + if existing == nil { + try await db.create(Admin( + id: .stagingPublicKeychainOwner, + email: "public-keychain-owner" |> self.testEmail, + password: try Bcrypt.hash("\(UUID())") + )) + } } @discardableResult @@ -158,10 +137,6 @@ enum Reset { )) } } - - enum Ids { - static let htcKeychain = Keychain.Id.from("AAA00000-0000-0000-0000-000000000000") - } } enum TimestampAdjustment { diff --git a/api/Sources/Api/Routes/Reset/ResetRoute.swift b/api/Sources/Api/Routes/Reset/ResetRoute.swift index 6c727ffd..acc8c380 100644 --- a/api/Sources/Api/Routes/Reset/ResetRoute.swift +++ b/api/Sources/Api/Routes/Reset/ResetRoute.swift @@ -1,6 +1,19 @@ import Dependencies import Vapor +struct ResetCommand: AsyncCommand { + struct Signature: CommandSignature {} + var help: String { "Reset staging data" } + + func run(using context: CommandContext, signature: Signature) async throws { + guard get(dependency: \.env).mode != .prod else { + fatalError("`reset` is only allowed in non-prod environments") + } + try await Reset.run() + try await SyncStagingDataCommand().run(using: context, signature: .init()) + } +} + enum ResetRoute { @Sendable static func handler(_ request: Request) async throws -> Response { guard request.env.mode != .prod else { @@ -8,7 +21,13 @@ enum ResetRoute { } try await Reset.run() + try await SyncStagingDataCommand() + .exec(client: request.application.http.client.shared) + .mapError { Abort(.internalServerError, reason: $0.message) } + .get() + let betsy = try await request.context.db.find(AdminBetsy.Ids.betsy) + return .init( status: .ok, headers: ["Content-Type": "text/html"], @@ -24,7 +43,7 @@ enum ResetRoute { in <code>./dashboard/.env</code> </p> <pre style="background: #eaeaea; padding: 1em 1em 0 1em;"> - SNOWPACK_PUBLIC_TEST_ADMIN_CREDS=\(betsy.id.lowercased):\(betsy.id.lowercased) + VITE_ADMIN_CREDS=\(betsy.id.lowercased):\(betsy.id.lowercased) </pre> <p> ...or use her email address: diff --git a/api/Sources/Api/Services/SyncStagingData.swift b/api/Sources/Api/Services/SyncStagingData.swift new file mode 100644 index 00000000..21bd37b7 --- /dev/null +++ b/api/Sources/Api/Services/SyncStagingData.swift @@ -0,0 +1,121 @@ +import AsyncHTTPClient +import Dependencies +import DuetSQL +import Foundation +import NIOFoundationCompat +import Vapor +import XCore + +enum DumpStagingDataRoute { + @Sendable static func handler(_ request: Request) async throws -> some AsyncResponseEncodable { + @Dependency(\.db) var db + @Dependency(\.env) var env + + guard let reqToken = request.parameters.getUUID("token"), + let envToken = env.getUUID("SYNC_STAGING_TOKEN"), + reqToken == envToken, + env.mode != .staging else { + throw Abort(.notFound) + } + + var keychains = try await Keychain.query() + .where(.isPublic == true) + .all(in: db) + for i in 0 ..< keychains.count { + keychains[i].authorId = .stagingPublicKeychainOwner + } + + return SyncStagingData( + keychains: keychains, + keys: try await Key.query() + .where(.keychainId |=| keychains.map(\.id)) + .all(in: db), + appBundleIds: try await AppBundleId.query().all(in: db), + appCategories: try await AppCategory.query().all(in: db), + identifiedApps: try await IdentifiedApp.query().all(in: db) + ) + } +} + +struct SyncStagingDataCommand: AsyncCommand { + struct Signature: CommandSignature {} + var help: String { "Sync staging data" } + + @Dependency(\.db) var db + @Dependency(\.env) var env + @Dependency(\.logger) var logger + + func run(using context: CommandContext, signature: Signature) async throws { + switch await self.exec(client: context.application.http.client.shared) { + case .success: + self.logger.info("Synced staging data") + case .failure(let error): + self.logger.error("Error syncing staging data: \(error)") + } + } + + func exec(client: HTTPClient) async -> Result<Void, StringError> { + guard self.env.mode != .prod, let token = env.getUUID("SYNC_STAGING_TOKEN") else { + return .failure("No SYNC_STAGING_TOKEN") + } + + let dataResult = await fetchData(client, token) + guard case .success(let data) = dataResult else { + return dataResult.map { _ in () } + } + + do { + try await self.db.delete(all: AppBundleId.self) + try await self.db.delete(all: IdentifiedApp.self) + try await self.db.delete(all: AppCategory.self) + try await self.db.create(data.appCategories) + try await self.db.create(data.identifiedApps) + try await self.db.create(data.appBundleIds) + + try await Reset.ensurePublicKeychainOwner() + try await Keychain.query() + .where(.id |=| data.keychains.map(\.id)) + .delete(in: self.db) + try await self.db.create(data.keychains) + try await self.db.create(data.keys) + return .success(()) + } catch { + return .failure("Error saving staging data: \(error)") + } + } + + func fetchData( + _ client: HTTPClient, + _ token: UUID + ) async -> Result<SyncStagingData, StringError> { + do { + let request = + HTTPClientRequest(url: "https://api.gertrude.app/dump-staging-data/\(token.lowercased)") + let response = try await client.execute(request, timeout: .seconds(5)) + guard response.status == .ok else { + return .failure("Unexpected status syncing staging data: \(response.status)") + } + let buf = try await response.body.collect(upTo: 1024 * 1024) // 1 MB + guard let json = buf.getString(at: 0, length: buf.readableBytes, encoding: .utf8) else { + return .failure("Error converting buffer to string") + } + let data = try JSON.decode(json, as: SyncStagingData.self, [.isoDates]) + return .success(data) + } catch { + return .failure("Error syncing staging data: \(error)") + } + } +} + +struct SyncStagingData: Content { + var keychains: [Keychain] + var keys: [Key] + var appBundleIds: [AppBundleId] + var appCategories: [AppCategory] + var identifiedApps: [IdentifiedApp] +} + +extension Admin.Id { + static let stagingPublicKeychainOwner = + Self(UUID(uuidString: "FFFFFFFF-FFFF-FFFF-FFFF-FFFFFFFFFFFF")!) +} diff --git a/justfile b/justfile index da769db6..266ec564 100644 --- a/justfile +++ b/justfile @@ -35,19 +35,25 @@ watch-api: @just watch-swift . 'just run-api' 'macapp/**/*' run-api: build-api - @just exec-api serve + @just exec-api serve run-api-ip: build-api - @just exec-api serve --hostname 192.168.10.227 + @just exec-api serve --hostname 192.168.10.227 build-api: - @cd api && swift build + @cd api && swift build migrate-up: build-api - @just exec-api migrate --yes + @just exec-api migrate --yes migrate-down: build-api - @just exec-api migrate --revert --yes + @just exec-api migrate --revert --yes + +reset: build-api + just exec-api reset + +sync-staging-data: build-api + @just exec-api sync-staging-data nuke-test-db: @killall -q Postico; dropdb --if-exists gertrude_test; createdb gertrude_test @@ -72,44 +78,44 @@ watch-web-email template: # infra db-sync: - @node ../infra/db-sync.mjs + @node ../infra/db-sync.mjs sync-env: - @node ../infra/sync-env.mjs + @node ../infra/sync-env.mjs # root build: - @just nx-run-many build + @just nx-run-many build test: - @just nx-run-many test + @just nx-run-many test check: - @just build - @just test + @just build + @just test exclude: - @find . -path '**/.build/**/swift-nio*/**/hash.txt' -delete - @find . -path '**/.build/**/swift-nio*/**/*_nasm.inc' -delete - @find . -path '**/.build/**/swift-nio*/**/*_sha1.sh' -delete - @find . -path '**/.build/**/swift-nio*/**/*_llhttp.sh' -delete - @find . -path '**/.build/**/swift-nio*/**/LICENSE-MIT' -delete + @find . -path '**/.build/**/swift-nio*/**/hash.txt' -delete + @find . -path '**/.build/**/swift-nio*/**/*_nasm.inc' -delete + @find . -path '**/.build/**/swift-nio*/**/*_sha1.sh' -delete + @find . -path '**/.build/**/swift-nio*/**/*_llhttp.sh' -delete + @find . -path '**/.build/**/swift-nio*/**/LICENSE-MIT' -delete nx-reset: - @pnpm nx reset + @pnpm nx reset clean: nx-reset - @rm -rf node_modules/.cache - @rm -rf api/.build - @rm -rf duet/.build - @rm -rf pairql/.build - @rm -rf pairql-macapp/.build - @rm -rf shared/.build - @rm -rf x-kit/.build + @rm -rf node_modules/.cache + @rm -rf api/.build + @rm -rf duet/.build + @rm -rf pairql/.build + @rm -rf pairql-macapp/.build + @rm -rf shared/.build + @rm -rf x-kit/.build clean-api-tests: - @cd api && find .build -name '*AppTests*' -delete + @cd api && find .build -name '*AppTests*' -delete # helpers diff --git a/x-kit/Sources/XCore/Error.swift b/x-kit/Sources/XCore/Error.swift index 552f456c..0fc811a2 100644 --- a/x-kit/Sources/XCore/Error.swift +++ b/x-kit/Sources/XCore/Error.swift @@ -14,3 +14,23 @@ public enum XCore { } } } + +public struct StringError: Error { + public var message: String + + public init(_ message: String) { + self.message = message + } +} + +extension StringError: ExpressibleByStringLiteral { + public init(stringLiteral value: String) { + self.message = value + } +} + +public extension Result where Failure == StringError { + static func failure(_ message: String) -> Self { + .failure(.init(message)) + } +}