Merge pull request #1977 from DataDog/feature/continuous-benchmarks

RUM-5551 Add Benchmark Tests Project

.gitlab-ci.yml

@@ -5,6 +5,7 @@ stages:
   - ui-test
   - smoke-test
   - e2e-test
+  - benchmark-test
   - dogfood
   - release-build
   - release-publish
@@ -266,6 +267,24 @@ E2E Test (upload to s8s):
     - export DRY_RUN=${DRY_RUN:-0} # default to 0 if not specified
     - make e2e-build-upload ARTIFACTS_PATH="artifacts/e2e"
 
+# ┌────────────────────────────┐
+# │ Benchmark Test app upload: │
+# └────────────────────────────┘
+
+Benchmark Test (upload to s8s):
+  stage: benchmark-test
+  rules:
+    - if: '$CI_COMMIT_BRANCH == $DEVELOP_BRANCH'
+  artifacts:
+    paths:
+      - artifacts
+    expire_in: 2 weeks
+  script:
+    - ./tools/runner-setup.sh --xcode "$DEFAULT_XCODE" --datadog-ci
+    - make clean
+    - export DRY_RUN=${DRY_RUN:-0} # default to 0 if not specified
+    - make benchmark-build-upload ARTIFACTS_PATH="artifacts/benchmark"
+
 # ┌─────────────────┐
 # │ SDK dogfooding: │
 # └─────────────────┘

BenchmarkTests/BenchmarkTests.xcodeproj/.xcodesamplecode.plist

@@ -0,0 +1 @@
+<?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> <plist version="1.0"> <array/> </plist>

BenchmarkTests/BenchmarkTests.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>IDEDidComputeMac32BitWarning</key>
+	<true/>
+</dict>
+</plist>

BenchmarkTests/BenchmarkTests.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme

@@ -0,0 +1,78 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Scheme
+   LastUpgradeVersion = "1540"
+   version = "1.7">
+   <BuildAction
+      parallelizeBuildables = "YES"
+      buildImplicitDependencies = "YES"
+      buildArchitectures = "Automatic">
+      <BuildActionEntries>
+         <BuildActionEntry
+            buildForTesting = "YES"
+            buildForRunning = "YES"
+            buildForProfiling = "YES"
+            buildForArchiving = "YES"
+            buildForAnalyzing = "YES">
+            <BuildableReference
+               BuildableIdentifier = "primary"
+               BlueprintIdentifier = "D29F754C2C4AA07E00288638"
+               BuildableName = "Runner.app"
+               BlueprintName = "Runner"
+               ReferencedContainer = "container:BenchmarkTests.xcodeproj">
+            </BuildableReference>
+         </BuildActionEntry>
+      </BuildActionEntries>
+   </BuildAction>
+   <TestAction
+      buildConfiguration = "Debug"
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      shouldUseLaunchSchemeArgsEnv = "YES"
+      shouldAutocreateTestPlan = "YES">
+   </TestAction>
+   <LaunchAction
+      buildConfiguration = "Debug"
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      launchStyle = "0"
+      useCustomWorkingDirectory = "NO"
+      ignoresPersistentStateOnLaunch = "NO"
+      debugDocumentVersioning = "YES"
+      debugServiceExtension = "internal"
+      allowLocationSimulation = "YES">
+      <BuildableProductRunnable
+         runnableDebuggingMode = "0">
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "D29F754C2C4AA07E00288638"
+            BuildableName = "Runner.app"
+            BlueprintName = "Runner"
+            ReferencedContainer = "container:BenchmarkTests.xcodeproj">
+         </BuildableReference>
+      </BuildableProductRunnable>
+   </LaunchAction>
+   <ProfileAction
+      buildConfiguration = "Release"
+      shouldUseLaunchSchemeArgsEnv = "YES"
+      savedToolIdentifier = ""
+      useCustomWorkingDirectory = "NO"
+      debugDocumentVersioning = "YES">
+      <BuildableProductRunnable
+         runnableDebuggingMode = "0">
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "D29F754C2C4AA07E00288638"
+            BuildableName = "Runner.app"
+            BlueprintName = "Runner"
+            ReferencedContainer = "container:BenchmarkTests.xcodeproj">
+         </BuildableReference>
+      </BuildableProductRunnable>
+   </ProfileAction>
+   <AnalyzeAction
+      buildConfiguration = "Debug">
+   </AnalyzeAction>
+   <ArchiveAction
+      buildConfiguration = "Release"
+      revealArchiveInOrganizer = "YES">
+   </ArchiveAction>
+</Scheme>

BenchmarkTests/Benchmarks/Package.swift

@@ -0,0 +1,57 @@
+// swift-tools-version: 5.9
+// The swift-tools-version declares the minimum version of Swift required to build this package.
+
+import PackageDescription
+import Foundation
+
+let package = Package(
+    name: "DatadogBenchmarks",
+    products: [
+        .library(
+            name: "DatadogBenchmarks",
+            targets: ["Benchmarks"]
+        )
+    ]
+)
+
+func addOpenTelemetryDependency(_ version: Version) {
+    // The project must be open with the 'OTEL_SWIFT' env variable.
+    // Please run 'make benchmark-tests-open' from the root directory.
+    //
+    // Note: Carthage will still try to resolve dependencies of Xcode projects in
+    // sub directories, in this case the project will depend on the default
+    // 'DataDog/opentelemetry-swift-packages' depedency.
+    if ProcessInfo.processInfo.environment["OTEL_SWIFT"] != nil {
+        package.dependencies = [
+            .package(url: "https://github.com/open-telemetry/opentelemetry-swift", exact: version)
+        ]
+
+        package.targets = [
+            .target(
+                name: "Benchmarks",
+                dependencies: [
+                    .product(name: "OpenTelemetryApi", package: "opentelemetry-swift"),
+                    .product(name: "OpenTelemetrySdk", package: "opentelemetry-swift")
+                ],
+                swiftSettings: [.unsafeFlags(["-DOTEL_SWIFT"])]
+            )
+        ]
+
+    } else {
+        package.dependencies = [
+            .package(url: "https://github.com/DataDog/opentelemetry-swift-packages", exact: version)
+        ]
+
+        package.targets = [
+            .target(
+                name: "Benchmarks",
+                dependencies: [
+                    .product(name: "OpenTelemetryApi", package: "opentelemetry-swift-packages")
+                ],
+                swiftSettings: [.unsafeFlags(["-DOTEL_API"])]
+            )
+        ]
+    }
+}
+
+addOpenTelemetryDependency("1.6.0")

BenchmarkTests/Benchmarks/Sources/DatadogExporter.swift

@@ -0,0 +1,46 @@
+/*
+ * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
+ * This product includes software developed at Datadog (https://www.datadoghq.com/).
+ * Copyright 2019-Present Datadog, Inc.
+ */
+
+import Foundation
+
+#if OTEL_API
+#error("Benchmarks depends on opentelemetry-swift. Please open the project with 'make benchmark-tests-open'.")
+#endif
+
+#if OTEL_SWIFT
+import OpenTelemetrySdk
+
+public final class DatadogExporter: SpanExporter, MetricExporter {
+    private let session: URLSession
+
+    public convenience init() {
+        let configuration: URLSessionConfiguration = .ephemeral
+        configuration.urlCache = nil
+        self.init(session: URLSession(configuration: configuration))
+    }
+
+    public init(session: URLSession) {
+        self.session = session
+    }
+
+    public func export(spans: [SpanData]) -> SpanExporterResultCode {
+        return .success
+    }
+
+    public func export(metrics: [Metric], shouldCancel: (() -> Bool)?) -> MetricExporterResultCode {
+        return .success
+    }
+
+    public func flush() -> SpanExporterResultCode {
+        return .success
+    }
+
+    public func shutdown() {
+
+    }
+}
+
+#endif

BenchmarkTests/Makefile

@@ -0,0 +1,61 @@
+.PHONY: clean archive export upload
+
+REPO_ROOT := ../
+include ../tools/utils/common.mk
+
+BUILD_DIR := .build
+ARCHIVE_PATH := $(BUILD_DIR)/Runner.xcarchive
+IPA_PATH := $(ARTIFACTS_PATH)/Runner.ipa
+
+clean:
+	@$(ECHO_SUBTITLE2) "make clean"
+	rm -rf "$(BUILD_DIR)"
+ifdef ARTIFACTS_PATH
+	rm -rf "$(IPA_PATH)"
+endif
+
+archive:
+	@:$(eval VERSION ?= $(CURRENT_GIT_COMMIT_SHORT))
+	@$(ECHO_SUBTITLE2) "make archive VERSION='$(VERSION)'"
+	@xcrun agvtool new-version "$(VERSION)"
+	set -eo pipefail; \
+	OTEL_SWIFT=1 xcodebuild \
+		-project BenchmarkTests.xcodeproj \
+		-scheme Runner \
+		-sdk iphoneos \
+		-configuration Synthetics \
+		-destination generic/platform=iOS \
+		-archivePath $(ARCHIVE_PATH) \
+		archive | xcbeautify
+	@$(ECHO_SUCCESS) "Archive ready in '$(ARCHIVE_PATH)'"
+
+export:
+	@$(call require_param,ARTIFACTS_PATH)
+	@:$(eval VERSION ?= $(CURRENT_GIT_COMMIT_SHORT))
+	@$(ECHO_SUBTITLE2) "make export VERSION='$(VERSION)' ARTIFACTS_PATH='$(ARTIFACTS_PATH)'"
+	set -o pipefaill; \
+	xcodebuild -exportArchive \
+		-archivePath $(ARCHIVE_PATH)  \
+		-exportOptionsPlist exportOptions.plist \
+		-exportPath $(BUILD_DIR) \
+		| xcbeautify
+	mkdir -p "$(ARTIFACTS_PATH)"
+	cp -v "$(BUILD_DIR)/Runner.ipa" "$(IPA_PATH)"
+	@$(ECHO_SUCCESS) "IPA exported to '$(IPA_PATH)'"
+
+upload:
+	@$(call require_param,ARTIFACTS_PATH)
+	@$(call require_param,DATADOG_API_KEY)
+	@$(call require_param,DATADOG_APP_KEY)
+	@$(call require_param,S8S_APPLICATION_ID)
+	@:$(eval VERSION ?= $(CURRENT_GIT_COMMIT_SHORT))
+	@$(ECHO_SUBTITLE2) "make upload VERSION='$(VERSION)' ARTIFACTS_PATH='$(ARTIFACTS_PATH)'"
+	datadog-ci synthetics upload-application \
+		--mobileApp "$(IPA_PATH)" \
+		--mobileApplicationId "${S8S_APPLICATION_ID}" \
+		--versionName "$(VERSION)" \
+		--latest
+
+open:
+	@$(ECHO_SUBTITLE2) "make open"
+	@open --new --env OTEL_SWIFT BenchmarkTests.xcodeproj

BenchmarkTests/README.md

@@ -0,0 +1,73 @@
+# Benchmark Tests
+
+[Synthetics for Mobile](https://docs.datadoghq.com/mobile_app_testing/) runs Benchmark test scenarios to collect metrics of the SDK performances.
+
+
+## CI
+
+CI continuously builds, signs, and uploads a runner application to Synthetics which runs predefined tests.
+
+### Build
+
+Before building the application, make sure the `BenchmarkTests/xcconfigs/Benchmark.local.xcconfig` configuration file is present and contains the `Mobile - Integration Org` client token, RUM application ID, and API Key. These values are sensitive and must be securely stored.
+
+```ini
+CLIENT_TOKEN=
+RUM_APPLICATION_ID=
+API_KEY=
+```
+
+### Sign
+
+To sign the runner application, the certificate and provision profile defined in [Synthetics.xcconfig](xcconfigs/Synthetics.xcconfig) and in [exportOptions.plist](exportOptions.plist) needs to be installed on the build machine. The certificate and profile are sensitive files and must be securely stored. Make sure to update both files when updating the certificate and provisioning profile, otherwise signing fails.
+
+> [!NOTE]
+> Certificate & Provisioning Profile are also available through the [App Store Connect API](https://developer.apple.com/documentation/appstoreconnectapi). But we don't have the tooling in place.
+
+### Upload
+
+The application version (build number) is set to the commit SHA of the current job, and the build is uploaded to Synthetics using the [datadog-ci](https://github.com/DataDog/datadog-ci) CLI. This step expects environment variables to authenticate with the `Mobile - Integration Org`:
+
+```bash
+export DATADOG_API_KEY=
+export DATADOG_APP_KEY=
+export S8S_APPLICATION_ID=
+```
+
+## Development
+
+Each scenario is independent and can be considered as an app within the runner.
+
+### Create a scenario
+
+A scenario must comply with the [`Scenario`](Runner/Scenarios/Scenario.swift) protocol. Upon start, a scenario initializes the SDK, enables features, and returns a root view-controller.
+
+Here is a simple example of a scenario using Logs:
+```swift
+import Foundation
+import UIKit
+
+import DatadogCore
+import DatadogLogs
+
+struct LogsScenario: Scenario {
+
+    func start(info: TestInfo) -> UIViewController {
+
+        Datadog.initialize(
+            with: .benchmark(info: info), // SDK init with the benchmark configuration
+            trackingConsent: .granted
+        )
+
+        Logs.enable()
+
+        return LoggerViewController()
+    }
+}
+```
+
+Add the test to the [`SyntheticScenario`](Runner/Scenarios/Scenario.swift) enumeration so it can be selected, either manually or by setting the `BENCHMARK_SCENARIO` environment variable.
+
+### Synthetics Configuration
+
+Please refer to [Confluence page (internal)](https://datadoghq.atlassian.net/wiki/spaces/RUMP/pages/3981476482/Benchmarks+iOS)