From f85fb8a46e13ccd0a070d82ae94caed573257fd3 Mon Sep 17 00:00:00 2001 From: Maxime Epain Date: Wed, 24 Jul 2024 17:42:52 +0200 Subject: [PATCH 01/13] RUM-5551 Add BenchmarkTests project --- .../.xcodesamplecode.plist | 1 + .../BenchmarkTests.xcodeproj/project.pbxproj | 567 ++++++++++++++++++ .../contents.xcworkspacedata | 7 + .../xcshareddata/IDEWorkspaceChecks.plist | 8 + .../xcshareddata/xcschemes/Runner.xcscheme | 78 +++ BenchmarkTests/Benchmarks/Package.swift | 24 + .../Benchmarks/Sources/DatadogExporter.swift | 38 ++ BenchmarkTests/Makefile | 57 ++ BenchmarkTests/Runner/AppConfiguration.swift | 76 +++ BenchmarkTests/Runner/AppDelegate.swift | 23 + BenchmarkTests/Runner/Info.plist | 19 + .../Runner/Scenarios/DefaultScenario.swift | 54 ++ .../Runner/Scenarios/Scenario.swift | 67 +++ .../SessionReplay/SessionReplay.storyboard | 33 + .../SessionReplayController.swift | 10 + .../SessionReplay/SessionReplayScenario.swift | 41 ++ BenchmarkTests/exportOptions.plist | 19 + BenchmarkTests/xcconfigs/Runner.xcconfig | 9 + BenchmarkTests/xcconfigs/Synthetics.xcconfig | 6 + Makefile | 8 + Package.swift | 9 +- tools/benchmark-build-upload.sh | 72 +++ tools/secrets/config.sh | 6 + tools/utils/code-sign.sh | 90 +++ 24 files changed, 1319 insertions(+), 3 deletions(-) create mode 100644 BenchmarkTests/BenchmarkTests.xcodeproj/.xcodesamplecode.plist create mode 100644 BenchmarkTests/BenchmarkTests.xcodeproj/project.pbxproj create mode 100644 BenchmarkTests/BenchmarkTests.xcodeproj/project.xcworkspace/contents.xcworkspacedata create mode 100644 BenchmarkTests/BenchmarkTests.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist create mode 100644 BenchmarkTests/BenchmarkTests.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme create mode 100644 BenchmarkTests/Benchmarks/Package.swift create mode 100644 BenchmarkTests/Benchmarks/Sources/DatadogExporter.swift create mode 100644 BenchmarkTests/Makefile create mode 100644 BenchmarkTests/Runner/AppConfiguration.swift create mode 100644 BenchmarkTests/Runner/AppDelegate.swift create mode 100644 BenchmarkTests/Runner/Info.plist create mode 100644 BenchmarkTests/Runner/Scenarios/DefaultScenario.swift create mode 100644 BenchmarkTests/Runner/Scenarios/Scenario.swift create mode 100644 BenchmarkTests/Runner/Scenarios/SessionReplay/SessionReplay.storyboard create mode 100644 BenchmarkTests/Runner/Scenarios/SessionReplay/SessionReplayController.swift create mode 100644 BenchmarkTests/Runner/Scenarios/SessionReplay/SessionReplayScenario.swift create mode 100644 BenchmarkTests/exportOptions.plist create mode 100644 BenchmarkTests/xcconfigs/Runner.xcconfig create mode 100644 BenchmarkTests/xcconfigs/Synthetics.xcconfig create mode 100755 tools/benchmark-build-upload.sh create mode 100755 tools/utils/code-sign.sh diff --git a/BenchmarkTests/BenchmarkTests.xcodeproj/.xcodesamplecode.plist b/BenchmarkTests/BenchmarkTests.xcodeproj/.xcodesamplecode.plist new file mode 100644 index 0000000000..4bc741ca64 --- /dev/null +++ b/BenchmarkTests/BenchmarkTests.xcodeproj/.xcodesamplecode.plist @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/BenchmarkTests/BenchmarkTests.xcodeproj/project.pbxproj b/BenchmarkTests/BenchmarkTests.xcodeproj/project.pbxproj new file mode 100644 index 0000000000..a95c31d33c --- /dev/null +++ b/BenchmarkTests/BenchmarkTests.xcodeproj/project.pbxproj @@ -0,0 +1,567 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 56; + objects = { + +/* Begin PBXBuildFile section */ + D276069F2C514F37002D2A14 /* SessionReplay.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = D27606962C514F37002D2A14 /* SessionReplay.storyboard */; }; + D27606A02C514F37002D2A14 /* SessionReplayController.swift in Sources */ = {isa = PBXBuildFile; fileRef = D27606972C514F37002D2A14 /* SessionReplayController.swift */; }; + D27606A12C514F37002D2A14 /* SessionReplayScenario.swift in Sources */ = {isa = PBXBuildFile; fileRef = D27606982C514F37002D2A14 /* SessionReplayScenario.swift */; }; + D27606A22C514F37002D2A14 /* DefaultScenario.swift in Sources */ = {isa = PBXBuildFile; fileRef = D276069A2C514F37002D2A14 /* DefaultScenario.swift */; }; + D27606A32C514F37002D2A14 /* Scenario.swift in Sources */ = {isa = PBXBuildFile; fileRef = D276069B2C514F37002D2A14 /* Scenario.swift */; }; + D27606A42C514F37002D2A14 /* AppConfiguration.swift in Sources */ = {isa = PBXBuildFile; fileRef = D276069D2C514F37002D2A14 /* AppConfiguration.swift */; }; + D27606A72C514F77002D2A14 /* DatadogCore in Frameworks */ = {isa = PBXBuildFile; productRef = D27606A62C514F77002D2A14 /* DatadogCore */; }; + D27606A92C514F77002D2A14 /* DatadogLogs in Frameworks */ = {isa = PBXBuildFile; productRef = D27606A82C514F77002D2A14 /* DatadogLogs */; }; + D27606AB2C514F77002D2A14 /* DatadogRUM in Frameworks */ = {isa = PBXBuildFile; productRef = D27606AA2C514F77002D2A14 /* DatadogRUM */; }; + D27606AD2C514F77002D2A14 /* DatadogSessionReplay in Frameworks */ = {isa = PBXBuildFile; productRef = D27606AC2C514F77002D2A14 /* DatadogSessionReplay */; }; + D27606AF2C514F77002D2A14 /* DatadogTrace in Frameworks */ = {isa = PBXBuildFile; productRef = D27606AE2C514F77002D2A14 /* DatadogTrace */; }; + D27606B12C514F77002D2A14 /* DatadogBenchmarks in Frameworks */ = {isa = PBXBuildFile; productRef = D27606B02C514F77002D2A14 /* DatadogBenchmarks */; }; + D29F75502C4AA07E00288638 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = D29F754F2C4AA07E00288638 /* AppDelegate.swift */; }; +/* End PBXBuildFile section */ + +/* Begin PBXCopyFilesBuildPhase section */ + D29F75872C4AA98F00288638 /* Embed Frameworks */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = ""; + dstSubfolderSpec = 10; + files = ( + ); + name = "Embed Frameworks"; + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXCopyFilesBuildPhase section */ + +/* Begin PBXFileReference section */ + D247578B2C53ECBB00DF1397 /* dd-sdk-ios */ = {isa = PBXFileReference; lastKnownFileType = wrapper; name = "dd-sdk-ios"; path = ..; sourceTree = ""; }; + D27606952C514EED002D2A14 /* Benchmarks */ = {isa = PBXFileReference; lastKnownFileType = wrapper; path = Benchmarks; sourceTree = SOURCE_ROOT; }; + D27606962C514F37002D2A14 /* SessionReplay.storyboard */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.storyboard; path = SessionReplay.storyboard; sourceTree = ""; }; + D27606972C514F37002D2A14 /* SessionReplayController.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SessionReplayController.swift; sourceTree = ""; }; + D27606982C514F37002D2A14 /* SessionReplayScenario.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SessionReplayScenario.swift; sourceTree = ""; }; + D276069A2C514F37002D2A14 /* DefaultScenario.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DefaultScenario.swift; sourceTree = ""; }; + D276069B2C514F37002D2A14 /* Scenario.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Scenario.swift; sourceTree = ""; }; + D276069D2C514F37002D2A14 /* AppConfiguration.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AppConfiguration.swift; sourceTree = ""; }; + D27606B22C526908002D2A14 /* Benchmarks.local.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Benchmarks.local.xcconfig; sourceTree = ""; }; + D27606B32C526908002D2A14 /* Runner.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Runner.xcconfig; sourceTree = ""; }; + D27606B42C526908002D2A14 /* Synthetics.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Synthetics.xcconfig; sourceTree = ""; }; + D29F754D2C4AA07E00288638 /* Runner.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = Runner.app; sourceTree = BUILT_PRODUCTS_DIR; }; + D29F754F2C4AA07E00288638 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; + D29F755D2C4AA08000288638 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; +/* End PBXFileReference section */ + +/* Begin PBXFrameworksBuildPhase section */ + D29F754A2C4AA07E00288638 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + D27606A92C514F77002D2A14 /* DatadogLogs in Frameworks */, + D27606AF2C514F77002D2A14 /* DatadogTrace in Frameworks */, + D27606AD2C514F77002D2A14 /* DatadogSessionReplay in Frameworks */, + D27606B12C514F77002D2A14 /* DatadogBenchmarks in Frameworks */, + D27606AB2C514F77002D2A14 /* DatadogRUM in Frameworks */, + D27606A72C514F77002D2A14 /* DatadogCore in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + D27606992C514F37002D2A14 /* SessionReplay */ = { + isa = PBXGroup; + children = ( + D27606962C514F37002D2A14 /* SessionReplay.storyboard */, + D27606972C514F37002D2A14 /* SessionReplayController.swift */, + D27606982C514F37002D2A14 /* SessionReplayScenario.swift */, + ); + path = SessionReplay; + sourceTree = ""; + }; + D276069C2C514F37002D2A14 /* Scenarios */ = { + isa = PBXGroup; + children = ( + D27606992C514F37002D2A14 /* SessionReplay */, + D276069A2C514F37002D2A14 /* DefaultScenario.swift */, + D276069B2C514F37002D2A14 /* Scenario.swift */, + ); + path = Scenarios; + sourceTree = ""; + }; + D27606B52C526908002D2A14 /* xcconfigs */ = { + isa = PBXGroup; + children = ( + D27606B22C526908002D2A14 /* Benchmarks.local.xcconfig */, + D27606B32C526908002D2A14 /* Runner.xcconfig */, + D27606B42C526908002D2A14 /* Synthetics.xcconfig */, + ); + path = xcconfigs; + sourceTree = ""; + }; + D29F75272C4A9EFA00288638 = { + isa = PBXGroup; + children = ( + D27606B52C526908002D2A14 /* xcconfigs */, + D29F754E2C4AA07E00288638 /* Runner */, + D29F75482C4A9F9500288638 /* Frameworks */, + D29F75312C4A9EFA00288638 /* Products */, + ); + sourceTree = ""; + }; + D29F75312C4A9EFA00288638 /* Products */ = { + isa = PBXGroup; + children = ( + D29F754D2C4AA07E00288638 /* Runner.app */, + ); + name = Products; + sourceTree = ""; + }; + D29F75482C4A9F9500288638 /* Frameworks */ = { + isa = PBXGroup; + children = ( + D27606952C514EED002D2A14 /* Benchmarks */, + D247578B2C53ECBB00DF1397 /* dd-sdk-ios */, + ); + name = Frameworks; + sourceTree = ""; + }; + D29F754E2C4AA07E00288638 /* Runner */ = { + isa = PBXGroup; + children = ( + D29F754F2C4AA07E00288638 /* AppDelegate.swift */, + D276069D2C514F37002D2A14 /* AppConfiguration.swift */, + D276069C2C514F37002D2A14 /* Scenarios */, + D29F755D2C4AA08000288638 /* Info.plist */, + ); + path = Runner; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + D29F754C2C4AA07E00288638 /* Runner */ = { + isa = PBXNativeTarget; + buildConfigurationList = D29F75602C4AA08000288638 /* Build configuration list for PBXNativeTarget "Runner" */; + buildPhases = ( + D29F75492C4AA07E00288638 /* Sources */, + D29F754A2C4AA07E00288638 /* Frameworks */, + D29F754B2C4AA07E00288638 /* Resources */, + D29F75872C4AA98F00288638 /* Embed Frameworks */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = Runner; + packageProductDependencies = ( + D27606A62C514F77002D2A14 /* DatadogCore */, + D27606A82C514F77002D2A14 /* DatadogLogs */, + D27606AA2C514F77002D2A14 /* DatadogRUM */, + D27606AC2C514F77002D2A14 /* DatadogSessionReplay */, + D27606AE2C514F77002D2A14 /* DatadogTrace */, + D27606B02C514F77002D2A14 /* DatadogBenchmarks */, + ); + productName = Runner; + productReference = D29F754D2C4AA07E00288638 /* Runner.app */; + productType = "com.apple.product-type.application"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + D29F75282C4A9EFA00288638 /* Project object */ = { + isa = PBXProject; + attributes = { + BuildIndependentTargetsInParallel = 1; + LastSwiftUpdateCheck = 1540; + LastUpgradeCheck = 1540; + TargetAttributes = { + D29F754C2C4AA07E00288638 = { + CreatedOnToolsVersion = 15.4; + }; + }; + }; + buildConfigurationList = D29F752B2C4A9EFA00288638 /* Build configuration list for PBXProject "BenchmarkTests" */; + compatibilityVersion = "Xcode 14.0"; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + en, + Base, + ); + mainGroup = D29F75272C4A9EFA00288638; + packageReferences = ( + D29F75612C4AA8C200288638 /* XCRemoteSwiftPackageReference "opentelemetry-swift" */, + ); + productRefGroup = D29F75312C4A9EFA00288638 /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + D29F754C2C4AA07E00288638 /* Runner */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXResourcesBuildPhase section */ + D29F754B2C4AA07E00288638 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + D276069F2C514F37002D2A14 /* SessionReplay.storyboard in Resources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXResourcesBuildPhase section */ + +/* Begin PBXSourcesBuildPhase section */ + D29F75492C4AA07E00288638 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + D27606A42C514F37002D2A14 /* AppConfiguration.swift in Sources */, + D29F75502C4AA07E00288638 /* AppDelegate.swift in Sources */, + D27606A22C514F37002D2A14 /* DefaultScenario.swift in Sources */, + D27606A12C514F37002D2A14 /* SessionReplayScenario.swift in Sources */, + D27606A32C514F37002D2A14 /* Scenario.swift in Sources */, + D27606A02C514F37002D2A14 /* SessionReplayController.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin XCBuildConfiguration section */ + D27606B62C526925002D2A14 /* Synthetics */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_USER_SCRIPT_SANDBOXING = YES; + GCC_C_LANGUAGE_STANDARD = gnu17; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 17.5; + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MTL_ENABLE_DEBUG_INFO = NO; + MTL_FAST_MATH = YES; + SDKROOT = iphoneos; + SWIFT_COMPILATION_MODE = wholemodule; + VALIDATE_PRODUCT = YES; + }; + name = Synthetics; + }; + D27606B72C526925002D2A14 /* Synthetics */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = D27606B42C526908002D2A14 /* Synthetics.xcconfig */; + buildSettings = { + ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = YES; + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CURRENT_PROJECT_VERSION = f34790fea; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_FILE = Runner/Info.plist; + INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; + INFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen; + INFOPLIST_KEY_UIMainStoryboardFile = Main; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = com.datadoghq.benchmarks.Runner; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = YES; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Synthetics; + }; + D29F75422C4A9EFB00288638 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = dwarf; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + ENABLE_USER_SCRIPT_SANDBOXING = YES; + GCC_C_LANGUAGE_STANDARD = gnu17; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 17.5; + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; + MTL_FAST_MATH = YES; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = iphoneos; + SWIFT_ACTIVE_COMPILATION_CONDITIONS = "DEBUG $(inherited)"; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + }; + name = Debug; + }; + D29F75432C4A9EFB00288638 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_USER_SCRIPT_SANDBOXING = YES; + GCC_C_LANGUAGE_STANDARD = gnu17; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 17.5; + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MTL_ENABLE_DEBUG_INFO = NO; + MTL_FAST_MATH = YES; + SDKROOT = iphoneos; + SWIFT_COMPILATION_MODE = wholemodule; + VALIDATE_PRODUCT = YES; + }; + name = Release; + }; + D29F755E2C4AA08000288638 /* Debug */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = D27606B32C526908002D2A14 /* Runner.xcconfig */; + buildSettings = { + ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = YES; + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CODE_SIGN_IDENTITY = "Apple Development"; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = f34790fea; + DEVELOPMENT_TEAM = JKFCB4CN7C; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_FILE = Runner/Info.plist; + INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; + INFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen; + INFOPLIST_KEY_UIMainStoryboardFile = Main; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = com.datadoghq.benchmarks.Runner; + PRODUCT_NAME = "$(TARGET_NAME)"; + PROVISIONING_PROFILE_SPECIFIER = ""; + SWIFT_EMIT_LOC_STRINGS = YES; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Debug; + }; + D29F755F2C4AA08000288638 /* Release */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = D27606B32C526908002D2A14 /* Runner.xcconfig */; + buildSettings = { + ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = YES; + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CODE_SIGN_IDENTITY = "Apple Development"; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = f34790fea; + DEVELOPMENT_TEAM = JKFCB4CN7C; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_FILE = Runner/Info.plist; + INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; + INFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen; + INFOPLIST_KEY_UIMainStoryboardFile = Main; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = com.datadoghq.benchmarks.Runner; + PRODUCT_NAME = "$(TARGET_NAME)"; + PROVISIONING_PROFILE_SPECIFIER = ""; + SWIFT_EMIT_LOC_STRINGS = YES; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + D29F752B2C4A9EFA00288638 /* Build configuration list for PBXProject "BenchmarkTests" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + D29F75422C4A9EFB00288638 /* Debug */, + D29F75432C4A9EFB00288638 /* Release */, + D27606B62C526925002D2A14 /* Synthetics */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + D29F75602C4AA08000288638 /* Build configuration list for PBXNativeTarget "Runner" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + D29F755E2C4AA08000288638 /* Debug */, + D29F755F2C4AA08000288638 /* Release */, + D27606B72C526925002D2A14 /* Synthetics */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + +/* Begin XCRemoteSwiftPackageReference section */ + D29F75612C4AA8C200288638 /* XCRemoteSwiftPackageReference "opentelemetry-swift" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/open-telemetry/opentelemetry-swift"; + requirement = { + kind = exactVersion; + version = 1.6.0; + }; + }; +/* End XCRemoteSwiftPackageReference section */ + +/* Begin XCSwiftPackageProductDependency section */ + D27606A62C514F77002D2A14 /* DatadogCore */ = { + isa = XCSwiftPackageProductDependency; + productName = DatadogCore; + }; + D27606A82C514F77002D2A14 /* DatadogLogs */ = { + isa = XCSwiftPackageProductDependency; + productName = DatadogLogs; + }; + D27606AA2C514F77002D2A14 /* DatadogRUM */ = { + isa = XCSwiftPackageProductDependency; + productName = DatadogRUM; + }; + D27606AC2C514F77002D2A14 /* DatadogSessionReplay */ = { + isa = XCSwiftPackageProductDependency; + productName = DatadogSessionReplay; + }; + D27606AE2C514F77002D2A14 /* DatadogTrace */ = { + isa = XCSwiftPackageProductDependency; + productName = DatadogTrace; + }; + D27606B02C514F77002D2A14 /* DatadogBenchmarks */ = { + isa = XCSwiftPackageProductDependency; + productName = DatadogBenchmarks; + }; +/* End XCSwiftPackageProductDependency section */ + }; + rootObject = D29F75282C4A9EFA00288638 /* Project object */; +} diff --git a/BenchmarkTests/BenchmarkTests.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/BenchmarkTests/BenchmarkTests.xcodeproj/project.xcworkspace/contents.xcworkspacedata new file mode 100644 index 0000000000..919434a625 --- /dev/null +++ b/BenchmarkTests/BenchmarkTests.xcodeproj/project.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,7 @@ + + + + + diff --git a/BenchmarkTests/BenchmarkTests.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist b/BenchmarkTests/BenchmarkTests.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist new file mode 100644 index 0000000000..18d981003d --- /dev/null +++ b/BenchmarkTests/BenchmarkTests.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist @@ -0,0 +1,8 @@ + + + + + IDEDidComputeMac32BitWarning + + + diff --git a/BenchmarkTests/BenchmarkTests.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme b/BenchmarkTests/BenchmarkTests.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme new file mode 100644 index 0000000000..47396a3dc5 --- /dev/null +++ b/BenchmarkTests/BenchmarkTests.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme @@ -0,0 +1,78 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/BenchmarkTests/Benchmarks/Package.swift b/BenchmarkTests/Benchmarks/Package.swift new file mode 100644 index 0000000000..4746338dbc --- /dev/null +++ b/BenchmarkTests/Benchmarks/Package.swift @@ -0,0 +1,24 @@ +// swift-tools-version: 5.10 +// The swift-tools-version declares the minimum version of Swift required to build this package. + +import PackageDescription + +let package = Package( + name: "DatadogBenchmarks", + products: [ + .library( + name: "DatadogBenchmarks", + targets: ["Benchmarks"]), + ], + dependencies: [ + .package(url: "https://github.com/open-telemetry/opentelemetry-swift", from: "1.0.0"), + ], + targets: [ + .target( + name: "Benchmarks", + dependencies: [ + .product(name: "OpenTelemetrySdk", package: "opentelemetry-swift") + ] + ) + ] +) diff --git a/BenchmarkTests/Benchmarks/Sources/DatadogExporter.swift b/BenchmarkTests/Benchmarks/Sources/DatadogExporter.swift new file mode 100644 index 0000000000..c9a79a48cd --- /dev/null +++ b/BenchmarkTests/Benchmarks/Sources/DatadogExporter.swift @@ -0,0 +1,38 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2019-Present Datadog, Inc. + */ + +import Foundation +import OpenTelemetrySdk + +public final class DatadogExporter: SpanExporter, MetricExporter { + private let session: URLSession + + convenience init() { + let configuration: URLSessionConfiguration = .ephemeral + configuration.urlCache = nil + self.init(session: URLSession(configuration: configuration)) + } + + init(session: URLSession) { + self.session = session + } + + public func export(spans: [SpanData]) -> SpanExporterResultCode { + return .success + } + + public func export(metrics: [Metric], shouldCancel: (() -> Bool)?) -> MetricExporterResultCode { + return .success + } + + public func flush() -> SpanExporterResultCode { + return .success + } + + public func shutdown() { + + } +} diff --git a/BenchmarkTests/Makefile b/BenchmarkTests/Makefile new file mode 100644 index 0000000000..890960fd02 --- /dev/null +++ b/BenchmarkTests/Makefile @@ -0,0 +1,57 @@ +.PHONY: clean archive export upload + +REPO_ROOT := ../ +include ../tools/utils/common.mk + +BUILD_DIR := .build +ARCHIVE_PATH := $(BUILD_DIR)/Runner.xcarchive +IPA_PATH := $(ARTIFACTS_PATH)/Runner.ipa + +clean: + @$(ECHO_SUBTITLE2) "make clean" + rm -rf "$(BUILD_DIR)" +ifdef ARTIFACTS_PATH + rm -rf "$(IPA_PATH)" +endif + +archive: + @:$(eval VERSION ?= $(CURRENT_GIT_COMMIT_SHORT)) + @$(ECHO_SUBTITLE2) "make archive VERSION='$(VERSION)'" + @xcrun agvtool new-version "$(VERSION)" + set -eo pipefail; \ + OTEL_SWIFT=1 xcodebuild \ + -project BenchmarkTests.xcodeproj \ + -scheme Runner \ + -sdk iphoneos \ + -configuration Synthetics \ + -destination generic/platform=iOS \ + -archivePath $(ARCHIVE_PATH) \ + archive | xcbeautify + @$(ECHO_SUCCESS) "Archive ready in '$(ARCHIVE_PATH)'" + +export: + @$(call require_param,ARTIFACTS_PATH) + @:$(eval VERSION ?= $(CURRENT_GIT_COMMIT_SHORT)) + @$(ECHO_SUBTITLE2) "make export VERSION='$(VERSION)' ARTIFACTS_PATH='$(ARTIFACTS_PATH)'" + set -o pipefaill; \ + xcodebuild -exportArchive \ + -archivePath $(ARCHIVE_PATH) \ + -exportOptionsPlist exportOptions.plist \ + -exportPath $(BUILD_DIR) \ + | xcbeautify + mkdir -p "$(ARTIFACTS_PATH)" + cp -v "$(BUILD_DIR)/Runner.ipa" "$(IPA_PATH)" + @$(ECHO_SUCCESS) "IPA exported to '$(IPA_PATH)'" + +upload: + @$(call require_param,ARTIFACTS_PATH) + @$(call require_param,DATADOG_API_KEY) + @$(call require_param,DATADOG_APP_KEY) + @$(call require_param,S8S_APPLICATION_ID) + @:$(eval VERSION ?= $(CURRENT_GIT_COMMIT_SHORT)) + @$(ECHO_SUBTITLE2) "make upload VERSION='$(VERSION)' ARTIFACTS_PATH='$(ARTIFACTS_PATH)'" + datadog-ci synthetics upload-application \ + --mobileApp "$(IPA_PATH)" \ + --mobileApplicationId "${S8S_APPLICATION_ID}" \ + --versionName "$(VERSION)" \ + --latest diff --git a/BenchmarkTests/Runner/AppConfiguration.swift b/BenchmarkTests/Runner/AppConfiguration.swift new file mode 100644 index 0000000000..d6e13ccdec --- /dev/null +++ b/BenchmarkTests/Runner/AppConfiguration.swift @@ -0,0 +1,76 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2019-Present Datadog, Inc. + */ + +import Foundation +import DatadogInternal +import DatadogCore + +/// Test info reads configuration from `Info.plist`. +/// +/// The expected format is as follow: +/// +/// +/// DatadogConfiguration +/// +/// ClientToken +/// $(CLIENT_TOKEN) +/// ApplicationID +/// $(RUM_APPLICATION_ID) +/// ApiKey +/// $(API_KEY) +/// Environment +/// $(DD_ENV) +/// Site +/// $(DD_SITE) +/// +/// +struct TestInfo: Decodable { + let clientToken: String + let applicationID: String + let apiKey: String + let site: DatadogSite + let env: String + + enum CodingKeys: String, CodingKey { + case clientToken = "ClientToken" + case applicationID = "ApplicationID" + case apiKey = "ApiKey" + case site = "Site" + case env = "Environment" + } +} + +extension TestInfo { + init(bundle: Bundle = .main) throws { + let decoder = AnyDecoder() + let obj = bundle.object(forInfoDictionaryKey: "DatadogConfiguration") + self = try decoder.decode(from: obj) + } +} + +extension TestInfo { + static var empty: Self { + .init( + clientToken: "", + applicationID: "", + apiKey: "", + site: .us1, + env: "benchmarks" + ) + } +} + +extension DatadogSite: Decodable {} + +extension Datadog.Configuration { + static func benchmark(info: TestInfo) -> Self { + .init( + clientToken: info.clientToken, + env: info.env, + site: info.site + ) + } +} diff --git a/BenchmarkTests/Runner/AppDelegate.swift b/BenchmarkTests/Runner/AppDelegate.swift new file mode 100644 index 0000000000..9b992f4fe0 --- /dev/null +++ b/BenchmarkTests/Runner/AppDelegate.swift @@ -0,0 +1,23 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2019-Present Datadog, Inc. + */ + +import UIKit + +@main +class AppDelegate: UIResponder, UIApplicationDelegate { + var window: UIWindow? + + func application(_ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?) -> Bool { + let info = try! TestInfo() // crash if test info are missing or malformed + + let scenario: Scenario = SyntheticScenario() ?? DefaultScenario() + + window = UIWindow(frame: UIScreen.main.bounds) + window?.rootViewController = scenario.start(info: info) + window?.makeKeyAndVisible() + return true + } +} diff --git a/BenchmarkTests/Runner/Info.plist b/BenchmarkTests/Runner/Info.plist new file mode 100644 index 0000000000..1c5a6ca83d --- /dev/null +++ b/BenchmarkTests/Runner/Info.plist @@ -0,0 +1,19 @@ + + + + + DatadogConfiguration + + ApiKey + $(API_KEY) + ApplicationID + $(RUM_APPLICATION_ID) + ClientToken + $(CLIENT_TOKEN) + Environment + $(DD_ENV) + Site + $(DD_SITE) + + + diff --git a/BenchmarkTests/Runner/Scenarios/DefaultScenario.swift b/BenchmarkTests/Runner/Scenarios/DefaultScenario.swift new file mode 100644 index 0000000000..d6760fed82 --- /dev/null +++ b/BenchmarkTests/Runner/Scenarios/DefaultScenario.swift @@ -0,0 +1,54 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2019-Present Datadog, Inc. + */ + +import Foundation +import UIKit +import SwiftUI + +/// The default scenario will present the list of Synthetic scenarios to run in development mode. +/// To skip this screen, you can set the `E2E_SCENARIO` environment variable with the name +/// the desired scenario. +struct DefaultScenario: Scenario { + func start(info: TestInfo) -> UIViewController { + UIHostingController(rootView: ContentView(info: info)) + } + + struct ContentView: View { + let info: TestInfo + + var body: some View { + NavigationView { + List(SyntheticScenario.allCases, id: \.rawValue) { scenario in + NavigationLink { + ScenarioView(info: info, scenario: scenario) + } label: { + Text(scenario.rawValue) + } + } + .navigationBarTitle("Scenarios") + } + } + } + + struct ScenarioView: UIViewControllerRepresentable { + let info: TestInfo + let scenario: Scenario + + func makeUIViewController(context: Context) -> UIViewController { + scenario.start(info: info) + } + + func updateUIViewController(_ uiViewController: UIViewController, context: Context) { } + } +} + +#if DEBUG +struct DefaultScenario_Previews: PreviewProvider { + static var previews: some View { + DefaultScenario.ContentView(info: .empty) + } +} +#endif diff --git a/BenchmarkTests/Runner/Scenarios/Scenario.swift b/BenchmarkTests/Runner/Scenarios/Scenario.swift new file mode 100644 index 0000000000..7a5abd9f85 --- /dev/null +++ b/BenchmarkTests/Runner/Scenarios/Scenario.swift @@ -0,0 +1,67 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2019-Present Datadog, Inc. + */ + +import Foundation +import UIKit + +/// A `Scenario` is the entry-point of the E2E runner application. +/// +/// The compliant objects are responsible for initializing the SDK, enabling +/// Features, and create the root view-controller. +protocol Scenario { + /// Starts the scenario. + /// + /// Starting the scenario should intialize the SDK and enable Features based on + /// the provided ``TestInfo`` and scenario's needs. + /// + /// The returned view-controller will be used as the root view controller of the + /// application window. + /// + /// - Parameter info: The test info for configuring the SDK. + /// - Returns: The root view-controller. + func start(info: TestInfo) -> UIViewController +} + +/// A Synthetic scenario can be initialized by defining a Synthetic Test Process Argument +/// named `BENCHMARK_SCENARIO`. +/// +/// Note: The raw value of enum case must match the test name defined in Synthetics. +enum SyntheticScenario: String, CaseIterable { + case sessionReplay + + /// Creates the scenario defined by the`BENCHMARK_SCENARIO` environment variable. + /// + /// - Parameter processInfo: The process info holding the environment variables. + init?(processInfo: ProcessInfo = .processInfo) { + guard + processInfo.environment["XCODE_RUNNING_FOR_PREVIEWS"] == nil, // skip SwiftUI preview + let rawValue = processInfo.environment["BENCHMARK_SCENARIO"], + let scenario = Self(rawValue: rawValue) + else { + return nil + } + + self = scenario + } + + /// Returns the scenario defined by the environment variable. + var scenario: Scenario { + switch self { + case .sessionReplay: + return SessionReplayScenario() + } + } +} + +extension SyntheticScenario: Scenario { + /// Starts the underlying scenario. + /// + /// - Parameter info: The test info for configuring the SDK. + /// - Returns: The root view-controller. + func start(info: TestInfo) -> UIViewController { + scenario.start(info: info) + } +} diff --git a/BenchmarkTests/Runner/Scenarios/SessionReplay/SessionReplay.storyboard b/BenchmarkTests/Runner/Scenarios/SessionReplay/SessionReplay.storyboard new file mode 100644 index 0000000000..53add6285f --- /dev/null +++ b/BenchmarkTests/Runner/Scenarios/SessionReplay/SessionReplay.storyboard @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/BenchmarkTests/Runner/Scenarios/SessionReplay/SessionReplayController.swift b/BenchmarkTests/Runner/Scenarios/SessionReplay/SessionReplayController.swift new file mode 100644 index 0000000000..8b7ebbad0c --- /dev/null +++ b/BenchmarkTests/Runner/Scenarios/SessionReplay/SessionReplayController.swift @@ -0,0 +1,10 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2019-Present Datadog, Inc. + */ + +import UIKit + +class SessionReplayController: UIViewController { +} diff --git a/BenchmarkTests/Runner/Scenarios/SessionReplay/SessionReplayScenario.swift b/BenchmarkTests/Runner/Scenarios/SessionReplay/SessionReplayScenario.swift new file mode 100644 index 0000000000..6b3b33ff96 --- /dev/null +++ b/BenchmarkTests/Runner/Scenarios/SessionReplay/SessionReplayScenario.swift @@ -0,0 +1,41 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2019-Present Datadog, Inc. + */ + +import Foundation +import UIKit + +import DatadogCore +import DatadogRUM +import DatadogSessionReplay + +struct SessionReplayScenario: Scenario { + func start(info: TestInfo) -> UIViewController { + Datadog.initialize( + with: .benchmark(info: info), + trackingConsent: .granted + ) + + RUM.enable( + with: RUM.Configuration( + applicationID: info.applicationID, + uiKitViewsPredicate: DefaultUIKitRUMViewsPredicate(), + uiKitActionsPredicate: DefaultUIKitRUMActionsPredicate() + ) + ) + + SessionReplay.enable( + with: SessionReplay.Configuration( + replaySampleRate: 100, + defaultPrivacyLevel: .allow + ) + ) + + RUMMonitor.shared().addAttribute(forKey: "scenario", value: "SessionReplay") + + let storyboard = UIStoryboard(name: "SessionReplay", bundle: nil) + return storyboard.instantiateInitialViewController()! + } +} diff --git a/BenchmarkTests/exportOptions.plist b/BenchmarkTests/exportOptions.plist new file mode 100644 index 0000000000..00cd98869b --- /dev/null +++ b/BenchmarkTests/exportOptions.plist @@ -0,0 +1,19 @@ + + + + + distributionBundleIdentifier + com.datadoghq.benchmarks.Runner + method + development + provisioningProfiles + + com.datadoghq.benchmarks.Runner + Datadog Benchmark Runner + + signingCertificate + Apple Development: Robot Bitrise (9HKDHCMCGH) + teamID + JKFCB4CN7C + + diff --git a/BenchmarkTests/xcconfigs/Runner.xcconfig b/BenchmarkTests/xcconfigs/Runner.xcconfig new file mode 100644 index 0000000000..251d60c004 --- /dev/null +++ b/BenchmarkTests/xcconfigs/Runner.xcconfig @@ -0,0 +1,9 @@ +CLIENT_TOKEN = // the Client Token on Mobile Integration Org +RUM_APPLICATION_ID = // the RUM Application ID on Mobile Integration Org +API_KEY = // the API Key on Mobile Integration Org + +DD_ENV[config=*] = benchmarks +DD_ENV[config=Debug] = development +DD_SITE = us1 + +#include? "Benchmarks.local.xcconfig" diff --git a/BenchmarkTests/xcconfigs/Synthetics.xcconfig b/BenchmarkTests/xcconfigs/Synthetics.xcconfig new file mode 100644 index 0000000000..b7e14c3c51 --- /dev/null +++ b/BenchmarkTests/xcconfigs/Synthetics.xcconfig @@ -0,0 +1,6 @@ +#include "Runner.xcconfig" + +CODE_SIGN_STYLE = Manual +CODE_SIGN_IDENTITY = Apple Development: Robot Bitrise (9HKDHCMCGH) +DEVELOPMENT_TEAM = JKFCB4CN7C +PROVISIONING_PROFILE_SPECIFIER = Datadog Benchmark Runner diff --git a/Makefile b/Makefile index 5105b5cb58..c6f2fc518e 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,7 @@ all: env-check repo-setup templates smoke-test smoke-test-ios smoke-test-ios-all smoke-test-tvos smoke-test-tvos-all \ spm-build spm-build-ios spm-build-tvos spm-build-visionos spm-build-macos spm-build-watchos \ e2e-build-upload \ + benchmark-build-upload \ models-generate rum-models-generate sr-models-generate models-verify rum-models-verify sr-models-verify \ dogfood-shopist dogfood-datadog-app \ release-build release-validate release-publish-github \ @@ -268,6 +269,13 @@ e2e-build-upload: @$(ECHO_TITLE) "make e2e-build-upload ARTIFACTS_PATH='$(ARTIFACTS_PATH)' DRY_RUN='$(DRY_RUN)'" DRY_RUN=$(DRY_RUN) ./tools/e2e-build-upload.sh --artifacts-path "$(ARTIFACTS_PATH)" +# Builds a new version of the Benchmark app and publishes it to synthetics. +benchmark-build-upload: + @$(call require_param,ARTIFACTS_PATH) + @:$(eval DRY_RUN ?= 1) + @$(ECHO_TITLE) "make benchmark-build-upload ARTIFACTS_PATH='$(ARTIFACTS_PATH)' DRY_RUN='$(DRY_RUN)'" + DRY_RUN=$(DRY_RUN) ./tools/benchmark-build-upload.sh --artifacts-path "$(ARTIFACTS_PATH)" + xcodeproj-session-replay: @echo "⚙️ Generating 'DatadogSessionReplay.xcodeproj'..." @cd DatadogSessionReplay/ && swift package generate-xcodeproj diff --git a/Package.swift b/Package.swift index 4175171f79..9a2b4c5bd7 100644 --- a/Package.swift +++ b/Package.swift @@ -3,6 +3,10 @@ import PackageDescription import Foundation +let opentelemetry = ProcessInfo.processInfo.environment["OTEL_SWIFT"] != nil ? + (name: "opentelemetry-swift", url: "https://github.com/open-telemetry/opentelemetry-swift.git") : + (name: "opentelemetry-swift-packages", url: "https://github.com/DataDog/opentelemetry-swift-packages.git") + let package = Package( name: "Datadog", platforms: [ @@ -47,7 +51,7 @@ let package = Package( ], dependencies: [ .package(url: "https://github.com/microsoft/plcrashreporter.git", from: "1.11.2"), - .package(url: "https://github.com/DataDog/opentelemetry-swift-packages.git", exact: "1.6.0") + .package(url: opentelemetry.url, exact: "1.6.0"), ], targets: [ .target( @@ -112,7 +116,7 @@ let package = Package( name: "DatadogTrace", dependencies: [ .target(name: "DatadogInternal"), - .product(name: "OpenTelemetryApi", package: "opentelemetry-swift-packages") + .product(name: "OpenTelemetryApi", package: opentelemetry.name) ], path: "DatadogTrace/Sources" ), @@ -207,7 +211,6 @@ let package = Package( ] ) - // If the `DD_TEST_UTILITIES_ENABLED` development ENV is set, export additional utility packages. // To set this ENV for Xcode projects that fetch this package locally, use `open --env DD_TEST_UTILITIES_ENABLED path/to/`. if ProcessInfo.processInfo.environment["DD_TEST_UTILITIES_ENABLED"] != nil { diff --git a/tools/benchmark-build-upload.sh b/tools/benchmark-build-upload.sh new file mode 100755 index 0000000000..e43bc7293a --- /dev/null +++ b/tools/benchmark-build-upload.sh @@ -0,0 +1,72 @@ +#!/bin/zsh + +# Usage: +# $ ./tools/benchmark-build-upload.sh -h +# Publishes IPA of a new version of the Benchmark app to synthetics. + +# Options: +# --artifacts-path: Path where the IPA artifact will be exported. + +# ENVs: +# - DRY_RUN: Set to '1' to do everything except uploading the IPA to synthetics. + +set +x +set -eo pipefail +source ./tools/utils/argparse.sh +source ./tools/utils/echo-color.sh +source ./tools/utils/code-sign.sh +source ./tools/secrets/get-secret.sh + +set_description "Publishes IPA a new version of the Benchamrk app to synthetics." +define_arg "artifacts-path" "" "Path where the IPA artifact will be exported." "string" "true" + +check_for_help "$@" +parse_args "$@" + +BENCHMARK_DIR="BenchmarkTests" +BENCHMARK_XCCONFIG_PATH="$BENCHMARK_DIR/xcconfigs/Benchmarks.local.xcconfig" +BENCHMARK_CODESIGN_DIR="$BENCHMARK_DIR/benchmark-signing" +P12_PATH="$BENCHMARK_CODESIGN_DIR/cert.p12" +PP_PATH="$BENCHMARK_CODESIGN_DIR/runner.mobileprovision" + +ARTIFACTS_PATH="$(realpath .)/$artifacts_path" + +create_xcconfig() { + echo_subtitle "Create '$BENCHMARK_XCCONFIG_PATH'" + get_secret $DD_IOS_SECRET__BENCHMARK_XCCONFIG_BASE64 | base64 --decode -o $BENCHMARK_XCCONFIG_PATH + echo_succ "▸ '$BENCHMARK_XCCONFIG_PATH' ready" +} + +create_codesign_files() { + echo_subtitle "Create codesign files in '$BENCHMARK_CODESIGN_DIR'" + rm -rf "$BENCHMARK_CODESIGN_DIR" + mkdir -p "$BENCHMARK_CODESIGN_DIR" + get_secret $DD_IOS_SECRET__E2E_CERTIFICATE_P12_BASE64 | base64 --decode -o $P12_PATH + echo_succ "▸ $P12_PATH - ready" + get_secret $DD_IOS_SECRET__BENCHMARK_PROVISIONING_PROFILE_BASE64 | base64 --decode -o $PP_PATH + echo_succ "▸ $PP_PATH - ready" +} + +trap cleanup_codesigning EXIT INT # clean up keychain on exit + +create_xcconfig +create_codesign_files +install_provisioning_profile $PP_PATH + +create_keychain +keychain_import \ + --p12 $P12_PATH \ + --p12-password $(get_secret "$DD_IOS_SECRET__E2E_CERTIFICATE_P12_PASSWORD") + +echo_subtitle "Run 'make clean archive export upload ARTIFACTS_PATH=\"$ARTIFACTS_PATH\"' in '$BENCHMARK_DIR'" +cd "$BENCHMARK_DIR" +make clean archive export ARTIFACTS_PATH="$ARTIFACTS_PATH" + +if [ "$DRY_RUN" = "1" ] || [ "$DRY_RUN" = "true" ]; then + echo_warn "Running in DRY RUN mode. Skipping 'make upload'." +else + export DATADOG_API_KEY=$(get_secret $DD_IOS_SECRET__E2E_S8S_API_KEY) + export DATADOG_APP_KEY=$(get_secret $DD_IOS_SECRET__E2E_S8S_APP_KEY) + export S8S_APPLICATION_ID=$(get_secret $DD_IOS_SECRET__BENCHMARK_S8S_APPLICATION_ID) + make upload ARTIFACTS_PATH="$ARTIFACTS_PATH" +fi diff --git a/tools/secrets/config.sh b/tools/secrets/config.sh index aa987198ac..754145b643 100644 --- a/tools/secrets/config.sh +++ b/tools/secrets/config.sh @@ -23,6 +23,9 @@ DD_IOS_SECRET__E2E_XCCONFIG_BASE64="e2e.xcconfig.base64" DD_IOS_SECRET__E2E_S8S_API_KEY="e2e.s8s.api.key" DD_IOS_SECRET__E2E_S8S_APP_KEY="e2e.s8s.app.key" DD_IOS_SECRET__E2E_S8S_APPLICATION_ID="e2e.s8s.app.id" +DD_IOS_SECRET__BENCHMARK_PROVISIONING_PROFILE_BASE64="benchmark.provisioning.profile.base64" +DD_IOS_SECRET__BENCHMARK_XCCONFIG_BASE64="benchmark.xcconfig.base64" +DD_IOS_SECRET__BENCHMARK_S8S_APPLICATION_ID="benchmark.s8s.app.id" idx=0 declare -A DD_IOS_SECRETS @@ -38,3 +41,6 @@ DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__E2E_XCCONFIG_BASE64 | Base64-encoded DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__E2E_S8S_API_KEY | DATADOG_API_KEY for uploading E2E app to synthetics" DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__E2E_S8S_APP_KEY | DATADOG_APP_KEY for uploading E2E app to synthetics" DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__E2E_S8S_APPLICATION_ID | Synthetics app ID for E2E tests" +DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__BENCHMARK_PROVISIONING_PROFILE_BASE64 | Base64-encoded provisioning profile file for signing Benchmark app" +DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__BENCHMARK_XCCONFIG_BASE64 | Base64-encoded xcconfig file for Benchmark app" +DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__BENCHMARK_S8S_APPLICATION_ID | Synthetics app ID for Benchmark tests" diff --git a/tools/utils/code-sign.sh b/tools/utils/code-sign.sh new file mode 100755 index 0000000000..04581dfcd2 --- /dev/null +++ b/tools/utils/code-sign.sh @@ -0,0 +1,90 @@ +#!/bin/zsh + +set +x +set -eo pipefail +source ./tools/utils/echo-color.sh + +KEYCHAIN=datadog.keychain +KEYCHAIN_PASSWORD="$(openssl rand -base64 32)" + +PROFILE=datadog.mobileprovision +USER_PP_DIR="$HOME/Library/MobileDevice/Provisioning Profiles" +USER_PP_PATH="$USER_PP_DIR/$PROFILE" + +cleanup_codesigning() { + echo_subtitle "Cleanup code signing" + + rm -f "$PP_PATH" + echo_info "▸ '$PP_PATH' deleted" + + if security delete-keychain "$KEYCHAIN" 2>/dev/null; then + echo_info "▸ '$KEYCHAIN' deleted" + else + echo_warn "▸ Keychain '$KEYCHAIN' not found or failed to delete" + fi +} + +create_keychain() { + # Create temporary keychain + if ! security delete-keychain "$KEYCHAIN" 2>/dev/null; then + echo_warn "▸ Keychain '$KEYCHAIN' not found, nothing to delete" + fi + if ! security create-keychain -p "$KEYCHAIN_PASSWORD" "$KEYCHAIN"; then + echo_err "▸ Error:" "Failed to create keychain '$KEYCHAIN'" + return 1 + fi + if ! security set-keychain-settings -lut 21600 "$KEYCHAIN"; then + echo_err "▸ Error:" "Failed to set keychain settings for '$KEYCHAIN'" + return 1 + fi + if ! security unlock-keychain -p "$KEYCHAIN_PASSWORD" "$KEYCHAIN"; then + echo "▸ Error:" "Failed to unlock keychain '$KEYCHAIN'" + return 1 + fi + echo_succ "▸ '$KEYCHAIN' created and unlocked" +} + +keychain_import() { + # read cmd arguments + while :; do + case $1 in + --p12) P12_PATH=$2 + shift + ;; + --p12-password) P12_PASSWORD=$2 + shift + ;; + *) break + esac + shift + done + + # Import certificate to keychain + if ! security import "$P12_PATH" -P "$P12_PASSWORD" -A -t cert -f pkcs12 -k "$KEYCHAIN"; then + echo_err "▸ Error:" "Failed to import certificate from '$p12' to '$KEYCHAIN'" + return 1 + fi + echo_succ "▸ '$P12_PATH' certificate imported to '$KEYCHAIN'" + + if ! security list-keychain -d user -s "$KEYCHAIN" "login.keychain" "System.keychain"; then + echo_err "▸ Error:" "Failed to configure keychain search list for '$KEYCHAIN'" + return 1 + fi + echo_succ "▸ '$KEYCHAIN' keychain search configured" + + if ! security set-key-partition-list -S apple-tool:,apple: -s -k "$KEYCHAIN_PASSWORD" "$KEYCHAIN" >/dev/null 2>&1; then + echo_err "▸ Error:" "Failed to set key partition list for '$KEYCHAIN'" + return 1 + fi + echo_succ "▸ Permission granted for '$KEYCHAIN' keychain" +} + +install_provisioning_profile() { + # Install provisioning profile + mkdir -p "$USER_PP_DIR" + if ! cp "$1" "$USER_PP_PATH"; then + echo_err "▸ Error:" "Failed to install provisioning profile from '$1' to '$USER_PP_PATH'" + return 1 + fi + echo_succ "▸ '$1' provisioning profile installed in '$USER_PP_PATH'" +} \ No newline at end of file From d1e3b5173c37da8cad090bf576cb9ef4c8f7a13b Mon Sep 17 00:00:00 2001 From: Maxime Epain Date: Fri, 26 Jul 2024 17:06:10 +0200 Subject: [PATCH 02/13] Update .gitlab-ci.yml --- .gitlab-ci.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5e067ec131..fa12d2e70b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,6 +5,7 @@ stages: - ui-test - smoke-test - e2e-test + - benchmark-test - dogfood - release-build - release-publish @@ -266,6 +267,27 @@ E2E Test (upload to s8s): - export DRY_RUN=${DRY_RUN:-0} # default to 0 if not specified - make e2e-build-upload ARTIFACTS_PATH="artifacts/e2e" +# ┌────────────────────────────┐ +# │ Benchmark Test app upload: │ +# └────────────────────────────┘ + +Benchmark Test (upload to s8s): + stage: benchmark-test + rules: + - if: '$CI_COMMIT_BRANCH == $DEVELOP_BRANCH' + variables: + XCODE: "15.3.0" + OS: "17.4" + artifacts: + paths: + - artifacts + expire_in: 2 weeks + script: + - ./tools/runner-setup.sh --xcode "$XCODE" --iOS --os "$OS" --datadog-ci # temporary, waiting for AMI + - make clean + - export DRY_RUN=${DRY_RUN:-0} # default to 0 if not specified + - make benchmark-build-upload ARTIFACTS_PATH="artifacts/benchmark" + # ┌─────────────────┐ # │ SDK dogfooding: │ # └─────────────────┘ From a32c6ab95664db0bb071b337b019f3f2192dc923 Mon Sep 17 00:00:00 2001 From: Maxime Epain Date: Fri, 26 Jul 2024 17:12:42 +0200 Subject: [PATCH 03/13] Update e2e-build-upload.sh --- tools/e2e-build-upload.sh | 82 +++++---------------------------------- 1 file changed, 10 insertions(+), 72 deletions(-) diff --git a/tools/e2e-build-upload.sh b/tools/e2e-build-upload.sh index 2064e39ff8..a17de8fc6b 100755 --- a/tools/e2e-build-upload.sh +++ b/tools/e2e-build-upload.sh @@ -14,6 +14,7 @@ set +x set -eo pipefail source ./tools/utils/argparse.sh source ./tools/utils/echo-color.sh +source ./tools/utils/code-sign.sh source ./tools/secrets/get-secret.sh set_description "Publishes IPA a new version of the E2E app to synthetics." @@ -22,21 +23,15 @@ define_arg "artifacts-path" "" "Path where the IPA artifact will be exported." " check_for_help "$@" parse_args "$@" -KEYCHAIN=datadog.e2e.keychain -KEYCHAIN_PASSWORD="$(openssl rand -base64 32)" -PROFILE=datadog.e2e.mobileprovision - E2E_DIR="E2ETests" E2E_XCCONFIG_PATH="$E2E_DIR/xcconfigs/E2E.local.xcconfig" E2E_CODESIGN_DIR="$E2E_DIR/code-signing" P12_PATH="$E2E_CODESIGN_DIR/e2e_cert.p12" PP_PATH="$E2E_CODESIGN_DIR/e2e.mobileprovision" -PP_INSTALL_DIR="$HOME/Library/MobileDevice/Provisioning Profiles" -PP_INSTALL_PATH="$PP_INSTALL_DIR/$PROFILE" ARTIFACTS_PATH="$(realpath .)/$artifacts_path" -create_e2e_xcconfig() { +create_xcconfig() { echo_subtitle "Create '$E2E_XCCONFIG_PATH'" get_secret $DD_IOS_SECRET__E2E_XCCONFIG_BASE64 | base64 --decode -o $E2E_XCCONFIG_PATH echo_succ "▸ '$E2E_XCCONFIG_PATH' ready" @@ -52,73 +47,16 @@ create_codesign_files() { echo_succ "▸ $PP_PATH - ready" } -setup_codesigning() { - echo_subtitle "Setup code signing" - - # Create temporary keychain - if ! security delete-keychain "$KEYCHAIN" 2>/dev/null; then - echo_warn "▸ Keychain '$KEYCHAIN' not found, nothing to delete" - fi - if ! security create-keychain -p "$KEYCHAIN_PASSWORD" "$KEYCHAIN"; then - echo_err "▸ Error:" "Failed to create keychain '$KEYCHAIN'" - return 1 - fi - if ! security set-keychain-settings -lut 21600 "$KEYCHAIN"; then - echo_err "▸ Error:" "Failed to set keychain settings for '$KEYCHAIN'" - return 1 - fi - if ! security unlock-keychain -p "$KEYCHAIN_PASSWORD" "$KEYCHAIN"; then - echo "▸ Error:" "Failed to unlock keychain '$KEYCHAIN'" - return 1 - fi - echo_succ "▸ '$KEYCHAIN' created and unlocked" - - # Import certificate to keychain - P12_PASSWORD=$(get_secret "$DD_IOS_SECRET__E2E_CERTIFICATE_P12_PASSWORD") - if ! security import "$P12_PATH" -P "$P12_PASSWORD" -A -t cert -f pkcs12 -k "$KEYCHAIN"; then - echo_err "▸ Error:" "Failed to import certificate from '$P12_PATH' to '$KEYCHAIN'" - return 1 - fi - echo_succ "▸ '$P12_PATH' certificate imported to '$KEYCHAIN'" - - if ! security list-keychain -d user -s "$KEYCHAIN" "login.keychain" "System.keychain"; then - echo_err "▸ Error:" "Failed to configure keychain search list for '$KEYCHAIN'" - return 1 - fi - echo_succ "▸ '$KEYCHAIN' keychain search configured" - - if ! security set-key-partition-list -S apple-tool:,apple: -s -k "$KEYCHAIN_PASSWORD" "$KEYCHAIN" >/dev/null 2>&1; then - echo_err "▸ Error:" "Failed to set key partition list for '$KEYCHAIN'" - return 1 - fi - echo_succ "▸ Permission granted for '$KEYCHAIN' keychain" - - # Install provisioning profile - mkdir -p "$PP_INSTALL_DIR" - if ! cp "$PP_PATH" "$PP_INSTALL_PATH"; then - echo_err "▸ Error:" "Failed to install provisioning profile from '$PP_PATH' to '$PP_INSTALL_PATH'" - return 1 - fi - echo_succ "▸ '$PP_PATH' provisioning profile installed in '$PP_INSTALL_PATH'" -} - -cleanup_codesigning() { - echo_subtitle "Cleanup code signing" - - rm -f "$PP_INSTALL_PATH" - echo_info "▸ '$PP_INSTALL_PATH' deleted" - - if security delete-keychain "$KEYCHAIN" 2>/dev/null; then - echo_info "▸ '$KEYCHAIN' deleted" - else - echo_warn "▸ Keychain '$KEYCHAIN' not found or failed to delete" - fi -} +trap cleanup_codesigning EXIT INT # clean up keychain on exit -create_e2e_xcconfig +create_xcconfig create_codesign_files -trap cleanup_codesigning EXIT INT # clean up keychain on exit -setup_codesigning +install_provisioning_profile $PP_PATH + +create_keychain +keychain_import \ + --p12 $P12_PATH \ + --p12-password $(get_secret "$DD_IOS_SECRET__E2E_CERTIFICATE_P12_PASSWORD") echo_subtitle "Run 'make clean archive export upload ARTIFACTS_PATH=\"$ARTIFACTS_PATH\"' in '$E2E_DIR'" cd "$E2E_DIR" From 1647db1a354ea37ea43d5a8033229179493b8773 Mon Sep 17 00:00:00 2001 From: Maxime Epain Date: Mon, 29 Jul 2024 13:15:39 +0200 Subject: [PATCH 04/13] swift-tools-version: 5.9 --- BenchmarkTests/Benchmarks/Package.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BenchmarkTests/Benchmarks/Package.swift b/BenchmarkTests/Benchmarks/Package.swift index 4746338dbc..4376a5177c 100644 --- a/BenchmarkTests/Benchmarks/Package.swift +++ b/BenchmarkTests/Benchmarks/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 5.10 +// swift-tools-version: 5.9 // The swift-tools-version declares the minimum version of Swift required to build this package. import PackageDescription From b7e6625a310b1ab9a99514a0fde223b4c62f7966 Mon Sep 17 00:00:00 2001 From: Maxime Epain Date: Mon, 29 Jul 2024 16:57:55 +0200 Subject: [PATCH 05/13] Update Package.swift --- .../BenchmarkTests.xcodeproj/project.pbxproj | 37 ++++--------- BenchmarkTests/Benchmarks/Package.swift | 55 +++++++++++++++---- .../Benchmarks/Sources/DatadogExporter.swift | 12 +++- BenchmarkTests/Makefile | 6 ++ BenchmarkTests/Runner/AppDelegate.swift | 1 + Makefile | 17 ++++-- 6 files changed, 83 insertions(+), 45 deletions(-) diff --git a/BenchmarkTests/BenchmarkTests.xcodeproj/project.pbxproj b/BenchmarkTests/BenchmarkTests.xcodeproj/project.pbxproj index a95c31d33c..bbd8e9b449 100644 --- a/BenchmarkTests/BenchmarkTests.xcodeproj/project.pbxproj +++ b/BenchmarkTests/BenchmarkTests.xcodeproj/project.pbxproj @@ -7,6 +7,7 @@ objects = { /* Begin PBXBuildFile section */ + D23DD32D2C58D80C00B90C4C /* DatadogBenchmarks in Frameworks */ = {isa = PBXBuildFile; productRef = D23DD32C2C58D80C00B90C4C /* DatadogBenchmarks */; }; D276069F2C514F37002D2A14 /* SessionReplay.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = D27606962C514F37002D2A14 /* SessionReplay.storyboard */; }; D27606A02C514F37002D2A14 /* SessionReplayController.swift in Sources */ = {isa = PBXBuildFile; fileRef = D27606972C514F37002D2A14 /* SessionReplayController.swift */; }; D27606A12C514F37002D2A14 /* SessionReplayScenario.swift in Sources */ = {isa = PBXBuildFile; fileRef = D27606982C514F37002D2A14 /* SessionReplayScenario.swift */; }; @@ -18,7 +19,6 @@ D27606AB2C514F77002D2A14 /* DatadogRUM in Frameworks */ = {isa = PBXBuildFile; productRef = D27606AA2C514F77002D2A14 /* DatadogRUM */; }; D27606AD2C514F77002D2A14 /* DatadogSessionReplay in Frameworks */ = {isa = PBXBuildFile; productRef = D27606AC2C514F77002D2A14 /* DatadogSessionReplay */; }; D27606AF2C514F77002D2A14 /* DatadogTrace in Frameworks */ = {isa = PBXBuildFile; productRef = D27606AE2C514F77002D2A14 /* DatadogTrace */; }; - D27606B12C514F77002D2A14 /* DatadogBenchmarks in Frameworks */ = {isa = PBXBuildFile; productRef = D27606B02C514F77002D2A14 /* DatadogBenchmarks */; }; D29F75502C4AA07E00288638 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = D29F754F2C4AA07E00288638 /* AppDelegate.swift */; }; /* End PBXBuildFile section */ @@ -36,8 +36,6 @@ /* End PBXCopyFilesBuildPhase section */ /* Begin PBXFileReference section */ - D247578B2C53ECBB00DF1397 /* dd-sdk-ios */ = {isa = PBXFileReference; lastKnownFileType = wrapper; name = "dd-sdk-ios"; path = ..; sourceTree = ""; }; - D27606952C514EED002D2A14 /* Benchmarks */ = {isa = PBXFileReference; lastKnownFileType = wrapper; path = Benchmarks; sourceTree = SOURCE_ROOT; }; D27606962C514F37002D2A14 /* SessionReplay.storyboard */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.storyboard; path = SessionReplay.storyboard; sourceTree = ""; }; D27606972C514F37002D2A14 /* SessionReplayController.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SessionReplayController.swift; sourceTree = ""; }; D27606982C514F37002D2A14 /* SessionReplayScenario.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SessionReplayScenario.swift; sourceTree = ""; }; @@ -47,9 +45,11 @@ D27606B22C526908002D2A14 /* Benchmarks.local.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Benchmarks.local.xcconfig; sourceTree = ""; }; D27606B32C526908002D2A14 /* Runner.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Runner.xcconfig; sourceTree = ""; }; D27606B42C526908002D2A14 /* Synthetics.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Synthetics.xcconfig; sourceTree = ""; }; + D277C84A2C58D3210072343C /* Benchmarks */ = {isa = PBXFileReference; lastKnownFileType = wrapper; path = Benchmarks; sourceTree = ""; }; D29F754D2C4AA07E00288638 /* Runner.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = Runner.app; sourceTree = BUILT_PRODUCTS_DIR; }; D29F754F2C4AA07E00288638 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; D29F755D2C4AA08000288638 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; + D2CA7E862C57F9B800AAB380 /* dd-sdk-ios */ = {isa = PBXFileReference; lastKnownFileType = wrapper; name = "dd-sdk-ios"; path = ..; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -60,7 +60,7 @@ D27606A92C514F77002D2A14 /* DatadogLogs in Frameworks */, D27606AF2C514F77002D2A14 /* DatadogTrace in Frameworks */, D27606AD2C514F77002D2A14 /* DatadogSessionReplay in Frameworks */, - D27606B12C514F77002D2A14 /* DatadogBenchmarks in Frameworks */, + D23DD32D2C58D80C00B90C4C /* DatadogBenchmarks in Frameworks */, D27606AB2C514F77002D2A14 /* DatadogRUM in Frameworks */, D27606A72C514F77002D2A14 /* DatadogCore in Frameworks */, ); @@ -120,8 +120,8 @@ D29F75482C4A9F9500288638 /* Frameworks */ = { isa = PBXGroup; children = ( - D27606952C514EED002D2A14 /* Benchmarks */, - D247578B2C53ECBB00DF1397 /* dd-sdk-ios */, + D277C84A2C58D3210072343C /* Benchmarks */, + D2CA7E862C57F9B800AAB380 /* dd-sdk-ios */, ); name = Frameworks; sourceTree = ""; @@ -160,7 +160,7 @@ D27606AA2C514F77002D2A14 /* DatadogRUM */, D27606AC2C514F77002D2A14 /* DatadogSessionReplay */, D27606AE2C514F77002D2A14 /* DatadogTrace */, - D27606B02C514F77002D2A14 /* DatadogBenchmarks */, + D23DD32C2C58D80C00B90C4C /* DatadogBenchmarks */, ); productName = Runner; productReference = D29F754D2C4AA07E00288638 /* Runner.app */; @@ -191,7 +191,6 @@ ); mainGroup = D29F75272C4A9EFA00288638; packageReferences = ( - D29F75612C4AA8C200288638 /* XCRemoteSwiftPackageReference "opentelemetry-swift" */, ); productRefGroup = D29F75312C4A9EFA00288638 /* Products */; projectDirPath = ""; @@ -298,7 +297,6 @@ INFOPLIST_FILE = Runner/Info.plist; INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; INFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen; - INFOPLIST_KEY_UIMainStoryboardFile = Main; INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; IPHONEOS_DEPLOYMENT_TARGET = 13.0; @@ -449,7 +447,6 @@ INFOPLIST_FILE = Runner/Info.plist; INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; INFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen; - INFOPLIST_KEY_UIMainStoryboardFile = Main; INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; IPHONEOS_DEPLOYMENT_TARGET = 13.0; @@ -482,7 +479,6 @@ INFOPLIST_FILE = Runner/Info.plist; INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; INFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen; - INFOPLIST_KEY_UIMainStoryboardFile = Main; INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; IPHONEOS_DEPLOYMENT_TARGET = 13.0; @@ -525,18 +521,11 @@ }; /* End XCConfigurationList section */ -/* Begin XCRemoteSwiftPackageReference section */ - D29F75612C4AA8C200288638 /* XCRemoteSwiftPackageReference "opentelemetry-swift" */ = { - isa = XCRemoteSwiftPackageReference; - repositoryURL = "https://github.com/open-telemetry/opentelemetry-swift"; - requirement = { - kind = exactVersion; - version = 1.6.0; - }; - }; -/* End XCRemoteSwiftPackageReference section */ - /* Begin XCSwiftPackageProductDependency section */ + D23DD32C2C58D80C00B90C4C /* DatadogBenchmarks */ = { + isa = XCSwiftPackageProductDependency; + productName = DatadogBenchmarks; + }; D27606A62C514F77002D2A14 /* DatadogCore */ = { isa = XCSwiftPackageProductDependency; productName = DatadogCore; @@ -557,10 +546,6 @@ isa = XCSwiftPackageProductDependency; productName = DatadogTrace; }; - D27606B02C514F77002D2A14 /* DatadogBenchmarks */ = { - isa = XCSwiftPackageProductDependency; - productName = DatadogBenchmarks; - }; /* End XCSwiftPackageProductDependency section */ }; rootObject = D29F75282C4A9EFA00288638 /* Project object */; diff --git a/BenchmarkTests/Benchmarks/Package.swift b/BenchmarkTests/Benchmarks/Package.swift index 4376a5177c..8f08cf4eca 100644 --- a/BenchmarkTests/Benchmarks/Package.swift +++ b/BenchmarkTests/Benchmarks/Package.swift @@ -2,23 +2,56 @@ // The swift-tools-version declares the minimum version of Swift required to build this package. import PackageDescription +import Foundation let package = Package( name: "DatadogBenchmarks", products: [ .library( name: "DatadogBenchmarks", - targets: ["Benchmarks"]), - ], - dependencies: [ - .package(url: "https://github.com/open-telemetry/opentelemetry-swift", from: "1.0.0"), - ], - targets: [ - .target( - name: "Benchmarks", - dependencies: [ - .product(name: "OpenTelemetrySdk", package: "opentelemetry-swift") - ] + targets: ["Benchmarks"] ) ] ) + +func addOpenTelemetryDependency(_ version: Version) { + // The project must be open with the 'OTEL_SWIFT' env variable. + // Please run 'make benchmark-tests-open' from the root directory. + // + // Note: Carthage will still try to resolve dependencies of Xcode projects in + // sub directories, in this case the project will depend on the default + // 'DataDog/opentelemetry-swift-packages' depedency. + if ProcessInfo.processInfo.environment["OTEL_SWIFT"] != nil { + package.dependencies = [ + .package(url: "https://github.com/open-telemetry/opentelemetry-swift", exact: version) + ] + + package.targets = [ + .target( + name: "Benchmarks", + dependencies: [ + .product(name: "OpenTelemetryApi", package: "opentelemetry-swift"), + .product(name: "OpenTelemetrySdk", package: "opentelemetry-swift") + ], + swiftSettings: [.unsafeFlags(["-DOTEL_SWIFT"])] + ) + ] + + } else { + package.dependencies = [ + .package(url: "https://github.com/DataDog/opentelemetry-swift-packages", exact: version) + ] + + package.targets = [ + .target( + name: "Benchmarks", + dependencies: [ + .product(name: "OpenTelemetryApi", package: "opentelemetry-swift-packages") + ], + swiftSettings: [.unsafeFlags(["-DOTEL_API"])] + ) + ] + } +} + +addOpenTelemetryDependency("1.6.0") diff --git a/BenchmarkTests/Benchmarks/Sources/DatadogExporter.swift b/BenchmarkTests/Benchmarks/Sources/DatadogExporter.swift index c9a79a48cd..9d32d57e4b 100644 --- a/BenchmarkTests/Benchmarks/Sources/DatadogExporter.swift +++ b/BenchmarkTests/Benchmarks/Sources/DatadogExporter.swift @@ -5,18 +5,24 @@ */ import Foundation + +#if OTEL_API +#error("Benchmarks depends on opentelemetry-swift. Please open the project with 'make benchmark-tests-open'.") +#endif + +#if OTEL_SWIFT import OpenTelemetrySdk public final class DatadogExporter: SpanExporter, MetricExporter { private let session: URLSession - convenience init() { + public convenience init() { let configuration: URLSessionConfiguration = .ephemeral configuration.urlCache = nil self.init(session: URLSession(configuration: configuration)) } - init(session: URLSession) { + public init(session: URLSession) { self.session = session } @@ -36,3 +42,5 @@ public final class DatadogExporter: SpanExporter, MetricExporter { } } + +#endif diff --git a/BenchmarkTests/Makefile b/BenchmarkTests/Makefile index 890960fd02..f8bb035ecb 100644 --- a/BenchmarkTests/Makefile +++ b/BenchmarkTests/Makefile @@ -55,3 +55,9 @@ upload: --mobileApplicationId "${S8S_APPLICATION_ID}" \ --versionName "$(VERSION)" \ --latest + +open: + @$(ECHO_SUBTITLE2) "make open" + @pgrep -q Xcode && killall Xcode && $(ECHO_WARNING) "- Xcode killed" || $(ECHO_SUCCESS) "- Xcode not running" + @sleep 0.5 && $(ECHO_INFO) "- launching" + @open --env OTEL_SWIFT BenchmarkTests.xcodeproj diff --git a/BenchmarkTests/Runner/AppDelegate.swift b/BenchmarkTests/Runner/AppDelegate.swift index 9b992f4fe0..212ec75bad 100644 --- a/BenchmarkTests/Runner/AppDelegate.swift +++ b/BenchmarkTests/Runner/AppDelegate.swift @@ -18,6 +18,7 @@ class AppDelegate: UIResponder, UIApplicationDelegate { window = UIWindow(frame: UIScreen.main.bounds) window?.rootViewController = scenario.start(info: info) window?.makeKeyAndVisible() + return true } } diff --git a/Makefile b/Makefile index c6f2fc518e..89ec96d028 100644 --- a/Makefile +++ b/Makefile @@ -276,6 +276,11 @@ benchmark-build-upload: @$(ECHO_TITLE) "make benchmark-build-upload ARTIFACTS_PATH='$(ARTIFACTS_PATH)' DRY_RUN='$(DRY_RUN)'" DRY_RUN=$(DRY_RUN) ./tools/benchmark-build-upload.sh --artifacts-path "$(ARTIFACTS_PATH)" +# Opens `BenchmarkTests` project with passing required ENV variables +benchmark-tests-open: + @$(ECHO_TITLE) "benchmark-tests-open" + @$(MAKE) -C BenchmarkTests open + xcodeproj-session-replay: @echo "⚙️ Generating 'DatadogSessionReplay.xcodeproj'..." @cd DatadogSessionReplay/ && swift package generate-xcodeproj @@ -453,9 +458,9 @@ set-ci-secret: @./tools/secrets/set-secret.sh bump: - @read -p "Enter version number: " version; \ - echo "// GENERATED FILE: Do not edit directly\n\ninternal let __sdkVersion = \"$$version\"" > DatadogCore/Sources/Versioning.swift; \ - ./tools/podspec_bump_version.sh $$version; \ - git add . ; \ - git commit -m "Bumped version to $$version"; \ - echo Bumped version to $$version + @read -p "Enter version number: " version; \ + echo "// GENERATED FILE: Do not edit directly\n\ninternal let __sdkVersion = \"$$version\"" > DatadogCore/Sources/Versioning.swift; \ + ./tools/podspec_bump_version.sh $$version; \ + git add . ; \ + git commit -m "Bumped version to $$version"; \ + echo Bumped version to $$version From 9f3bd620e6f5d4991b88c260c2c85173d3e94d42 Mon Sep 17 00:00:00 2001 From: Maxime Epain Date: Thu, 1 Aug 2024 13:58:01 +0200 Subject: [PATCH 06/13] Create README.md --- BenchmarkTests/README.md | 83 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 BenchmarkTests/README.md diff --git a/BenchmarkTests/README.md b/BenchmarkTests/README.md new file mode 100644 index 0000000000..e1a9d046eb --- /dev/null +++ b/BenchmarkTests/README.md @@ -0,0 +1,83 @@ +# Benchmark Tests + +[Synthetics for Mobile](https://docs.datadoghq.com/mobile_app_testing/) runs Benchmark test scenarios to collect metrics of the SDK performances. + + +## CI + +CI continuously builds, signs, and uploads a runner application to Synthetics which runs predefined tests daily. + +### Build + +Before building the application, the `BenchmarkTests/xcconfigs/Benchmark.local.xcconfig` configuration file must be present and contain the `Mobile - Integration Org` client token and RUM application ID, and API Key. These values are sensitive and must be securely stored. + +```ini +CLIENT_TOKEN= +RUM_APPLICATION_ID= +API_KEY= +``` + +### Sign + +To sign the runner application, the certificate and provision profile defined in [Synthetics.xcconfig](xcconfigs/Synthetics.xcconfig) and in [exportOptions.plist](exportOptions.plist) needs to be installed on the build machine. These files are sensitive and must be securely stored. Make sure to update both files when updating the certificate and provisioning profile, otherwise signing fails. + +> [!NOTE] +> Certificate & Provisioning Profile are also available through the [App Store Connect API](https://developer.apple.com/documentation/appstoreconnectapi). But we don't have the tooling in place. + +### Upload + +The application version (build number) is set to the commit SHA of the current job, and the build is uploaded to Synthetics using the [datadog-ci](https://github.com/DataDog/datadog-ci) CLI. This step expects environment variables to authenticate with the `Mobile - Integration Org`: + +```bash +export DATADOG_API_KEY= +export DATADOG_APP_KEY= +export S8S_APPLICATION_ID= +``` + +## Development + +Each scenario is independent and can be considered as an app within the runner. + +### Create a scenario + +A scenario must comply with the [`Scenario`](Runner/Scenarios/Scenario.swift) protocol. Upon start, a scenario initializes the SDK, enables features, and returns a root view-controller. + +Here is a simple example of a scenario using Logs: +```swift +import Foundation +import UIKit + +import DatadogCore +import DatadogLogs + +struct SessionReplayWebViewScenario: Scenario { + + func start(info: TestInfo) -> UIViewController { + + Datadog.initialize( + with: .benchmark(info: info), // SDK init with the benchmark configuration + trackingConsent: .granted + ) + + Logs.enable() + + return LoggerViewController() + } +} +``` + +The test should then be added to the [`SyntheticScenario`](Runner/Scenarios/Scenario.swift) enumeration so it can be selected, either manually or by setting the `BENCHMARK_SCENARIO` environment variable. + + +### Adding the test in synthetics + +**Note:** When creating a test in Synthetics, make sure to always run on the _latest version_. + +You can skip the scenario selection screen by setting the **Process Arguments** of the Synthetic test: +```json +{ + "BENCHMARK_SCENARIO": "" +} +``` + +The test's name must match the [`SyntheticScenario`](Runner/Scenarios/Scenario.swift) enum case. \ No newline at end of file From 29bedde9b381bdb404a4ce66bddbc153265bf2a3 Mon Sep 17 00:00:00 2001 From: Maxime Epain Date: Thu, 1 Aug 2024 15:11:39 +0200 Subject: [PATCH 07/13] Rename secrets --- tools/benchmark-build-upload.sh | 8 ++++---- tools/e2e-build-upload.sh | 8 ++++---- tools/secrets/check-secrets.sh | 2 +- tools/secrets/config.sh | 28 ++++++++++++++++++++-------- 4 files changed, 29 insertions(+), 17 deletions(-) diff --git a/tools/benchmark-build-upload.sh b/tools/benchmark-build-upload.sh index e43bc7293a..5b44778242 100755 --- a/tools/benchmark-build-upload.sh +++ b/tools/benchmark-build-upload.sh @@ -41,7 +41,7 @@ create_codesign_files() { echo_subtitle "Create codesign files in '$BENCHMARK_CODESIGN_DIR'" rm -rf "$BENCHMARK_CODESIGN_DIR" mkdir -p "$BENCHMARK_CODESIGN_DIR" - get_secret $DD_IOS_SECRET__E2E_CERTIFICATE_P12_BASE64 | base64 --decode -o $P12_PATH + get_secret $DD_IOS_SECRET__DEV_CERTIFICATE_P12_BASE64 | base64 --decode -o $P12_PATH echo_succ "▸ $P12_PATH - ready" get_secret $DD_IOS_SECRET__BENCHMARK_PROVISIONING_PROFILE_BASE64 | base64 --decode -o $PP_PATH echo_succ "▸ $PP_PATH - ready" @@ -56,7 +56,7 @@ install_provisioning_profile $PP_PATH create_keychain keychain_import \ --p12 $P12_PATH \ - --p12-password $(get_secret "$DD_IOS_SECRET__E2E_CERTIFICATE_P12_PASSWORD") + --p12-password $(get_secret "$DD_IOS_SECRET__DEV_CERTIFICATE_P12_PASSWORD") echo_subtitle "Run 'make clean archive export upload ARTIFACTS_PATH=\"$ARTIFACTS_PATH\"' in '$BENCHMARK_DIR'" cd "$BENCHMARK_DIR" @@ -65,8 +65,8 @@ make clean archive export ARTIFACTS_PATH="$ARTIFACTS_PATH" if [ "$DRY_RUN" = "1" ] || [ "$DRY_RUN" = "true" ]; then echo_warn "Running in DRY RUN mode. Skipping 'make upload'." else - export DATADOG_API_KEY=$(get_secret $DD_IOS_SECRET__E2E_S8S_API_KEY) - export DATADOG_APP_KEY=$(get_secret $DD_IOS_SECRET__E2E_S8S_APP_KEY) + export DATADOG_API_KEY=$(get_secret $DD_IOS_SECRET__MI_S8S_API_KEY) + export DATADOG_APP_KEY=$(get_secret $DD_IOS_SECRET__MI_S8S_APP_KEY) export S8S_APPLICATION_ID=$(get_secret $DD_IOS_SECRET__BENCHMARK_S8S_APPLICATION_ID) make upload ARTIFACTS_PATH="$ARTIFACTS_PATH" fi diff --git a/tools/e2e-build-upload.sh b/tools/e2e-build-upload.sh index a17de8fc6b..fc1feca3cb 100755 --- a/tools/e2e-build-upload.sh +++ b/tools/e2e-build-upload.sh @@ -41,7 +41,7 @@ create_codesign_files() { echo_subtitle "Create codesign files in '$E2E_CODESIGN_DIR'" rm -rf "$E2E_CODESIGN_DIR" mkdir -p "$E2E_CODESIGN_DIR" - get_secret $DD_IOS_SECRET__E2E_CERTIFICATE_P12_BASE64 | base64 --decode -o $P12_PATH + get_secret $DD_IOS_SECRET__DEV_CERTIFICATE_P12_BASE64 | base64 --decode -o $P12_PATH echo_succ "▸ $P12_PATH - ready" get_secret $DD_IOS_SECRET__E2E_PROVISIONING_PROFILE_BASE64 | base64 --decode -o $PP_PATH echo_succ "▸ $PP_PATH - ready" @@ -56,7 +56,7 @@ install_provisioning_profile $PP_PATH create_keychain keychain_import \ --p12 $P12_PATH \ - --p12-password $(get_secret "$DD_IOS_SECRET__E2E_CERTIFICATE_P12_PASSWORD") + --p12-password $(get_secret "$DD_IOS_SECRET__DEV_CERTIFICATE_P12_PASSWORD") echo_subtitle "Run 'make clean archive export upload ARTIFACTS_PATH=\"$ARTIFACTS_PATH\"' in '$E2E_DIR'" cd "$E2E_DIR" @@ -65,8 +65,8 @@ make clean archive export ARTIFACTS_PATH="$ARTIFACTS_PATH" if [ "$DRY_RUN" = "1" ] || [ "$DRY_RUN" = "true" ]; then echo_warn "Running in DRY RUN mode. Skipping 'make upload'." else - export DATADOG_API_KEY=$(get_secret $DD_IOS_SECRET__E2E_S8S_API_KEY) - export DATADOG_APP_KEY=$(get_secret $DD_IOS_SECRET__E2E_S8S_APP_KEY) + export DATADOG_API_KEY=$(get_secret $DD_IOS_SECRET__MI_S8S_API_KEY) + export DATADOG_APP_KEY=$(get_secret $DD_IOS_SECRET__MI_S8S_APP_KEY) export S8S_APPLICATION_ID=$(get_secret $DD_IOS_SECRET__E2E_S8S_APPLICATION_ID) make upload ARTIFACTS_PATH="$ARTIFACTS_PATH" fi diff --git a/tools/secrets/check-secrets.sh b/tools/secrets/check-secrets.sh index 76e87b28df..794509b378 100755 --- a/tools/secrets/check-secrets.sh +++ b/tools/secrets/check-secrets.sh @@ -3,7 +3,7 @@ # Checks if all secret values are available in current env. # # Usage: -# $ ./tools/secrets/set-secret.sh +# $ ./tools/secrets/check-secrets.sh # # Note: # - Requires `vault` to be installed diff --git a/tools/secrets/config.sh b/tools/secrets/config.sh index 754145b643..56c11b13cb 100644 --- a/tools/secrets/config.sh +++ b/tools/secrets/config.sh @@ -16,17 +16,23 @@ DD_IOS_SECRET__GH_CLI_TOKEN="gh.cli.token" DD_IOS_SECRET__CARTHAGE_GH_TOKEN="carthage.gh.token" DD_IOS_SECRET__CP_TRUNK_TOKEN="cocoapods.trunk.token" DD_IOS_SECRET__SSH_KEY="ssh.key" -DD_IOS_SECRET__E2E_CERTIFICATE_P12_BASE64="e2e.certificate.p12.base64" -DD_IOS_SECRET__E2E_CERTIFICATE_P12_PASSWORD="e2e.certificate.p12.password" +DD_IOS_SECRET__DEV_CERTIFICATE_P12_BASE64="dev.certificate.p12.base64" +DD_IOS_SECRET__DEV_CERTIFICATE_P12_PASSWORD="dev.certificate.p12.password" +DD_IOS_SECRET__MI_S8S_API_KEY="mi.s8s.api.key" +DD_IOS_SECRET__MI_S8S_APP_KEY="mi.s8s.app.key" DD_IOS_SECRET__E2E_PROVISIONING_PROFILE_BASE64="e2e.provisioning.profile.base64" DD_IOS_SECRET__E2E_XCCONFIG_BASE64="e2e.xcconfig.base64" -DD_IOS_SECRET__E2E_S8S_API_KEY="e2e.s8s.api.key" -DD_IOS_SECRET__E2E_S8S_APP_KEY="e2e.s8s.app.key" DD_IOS_SECRET__E2E_S8S_APPLICATION_ID="e2e.s8s.app.id" DD_IOS_SECRET__BENCHMARK_PROVISIONING_PROFILE_BASE64="benchmark.provisioning.profile.base64" DD_IOS_SECRET__BENCHMARK_XCCONFIG_BASE64="benchmark.xcconfig.base64" DD_IOS_SECRET__BENCHMARK_S8S_APPLICATION_ID="benchmark.s8s.app.id" +# To remove +DD_IOS_SECRET__E2E_CERTIFICATE_P12_BASE64="e2e.certificate.p12.base64" +DD_IOS_SECRET__E2E_CERTIFICATE_P12_PASSWORD="e2e.certificate.p12.password" +DD_IOS_SECRET__E2E_S8S_API_KEY="e2e.s8s.api.key" +DD_IOS_SECRET__E2E_S8S_APP_KEY="e2e.s8s.app.key" + idx=0 declare -A DD_IOS_SECRETS DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__TEST_SECRET | test secret to see if things work, free to change but not delete" @@ -34,13 +40,19 @@ DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__GH_CLI_TOKEN | GitHub token to authe DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__CARTHAGE_GH_TOKEN | GitHub token to avoid rate limiting Carthage commands (https://github.com/Carthage/Carthage/pull/605)" DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__CP_TRUNK_TOKEN | Cocoapods token to authenticate 'pod trunk' operations (https://guides.cocoapods.org/terminal/commands.html)" DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__SSH_KEY | SSH key to authenticate 'git clone git@github.com:...' operations" -DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__E2E_CERTIFICATE_P12_BASE64 | Base64-encoded '.p12' certificate file for signing E2E app" -DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__E2E_CERTIFICATE_P12_PASSWORD | Password to '$DD_IOS_SECRET__E2E_CERTIFICATE_P12_BASE64' certificate" +DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__DEV_CERTIFICATE_P12_BASE64 | Base64-encoded '.p12' developer certificate file for signing apps" +DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__DEV_CERTIFICATE_P12_PASSWORD | Password to '$DD_IOS_SECRET__DEV_CERTIFICATE_P12_PASSWORD' certificate" +DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__MI_S8S_API_KEY | DATADOG_API_KEY for uploading app to synthetics in Mobile - Integration org" +DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__MI_S8S_APP_KEY | DATADOG_APP_KEY for uploading E2E app to synthetics in Mobile - Integration org" DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__E2E_PROVISIONING_PROFILE_BASE64 | Base64-encoded provisioning profile file for signing E2E app" DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__E2E_XCCONFIG_BASE64 | Base64-encoded xcconfig file for E2E app" -DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__E2E_S8S_API_KEY | DATADOG_API_KEY for uploading E2E app to synthetics" -DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__E2E_S8S_APP_KEY | DATADOG_APP_KEY for uploading E2E app to synthetics" DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__E2E_S8S_APPLICATION_ID | Synthetics app ID for E2E tests" DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__BENCHMARK_PROVISIONING_PROFILE_BASE64 | Base64-encoded provisioning profile file for signing Benchmark app" DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__BENCHMARK_XCCONFIG_BASE64 | Base64-encoded xcconfig file for Benchmark app" DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__BENCHMARK_S8S_APPLICATION_ID | Synthetics app ID for Benchmark tests" + +# To remove +DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__E2E_CERTIFICATE_P12_BASE64 | Base64-encoded '.p12' certificate file for signing E2E app (to be removed, use dev.certificate.p12.base64 instead)" +DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__E2E_CERTIFICATE_P12_PASSWORD | Password to '$DD_IOS_SECRET__E2E_CERTIFICATE_P12_BASE64' certificate(to be removed, use dev.certificate.p12.password instead)" +DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__E2E_S8S_API_KEY | DATADOG_API_KEY for uploading E2E app to synthetics" +DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__E2E_S8S_APP_KEY | DATADOG_APP_KEY for uploading E2E app to synthetics" From d588ab07be0e4d885e59e5f588af57b2b743957f Mon Sep 17 00:00:00 2001 From: Maxime Epain Date: Wed, 7 Aug 2024 13:50:40 +0200 Subject: [PATCH 08/13] Remove code sign in pbxproj --- BenchmarkTests/BenchmarkTests.xcodeproj/project.pbxproj | 8 -------- 1 file changed, 8 deletions(-) diff --git a/BenchmarkTests/BenchmarkTests.xcodeproj/project.pbxproj b/BenchmarkTests/BenchmarkTests.xcodeproj/project.pbxproj index bbd8e9b449..4cb16fa4c4 100644 --- a/BenchmarkTests/BenchmarkTests.xcodeproj/project.pbxproj +++ b/BenchmarkTests/BenchmarkTests.xcodeproj/project.pbxproj @@ -439,10 +439,7 @@ ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = YES; ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; - CODE_SIGN_IDENTITY = "Apple Development"; - CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = f34790fea; - DEVELOPMENT_TEAM = JKFCB4CN7C; GENERATE_INFOPLIST_FILE = YES; INFOPLIST_FILE = Runner/Info.plist; INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; @@ -457,7 +454,6 @@ MARKETING_VERSION = 1.0; PRODUCT_BUNDLE_IDENTIFIER = com.datadoghq.benchmarks.Runner; PRODUCT_NAME = "$(TARGET_NAME)"; - PROVISIONING_PROFILE_SPECIFIER = ""; SWIFT_EMIT_LOC_STRINGS = YES; SWIFT_VERSION = 5.0; TARGETED_DEVICE_FAMILY = "1,2"; @@ -471,10 +467,7 @@ ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = YES; ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; - CODE_SIGN_IDENTITY = "Apple Development"; - CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = f34790fea; - DEVELOPMENT_TEAM = JKFCB4CN7C; GENERATE_INFOPLIST_FILE = YES; INFOPLIST_FILE = Runner/Info.plist; INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; @@ -489,7 +482,6 @@ MARKETING_VERSION = 1.0; PRODUCT_BUNDLE_IDENTIFIER = com.datadoghq.benchmarks.Runner; PRODUCT_NAME = "$(TARGET_NAME)"; - PROVISIONING_PROFILE_SPECIFIER = ""; SWIFT_EMIT_LOC_STRINGS = YES; SWIFT_VERSION = 5.0; TARGETED_DEVICE_FAMILY = "1,2"; From 46c8038b386230dbf3cff2b1e79b051086f07102 Mon Sep 17 00:00:00 2001 From: Maxime Epain Date: Thu, 8 Aug 2024 11:29:45 +0200 Subject: [PATCH 09/13] Update README.md --- BenchmarkTests/README.md | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/BenchmarkTests/README.md b/BenchmarkTests/README.md index e1a9d046eb..3d40e49f35 100644 --- a/BenchmarkTests/README.md +++ b/BenchmarkTests/README.md @@ -5,11 +5,11 @@ ## CI -CI continuously builds, signs, and uploads a runner application to Synthetics which runs predefined tests daily. +CI continuously builds, signs, and uploads a runner application to Synthetics which runs predefined tests. ### Build -Before building the application, the `BenchmarkTests/xcconfigs/Benchmark.local.xcconfig` configuration file must be present and contain the `Mobile - Integration Org` client token and RUM application ID, and API Key. These values are sensitive and must be securely stored. +Before building the application, make sure the `BenchmarkTests/xcconfigs/Benchmark.local.xcconfig` configuration file is present and contains the `Mobile - Integration Org` client token, RUM application ID, and API Key. These values are sensitive and must be securely stored. ```ini CLIENT_TOKEN= @@ -19,7 +19,7 @@ API_KEY= ### Sign -To sign the runner application, the certificate and provision profile defined in [Synthetics.xcconfig](xcconfigs/Synthetics.xcconfig) and in [exportOptions.plist](exportOptions.plist) needs to be installed on the build machine. These files are sensitive and must be securely stored. Make sure to update both files when updating the certificate and provisioning profile, otherwise signing fails. +To sign the runner application, the certificate and provision profile defined in [Synthetics.xcconfig](xcconfigs/Synthetics.xcconfig) and in [exportOptions.plist](exportOptions.plist) needs to be installed on the build machine. The certificate and profile are sensitive files and must be securely stored. Make sure to update both files when updating the certificate and provisioning profile, otherwise signing fails. > [!NOTE] > Certificate & Provisioning Profile are also available through the [App Store Connect API](https://developer.apple.com/documentation/appstoreconnectapi). But we don't have the tooling in place. @@ -50,7 +50,7 @@ import UIKit import DatadogCore import DatadogLogs -struct SessionReplayWebViewScenario: Scenario { +struct LogsScenario: Scenario { func start(info: TestInfo) -> UIViewController { @@ -66,18 +66,8 @@ struct SessionReplayWebViewScenario: Scenario { } ``` -The test should then be added to the [`SyntheticScenario`](Runner/Scenarios/Scenario.swift) enumeration so it can be selected, either manually or by setting the `BENCHMARK_SCENARIO` environment variable. +Add the test to the [`SyntheticScenario`](Runner/Scenarios/Scenario.swift) enumeration so it can be selected, either manually or by setting the `BENCHMARK_SCENARIO` environment variable. +### Synthetics Configuration -### Adding the test in synthetics - -**Note:** When creating a test in Synthetics, make sure to always run on the _latest version_. - -You can skip the scenario selection screen by setting the **Process Arguments** of the Synthetic test: -```json -{ - "BENCHMARK_SCENARIO": "" -} -``` - -The test's name must match the [`SyntheticScenario`](Runner/Scenarios/Scenario.swift) enum case. \ No newline at end of file +Please refer to [Confluence page (internal)](https://datadoghq.atlassian.net/wiki/spaces/RUMP/pages/3981476482/Benchmarks+iOS) \ No newline at end of file From d76d2a2d3bd9157ee3c2e3e07aa612c9dd025e7f Mon Sep 17 00:00:00 2001 From: Maxime Epain Date: Thu, 8 Aug 2024 11:30:31 +0200 Subject: [PATCH 10/13] Apply CR suggestions --- Makefile | 2 +- tools/secrets/config.sh | 2 +- tools/utils/code-sign.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 89ec96d028..e6fd5172d3 100644 --- a/Makefile +++ b/Makefile @@ -278,7 +278,7 @@ benchmark-build-upload: # Opens `BenchmarkTests` project with passing required ENV variables benchmark-tests-open: - @$(ECHO_TITLE) "benchmark-tests-open" + @$(ECHO_TITLE) "make benchmark-tests-open" @$(MAKE) -C BenchmarkTests open xcodeproj-session-replay: diff --git a/tools/secrets/config.sh b/tools/secrets/config.sh index 56c11b13cb..f5f2f70818 100644 --- a/tools/secrets/config.sh +++ b/tools/secrets/config.sh @@ -43,7 +43,7 @@ DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__SSH_KEY | SSH key to authenticate 'g DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__DEV_CERTIFICATE_P12_BASE64 | Base64-encoded '.p12' developer certificate file for signing apps" DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__DEV_CERTIFICATE_P12_PASSWORD | Password to '$DD_IOS_SECRET__DEV_CERTIFICATE_P12_PASSWORD' certificate" DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__MI_S8S_API_KEY | DATADOG_API_KEY for uploading app to synthetics in Mobile - Integration org" -DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__MI_S8S_APP_KEY | DATADOG_APP_KEY for uploading E2E app to synthetics in Mobile - Integration org" +DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__MI_S8S_APP_KEY | DATADOG_APP_KEY for uploading app to synthetics in Mobile - Integration org" DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__E2E_PROVISIONING_PROFILE_BASE64 | Base64-encoded provisioning profile file for signing E2E app" DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__E2E_XCCONFIG_BASE64 | Base64-encoded xcconfig file for E2E app" DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__E2E_S8S_APPLICATION_ID | Synthetics app ID for E2E tests" diff --git a/tools/utils/code-sign.sh b/tools/utils/code-sign.sh index 04581dfcd2..4a7cc53c06 100755 --- a/tools/utils/code-sign.sh +++ b/tools/utils/code-sign.sh @@ -87,4 +87,4 @@ install_provisioning_profile() { return 1 fi echo_succ "▸ '$1' provisioning profile installed in '$USER_PP_PATH'" -} \ No newline at end of file +} From f0de4ea204505fee835632218808edba879b256f Mon Sep 17 00:00:00 2001 From: Maxime Epain Date: Thu, 8 Aug 2024 13:55:11 +0200 Subject: [PATCH 11/13] Use open --new --env --- BenchmarkTests/Makefile | 4 +--- tools/sr-snapshot-test.sh | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/BenchmarkTests/Makefile b/BenchmarkTests/Makefile index f8bb035ecb..9606e4187f 100644 --- a/BenchmarkTests/Makefile +++ b/BenchmarkTests/Makefile @@ -58,6 +58,4 @@ upload: open: @$(ECHO_SUBTITLE2) "make open" - @pgrep -q Xcode && killall Xcode && $(ECHO_WARNING) "- Xcode killed" || $(ECHO_SUCCESS) "- Xcode not running" - @sleep 0.5 && $(ECHO_INFO) "- launching" - @open --env OTEL_SWIFT BenchmarkTests.xcodeproj + @open --new --env OTEL_SWIFT BenchmarkTests.xcodeproj diff --git a/tools/sr-snapshot-test.sh b/tools/sr-snapshot-test.sh index 380b438776..63a1c30828 100755 --- a/tools/sr-snapshot-test.sh +++ b/tools/sr-snapshot-test.sh @@ -75,9 +75,7 @@ test_snapshots() { open_snapshot_tests_project() { echo_info "Opening SRSnapshotTests with DD_TEST_UTILITIES_ENABLED ..." - pgrep -q Xcode && killall Xcode && echo_warn "- Xcode killed" || echo_succ "- Xcode not running" - sleep 0.5 && echo "- launching" # Sleep, otherwise, if Xcode was running it often fails with "procNotFound: no eligible process with specified descriptor" - open --env DD_TEST_UTILITIES_ENABLED "$TEST_WORKSPACE" + open --new --env DD_TEST_UTILITIES_ENABLED "$TEST_WORKSPACE" } if [ "$open_project" = "true" ]; then From b332ab2760d0728d939935493209d8fe2818e4df Mon Sep 17 00:00:00 2001 From: Maxime Epain Date: Thu, 8 Aug 2024 17:03:04 +0200 Subject: [PATCH 12/13] Update .gitlab-ci.yml Co-authored-by: Maciek Grzybowski --- .gitlab-ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fa12d2e70b..143c90cb0e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -275,15 +275,12 @@ Benchmark Test (upload to s8s): stage: benchmark-test rules: - if: '$CI_COMMIT_BRANCH == $DEVELOP_BRANCH' - variables: - XCODE: "15.3.0" - OS: "17.4" artifacts: paths: - artifacts expire_in: 2 weeks script: - - ./tools/runner-setup.sh --xcode "$XCODE" --iOS --os "$OS" --datadog-ci # temporary, waiting for AMI + - ./tools/runner-setup.sh --xcode "$DEFAULT_XCODE" --datadog-ci - make clean - export DRY_RUN=${DRY_RUN:-0} # default to 0 if not specified - make benchmark-build-upload ARTIFACTS_PATH="artifacts/benchmark" From 092ae30fff2f666203636107564c1e36f05838b6 Mon Sep 17 00:00:00 2001 From: Maxime Epain Date: Tue, 13 Aug 2024 16:12:40 +0200 Subject: [PATCH 13/13] Remove secrets from config file --- tools/secrets/config.sh | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tools/secrets/config.sh b/tools/secrets/config.sh index f5f2f70818..8b281fc216 100644 --- a/tools/secrets/config.sh +++ b/tools/secrets/config.sh @@ -27,12 +27,6 @@ DD_IOS_SECRET__BENCHMARK_PROVISIONING_PROFILE_BASE64="benchmark.provisioning.pro DD_IOS_SECRET__BENCHMARK_XCCONFIG_BASE64="benchmark.xcconfig.base64" DD_IOS_SECRET__BENCHMARK_S8S_APPLICATION_ID="benchmark.s8s.app.id" -# To remove -DD_IOS_SECRET__E2E_CERTIFICATE_P12_BASE64="e2e.certificate.p12.base64" -DD_IOS_SECRET__E2E_CERTIFICATE_P12_PASSWORD="e2e.certificate.p12.password" -DD_IOS_SECRET__E2E_S8S_API_KEY="e2e.s8s.api.key" -DD_IOS_SECRET__E2E_S8S_APP_KEY="e2e.s8s.app.key" - idx=0 declare -A DD_IOS_SECRETS DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__TEST_SECRET | test secret to see if things work, free to change but not delete" @@ -50,9 +44,3 @@ DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__E2E_S8S_APPLICATION_ID | Synthetics DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__BENCHMARK_PROVISIONING_PROFILE_BASE64 | Base64-encoded provisioning profile file for signing Benchmark app" DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__BENCHMARK_XCCONFIG_BASE64 | Base64-encoded xcconfig file for Benchmark app" DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__BENCHMARK_S8S_APPLICATION_ID | Synthetics app ID for Benchmark tests" - -# To remove -DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__E2E_CERTIFICATE_P12_BASE64 | Base64-encoded '.p12' certificate file for signing E2E app (to be removed, use dev.certificate.p12.base64 instead)" -DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__E2E_CERTIFICATE_P12_PASSWORD | Password to '$DD_IOS_SECRET__E2E_CERTIFICATE_P12_BASE64' certificate(to be removed, use dev.certificate.p12.password instead)" -DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__E2E_S8S_API_KEY | DATADOG_API_KEY for uploading E2E app to synthetics" -DD_IOS_SECRETS[$((idx++))]="$DD_IOS_SECRET__E2E_S8S_APP_KEY | DATADOG_APP_KEY for uploading E2E app to synthetics"