KTOR-8043 Add retry to server tests by default

ktor-server/ktor-server-core/jsAndWasmShared/src/io/ktor/server/config/ConfigLoaders.jsAndWasmShared.kt

@@ -1,11 +1,10 @@
 /*
- * Copyright 2014-2022 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+ * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
  */
 
 package io.ktor.server.config
 
 import io.ktor.server.engine.*
-import io.ktor.utils.io.*
 
 internal actual val CONFIG_PATH: List<String>
     get() = listOfNotNull(
@@ -18,6 +17,7 @@ internal actual val CONFIG_PATH: List<String>
 public actual val configLoaders: List<ConfigLoader>
     get() = _configLoaders
 
+@Suppress("ObjectPropertyName")
 private val _configLoaders: MutableList<ConfigLoader> = mutableListOf()
 
 public fun addConfigLoader(loader: ConfigLoader) {

ktor-server/ktor-server-test-base/build.gradle.kts

@@ -1,5 +1,5 @@
 /*
- * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+ * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
  */
 
 description = ""
@@ -10,7 +10,7 @@ kotlin.sourceSets {
     commonMain {
         dependencies {
             api(project(":ktor-server:ktor-server-test-host"))
-            api(libs.kotlin.test)
+            api(project(":ktor-shared:ktor-test-base"))
         }
     }
 
@@ -21,7 +21,6 @@ kotlin.sourceSets {
             api(project(":ktor-client:ktor-client-apache"))
             api(project(":ktor-network:ktor-network-tls:ktor-network-tls-certificates"))
             api(project(":ktor-server:ktor-server-plugins:ktor-server-call-logging"))
-            api(project(":ktor-shared:ktor-test-base"))
 
             if (jetty_alpn_boot_version != null) {
                 api(libs.jetty.alpn.boot)

ktor-server/ktor-server-test-base/common/src/io/ktor/server/test/base/BaseTest.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+ * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
  */
 
 package io.ktor.server.test.base
@@ -16,21 +16,15 @@ expect abstract class BaseTest() {
     open fun afterTest()
 
     fun collectUnhandledException(error: Throwable) // TODO: better name?
-    fun runTest(timeout: Duration = 60.seconds, block: suspend CoroutineScope.() -> Unit): TestResult
+    fun runTest(
+        timeout: Duration = 60.seconds,
+        retries: Int = DEFAULT_RETRIES,
+        block: suspend CoroutineScope.() -> Unit
+    ): TestResult
 }
 
-fun BaseTest.runTest(
-    retry: Int,
-    timeout: Duration = this.timeout,
-    block: suspend CoroutineScope.() -> Unit
-): TestResult {
-    lateinit var lastCause: Throwable
-    repeat(retry) {
-        try {
-            return runTest(timeout, block)
-        } catch (cause: Throwable) {
-            lastCause = cause
-        }
-    }
-    throw lastCause
-}
+/**
+ * Defaults to `1` on all platforms except for JVM.
+ * On JVM retries are disabled as we use test-retry Gradle plugin instead.
+ */
+internal expect val DEFAULT_RETRIES: Int

ktor-server/ktor-server-test-base/common/test/io/ktor/server/test/base/BaseTestTest.kt

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package io.ktor.server.test.base
+
+import io.ktor.util.*
+import kotlinx.coroutines.test.TestResult
+import kotlin.test.Test
+import kotlin.test.fail
+
+class BaseTestTest : BaseTest() {
+
+    @Test
+    fun `runTest - retry test by default on non-JVM platform`(): TestResult {
+        var retryCount = 0
+        return runTest {
+            if (!PlatformUtils.IS_JVM && retryCount++ < 1) fail("This test should be retried")
+        }
+    }
+
+    @Test
+    fun `runTest - don't retry test by default on JVM platform`(): TestResult {
+        var retryCount = 0
+        return runTest {
+            if (PlatformUtils.IS_JVM && retryCount++ > 0) fail("This test should not be retried")
+        }
+    }
+
+    @Test
+    fun `runTest - more than one retry`(): TestResult {
+        var retryCount = 0
+        return runTest(retries = 3) {
+            if (retryCount++ < 3) fail("This test should be retried")
+        }
+    }
+
+    @Test
+    fun `runTest - retry should work collected exceptions`(): TestResult {
+        var retryCount = 0
+        return runTest(retries = 1) {
+            if (retryCount++ < 1) collectUnhandledException(Exception("This test should be retried"))
+        }
+    }
+}

ktor-server/ktor-server-test-base/jvm/src/io/ktor/server/test/base/BaseTestJvm.kt

@@ -1,9 +1,10 @@
 /*
- * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+ * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
  */
 
 package io.ktor.server.test.base
 
+import io.ktor.test.*
 import io.ktor.test.dispatcher.*
 import io.ktor.test.junit.*
 import io.ktor.test.junit.coroutines.*
@@ -47,13 +48,20 @@ actual abstract class BaseTest actual constructor() {
 
     actual fun runTest(
         timeout: Duration,
+        retries: Int,
         block: suspend CoroutineScope.() -> Unit
-    ): TestResult = runTestWithRealTime(CoroutineName("test-$testName"), timeout) {
-        beforeTest()
-        try {
-            block()
-        } finally {
-            afterTest()
+    ): TestResult = retryTest(retries) { retry ->
+        runTestWithRealTime(CoroutineName("test-$testName"), timeout) {
+            if (retry > 0) println("[Retry $retry/$retries]")
+            beforeTest()
+            try {
+                block()
+            } finally {
+                afterTest()
+            }
         }
     }
 }
+
+/** On JVM retries are disabled as we use test-retry Gradle plugin instead. */
+internal actual const val DEFAULT_RETRIES: Int = 0

ktor-server/ktor-server-test-base/posix/src/io/ktor/server/test/base/BaseTestNix.kt → ktor-server/ktor-server-test-base/nonJvm/src/io/ktor/server/test/base/BaseTest.nonJvm.kt

@@ -1,9 +1,10 @@
 /*
- * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+ * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
  */
 
 package io.ktor.server.test.base
 
+import io.ktor.test.*
 import io.ktor.test.dispatcher.*
 import io.ktor.utils.io.*
 import io.ktor.utils.io.locks.*
@@ -12,15 +13,14 @@ import kotlinx.coroutines.test.TestResult
 import kotlin.time.Duration
 import kotlin.time.Duration.Companion.seconds
 
+@OptIn(InternalAPI::class)
 actual abstract class BaseTest actual constructor() {
     actual open val timeout: Duration = 10.seconds
 
     private val errors = mutableListOf<Throwable>()
 
-    @OptIn(InternalAPI::class)
     private val errorsLock = SynchronizedObject()
 
-    @OptIn(InternalAPI::class)
     actual fun collectUnhandledException(error: Throwable) {
         synchronized(errorsLock) {
             errors.add(error)
@@ -31,6 +31,9 @@ actual abstract class BaseTest actual constructor() {
     }
 
     actual open fun afterTest() {
+        val errors = synchronized(errorsLock) { errors.toList() }
+        this.errors.clear()
+
         if (errors.isEmpty()) return
 
         val error = UnhandledErrorsException(
@@ -46,15 +49,21 @@ actual abstract class BaseTest actual constructor() {
 
     actual fun runTest(
         timeout: Duration,
+        retries: Int,
         block: suspend CoroutineScope.() -> Unit
-    ): TestResult = runTestWithRealTime(timeout = timeout) {
-        beforeTest()
-        try {
-            block()
-        } finally {
-            afterTest()
+    ): TestResult = retryTest(retries) { retry ->
+        runTestWithRealTime(timeout = timeout) {
+            if (retry > 0) println("[Retry $retry/$retries]")
+            beforeTest()
+            try {
+                block()
+            } finally {
+                afterTest()
+            }
         }
     }
 }
 
+internal actual const val DEFAULT_RETRIES: Int = 1
+
 private class UnhandledErrorsException(override val message: String) : Exception()

ktor-shared/ktor-test-base/common/src/io/ktor/test/TestResult.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+ * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
  */
 
 package io.ktor.test
@@ -18,4 +18,21 @@ expect inline fun TestResult.andThen(crossinline block: () -> Any): TestResult
 internal expect inline fun testWithRecover(noinline recover: (Throwable) -> Unit, test: () -> TestResult): TestResult
 
 internal expect inline fun <T> runTestForEach(items: Iterable<T>, crossinline test: (T) -> TestResult): TestResult
-internal expect inline fun retryTest(retries: Int, crossinline test: (Int) -> TestResult): TestResult
+
+/**
+ * Executes a test function with retry capabilities.
+ *
+ * ```
+ * retryTest(retires = 2) { retry ->
+ *     runTest {
+ *         println("This test passes only on second retry. Current retry is $retry")
+ *         assertEquals(2, retry)
+ *     }
+ * }
+ * ```
+ *
+ * @param retries The number of retries to attempt after an initial failure. Must be a non-negative integer.
+ * @param test A test to execute, which accepts the current retry attempt (starting at 0) as an argument.
+ * @return A [TestResult] representing the outcome of the test after all attempts.
+ */
+expect inline fun retryTest(retries: Int, crossinline test: (Int) -> TestResult): TestResult

ktor-shared/ktor-test-base/common/test/io/ktor/test/RunTestWithDataTest.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+ * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
  */
 
 package io.ktor.test
@@ -101,18 +101,18 @@ class RunTestWithDataTest {
     fun testRetriesHaveIndependentTimeout() = runTestWithData(
         singleTestCase,
         retries = 1,
-        timeout = 30.milliseconds,
+        timeout = 50.milliseconds,
         test = { (_, retry) ->
-            realTimeDelay(20.milliseconds)
+            realTimeDelay(30.milliseconds)
             if (retry == 0) fail("Try again, please")
         },
     )
 
     @Test
     fun testDifferentItemsHaveIndependentTimeout() = runTestWithData(
         testCases = 1..2,
-        timeout = 30.milliseconds,
-        test = { realTimeDelay(20.milliseconds) },
+        timeout = 50.milliseconds,
+        test = { realTimeDelay(30.milliseconds) },
     )
 
     @Test

ktor-shared/ktor-test-base/jsAndWasmShared/src/io/ktor/test/TestResult.jsAndWasmShared.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+ * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
  */
 
 package io.ktor.test
@@ -14,7 +14,8 @@ internal actual inline fun testWithRecover(
 internal actual inline fun <T> runTestForEach(items: Iterable<T>, crossinline test: (T) -> TestResult): TestResult =
     items.fold(DummyTestResult) { acc, item -> acc.andThen { test(item) } }
 
-internal actual inline fun retryTest(retries: Int, crossinline test: (Int) -> TestResult): TestResult =
+actual inline fun retryTest(retries: Int, crossinline test: (Int) -> TestResult): TestResult =
     (1..retries).fold(test(0)) { acc, retry -> acc.catch { test(retry) } }
 
+@PublishedApi
 internal expect fun TestResult.catch(action: (Throwable) -> Any): TestResult

ktor-shared/ktor-test-base/jvm/src/io/ktor/test/junit/ErrorCollector.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+ * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
  */
 
 package io.ktor.test.junit

ktor-shared/ktor-test-base/jvmAndPosix/src/io/ktor/test/TestResult.jvmAndPosix.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+ * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
  */
 
 package io.ktor.test
@@ -25,7 +25,7 @@ internal actual inline fun <T> runTestForEach(items: Iterable<T>, test: (T) -> T
     return DummyTestResult
 }
 
-internal actual inline fun retryTest(retries: Int, test: (Int) -> TestResult): TestResult {
+actual inline fun retryTest(retries: Int, test: (Int) -> TestResult): TestResult {
     lateinit var lastCause: Throwable
     repeat(retries + 1) { attempt ->
         try {