refactor(vcs): Enable to provide VCS-specific configuration options

 While VCS implementations are already plugins, they are not yet
 configurable. VCS implementations require common configurations
 (e.g., `revision`, `recursive`) and should support also
 VCS-specific configurations if they are consumed via their API.
 This allows to add functionality to individual VCS implementations
 without the need to implement them for all of them.

 This refactoring keeps the common configurations attributes as
 they are, while VCS-specific configurations are stored
 generically in an `options` attribute.

Fixes oss-review-toolkit#8556.

Signed-off-by: Wolfgang Klenk <[email protected]>

downloader/src/main/kotlin/Downloader.kt

@@ -270,8 +270,13 @@ class Downloader(private val config: DownloaderConfiguration) {
             return RepositoryProvenance(pkg.vcsProcessed, pkg.vcsProcessed.revision)
         }
 
+        val vcsConfig = config.getVersionControlSystemConfiguration(applicableVcs.type)
+        if (vcsConfig != null) {
+            logger.info { "Using ${applicableVcs.type}-specific configuration $vcsConfig." }
+        }
+
         val workingTree = try {
-            applicableVcs.download(pkg, outputDirectory, config.allowMovingRevisions, recursive)
+            applicableVcs.download(pkg, outputDirectory, config.allowMovingRevisions, recursive, vcsConfig)
         } catch (e: DownloadException) {
             // TODO: Introduce something like a "strict" mode and only do these kind of fallbacks in non-strict mode.
             val vcsUrlNoCredentials = pkg.vcsProcessed.url.replaceCredentialsInUri()
@@ -285,7 +290,7 @@ class Downloader(private val config: DownloaderConfiguration) {
                 outputDirectory.safeDeleteRecursively(baseDirectory = outputDirectory)
 
                 val fallbackPkg = pkg.copy(vcsProcessed = pkg.vcsProcessed.copy(url = vcsUrlNoCredentials))
-                applicableVcs.download(fallbackPkg, outputDirectory, config.allowMovingRevisions, recursive)
+                applicableVcs.download(fallbackPkg, outputDirectory, config.allowMovingRevisions, recursive, vcsConfig)
             } else {
                 throw e
             }

downloader/src/main/kotlin/VersionControlSystem.kt

@@ -28,6 +28,7 @@ import org.ossreviewtoolkit.model.Package
 import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.VcsType
 import org.ossreviewtoolkit.model.config.LicenseFilePatterns
+import org.ossreviewtoolkit.model.config.VersionControlSystemConfiguration
 import org.ossreviewtoolkit.model.orEmpty
 import org.ossreviewtoolkit.utils.common.CommandLineTool
 import org.ossreviewtoolkit.utils.common.Plugin
@@ -223,6 +224,7 @@ abstract class VersionControlSystem(
      * Download the source code as specified by the [pkg] information to [targetDir]. [allowMovingRevisions] toggles
      * whether to allow downloads using symbolic names that point to moving revisions, like Git branches. If [recursive]
      * is `true`, any nested repositories (like Git submodules or Mercurial subrepositories) are downloaded, too.
+     * VCS-specific configurations can be supplied via [vcsConfiguration].
      *
      * @return An object describing the downloaded working tree.
      *
@@ -232,7 +234,8 @@ abstract class VersionControlSystem(
         pkg: Package,
         targetDir: File,
         allowMovingRevisions: Boolean = false,
-        recursive: Boolean = true
+        recursive: Boolean = true,
+        vcsConfiguration: VersionControlSystemConfiguration? = null
     ): WorkingTree {
         val workingTree = try {
             initWorkingTree(targetDir, pkg.vcsProcessed)
@@ -248,7 +251,7 @@ abstract class VersionControlSystem(
 
         for ((index, revision) in revisionCandidates.withIndex()) {
             logger.info { "Trying revision candidate '$revision' (${index + 1} of ${revisionCandidates.size})..." }
-            results += updateWorkingTree(workingTree, revision, pkg.vcsProcessed.path, recursive)
+            results += updateWorkingTree(workingTree, revision, pkg.vcsProcessed.path, recursive, vcsConfiguration)
             if (results.last().isSuccess) break
         }
 
@@ -380,14 +383,17 @@ abstract class VersionControlSystem(
 
     /**
      * Update the [working tree][workingTree] by checking out the given [revision], optionally limited to the given
-     * [path] and [recursively][recursive] updating any nested working trees. Return a [Result] that encapsulates the
-     * originally requested [revision] on success, or the occurred exception on failure.
+     * [path] and [recursively][recursive] updating any nested working trees.
+     * Optionally, a VCS-specific configuration can be supplied via [vcsConfiguration].
+     * Return a [Result] that encapsulates the originally requested [revision] on success,
+     * or the occurred exception on failure.
      */
     abstract fun updateWorkingTree(
         workingTree: WorkingTree,
         revision: String,
         path: String = "",
-        recursive: Boolean = false
+        recursive: Boolean = false,
+        vcsConfiguration: VersionControlSystemConfiguration? = null
     ): Result<String>
 
     /**

downloader/src/test/kotlin/VersionControlSystemTest.kt

@@ -33,6 +33,7 @@ import java.lang.UnsupportedOperationException
 import org.ossreviewtoolkit.model.Package
 import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.VcsType
+import org.ossreviewtoolkit.model.config.VersionControlSystemConfiguration
 import org.ossreviewtoolkit.plugins.versioncontrolsystems.git.Git
 import org.ossreviewtoolkit.utils.common.CommandLineTool
 
@@ -168,6 +169,7 @@ private class VersionControlSystemTestImpl(
         workingTree: WorkingTree,
         revision: String,
         path: String,
-        recursive: Boolean
+        recursive: Boolean,
+        vcsConfiguration: VersionControlSystemConfiguration?
     ): Result<String> = Result.failure(UnsupportedOperationException("Unexpected invocation."))
 }

model/src/main/kotlin/config/DownloaderConfiguration.kt

@@ -44,9 +44,35 @@ data class DownloaderConfiguration(
      * Configuration of the considered source code origins and their priority order. This must not be empty and not
      * contain any duplicates.
      */
-    val sourceCodeOrigins: List<SourceCodeOrigin> = listOf(SourceCodeOrigin.VCS, SourceCodeOrigin.ARTIFACT)
+    val sourceCodeOrigins: List<SourceCodeOrigin> = listOf(SourceCodeOrigin.VCS, SourceCodeOrigin.ARTIFACT),
+
+    /**
+     * Version control system specific configurations. The key needs to match VCS type,
+     * e.g. "Git" for the Git version control system.
+     */
+    val versionControlSystems: Map<String, VersionControlSystemConfiguration>? = null
 ) {
+    /**
+     * A copy of [versionControlSystems] with case-insensitive keys.
+     */
+    private val versionControlSystemsCaseInsensitive: Map<String, VersionControlSystemConfiguration>? =
+        versionControlSystems?.toSortedMap(String.CASE_INSENSITIVE_ORDER)
+
     init {
         sourceCodeOrigins.requireNotEmptyNoDuplicates()
+
+        val duplicateVersionControlSystems =
+            versionControlSystems?.keys.orEmpty() - versionControlSystemsCaseInsensitive?.keys?.toSet().orEmpty()
+
+        require(duplicateVersionControlSystems.isEmpty()) {
+            "The following version control systems have duplicate configuration: " +
+                "${duplicateVersionControlSystems.joinToString()}."
+        }
     }
+
+    /**
+     * Get a [VersionControlSystemConfiguration] from [versionControlSystems].
+     * The difference to accessing the map directly is that [vcsType] can be case-insensitive.
+     */
+    fun getVersionControlSystemConfiguration(vcsType: String) = versionControlSystemsCaseInsensitive?.get(vcsType)
 }

model/src/main/kotlin/config/VersionControlSystemConfiguration.kt

@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package org.ossreviewtoolkit.model.config
+
+import com.fasterxml.jackson.annotation.JsonInclude
+
+import org.ossreviewtoolkit.utils.common.Options
+
+/**
+ * The configuration for a Version Control System (VCS).
+ */
+@JsonInclude(JsonInclude.Include.NON_NULL)
+data class VersionControlSystemConfiguration(
+    /**
+     * Custom configuration options. See the documentation of the respective class for available options.
+     */
+    val options: Options? = null
+)

model/src/main/resources/reference.yml

@@ -167,6 +167,17 @@ ort:
 
     sourceCodeOrigins: [VCS, ARTIFACT]
 
+    # Optional VCS-specific configuration options.
+    versionControlSystems:
+      Git:
+        options:
+          # Depth of the commit history to fetch when updating submodules
+          submoduleHistoryDepth: 10
+
+          # A flag to control whether nested submodules should be updated (true), or if only the submodules
+          # on the first layer should be considered (false).
+          updateNestedSubmodules: true
+
   scanner:
     skipConcluded: true
     skipExcluded: true

model/src/test/kotlin/config/DownloaderConfigurationTest.kt

@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package org.ossreviewtoolkit.model.config
+
+import io.kotest.assertions.throwables.shouldThrow
+import io.kotest.core.spec.style.WordSpec
+import io.kotest.matchers.shouldBe
+
+class DownloaderConfigurationTest : WordSpec({
+    "DownloaderConfiguration()" should {
+        "throw an exception on duplicate VCS configurations" {
+            shouldThrow<IllegalArgumentException> {
+                DownloaderConfiguration(
+                    versionControlSystems = mapOf(
+                        "Git" to VersionControlSystemConfiguration(),
+                        "git" to VersionControlSystemConfiguration()
+                    )
+                )
+            }
+        }
+    }
+
+    "getVersionControlSystemConfiguration" should {
+        "return configuration case-insensitively" {
+            val vcsConfiguration = VersionControlSystemConfiguration()
+            val downloaderConfiguration = DownloaderConfiguration(
+                versionControlSystems = mapOf(
+                    "MyVCS" to vcsConfiguration
+                )
+            )
+
+            downloaderConfiguration.getVersionControlSystemConfiguration("MyVCS") shouldBe vcsConfiguration
+            downloaderConfiguration.getVersionControlSystemConfiguration("myvcs") shouldBe vcsConfiguration
+            downloaderConfiguration.getVersionControlSystemConfiguration("MYVCS") shouldBe vcsConfiguration
+        }
+    }
+})

plugins/version-control-systems/git/src/main/kotlin/Git.kt

@@ -48,6 +48,7 @@ import org.ossreviewtoolkit.downloader.VersionControlSystem
 import org.ossreviewtoolkit.downloader.WorkingTree
 import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.VcsType
+import org.ossreviewtoolkit.model.config.VersionControlSystemConfiguration
 import org.ossreviewtoolkit.utils.common.CommandLineTool
 import org.ossreviewtoolkit.utils.common.Os
 import org.ossreviewtoolkit.utils.common.collectMessages
@@ -170,7 +171,8 @@ class Git : VersionControlSystem(GitCommand) {
         workingTree: WorkingTree,
         revision: String,
         path: String,
-        recursive: Boolean
+        recursive: Boolean,
+        vcsConfiguration: VersionControlSystemConfiguration? // Git-specific configuration only
     ): Result<String> =
         (workingTree as GitWorkingTree).useRepo {
             Git(this).use { git ->

plugins/version-control-systems/git/src/main/kotlin/GitRepo.kt

@@ -36,6 +36,7 @@ import org.ossreviewtoolkit.downloader.VersionControlSystem
 import org.ossreviewtoolkit.downloader.WorkingTree
 import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.VcsType
+import org.ossreviewtoolkit.model.config.VersionControlSystemConfiguration
 import org.ossreviewtoolkit.model.utils.parseRepoManifestPath
 import org.ossreviewtoolkit.utils.common.CommandLineTool
 import org.ossreviewtoolkit.utils.common.Os
@@ -192,7 +193,8 @@ class GitRepo : VersionControlSystem(GitRepoCommand) {
         workingTree: WorkingTree,
         revision: String,
         path: String,
-        recursive: Boolean
+        recursive: Boolean,
+        vcsConfiguration: VersionControlSystemConfiguration? // GitRepo-specific configuration only.
     ): Result<String> {
         val manifestRevision = revision.takeUnless { it.isBlank() }
         val manifestPath = workingTree.getInfo().url.parseRepoManifestPath()

plugins/version-control-systems/mercurial/src/main/kotlin/Mercurial.kt

@@ -27,6 +27,7 @@ import org.ossreviewtoolkit.downloader.VersionControlSystem
 import org.ossreviewtoolkit.downloader.WorkingTree
 import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.VcsType
+import org.ossreviewtoolkit.model.config.VersionControlSystemConfiguration
 import org.ossreviewtoolkit.utils.common.CommandLineTool
 import org.ossreviewtoolkit.utils.common.ProcessCapture
 
@@ -93,7 +94,13 @@ class Mercurial : VersionControlSystem(MercurialCommand) {
         return getWorkingTree(targetDir)
     }
 
-    override fun updateWorkingTree(workingTree: WorkingTree, revision: String, path: String, recursive: Boolean) =
+    override fun updateWorkingTree(
+        workingTree: WorkingTree,
+        revision: String,
+        path: String,
+        recursive: Boolean,
+        vcsConfiguration: VersionControlSystemConfiguration? // Mercurial-specific configuration only
+    ) =
         runCatching {
             // To safe network bandwidth, only pull exactly the revision we want. Do not use "-u" to update the
             // working tree just yet, as Mercurial would only update if new changesets were pulled. But that might

plugins/version-control-systems/subversion/src/main/kotlin/Subversion.kt

@@ -31,6 +31,7 @@ import org.ossreviewtoolkit.downloader.VersionControlSystem
 import org.ossreviewtoolkit.downloader.WorkingTree
 import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.VcsType
+import org.ossreviewtoolkit.model.config.VersionControlSystemConfiguration
 import org.ossreviewtoolkit.utils.common.collectMessages
 import org.ossreviewtoolkit.utils.ort.OrtAuthenticator
 import org.ossreviewtoolkit.utils.ort.OrtProxySelector
@@ -126,7 +127,13 @@ class Subversion : VersionControlSystem() {
         return pathRevisions.single()
     }
 
-    override fun updateWorkingTree(workingTree: WorkingTree, revision: String, path: String, recursive: Boolean) =
+    override fun updateWorkingTree(
+        workingTree: WorkingTree,
+        revision: String,
+        path: String,
+        recursive: Boolean,
+        vcsConfiguration: VersionControlSystemConfiguration? // Subversion-specific configuration only.
+    ) =
         runCatching {
             // Note that the path should never be part of the URL as that would root the working tree at that path, but
             // the path should be available in the working tree.