refactor(vcs): Enable to provide VCS-specific configuration options

While VCS implementations are already plugins, they are not yet configurable. VCS implementations require common configurations (e.g., `revision`, `recursive`) and should support also VCS-specific configurations if they are consumed via their API. This allows to add functionality to individual VCS implementations without the need to implement them for all of them. Fixes oss-review-toolkit#8556. Signed-off-by: Wolfgang Klenk <[email protected]>
boschglobal · Nov 13, 2024 · f8e6bb6 · f8e6bb6
1 parent f8a0c39
commit f8e6bb6
Show file tree

Hide file tree

Showing 20 changed files with 397 additions and 79 deletions.
diff --git a/cli/src/funTest/kotlin/AnalyzerFunTest.kt b/cli/src/funTest/kotlin/AnalyzerFunTest.kt
@@ -29,6 +29,7 @@ import java.util.concurrent.TimeUnit
 import org.ossreviewtoolkit.analyzer.Analyzer
 import org.ossreviewtoolkit.analyzer.PackageManagerFactory
 import org.ossreviewtoolkit.analyzer.analyze
+import org.ossreviewtoolkit.downloader.VersionControlSystemConfiguration
 import org.ossreviewtoolkit.model.Package
 import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.VcsType
@@ -52,7 +53,10 @@ class AnalyzerFunTest : WordSpec({
                     revision = "31588aa8f8555474e1c3c66a359ec99e4cd4b1fa"
                 )
             )
-            val outputDir = tempdir().also { GitRepo().download(pkg, it) }
+            val outputDir = tempdir().also {
+                GitRepo.Factory().create(VersionControlSystemConfiguration())
+                    .download(pkg, it)
+            }
 
             val result = analyze(outputDir, packageManagers = emptySet()).toYaml()
 

diff --git a/downloader/src/main/kotlin/Downloader.kt b/downloader/src/main/kotlin/Downloader.kt
@@ -232,7 +232,7 @@ class Downloader(private val config: DownloaderConfiguration) {
         var applicableVcs: VersionControlSystem? = null
 
         if (pkg.vcsProcessed.type != VcsType.UNKNOWN) {
-            applicableVcs = VersionControlSystem.forType(pkg.vcsProcessed.type)
+            applicableVcs = VersionControlSystem.forType(pkg.vcsProcessed.type, config.versionControlSystems)
             logger.info {
                 applicableVcs?.let {
                     "Detected VCS type '${it.type}' from type name '${pkg.vcsProcessed.type}'."
@@ -241,7 +241,7 @@ class Downloader(private val config: DownloaderConfiguration) {
         }
 
         if (applicableVcs == null) {
-            applicableVcs = VersionControlSystem.forUrl(pkg.vcsProcessed.url)
+            applicableVcs = VersionControlSystem.forUrl(pkg.vcsProcessed.url, config.versionControlSystems)
             logger.info {
                 applicableVcs?.let {
                     "Detected VCS type '${it.type}' from URL ${pkg.vcsProcessed.url}."

diff --git a/downloader/src/main/kotlin/VersionControlSystem.kt b/downloader/src/main/kotlin/VersionControlSystem.kt
@@ -24,13 +24,14 @@ import java.io.IOException
 
 import org.apache.logging.log4j.kotlin.logger
 
+import org.ossreviewtoolkit.downloader.VersionControlSystemFactory.Companion.ALL
 import org.ossreviewtoolkit.model.Package
 import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.VcsType
 import org.ossreviewtoolkit.model.config.LicenseFilePatterns
+import org.ossreviewtoolkit.model.config.VersionControlSystemConfiguration
 import org.ossreviewtoolkit.model.orEmpty
 import org.ossreviewtoolkit.utils.common.CommandLineTool
-import org.ossreviewtoolkit.utils.common.Plugin
 import org.ossreviewtoolkit.utils.common.collectMessages
 import org.ossreviewtoolkit.utils.common.uppercaseFirstChar
 import org.ossreviewtoolkit.utils.ort.ORT_REPO_CONFIG_FILENAME
@@ -44,22 +45,23 @@ abstract class VersionControlSystem(
      * the version control system is available.
      */
     private val commandLineTool: CommandLineTool? = null
-) : Plugin {
+) {
     companion object {
-        /**
-         * All [version control systems][VersionControlSystem] available in the classpath, sorted by their priority.
-         */
-        val ALL by lazy {
-            Plugin.getAll<VersionControlSystem>().toList().sortedByDescending { (_, vcs) -> vcs.priority }.toMap()
-        }
-
         /**
          * Return the applicable VCS for the given [vcsType], or null if none is applicable.
          */
-        fun forType(vcsType: VcsType) =
-            ALL.values.find {
-                it.isAvailable() && it.isApplicableType(vcsType)
+        fun forType(
+            vcsType: VcsType,
+            versionControlSystems: Map<String, VersionControlSystemConfiguration> = emptyMap()
+        ) = ALL.values.filter { vcsFactory -> vcsFactory.type == vcsType.toString() }
+            .map { vcsFactory ->
+                // If there is a configuration for the VCS type, use it, otherwise create
+                // the VCS with an empty configuration.
+                versionControlSystems[vcsFactory.type]?.let { vcsConfig ->
+                    vcsFactory.create(options = vcsConfig.options, secrets = emptyMap())
+                } ?: vcsFactory.create(options = emptyMap(), secrets = emptyMap())
             }
+            .firstOrNull { vcs -> vcs.isAvailable() }
 
         /**
          * A map to cache the [VersionControlSystem], if any, for previously queried URLs. This helps to speed up
@@ -72,7 +74,7 @@ abstract class VersionControlSystem(
          * Return the applicable VCS for the given [vcsUrl], or null if none is applicable.
          */
         @Synchronized
-        fun forUrl(vcsUrl: String) =
+        fun forUrl(vcsUrl: String, versionControlSystems: Map<String, VersionControlSystemConfiguration> = emptyMap()) =
             // Do not use getOrPut() here as it cannot handle null values, also see
             // https://youtrack.jetbrains.com/issue/KT-21392.
             if (vcsUrl in urlToVcsMap) {
@@ -82,12 +84,17 @@ abstract class VersionControlSystem(
                 when (val type = VcsHost.parseUrl(vcsUrl).type) {
                     VcsType.UNKNOWN -> {
                         // ...then eventually try to determine the type also dynamically.
-                        ALL.values.find {
-                            it.isAvailable() && it.isApplicableUrl(vcsUrl)
-                        }
+                        ALL.values
+                            .map { vcsFactory ->
+                                // If there is a configuration for the VCS type, use it, otherwise create
+                                // the VCS with an empty configuration.
+                                versionControlSystems[vcsFactory.type]?.let { vcsConfig ->
+                                    vcsFactory.create(options = vcsConfig.options, secrets = emptyMap())
+                                } ?: vcsFactory.create(options = emptyMap(), secrets = emptyMap())
+                            }.firstOrNull { vcs -> vcs.isAvailable() && vcs.isApplicableUrl(vcsUrl) }
                     }
 
-                    else -> forType(type)
+                    else -> forType(type, versionControlSystems)
                 }.also {
                     urlToVcsMap[vcsUrl] = it
                 }
@@ -109,28 +116,31 @@ abstract class VersionControlSystem(
             return if (absoluteVcsDirectory in dirToVcsMap) {
                 dirToVcsMap[absoluteVcsDirectory]
             } else {
-                ALL.values.asSequence().mapNotNull {
-                    if (it is CommandLineTool && !it.isInPath()) {
-                        null
-                    } else {
-                        it.getWorkingTree(absoluteVcsDirectory)
-                    }
-                }.find {
-                    try {
-                        it.isValid()
-                    } catch (e: IOException) {
-                        e.showStackTrace()
-
-                        logger.debug {
-                            "Exception while validating ${it.vcsType} working tree, treating it as non-applicable: " +
-                                e.collectMessages()
+                ALL.values.asSequence()
+                    .map { vcsFactory -> vcsFactory.create(options = emptyMap(), secrets = emptyMap()) }
+                    .mapNotNull {
+                        if (it is CommandLineTool && !it.isInPath()) {
+                            null
+                        } else {
+                            it.getWorkingTree(absoluteVcsDirectory)
                         }
-
-                        false
+                    }.find {
+                        try {
+                            it.isValid()
+                        } catch (e: IOException) {
+                            e.showStackTrace()
+
+                            logger.debug {
+                                "Exception while validating ${it.vcsType} working tree, " +
+                                    "treating it as non-applicable: " +
+                                    e.collectMessages()
+                            }
+
+                            false
+                        }
+                    }.also {
+                        dirToVcsMap[absoluteVcsDirectory] = it
                     }
-                }.also {
-                    dirToVcsMap[absoluteVcsDirectory] = it
-                }
             }
         }
 
@@ -165,9 +175,9 @@ abstract class VersionControlSystem(
     }
 
     /**
-     * The priority in which this VCS should be probed. A higher value means a higher priority.
+     * The type of CVS that is supported by this VCS plugin.
      */
-    protected open val priority: Int = 0
+    abstract val type: String
 
     /**
      * A list of symbolic names that point to the latest revision.

diff --git a/downloader/src/main/kotlin/VersionControlSystemFactory.kt b/downloader/src/main/kotlin/VersionControlSystemFactory.kt
@@ -0,0 +1,30 @@
+package org.ossreviewtoolkit.downloader
+
+import org.ossreviewtoolkit.utils.common.Plugin
+import org.ossreviewtoolkit.utils.common.TypedConfigurablePluginFactory
+
+/**
+ * An abstract class to be implemented by factories for [version contral systems][VersionControlSystem].
+ * The constructor parameter [type] denotes which VCS type is supported by this plugin.
+ * The constructor parameter [priority] is used to determine the order in which the VCS plugins are used.
+ */
+abstract class VersionControlSystemFactory<CONFIG>(override val type: String, val priority: Int) :
+    TypedConfigurablePluginFactory<CONFIG, VersionControlSystem> {
+    companion object {
+        /**
+         * All [version control system factories][VersionControlSystemFactory] available in the classpath,
+         * associated by their names, sorted by priority.
+         */
+        val ALL by lazy {
+            Plugin.getAll<VersionControlSystemFactory<*>>()
+                .toList()
+                .sortedByDescending { (_, vcsFactory) -> vcsFactory.priority }
+                .toMap()
+        }
+    }
+}
+
+/**
+ * A base class for specific version control system configurations.
+ */
+open class VersionControlSystemConfiguration
diff --git a/downloader/src/test/kotlin/VersionControlSystemTest.kt b/downloader/src/test/kotlin/VersionControlSystemTest.kt
@@ -34,6 +34,7 @@ import org.ossreviewtoolkit.model.Package
 import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.VcsType
 import org.ossreviewtoolkit.plugins.versioncontrolsystems.git.Git
+import org.ossreviewtoolkit.plugins.versioncontrolsystems.git.GitConfiguration
 import org.ossreviewtoolkit.utils.common.CommandLineTool
 
 class VersionControlSystemTest : WordSpec({
@@ -87,7 +88,8 @@ class VersionControlSystemTest : WordSpec({
 
             every { workingTree.guessRevisionName(any(), any()) } returns "v1.6.0"
 
-            Git().getRevisionCandidates(workingTree, pkg, allowMovingRevisions = true) shouldBeSuccess listOf(
+            Git.Factory().create(GitConfiguration())
+                .getRevisionCandidates(workingTree, pkg, allowMovingRevisions = true) shouldBeSuccess listOf(
                 "v1.6.0"
             )
         }
@@ -110,7 +112,8 @@ class VersionControlSystemTest : WordSpec({
             every { workingTree.listRemoteBranches() } returns listOf("main")
             every { workingTree.listRemoteTags() } returns emptyList()
 
-            Git().getRevisionCandidates(workingTree, pkg, allowMovingRevisions = true) shouldBeSuccess listOf(
+            Git.Factory().create(GitConfiguration())
+                .getRevisionCandidates(workingTree, pkg, allowMovingRevisions = true) shouldBeSuccess listOf(
                 "master",
                 "main"
             )

diff --git a/integrations/schemas/ort-configuration-schema.json b/integrations/schemas/ort-configuration-schema.json
@@ -89,6 +89,28 @@
                     "items": {
                         "$ref": "#/definitions/SourceCodeOrigins"
                     }
+                },
+                "versionControlSystems": {
+                    "type": "object",
+                    "properties": {
+                        "Git": {
+                            "type": "object",
+                            "properties": {
+                                "options": {
+                                    "type": "object",
+                                    "properties": {
+                                        "submoduleHistoryDepth": {
+                                            "type": "integer",
+                                            "minimum": 1
+                                        },
+                                        "updateNestedSubmodules": {
+                                            "type": "boolean"
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
                 }
             }
         },

diff --git a/model/src/main/kotlin/config/DownloaderConfiguration.kt b/model/src/main/kotlin/config/DownloaderConfiguration.kt
@@ -44,9 +44,35 @@ data class DownloaderConfiguration(
      * Configuration of the considered source code origins and their priority order. This must not be empty and not
      * contain any duplicates.
      */
-    val sourceCodeOrigins: List<SourceCodeOrigin> = listOf(SourceCodeOrigin.VCS, SourceCodeOrigin.ARTIFACT)
+    val sourceCodeOrigins: List<SourceCodeOrigin> = listOf(SourceCodeOrigin.VCS, SourceCodeOrigin.ARTIFACT),
+
+    /**
+     * Version control system specific configurations. The key needs to match VCS type,
+     * e.g. "Git" for the Git version control system.
+     */
+    val versionControlSystems: Map<String, VersionControlSystemConfiguration> = emptyMap()
 ) {
+    /**
+     * A copy of [versionControlSystems] with case-insensitive keys.
+     */
+    private val versionControlSystemsCaseInsensitive: Map<String, VersionControlSystemConfiguration> =
+        versionControlSystems.toSortedMap(String.CASE_INSENSITIVE_ORDER)
+
     init {
         sourceCodeOrigins.requireNotEmptyNoDuplicates()
+
+        val duplicateVersionControlSystems =
+            versionControlSystems.keys - versionControlSystemsCaseInsensitive.keys.toSet()
+
+        require(duplicateVersionControlSystems.isEmpty()) {
+            "The following version control systems have duplicate configuration: " +
+                "${duplicateVersionControlSystems.joinToString()}."
+        }
     }
+
+    /**
+     * Get a [VersionControlSystemConfiguration] from [versionControlSystems].
+     * The difference to accessing the map directly is that [vcsType] can be case-insensitive.
+     */
+    fun getVersionControlSystemConfiguration(vcsType: String) = versionControlSystemsCaseInsensitive[vcsType]
 }
diff --git a/model/src/main/kotlin/config/VersionControlSystemConfiguration.kt b/model/src/main/kotlin/config/VersionControlSystemConfiguration.kt
@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package org.ossreviewtoolkit.model.config
+
+import com.fasterxml.jackson.annotation.JsonInclude
+
+import org.ossreviewtoolkit.utils.common.Options
+
+/**
+ * The configuration for a Version Control System (VCS).
+ */
+@JsonInclude(JsonInclude.Include.NON_NULL)
+data class VersionControlSystemConfiguration(
+    /**
+     * Custom configuration options. See the documentation of the respective class for available options.
+     */
+    val options: Options = emptyMap()
+)
diff --git a/model/src/main/resources/reference.yml b/model/src/main/resources/reference.yml
@@ -167,6 +167,17 @@ ort:
 
     sourceCodeOrigins: [VCS, ARTIFACT]
 
+    # Optional VCS-specific configuration options.
+    versionControlSystems:
+      Git:
+        options:
+          # Depth of the commit history to fetch when updating submodules
+          submoduleHistoryDepth: 10
+
+          # A flag to control whether nested submodules should be updated (true), or if only the submodules
+          # on the first layer should be considered (false).
+          updateNestedSubmodules: true
+
   scanner:
     skipConcluded: true
     skipExcluded: true