diff --git a/downloader/src/main/kotlin/Downloader.kt b/downloader/src/main/kotlin/Downloader.kt index 6e5b49d87988f..9d191a0d31aa9 100644 --- a/downloader/src/main/kotlin/Downloader.kt +++ b/downloader/src/main/kotlin/Downloader.kt @@ -270,8 +270,13 @@ class Downloader(private val config: DownloaderConfiguration) { return RepositoryProvenance(pkg.vcsProcessed, pkg.vcsProcessed.revision) } + val vcsConfig = config.getVersionControlSystemConfiguration(applicableVcs.type) + if (vcsConfig != null) { + logger.info { "Using ${applicableVcs.type}-specific configuration $vcsConfig." } + } + val workingTree = try { - applicableVcs.download(pkg, outputDirectory, config.allowMovingRevisions, recursive) + applicableVcs.download(pkg, outputDirectory, config.allowMovingRevisions, recursive, vcsConfig) } catch (e: DownloadException) { // TODO: Introduce something like a "strict" mode and only do these kind of fallbacks in non-strict mode. val vcsUrlNoCredentials = pkg.vcsProcessed.url.replaceCredentialsInUri() @@ -285,7 +290,7 @@ class Downloader(private val config: DownloaderConfiguration) { outputDirectory.safeDeleteRecursively(baseDirectory = outputDirectory) val fallbackPkg = pkg.copy(vcsProcessed = pkg.vcsProcessed.copy(url = vcsUrlNoCredentials)) - applicableVcs.download(fallbackPkg, outputDirectory, config.allowMovingRevisions, recursive) + applicableVcs.download(fallbackPkg, outputDirectory, config.allowMovingRevisions, recursive, vcsConfig) } else { throw e } diff --git a/downloader/src/main/kotlin/VersionControlSystem.kt b/downloader/src/main/kotlin/VersionControlSystem.kt index 4d16e31b4861d..bdd9f43e173a8 100644 --- a/downloader/src/main/kotlin/VersionControlSystem.kt +++ b/downloader/src/main/kotlin/VersionControlSystem.kt @@ -28,6 +28,7 @@ import org.ossreviewtoolkit.model.Package import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType import org.ossreviewtoolkit.model.config.LicenseFilePatterns +import org.ossreviewtoolkit.model.config.VersionControlSystemConfiguration import org.ossreviewtoolkit.model.orEmpty import org.ossreviewtoolkit.utils.common.CommandLineTool import org.ossreviewtoolkit.utils.common.Plugin @@ -223,6 +224,7 @@ abstract class VersionControlSystem( * Download the source code as specified by the [pkg] information to [targetDir]. [allowMovingRevisions] toggles * whether to allow downloads using symbolic names that point to moving revisions, like Git branches. If [recursive] * is `true`, any nested repositories (like Git submodules or Mercurial subrepositories) are downloaded, too. + * VCS-specific configurations can be supplied via [vcsConfiguration]. * * @return An object describing the downloaded working tree. * @@ -232,7 +234,8 @@ abstract class VersionControlSystem( pkg: Package, targetDir: File, allowMovingRevisions: Boolean = false, - recursive: Boolean = true + recursive: Boolean = true, + vcsConfiguration: VersionControlSystemConfiguration? = null ): WorkingTree { val workingTree = try { initWorkingTree(targetDir, pkg.vcsProcessed) @@ -248,7 +251,7 @@ abstract class VersionControlSystem( for ((index, revision) in revisionCandidates.withIndex()) { logger.info { "Trying revision candidate '$revision' (${index + 1} of ${revisionCandidates.size})..." } - results += updateWorkingTree(workingTree, revision, pkg.vcsProcessed.path, recursive) + results += updateWorkingTree(workingTree, revision, pkg.vcsProcessed.path, recursive, vcsConfiguration) if (results.last().isSuccess) break } @@ -380,14 +383,17 @@ abstract class VersionControlSystem( /** * Update the [working tree][workingTree] by checking out the given [revision], optionally limited to the given - * [path] and [recursively][recursive] updating any nested working trees. Return a [Result] that encapsulates the - * originally requested [revision] on success, or the occurred exception on failure. + * [path] and [recursively][recursive] updating any nested working trees. + * Optionally, a VCS-specific configuration can be supplied via [vcsConfiguration]. + * Return a [Result] that encapsulates the originally requested [revision] on success, + * or the occurred exception on failure. */ abstract fun updateWorkingTree( workingTree: WorkingTree, revision: String, path: String = "", - recursive: Boolean = false + recursive: Boolean = false, + vcsConfiguration: VersionControlSystemConfiguration? = null ): Result /** diff --git a/downloader/src/test/kotlin/VersionControlSystemTest.kt b/downloader/src/test/kotlin/VersionControlSystemTest.kt index 9349902a37a18..831789273fe7e 100644 --- a/downloader/src/test/kotlin/VersionControlSystemTest.kt +++ b/downloader/src/test/kotlin/VersionControlSystemTest.kt @@ -33,6 +33,7 @@ import java.lang.UnsupportedOperationException import org.ossreviewtoolkit.model.Package import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType +import org.ossreviewtoolkit.model.config.VersionControlSystemConfiguration import org.ossreviewtoolkit.plugins.versioncontrolsystems.git.Git import org.ossreviewtoolkit.utils.common.CommandLineTool @@ -168,6 +169,7 @@ private class VersionControlSystemTestImpl( workingTree: WorkingTree, revision: String, path: String, - recursive: Boolean + recursive: Boolean, + vcsConfiguration: VersionControlSystemConfiguration? ): Result = Result.failure(UnsupportedOperationException("Unexpected invocation.")) } diff --git a/model/src/main/kotlin/config/DownloaderConfiguration.kt b/model/src/main/kotlin/config/DownloaderConfiguration.kt index 86245caeabeab..962696b4b07ff 100644 --- a/model/src/main/kotlin/config/DownloaderConfiguration.kt +++ b/model/src/main/kotlin/config/DownloaderConfiguration.kt @@ -44,9 +44,35 @@ data class DownloaderConfiguration( * Configuration of the considered source code origins and their priority order. This must not be empty and not * contain any duplicates. */ - val sourceCodeOrigins: List = listOf(SourceCodeOrigin.VCS, SourceCodeOrigin.ARTIFACT) + val sourceCodeOrigins: List = listOf(SourceCodeOrigin.VCS, SourceCodeOrigin.ARTIFACT), + + /** + * Version control system specific configurations. The key needs to match VCS type, + * e.g. "Git" for the Git version control system. + */ + val versionControlSystems: Map? = null ) { + /** + * A copy of [versionControlSystems] with case-insensitive keys. + */ + private val versionControlSystemsCaseInsensitive: Map? = + versionControlSystems?.toSortedMap(String.CASE_INSENSITIVE_ORDER) + init { sourceCodeOrigins.requireNotEmptyNoDuplicates() + + val duplicateVersionControlSystems = + versionControlSystems?.keys.orEmpty() - versionControlSystemsCaseInsensitive?.keys?.toSet().orEmpty() + + require(duplicateVersionControlSystems.isEmpty()) { + "The following version control systems have duplicate configuration: " + + "${duplicateVersionControlSystems.joinToString()}." + } } + + /** + * Get a [VersionControlSystemConfiguration] from [versionControlSystems]. + * The difference to accessing the map directly is that [vcsType] can be case-insensitive. + */ + fun getVersionControlSystemConfiguration(vcsType: String) = versionControlSystemsCaseInsensitive?.get(vcsType) } diff --git a/model/src/main/kotlin/config/VersionControlSystemConfiguration.kt b/model/src/main/kotlin/config/VersionControlSystemConfiguration.kt new file mode 100644 index 0000000000000..ed198e64461cd --- /dev/null +++ b/model/src/main/kotlin/config/VersionControlSystemConfiguration.kt @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2024 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.model.config + +import com.fasterxml.jackson.annotation.JsonInclude + +import org.ossreviewtoolkit.utils.common.Options + +/** + * The configuration for a Version Control System (VCS). + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +data class VersionControlSystemConfiguration( + /** + * Custom configuration options. See the documentation of the respective class for available options. + */ + val options: Options? = null +) diff --git a/model/src/main/resources/reference.yml b/model/src/main/resources/reference.yml index 8f76fe92fb783..32aead0119e78 100644 --- a/model/src/main/resources/reference.yml +++ b/model/src/main/resources/reference.yml @@ -167,6 +167,17 @@ ort: sourceCodeOrigins: [VCS, ARTIFACT] + # Optional VCS-specific configuration options. + versionControlSystems: + Git: + options: + # Depth of the commit history to fetch when updating submodules + submoduleHistoryDepth: 10 + + # A flag to control whether nested submodules should be updated (true), or if only the submodules + # on the first layer should be considered (false). + updateNestedSubmodules: true + scanner: skipConcluded: true skipExcluded: true diff --git a/model/src/test/kotlin/config/DownloaderConfigurationTest.kt b/model/src/test/kotlin/config/DownloaderConfigurationTest.kt new file mode 100644 index 0000000000000..cc8fbc9cba07c --- /dev/null +++ b/model/src/test/kotlin/config/DownloaderConfigurationTest.kt @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2024 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.model.config + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.WordSpec +import io.kotest.matchers.shouldBe + +class DownloaderConfigurationTest : WordSpec({ + "DownloaderConfiguration()" should { + "throw an exception on duplicate VCS configurations" { + shouldThrow { + DownloaderConfiguration( + versionControlSystems = mapOf( + "Git" to VersionControlSystemConfiguration(), + "git" to VersionControlSystemConfiguration() + ) + ) + } + } + } + + "getVersionControlSystemConfiguration" should { + "return configuration case-insensitively" { + val vcsConfiguration = VersionControlSystemConfiguration() + val downloaderConfiguration = DownloaderConfiguration( + versionControlSystems = mapOf( + "MyVCS" to vcsConfiguration + ) + ) + + downloaderConfiguration.getVersionControlSystemConfiguration("MyVCS") shouldBe vcsConfiguration + downloaderConfiguration.getVersionControlSystemConfiguration("myvcs") shouldBe vcsConfiguration + downloaderConfiguration.getVersionControlSystemConfiguration("MYVCS") shouldBe vcsConfiguration + } + } +}) diff --git a/plugins/version-control-systems/git/src/main/kotlin/Git.kt b/plugins/version-control-systems/git/src/main/kotlin/Git.kt index 1f77eec9d153d..e4ae61922eef7 100644 --- a/plugins/version-control-systems/git/src/main/kotlin/Git.kt +++ b/plugins/version-control-systems/git/src/main/kotlin/Git.kt @@ -48,6 +48,7 @@ import org.ossreviewtoolkit.downloader.VersionControlSystem import org.ossreviewtoolkit.downloader.WorkingTree import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType +import org.ossreviewtoolkit.model.config.VersionControlSystemConfiguration import org.ossreviewtoolkit.utils.common.CommandLineTool import org.ossreviewtoolkit.utils.common.Os import org.ossreviewtoolkit.utils.common.collectMessages @@ -170,7 +171,8 @@ class Git : VersionControlSystem(GitCommand) { workingTree: WorkingTree, revision: String, path: String, - recursive: Boolean + recursive: Boolean, + vcsConfiguration: VersionControlSystemConfiguration? // Git-specific configuration only ): Result = (workingTree as GitWorkingTree).useRepo { Git(this).use { git -> diff --git a/plugins/version-control-systems/git/src/main/kotlin/GitRepo.kt b/plugins/version-control-systems/git/src/main/kotlin/GitRepo.kt index 26cdcb7e28498..96ba5994b8b37 100644 --- a/plugins/version-control-systems/git/src/main/kotlin/GitRepo.kt +++ b/plugins/version-control-systems/git/src/main/kotlin/GitRepo.kt @@ -36,6 +36,7 @@ import org.ossreviewtoolkit.downloader.VersionControlSystem import org.ossreviewtoolkit.downloader.WorkingTree import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType +import org.ossreviewtoolkit.model.config.VersionControlSystemConfiguration import org.ossreviewtoolkit.model.utils.parseRepoManifestPath import org.ossreviewtoolkit.utils.common.CommandLineTool import org.ossreviewtoolkit.utils.common.Os @@ -192,7 +193,8 @@ class GitRepo : VersionControlSystem(GitRepoCommand) { workingTree: WorkingTree, revision: String, path: String, - recursive: Boolean + recursive: Boolean, + vcsConfiguration: VersionControlSystemConfiguration? // GitRepo-specific configuration only. ): Result { val manifestRevision = revision.takeUnless { it.isBlank() } val manifestPath = workingTree.getInfo().url.parseRepoManifestPath() diff --git a/plugins/version-control-systems/mercurial/src/main/kotlin/Mercurial.kt b/plugins/version-control-systems/mercurial/src/main/kotlin/Mercurial.kt index 613ab0e117b1b..776d935feb899 100644 --- a/plugins/version-control-systems/mercurial/src/main/kotlin/Mercurial.kt +++ b/plugins/version-control-systems/mercurial/src/main/kotlin/Mercurial.kt @@ -27,6 +27,7 @@ import org.ossreviewtoolkit.downloader.VersionControlSystem import org.ossreviewtoolkit.downloader.WorkingTree import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType +import org.ossreviewtoolkit.model.config.VersionControlSystemConfiguration import org.ossreviewtoolkit.utils.common.CommandLineTool import org.ossreviewtoolkit.utils.common.ProcessCapture @@ -93,7 +94,13 @@ class Mercurial : VersionControlSystem(MercurialCommand) { return getWorkingTree(targetDir) } - override fun updateWorkingTree(workingTree: WorkingTree, revision: String, path: String, recursive: Boolean) = + override fun updateWorkingTree( + workingTree: WorkingTree, + revision: String, + path: String, + recursive: Boolean, + vcsConfiguration: VersionControlSystemConfiguration? // Mercurial-specific configuration only + ) = runCatching { // To safe network bandwidth, only pull exactly the revision we want. Do not use "-u" to update the // working tree just yet, as Mercurial would only update if new changesets were pulled. But that might diff --git a/plugins/version-control-systems/subversion/src/main/kotlin/Subversion.kt b/plugins/version-control-systems/subversion/src/main/kotlin/Subversion.kt index 5a4a217fb32f8..4c897a4eeb3f1 100644 --- a/plugins/version-control-systems/subversion/src/main/kotlin/Subversion.kt +++ b/plugins/version-control-systems/subversion/src/main/kotlin/Subversion.kt @@ -31,6 +31,7 @@ import org.ossreviewtoolkit.downloader.VersionControlSystem import org.ossreviewtoolkit.downloader.WorkingTree import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType +import org.ossreviewtoolkit.model.config.VersionControlSystemConfiguration import org.ossreviewtoolkit.utils.common.collectMessages import org.ossreviewtoolkit.utils.ort.OrtAuthenticator import org.ossreviewtoolkit.utils.ort.OrtProxySelector @@ -126,7 +127,13 @@ class Subversion : VersionControlSystem() { return pathRevisions.single() } - override fun updateWorkingTree(workingTree: WorkingTree, revision: String, path: String, recursive: Boolean) = + override fun updateWorkingTree( + workingTree: WorkingTree, + revision: String, + path: String, + recursive: Boolean, + vcsConfiguration: VersionControlSystemConfiguration? // Subversion-specific configuration only. + ) = runCatching { // Note that the path should never be part of the URL as that would root the working tree at that path, but // the path should be available in the working tree.