refactor(vcs): Enable to provide VCS-specific configuration options

While VCS implementations are already plugins, they are not yet
configurable. VCS implementations require common configurations
(e.g., `revision`, `recursive`) and should support also
VCS-specific configurations if they are consumed via their API.
This allows to add functionality to individual VCS implementations
without the need to implement them for all of them.

The effects are limited to usage through the command-line interface
of the Downloader tool, designed primarily for scripted operations.
In other stages of the ORT pipeline these configuration options
have no effect.

Fixes #8556.

Signed-off-by: Wolfgang Klenk <[email protected]>

Author: wkl3nk

cli/src/funTest/kotlin/AnalyzerFunTest.kt

@@ -29,6 +29,7 @@ import java.util.concurrent.TimeUnit
 import org.ossreviewtoolkit.analyzer.Analyzer
 import org.ossreviewtoolkit.analyzer.PackageManagerFactory
 import org.ossreviewtoolkit.analyzer.analyze
+import org.ossreviewtoolkit.downloader.VersionControlSystemConfiguration
 import org.ossreviewtoolkit.model.Package
 import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.VcsType
@@ -52,7 +53,10 @@ class AnalyzerFunTest : WordSpec({
                     revision = "31588aa8f8555474e1c3c66a359ec99e4cd4b1fa"
                 )
             )
-            val outputDir = tempdir().also { GitRepo().download(pkg, it) }
+            val outputDir = tempdir().also {
+                GitRepo.Factory().create(VersionControlSystemConfiguration())
+                    .download(pkg, it)
+            }
 
             val result = analyze(outputDir, packageManagers = emptySet()).toYaml()
 

downloader/src/main/kotlin/Downloader.kt

@@ -232,7 +232,9 @@ class Downloader(private val config: DownloaderConfiguration) {
         var applicableVcs: VersionControlSystem? = null
 
         if (pkg.vcsProcessed.type != VcsType.UNKNOWN) {
-            applicableVcs = VersionControlSystem.forType(pkg.vcsProcessed.type)
+            applicableVcs = VersionControlSystem.forType(
+                pkg.vcsProcessed.type, config.getCaseInsensitiveVersionControlSystems()
+            )
             logger.info {
                 applicableVcs?.let {
                     "Detected VCS type '${it.type}' from type name '${pkg.vcsProcessed.type}'."
@@ -241,7 +243,9 @@ class Downloader(private val config: DownloaderConfiguration) {
         }
 
         if (applicableVcs == null) {
-            applicableVcs = VersionControlSystem.forUrl(pkg.vcsProcessed.url)
+            applicableVcs = VersionControlSystem.forUrl(
+                pkg.vcsProcessed.url, config.getCaseInsensitiveVersionControlSystems()
+            )
             logger.info {
                 applicableVcs?.let {
                     "Detected VCS type '${it.type}' from URL ${pkg.vcsProcessed.url}."

downloader/src/main/kotlin/VersionControlSystem.kt

@@ -24,13 +24,14 @@ import java.io.IOException
 
 import org.apache.logging.log4j.kotlin.logger
 
+import org.ossreviewtoolkit.downloader.VersionControlSystemFactory.Companion.ALL
 import org.ossreviewtoolkit.model.Package
 import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.VcsType
 import org.ossreviewtoolkit.model.config.LicenseFilePatterns
+import org.ossreviewtoolkit.model.config.VersionControlSystemConfiguration
 import org.ossreviewtoolkit.model.orEmpty
 import org.ossreviewtoolkit.utils.common.CommandLineTool
-import org.ossreviewtoolkit.utils.common.Plugin
 import org.ossreviewtoolkit.utils.common.collectMessages
 import org.ossreviewtoolkit.utils.common.uppercaseFirstChar
 import org.ossreviewtoolkit.utils.ort.ORT_REPO_CONFIG_FILENAME
@@ -44,22 +45,23 @@ abstract class VersionControlSystem(
      * the version control system is available.
      */
     private val commandLineTool: CommandLineTool? = null
-) : Plugin {
+) {
     companion object {
-        /**
-         * All [version control systems][VersionControlSystem] available in the classpath, sorted by their priority.
-         */
-        val ALL by lazy {
-            Plugin.getAll<VersionControlSystem>().toList().sortedByDescending { (_, vcs) -> vcs.priority }.toMap()
-        }
-
         /**
          * Return the applicable VCS for the given [vcsType], or null if none is applicable.
          */
-        fun forType(vcsType: VcsType) =
-            ALL.values.find {
-                it.isAvailable() && it.isApplicableType(vcsType)
+        fun forType(
+            vcsType: VcsType,
+            versionControlSystemsConfiguration: Map<String, VersionControlSystemConfiguration> = emptyMap()
+        ) = ALL.values.filter { vcsFactory -> vcsFactory.type == vcsType.toString() }
+            .map { vcsFactory ->
+                // If there is a configuration for the VCS type, use it, otherwise create
+                // the VCS with an empty configuration.
+                versionControlSystemsConfiguration[vcsFactory.type]?.let { vcsConfig ->
+                    vcsFactory.create(options = vcsConfig.options, secrets = emptyMap())
+                } ?: vcsFactory.create(options = emptyMap(), secrets = emptyMap())
             }
+            .firstOrNull { vcs -> vcs.isAvailable() }
 
         /**
          * A map to cache the [VersionControlSystem], if any, for previously queried URLs. This helps to speed up
@@ -72,8 +74,10 @@ abstract class VersionControlSystem(
          * Return the applicable VCS for the given [vcsUrl], or null if none is applicable.
          */
         @Synchronized
-        fun forUrl(vcsUrl: String) =
-            // Do not use getOrPut() here as it cannot handle null values, also see
+        fun forUrl(
+            vcsUrl: String,
+            versionControlSystemsConfiguration: Map<String, VersionControlSystemConfiguration> = emptyMap()
+        ) = // Do not use getOrPut() here as it cannot handle null values, also see
             // https://youtrack.jetbrains.com/issue/KT-21392.
             if (vcsUrl in urlToVcsMap) {
                 urlToVcsMap[vcsUrl]
@@ -82,12 +86,18 @@ abstract class VersionControlSystem(
                 when (val type = VcsHost.parseUrl(vcsUrl).type) {
                     VcsType.UNKNOWN -> {
                         // ...then eventually try to determine the type also dynamically.
-                        ALL.values.find {
-                            it.isAvailable() && it.isApplicableUrl(vcsUrl)
-                        }
+                        ALL.values
+                            .map { vcsFactory ->
+                                // If there is a configuration for the VCS type, use it, otherwise create
+                                // the VCS with an empty configuration.
+                                versionControlSystemsConfiguration[vcsFactory.type]
+                                    ?.let { vcsConfig ->
+                                        vcsFactory.create(options = vcsConfig.options, secrets = emptyMap())
+                                    } ?: vcsFactory.create(options = emptyMap(), secrets = emptyMap())
+                            }.firstOrNull { vcs -> vcs.isAvailable() && vcs.isApplicableUrl(vcsUrl) }
                     }
 
-                    else -> forType(type)
+                    else -> forType(type, versionControlSystemsConfiguration)
                 }.also {
                     urlToVcsMap[vcsUrl] = it
                 }
@@ -109,28 +119,31 @@ abstract class VersionControlSystem(
             return if (absoluteVcsDirectory in dirToVcsMap) {
                 dirToVcsMap[absoluteVcsDirectory]
             } else {
-                ALL.values.asSequence().mapNotNull {
-                    if (it is CommandLineTool && !it.isInPath()) {
-                        null
-                    } else {
-                        it.getWorkingTree(absoluteVcsDirectory)
-                    }
-                }.find {
-                    try {
-                        it.isValid()
-                    } catch (e: IOException) {
-                        e.showStackTrace()
-
-                        logger.debug {
-                            "Exception while validating ${it.vcsType} working tree, treating it as non-applicable: " +
-                                e.collectMessages()
+                ALL.values.asSequence()
+                    .map { vcsFactory -> vcsFactory.create(options = emptyMap(), secrets = emptyMap()) }
+                    .mapNotNull {
+                        if (it is CommandLineTool && !it.isInPath()) {
+                            null
+                        } else {
+                            it.getWorkingTree(absoluteVcsDirectory)
                         }
-
-                        false
+                    }.find {
+                        try {
+                            it.isValid()
+                        } catch (e: IOException) {
+                            e.showStackTrace()
+
+                            logger.debug {
+                                "Exception while validating ${it.vcsType} working tree, " +
+                                    "treating it as non-applicable: " +
+                                    e.collectMessages()
+                            }
+
+                            false
+                        }
+                    }.also {
+                        dirToVcsMap[absoluteVcsDirectory] = it
                     }
-                }.also {
-                    dirToVcsMap[absoluteVcsDirectory] = it
-                }
             }
         }
 
@@ -165,9 +178,9 @@ abstract class VersionControlSystem(
     }
 
     /**
-     * The priority in which this VCS should be probed. A higher value means a higher priority.
+     * The type of CVS that is supported by this VCS plugin.
      */
-    protected open val priority: Int = 0
+    abstract val type: String
 
     /**
      * A list of symbolic names that point to the latest revision.

downloader/src/main/kotlin/VersionControlSystemFactory.kt

@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package org.ossreviewtoolkit.downloader
+
+import org.ossreviewtoolkit.utils.common.Plugin
+import org.ossreviewtoolkit.utils.common.TypedConfigurablePluginFactory
+
+/**
+ * An abstract class to be implemented by factories for [version contral systems][VersionControlSystem].
+ * The constructor parameter [type] denotes which VCS type is supported by this plugin.
+ * The constructor parameter [priority] is used to determine the order in which the VCS plugins are used.
+ */
+abstract class VersionControlSystemFactory<CONFIG>(override val type: String, val priority: Int) :
+    TypedConfigurablePluginFactory<CONFIG, VersionControlSystem> {
+    companion object {
+        /**
+         * All [version control system factories][VersionControlSystemFactory] available in the classpath,
+         * associated by their names, sorted by priority.
+         */
+        val ALL by lazy {
+            Plugin.getAll<VersionControlSystemFactory<*>>()
+                .toList()
+                .sortedByDescending { (_, vcsFactory) -> vcsFactory.priority }
+                .toMap()
+        }
+    }
+}
+
+/**
+ * A base class for specific version control system configurations.
+ */
+open class VersionControlSystemConfiguration

downloader/src/test/kotlin/VersionControlSystemTest.kt

@@ -87,7 +87,8 @@ class VersionControlSystemTest : WordSpec({
 
             every { workingTree.guessRevisionName(any(), any()) } returns "v1.6.0"
 
-            Git().getRevisionCandidates(workingTree, pkg, allowMovingRevisions = true) shouldBeSuccess listOf(
+            Git.Factory().create(VersionControlSystemConfiguration())
+                .getRevisionCandidates(workingTree, pkg, allowMovingRevisions = true) shouldBeSuccess listOf(
                 "v1.6.0"
             )
         }
@@ -110,7 +111,8 @@ class VersionControlSystemTest : WordSpec({
             every { workingTree.listRemoteBranches() } returns listOf("main")
             every { workingTree.listRemoteTags() } returns emptyList()
 
-            Git().getRevisionCandidates(workingTree, pkg, allowMovingRevisions = true) shouldBeSuccess listOf(
+            Git.Factory().create(VersionControlSystemConfiguration())
+                .getRevisionCandidates(workingTree, pkg, allowMovingRevisions = true) shouldBeSuccess listOf(
                 "master",
                 "main"
             )

integrations/schemas/ort-configuration-schema.json

@@ -89,6 +89,28 @@
                     "items": {
                         "$ref": "#/definitions/SourceCodeOrigins"
                     }
+                },
+                "versionControlSystems": {
+                    "type": "object",
+                    "properties": {
+                        "Git": {
+                            "type": "object",
+                            "properties": {
+                                "options": {
+                                    "type": "object",
+                                    "properties": {
+                                        "submoduleHistoryDepth": {
+                                            "type": "integer",
+                                            "minimum": 1
+                                        },
+                                        "updateNestedSubmodules": {
+                                            "type": "boolean"
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
                 }
             }
         },

model/src/main/kotlin/config/DownloaderConfiguration.kt

@@ -44,9 +44,35 @@ data class DownloaderConfiguration(
      * Configuration of the considered source code origins and their priority order. This must not be empty and not
      * contain any duplicates.
      */
-    val sourceCodeOrigins: List<SourceCodeOrigin> = listOf(SourceCodeOrigin.VCS, SourceCodeOrigin.ARTIFACT)
+    val sourceCodeOrigins: List<SourceCodeOrigin> = listOf(SourceCodeOrigin.VCS, SourceCodeOrigin.ARTIFACT),
+
+    /**
+     * Version control system specific configurations. The key needs to match VCS type,
+     * e.g. "Git" for the Git version control system.
+     */
+    val versionControlSystems: Map<String, VersionControlSystemConfiguration> = emptyMap()
 ) {
+    /**
+     * A copy of [versionControlSystems] with case-insensitive keys.
+     */
+    private val versionControlSystemsCaseInsensitive: Map<String, VersionControlSystemConfiguration> =
+        versionControlSystems.toSortedMap(String.CASE_INSENSITIVE_ORDER)
+
     init {
         sourceCodeOrigins.requireNotEmptyNoDuplicates()
+
+        val duplicateVersionControlSystems =
+            versionControlSystems.keys - versionControlSystemsCaseInsensitive.keys.toSet()
+
+        require(duplicateVersionControlSystems.isEmpty()) {
+            "The following version control systems have duplicate configuration: " +
+                "${duplicateVersionControlSystems.joinToString()}."
+        }
     }
+
+    /**
+     * Get a [VersionControlSystemConfiguration] from [versionControlSystems].
+     * The difference to accessing the map directly is that VCS type can be case-insensitive.
+     */
+    fun getCaseInsensitiveVersionControlSystems() = versionControlSystemsCaseInsensitive
 }

model/src/main/kotlin/config/VersionControlSystemConfiguration.kt

@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package org.ossreviewtoolkit.model.config
+
+import com.fasterxml.jackson.annotation.JsonInclude
+
+import org.ossreviewtoolkit.utils.common.Options
+
+/**
+ * The configuration for a Version Control System (VCS).
+ */
+@JsonInclude(JsonInclude.Include.NON_NULL)
+data class VersionControlSystemConfiguration(
+    /**
+     * Custom configuration options. See the documentation of the respective class for available options.
+     */
+    val options: Options = emptyMap()
+)

model/src/main/resources/reference.yml

@@ -167,6 +167,20 @@ ort:
 
     sourceCodeOrigins: [VCS, ARTIFACT]
 
+    # Optional VCS-specific configuration options that allow to add VCS-specific configurations to leverage certain
+    # features directly in a VCS plugin. The effects are limited to usage through the command-line interface of the
+    # Downloader tool, designed primarily for scripted operations. In other stages of the ORT pipeline these
+    # configuration options have no effect.
+    versionControlSystems:
+      Git:
+        options:
+          # Depth of the commit history to fetch when updating submodules
+          submoduleHistoryDepth: 10
+
+          # A flag to control whether nested submodules should be updated (true), or if only the submodules
+          # on the first layer should be considered (false).
+          updateNestedSubmodules: true
+
   scanner:
     skipConcluded: true
     skipExcluded: true