From 9a35bd6bfbdf8db12400823f22c1afbd13103336 Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Wed, 24 Jan 2024 18:03:16 +0100 Subject: [PATCH 1/9] chore(scancode): Specify the timeout as a duration for convenience Signed-off-by: Sebastian Schuberth --- plugins/scanners/scancode/src/main/kotlin/ScanCode.kt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt index e06d18727f903..6493a73576a83 100644 --- a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt +++ b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt @@ -23,6 +23,7 @@ import java.io.File import java.time.Instant import kotlin.math.max +import kotlin.time.Duration.Companion.minutes import org.apache.logging.log4j.kotlin.logger @@ -72,7 +73,8 @@ class ScanCode internal constructor( private const val LICENSE_REFERENCES_OPTION_VERSION = "32.0.0" private const val OUTPUT_FORMAT = "json-pp" - private const val TIMEOUT = 300 + + private val TIMEOUT = 5.minutes /** * Configuration options that are relevant for [configuration] because they change the result file. @@ -82,7 +84,7 @@ class ScanCode internal constructor( "--license", "--info", "--strip-root", - "--timeout", TIMEOUT.toString() + "--timeout", "${TIMEOUT.inWholeSeconds}" ) /** From bf8a138705a17d145fc4cb1eb8e14b0b4ee8fd26 Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Sun, 21 Jan 2024 15:17:06 +0100 Subject: [PATCH 2/9] refactor(scancode)!: Move default configuration Move the knowledge about the default configuration from `ScanCode` to `ScanCodeConfig`. This allows to make properties of the latter non-nullable and more strongly typed (lists of strings instead of unparsed strings). Give some related variables more appropriate names in that context. Signed-off-by: Sebastian Schuberth --- .../funTest/kotlin/ScanCodeScannerFunTest.kt | 2 +- .../scancode/src/main/kotlin/ScanCode.kt | 44 ++++--------------- .../src/main/kotlin/ScanCodeConfig.kt | 40 ++++++++++++++--- .../scancode/src/test/kotlin/ScanCodeTest.kt | 44 +++++++++---------- 4 files changed, 65 insertions(+), 65 deletions(-) diff --git a/plugins/scanners/scancode/src/funTest/kotlin/ScanCodeScannerFunTest.kt b/plugins/scanners/scancode/src/funTest/kotlin/ScanCodeScannerFunTest.kt index ee894e7f2bd13..9ec252fe41f92 100644 --- a/plugins/scanners/scancode/src/funTest/kotlin/ScanCodeScannerFunTest.kt +++ b/plugins/scanners/scancode/src/funTest/kotlin/ScanCodeScannerFunTest.kt @@ -33,7 +33,7 @@ import org.ossreviewtoolkit.utils.spdx.getLicenseText import org.ossreviewtoolkit.utils.test.ExpensiveTag class ScanCodeScannerFunTest : AbstractPathScannerWrapperFunTest(setOf(ExpensiveTag)) { - override val scanner = ScanCode("ScanCode", ScanCodeConfig.EMPTY, ScannerWrapperConfig.EMPTY) + override val scanner = ScanCode("ScanCode", ScanCodeConfig.DEFAULT, ScannerWrapperConfig.EMPTY) override val expectedFileLicenses = listOf( LicenseFinding("Apache-2.0", TextLocation("LICENSE", 1, 187), 100.0f), diff --git a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt index 6493a73576a83..704f390854bc8 100644 --- a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt +++ b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt @@ -22,9 +22,6 @@ package org.ossreviewtoolkit.plugins.scanners.scancode import java.io.File import java.time.Instant -import kotlin.math.max -import kotlin.time.Duration.Companion.minutes - import org.apache.logging.log4j.kotlin.logger import org.ossreviewtoolkit.model.ScanSummary @@ -41,7 +38,6 @@ import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.Os import org.ossreviewtoolkit.utils.common.ProcessCapture import org.ossreviewtoolkit.utils.common.safeDeleteRecursively -import org.ossreviewtoolkit.utils.common.splitOnWhitespace import org.ossreviewtoolkit.utils.common.withoutPrefix import org.ossreviewtoolkit.utils.ort.createOrtTempDir @@ -56,17 +52,18 @@ import org.semver4j.Semver * configuration [options][PluginConfiguration.options]: * * * **"commandLine":** Command line options that modify the result. These are added to the [ScannerDetails] when - * looking up results from the [ScanResultsStorage]. Defaults to [DEFAULT_CONFIGURATION_OPTIONS]. + * looking up results from the [ScanResultsStorage]. Defaults to [ScanCodeConfig.DEFAULT_COMMAND_LINE_OPTIONS]. * * **"commandLineNonConfig":** Command line options that do not modify the result and should therefore not be - * considered in [configuration], like "--processes". Defaults to [DEFAULT_NON_CONFIGURATION_OPTIONS]. + * considered in [configuration], like "--processes". Defaults to + * [ScanCodeConfig.DEFAULT_COMMAND_LINE_NON_CONFIG_OPTIONS]. */ class ScanCode internal constructor( name: String, - config: ScanCodeConfig, + private val config: ScanCodeConfig, private val wrapperConfig: ScannerWrapperConfig ) : CommandLinePathScannerWrapper(name) { // This constructor is required by the `RequirementsCommand`. - constructor(name: String, wrapperConfig: ScannerWrapperConfig) : this(name, ScanCodeConfig.EMPTY, wrapperConfig) + constructor(name: String, wrapperConfig: ScannerWrapperConfig) : this(name, ScanCodeConfig.DEFAULT, wrapperConfig) companion object { const val SCANNER_NAME = "ScanCode" @@ -74,27 +71,6 @@ class ScanCode internal constructor( private const val LICENSE_REFERENCES_OPTION_VERSION = "32.0.0" private const val OUTPUT_FORMAT = "json-pp" - private val TIMEOUT = 5.minutes - - /** - * Configuration options that are relevant for [configuration] because they change the result file. - */ - private val DEFAULT_CONFIGURATION_OPTIONS = listOf( - "--copyright", - "--license", - "--info", - "--strip-root", - "--timeout", "${TIMEOUT.inWholeSeconds}" - ) - - /** - * Configuration options that are not relevant for [configuration] because they do not change the result - * file. - */ - private val DEFAULT_NON_CONFIGURATION_OPTIONS = listOf( - "--processes", max(1, Runtime.getRuntime().availableProcessors() - 1).toString() - ) - private val OUTPUT_FORMAT_OPTION = if (OUTPUT_FORMAT.startsWith("json")) { "--$OUTPUT_FORMAT" } else { @@ -117,19 +93,15 @@ class ScanCode internal constructor( override val configuration by lazy { buildList { - addAll(configurationOptions) + addAll(config.commandLine) add(OUTPUT_FORMAT_OPTION) }.joinToString(" ") } - private val configurationOptions = config.commandLine?.splitOnWhitespace() ?: DEFAULT_CONFIGURATION_OPTIONS - private val nonConfigurationOptions = config.commandLineNonConfig?.splitOnWhitespace() - ?: DEFAULT_NON_CONFIGURATION_OPTIONS - internal fun getCommandLineOptions(version: String) = buildList { - addAll(configurationOptions) - addAll(nonConfigurationOptions) + addAll(config.commandLine) + addAll(config.commandLineNonConfig) if (Semver(version).isGreaterThanOrEqualTo(LICENSE_REFERENCES_OPTION_VERSION)) { // Required to be able to map ScanCode license keys to SPDX IDs. diff --git a/plugins/scanners/scancode/src/main/kotlin/ScanCodeConfig.kt b/plugins/scanners/scancode/src/main/kotlin/ScanCodeConfig.kt index 715dc394396b2..0fdba0f820808 100644 --- a/plugins/scanners/scancode/src/main/kotlin/ScanCodeConfig.kt +++ b/plugins/scanners/scancode/src/main/kotlin/ScanCodeConfig.kt @@ -19,19 +19,47 @@ package org.ossreviewtoolkit.plugins.scanners.scancode +import kotlin.math.max +import kotlin.time.Duration.Companion.minutes + import org.ossreviewtoolkit.utils.common.Options +import org.ossreviewtoolkit.utils.common.splitOnWhitespace data class ScanCodeConfig( - val commandLine: String?, - val commandLineNonConfig: String? + val commandLine: List, + val commandLineNonConfig: List ) { companion object { - val EMPTY = ScanCodeConfig(null, null) + /** + * The default time after which scanning a file is aborted. + */ + private val DEFAULT_TIMEOUT = 5.minutes + + /** + * The default list of command line options that might have an impact on the scan results. + */ + private val DEFAULT_COMMAND_LINE_OPTIONS = listOf( + "--copyright", + "--license", + "--info", + "--strip-root", + "--timeout", "${DEFAULT_TIMEOUT.inWholeSeconds}" + ) + + /** + * The default list of command line options that cannot have an impact on the scan results. + */ + private val DEFAULT_COMMAND_LINE_NON_CONFIG_OPTIONS = listOf( + "--processes", max(1, Runtime.getRuntime().availableProcessors() - 1).toString() + ) - private const val COMMAND_LINE_PROPERTY = "commandLine" - private const val COMMAND_LINE_NON_CONFIG_PROPERTY = "commandLineNonConfig" + val DEFAULT = create(emptyMap()) fun create(options: Options) = - ScanCodeConfig(options[COMMAND_LINE_PROPERTY], options[COMMAND_LINE_NON_CONFIG_PROPERTY]) + ScanCodeConfig( + options["commandLine"]?.splitOnWhitespace() ?: DEFAULT_COMMAND_LINE_OPTIONS, + options["commandLineNonConfig"]?.splitOnWhitespace() + ?: DEFAULT_COMMAND_LINE_NON_CONFIG_OPTIONS + ) } } diff --git a/plugins/scanners/scancode/src/test/kotlin/ScanCodeTest.kt b/plugins/scanners/scancode/src/test/kotlin/ScanCodeTest.kt index 271af10181f07..64b38445ce2af 100644 --- a/plugins/scanners/scancode/src/test/kotlin/ScanCodeTest.kt +++ b/plugins/scanners/scancode/src/test/kotlin/ScanCodeTest.kt @@ -39,7 +39,7 @@ import org.ossreviewtoolkit.scanner.ScannerWrapperConfig import org.ossreviewtoolkit.utils.common.ProcessCapture class ScanCodeTest : WordSpec({ - val scanner = ScanCode("ScanCode", ScanCodeConfig.EMPTY, ScannerWrapperConfig.EMPTY) + val scanner = ScanCode("ScanCode", ScanCodeConfig.DEFAULT, ScannerWrapperConfig.EMPTY) "configuration" should { "return the default values if the scanner configuration is empty" { @@ -47,15 +47,15 @@ class ScanCodeTest : WordSpec({ } "return the non-config values from the scanner configuration" { - val scannerWithConfig = ScanCode( - "ScanCode", - ScanCodeConfig( - commandLine = "--command --line", - commandLineNonConfig = "--commandLineNonConfig" - ), - ScannerWrapperConfig.EMPTY + val config = ScanCodeConfig.create( + mapOf( + "commandLine" to "--command --line", + "commandLineNonConfig" to "--commandLineNonConfig" + ) ) + val scannerWithConfig = ScanCode("ScanCode", config, ScannerWrapperConfig.EMPTY) + scannerWithConfig.configuration shouldBe "--command --line --json-pp" } } @@ -69,29 +69,29 @@ class ScanCodeTest : WordSpec({ } "contain the values from the scanner configuration" { - val scannerWithConfig = ScanCode( - "ScanCode", - ScanCodeConfig( - commandLine = "--command --line", - commandLineNonConfig = "--commandLineNonConfig" - ), - ScannerWrapperConfig.EMPTY + val config = ScanCodeConfig.create( + mapOf( + "commandLine" to "--command --line", + "commandLineNonConfig" to "--commandLineNonConfig" + ) ) + val scannerWithConfig = ScanCode("ScanCode", config, ScannerWrapperConfig.EMPTY) + scannerWithConfig.getCommandLineOptions("31.2.4").joinToString(" ") shouldBe "--command --line --commandLineNonConfig" } "be handled correctly when containing multiple spaces" { - val scannerWithConfig = ScanCode( - "ScanCode", - ScanCodeConfig( - commandLine = " --command --line ", - commandLineNonConfig = " -n -c " - ), - ScannerWrapperConfig.EMPTY + val config = ScanCodeConfig.create( + mapOf( + "commandLine" to " --command --line ", + "commandLineNonConfig" to " -n -c " + ) ) + val scannerWithConfig = ScanCode("ScanCode", config, ScannerWrapperConfig.EMPTY) + scannerWithConfig.getCommandLineOptions("31.2.4") shouldBe listOf("--command", "--line", "-n", "-c") } } From ae200311cd5790e80db33ff1b5c4a5ee20edaa96 Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Sun, 21 Jan 2024 15:42:42 +0100 Subject: [PATCH 3/9] refactor(scancode): Disregard the output format in scanner configuration The chosen output does not impact the scanner results, so it should not be considered as part of scanner configuration. Signed-off-by: Sebastian Schuberth --- plugins/scanners/scancode/src/main/kotlin/ScanCode.kt | 7 +------ plugins/scanners/scancode/src/test/kotlin/ScanCodeTest.kt | 4 ++-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt index 704f390854bc8..cf839c998f455 100644 --- a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt +++ b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt @@ -91,12 +91,7 @@ class ScanCode internal constructor( override val writeToStorage by lazy { wrapperConfig.writeToStorageWithDefault(matcher) } - override val configuration by lazy { - buildList { - addAll(config.commandLine) - add(OUTPUT_FORMAT_OPTION) - }.joinToString(" ") - } + override val configuration by lazy { config.commandLine.joinToString(" ") } internal fun getCommandLineOptions(version: String) = buildList { diff --git a/plugins/scanners/scancode/src/test/kotlin/ScanCodeTest.kt b/plugins/scanners/scancode/src/test/kotlin/ScanCodeTest.kt index 64b38445ce2af..41cea589096b3 100644 --- a/plugins/scanners/scancode/src/test/kotlin/ScanCodeTest.kt +++ b/plugins/scanners/scancode/src/test/kotlin/ScanCodeTest.kt @@ -43,7 +43,7 @@ class ScanCodeTest : WordSpec({ "configuration" should { "return the default values if the scanner configuration is empty" { - scanner.configuration shouldBe "--copyright --license --info --strip-root --timeout 300 --json-pp" + scanner.configuration shouldBe "--copyright --license --info --strip-root --timeout 300" } "return the non-config values from the scanner configuration" { @@ -56,7 +56,7 @@ class ScanCodeTest : WordSpec({ val scannerWithConfig = ScanCode("ScanCode", config, ScannerWrapperConfig.EMPTY) - scannerWithConfig.configuration shouldBe "--command --line --json-pp" + scannerWithConfig.configuration shouldBe "--command --line" } } From 7245f600e728739461dc9f3bc6cba84ed5472584 Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Wed, 24 Jan 2024 18:23:31 +0100 Subject: [PATCH 4/9] chore(scancode): Reorder command line options when running ScanCode Make the output format option directly follow other command line options and add a commend that it needs to precede the result file path. To emphasize even more that the latter two belong together, put them into the same line of code. Signed-off-by: Sebastian Schuberth --- plugins/scanners/scancode/src/main/kotlin/ScanCode.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt index cf839c998f455..4ac332380120e 100644 --- a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt +++ b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt @@ -157,8 +157,8 @@ class ScanCode internal constructor( ProcessCapture( command(), *commandLineOptions.toTypedArray(), - path.absolutePath, - OUTPUT_FORMAT_OPTION, - resultFile.absolutePath + // The output format option needs to directly precede the result file path. + OUTPUT_FORMAT_OPTION, resultFile.absolutePath, + path.absolutePath ) } From e6deb43d5cb2c91a199a7bbac2676cc3d9be29a6 Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Sun, 21 Jan 2024 15:47:40 +0100 Subject: [PATCH 5/9] refactor(scancode): Inline the output format option The output format was hard-coded, so the derived option name was constant. Simplify the logic by inlining the whole output format option. Signed-off-by: Sebastian Schuberth --- plugins/scanners/scancode/src/main/kotlin/ScanCode.kt | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt index 4ac332380120e..8169eba1279c0 100644 --- a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt +++ b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt @@ -69,13 +69,6 @@ class ScanCode internal constructor( const val SCANNER_NAME = "ScanCode" private const val LICENSE_REFERENCES_OPTION_VERSION = "32.0.0" - private const val OUTPUT_FORMAT = "json-pp" - - private val OUTPUT_FORMAT_OPTION = if (OUTPUT_FORMAT.startsWith("json")) { - "--$OUTPUT_FORMAT" - } else { - "--output-$OUTPUT_FORMAT" - } } class Factory : ScannerWrapperFactory(SCANNER_NAME) { @@ -158,7 +151,7 @@ class ScanCode internal constructor( command(), *commandLineOptions.toTypedArray(), // The output format option needs to directly precede the result file path. - OUTPUT_FORMAT_OPTION, resultFile.absolutePath, + "--json-pp", resultFile.absolutePath, path.absolutePath ) } From 0046a2e62530494a2223ccda5ad1e053a4b1d0bc Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Sun, 21 Jan 2024 15:52:50 +0100 Subject: [PATCH 6/9] refactor(scanner)!: Make `commandLineOptions` private This should not be part of the public API. Signed-off-by: Sebastian Schuberth --- plugins/scanners/scancode/src/main/kotlin/ScanCode.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt index 8169eba1279c0..f83a6e1ea5b3b 100644 --- a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt +++ b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt @@ -97,7 +97,7 @@ class ScanCode internal constructor( } } - val commandLineOptions by lazy { getCommandLineOptions(version) } + private val commandLineOptions by lazy { getCommandLineOptions(version) } override fun command(workingDir: File?) = listOfNotNull(workingDir, if (Os.isWindows) "scancode.bat" else "scancode").joinToString(File.separator) From b4059299b23b1ceff0c2dae37e9fb2242488590a Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Sun, 21 Jan 2024 15:56:24 +0100 Subject: [PATCH 7/9] chore(scancode): Reorder functions for a better overview Make the property go first and group command line related code. Signed-off-by: Sebastian Schuberth --- .../scancode/src/main/kotlin/ScanCode.kt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt index f83a6e1ea5b3b..4671c4105459f 100644 --- a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt +++ b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt @@ -78,13 +78,7 @@ class ScanCode internal constructor( override fun parseConfig(options: Options, secrets: Options) = ScanCodeConfig.create(options) } - override val matcher by lazy { ScannerMatcher.create(details, wrapperConfig.matcherConfig) } - - override val readFromStorage by lazy { wrapperConfig.readFromStorageWithDefault(matcher) } - - override val writeToStorage by lazy { wrapperConfig.writeToStorageWithDefault(matcher) } - - override val configuration by lazy { config.commandLine.joinToString(" ") } + private val commandLineOptions by lazy { getCommandLineOptions(version) } internal fun getCommandLineOptions(version: String) = buildList { @@ -97,7 +91,13 @@ class ScanCode internal constructor( } } - private val commandLineOptions by lazy { getCommandLineOptions(version) } + override val configuration by lazy { config.commandLine.joinToString(" ") } + + override val matcher by lazy { ScannerMatcher.create(details, wrapperConfig.matcherConfig) } + + override val readFromStorage by lazy { wrapperConfig.readFromStorageWithDefault(matcher) } + + override val writeToStorage by lazy { wrapperConfig.writeToStorageWithDefault(matcher) } override fun command(workingDir: File?) = listOfNotNull(workingDir, if (Os.isWindows) "scancode.bat" else "scancode").joinToString(File.separator) From 24e716330ad07ccf50ab4bdf50a50a5e497b8d33 Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Mon, 22 Jan 2024 13:29:15 +0100 Subject: [PATCH 8/9] test(scancode): Also assert the number of license findings in a test This is a preparation for highlighting the different number of findings compared to when enabling a new scanner option introduced in the next commit. Signed-off-by: Sebastian Schuberth --- .../scancode/src/test/kotlin/ScanCodeResultParserTest.kt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/plugins/scanners/scancode/src/test/kotlin/ScanCodeResultParserTest.kt b/plugins/scanners/scancode/src/test/kotlin/ScanCodeResultParserTest.kt index 92faf13153bee..3005cf110c409 100644 --- a/plugins/scanners/scancode/src/test/kotlin/ScanCodeResultParserTest.kt +++ b/plugins/scanners/scancode/src/test/kotlin/ScanCodeResultParserTest.kt @@ -80,9 +80,11 @@ class ScanCodeResultParserTest : FreeSpec({ val summary = parseResult(resultFile).toScanSummary() - summary.licenseFindings.find { - it.location == TextLocation("README.md", 100) && it.score == 100.0f - }?.license.toString() shouldBe "GPL-2.0-only WITH GCC-exception-2.0" + with(summary.licenseFindings) { + shouldHaveSize(18) + find { it.location == TextLocation("README.md", 100) && it.score == 100.0f } + ?.license.toString() shouldBe "GPL-2.0-only WITH GCC-exception-2.0" + } } } From ead5b94aeabe237180954676bede18733a19828a Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Sun, 21 Jan 2024 14:37:18 +0100 Subject: [PATCH 9/9] feat(scancode): Add an option to prefer file- over line-level findings See [1] for discussions about the `detected_license_expression_spdx`, in particular that it "is not merely the accumulation of the underlying matches". Optionally making use of this file-level license aligns ORT's behavior with that of the Double Open Scanner (DOS), see [2], which is useful for comparison of results. [1]: https://github.com/nexB/scancode-toolkit/issues/3458 [2]: https://github.com/doubleopen-project/dos/blob/616c582/apps/api/src/helpers/db_operations.ts#L55-L78 Signed-off-by: Sebastian Schuberth --- model/src/main/resources/reference.yml | 3 ++ .../kotlin/config/OrtConfigurationTest.kt | 1 + .../scancode/src/main/kotlin/ScanCode.kt | 18 +++++++++-- .../src/main/kotlin/ScanCodeConfig.kt | 6 ++-- .../main/kotlin/ScanCodeResultModelMapper.kt | 32 +++++++++++++------ .../test/kotlin/ScanCodeResultParserTest.kt | 15 +++++++++ 6 files changed, 61 insertions(+), 14 deletions(-) diff --git a/model/src/main/resources/reference.yml b/model/src/main/resources/reference.yml index fedd2b1dd4268..c1b7c2b5ca1eb 100644 --- a/model/src/main/resources/reference.yml +++ b/model/src/main/resources/reference.yml @@ -229,6 +229,9 @@ ort: # Command line options that do not affect the ScanCode output. commandLineNonConfig: '--processes 4' + # Use per-file license findings instead of per-line ones. + preferFileLicense: false + # Criteria for matching stored scan results. These can be configured for any scanner that uses semantic # versioning. Note that the 'maxVersion' is exclusive and not part of the range of accepted versions. minVersion: '3.2.1-rc2' diff --git a/model/src/test/kotlin/config/OrtConfigurationTest.kt b/model/src/test/kotlin/config/OrtConfigurationTest.kt index c5439b95cae05..5daa5a4880964 100644 --- a/model/src/test/kotlin/config/OrtConfigurationTest.kt +++ b/model/src/test/kotlin/config/OrtConfigurationTest.kt @@ -255,6 +255,7 @@ class OrtConfigurationTest : WordSpec({ options shouldContainExactly mapOf( "commandLine" to "--copyright --license --info --strip-root --timeout 300", "commandLineNonConfig" to "--processes 4", + "preferFileLicense" to "false", "minVersion" to "3.2.1-rc2", "maxVersion" to "32.0.0" ) diff --git a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt index 4671c4105459f..ec0bf1478b18a 100644 --- a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt +++ b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt @@ -56,6 +56,13 @@ import org.semver4j.Semver * * **"commandLineNonConfig":** Command line options that do not modify the result and should therefore not be * considered in [configuration], like "--processes". Defaults to * [ScanCodeConfig.DEFAULT_COMMAND_LINE_NON_CONFIG_OPTIONS]. + * * **preferFileLicense**: A flag to indicate whether the "high-level" per-file license reported by ScanCode starting + * with version 32 should be used instead of the individual "low-level" per-line license findings. The per-file + * license may be different from the conjunction of per-line licenses and is supposed to contain fewer + * false-positives. However, no exact line numbers can be associated to the per-file license anymore. If enabled, the + * start line of the per-file license finding is set to the minimum of all start lines for per-line findings in that + * file, the end line is set to the maximum of all end lines for per-line findings in that file, and the score is set + * to the arithmetic average of the scores of all per-line findings in that file. */ class ScanCode internal constructor( name: String, @@ -91,7 +98,14 @@ class ScanCode internal constructor( } } - override val configuration by lazy { config.commandLine.joinToString(" ") } + override val configuration by lazy { + buildList { + addAll(config.commandLine) + + // Add this in the style of a fake command line option for consistency with the above. + if (config.preferFileLicense) add("--prefer-file-license") + }.joinToString(" ") + } override val matcher by lazy { ScannerMatcher.create(details, wrapperConfig.matcherConfig) } @@ -141,7 +155,7 @@ class ScanCode internal constructor( } override fun createSummary(result: String, startTime: Instant, endTime: Instant): ScanSummary = - parseResult(result).toScanSummary() + parseResult(result).toScanSummary(config.preferFileLicense) /** * Execute ScanCode with the configured arguments to scan the given [path] and produce [resultFile]. diff --git a/plugins/scanners/scancode/src/main/kotlin/ScanCodeConfig.kt b/plugins/scanners/scancode/src/main/kotlin/ScanCodeConfig.kt index 0fdba0f820808..0a7e5c216d5c7 100644 --- a/plugins/scanners/scancode/src/main/kotlin/ScanCodeConfig.kt +++ b/plugins/scanners/scancode/src/main/kotlin/ScanCodeConfig.kt @@ -27,7 +27,8 @@ import org.ossreviewtoolkit.utils.common.splitOnWhitespace data class ScanCodeConfig( val commandLine: List, - val commandLineNonConfig: List + val commandLineNonConfig: List, + val preferFileLicense: Boolean ) { companion object { /** @@ -59,7 +60,8 @@ data class ScanCodeConfig( ScanCodeConfig( options["commandLine"]?.splitOnWhitespace() ?: DEFAULT_COMMAND_LINE_OPTIONS, options["commandLineNonConfig"]?.splitOnWhitespace() - ?: DEFAULT_COMMAND_LINE_NON_CONFIG_OPTIONS + ?: DEFAULT_COMMAND_LINE_NON_CONFIG_OPTIONS, + options["preferFileLicense"].toBoolean() ) } } diff --git a/plugins/scanners/scancode/src/main/kotlin/ScanCodeResultModelMapper.kt b/plugins/scanners/scancode/src/main/kotlin/ScanCodeResultModelMapper.kt index 4f925e9b9a45e..6ec839b9880ae 100644 --- a/plugins/scanners/scancode/src/main/kotlin/ScanCodeResultModelMapper.kt +++ b/plugins/scanners/scancode/src/main/kotlin/ScanCodeResultModelMapper.kt @@ -58,7 +58,7 @@ private data class LicenseMatch( val score: Float ) -fun ScanCodeResult.toScanSummary(): ScanSummary { +fun ScanCodeResult.toScanSummary(preferFileLicense: Boolean = false): ScanSummary { val licenseFindings = mutableSetOf() val copyrightFindings = mutableSetOf() val issues = mutableListOf() @@ -91,19 +91,31 @@ fun ScanCodeResult.toScanSummary(): ScanSummary { it.value.first() } - licenses.mapTo(licenseFindings) { license -> - // ScanCode uses its own license keys as identifiers in license expressions. - val spdxLicenseExpression = license.licenseExpression.mapLicense(scanCodeKeyToSpdxIdMappings) - - LicenseFinding( - license = spdxLicenseExpression, + if (preferFileLicense && file is FileEntry.Version3 && file.detectedLicenseExpressionSpdx != null) { + licenseFindings += LicenseFinding( + license = file.detectedLicenseExpressionSpdx, location = TextLocation( path = file.path, - startLine = license.startLine, - endLine = license.endLine + startLine = licenses.minOf { it.startLine }, + endLine = licenses.maxOf { it.endLine } ), - score = license.score + score = licenses.map { it.score }.average().toFloat() ) + } else { + licenses.mapTo(licenseFindings) { license -> + // ScanCode uses its own license keys as identifiers in license expressions. + val spdxLicenseExpression = license.licenseExpression.mapLicense(scanCodeKeyToSpdxIdMappings) + + LicenseFinding( + license = spdxLicenseExpression, + location = TextLocation( + path = file.path, + startLine = license.startLine, + endLine = license.endLine + ), + score = license.score + ) + } } file.copyrights.mapTo(copyrightFindings) { copyright -> diff --git a/plugins/scanners/scancode/src/test/kotlin/ScanCodeResultParserTest.kt b/plugins/scanners/scancode/src/test/kotlin/ScanCodeResultParserTest.kt index 3005cf110c409..3828200795f54 100644 --- a/plugins/scanners/scancode/src/test/kotlin/ScanCodeResultParserTest.kt +++ b/plugins/scanners/scancode/src/test/kotlin/ScanCodeResultParserTest.kt @@ -23,6 +23,7 @@ import io.kotest.core.spec.style.FreeSpec import io.kotest.matchers.Matcher import io.kotest.matchers.collections.beEmpty import io.kotest.matchers.collections.containExactlyInAnyOrder +import io.kotest.matchers.collections.shouldContainExactlyInAnyOrder import io.kotest.matchers.collections.shouldHaveSingleElement import io.kotest.matchers.collections.shouldHaveSize import io.kotest.matchers.should @@ -86,6 +87,20 @@ class ScanCodeResultParserTest : FreeSpec({ ?.license.toString() shouldBe "GPL-2.0-only WITH GCC-exception-2.0" } } + + "get file-level findings with the 'preferFileLicense' option" { + val resultFile = getAssetFile("scancode-32.0.8_spdx-expression-parse_no-license-references.json") + + val summary = parseResult(resultFile).toScanSummary(preferFileLicense = true) + + summary.licenseFindings.map { it.license.toString() }.shouldContainExactlyInAnyOrder( + "LicenseRef-scancode-generic-cla AND MIT", + "MIT", + "MIT", + "GPL-2.0-only WITH GCC-exception-2.0 AND JSON AND BSD-2-Clause AND CC-BY-3.0 AND MIT", + "GPL-2.0-only WITH GCC-exception-2.0 AND BSD-3-Clause" + ) + } } for (version in 1..MAX_SUPPORTED_OUTPUT_FORMAT_MAJOR_VERSION) {