Skip to content

Commit 41e0a7a

Browse files
fviernaufviernau
committed
feat(helper-cli): Add a command to re-serialize the test assets
This can be helpful for development. Signed-off-by: Frank Viernau <[email protected]>
1 parent c0015b4 commit 41e0a7a

File tree

3 files changed

+242
-0
lines changed

3 files changed

+242
-0
lines changed

‎helper-cli/src/main/kotlin/HelperMain.kt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import kotlin.system.exitProcess
3535

3636
import org.ossreviewtoolkit.helper.commands.*
3737
import org.ossreviewtoolkit.helper.commands.classifications.LicenseClassificationsCommand
38+
import org.ossreviewtoolkit.helper.commands.dev.DevCommand
3839
import org.ossreviewtoolkit.helper.commands.packageconfig.PackageConfigurationCommand
3940
import org.ossreviewtoolkit.helper.commands.packagecuration.PackageCurationsCommand
4041
import org.ossreviewtoolkit.helper.commands.repoconfig.RepositoryConfigurationCommand
@@ -73,6 +74,7 @@ internal class HelperMain : CliktCommand(name = ORTH_NAME, epilog = "* denotes r
7374
subcommands(
7475
ConvertOrtFileCommand(),
7576
CreateAnalyzerResultCommand(),
77+
DevCommand(),
7678
ExtractRepositoryConfigurationCommand(),
7779
GenerateTimeoutErrorResolutionsCommand(),
7880
GetPackageLicensesCommand(),

‎helper-cli/src/main/kotlin/commands/dev/DevCommand.kt‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
/*
2+
* Copyright (C) 2023 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*
16+
* SPDX-License-Identifier: Apache-2.0
17+
* License-Filename: LICENSE
18+
*/
19+
20+
package org.ossreviewtoolkit.helper.commands.dev
21+
22+
import com.github.ajalt.clikt.core.NoOpCliktCommand
23+
import com.github.ajalt.clikt.core.subcommands
24+
25+
internal class DevCommand : NoOpCliktCommand(
26+
help = "Commands for development."
27+
) {
28+
init {
29+
subcommands(
30+
RewriteTestAssetsCommand()
31+
)
32+
}
33+
}

‎helper-cli/src/main/kotlin/commands/dev/RewriteTestAssetsCommand.kt‎

Lines changed: 207 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,207 @@
1+
/*
2+
* Copyright (C) 2023 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*
16+
* SPDX-License-Identifier: Apache-2.0
17+
* License-Filename: LICENSE
18+
*/
19+
20+
package org.ossreviewtoolkit.helper.commands.dev
21+
22+
import com.fasterxml.jackson.databind.JsonNode
23+
import com.fasterxml.jackson.databind.node.ObjectNode
24+
25+
import com.github.ajalt.clikt.core.CliktCommand
26+
import com.github.ajalt.clikt.parameters.options.convert
27+
import com.github.ajalt.clikt.parameters.options.option
28+
import com.github.ajalt.clikt.parameters.options.required
29+
import com.github.ajalt.clikt.parameters.types.file
30+
31+
import java.io.File
32+
33+
import kotlin.reflect.KClass
34+
35+
import org.ossreviewtoolkit.model.AnalyzerResult
36+
import org.ossreviewtoolkit.model.AnalyzerRun
37+
import org.ossreviewtoolkit.model.EvaluatorRun
38+
import org.ossreviewtoolkit.model.FileFormat
39+
import org.ossreviewtoolkit.model.OrtResult
40+
import org.ossreviewtoolkit.model.ProjectAnalyzerResult
41+
import org.ossreviewtoolkit.model.ScanResult
42+
import org.ossreviewtoolkit.model.ScannerRun
43+
import org.ossreviewtoolkit.utils.common.expandTilde
44+
45+
internal class RewriteTestAssetsCommand : CliktCommand(
46+
help = "Searches all test assets directories in the given ORT sources directory for recognized serialized files " +
47+
"and tries to de-serialize and serialize the file. The command can be used to update the test assets " +
48+
"after making changes to the corresponding model classes or serializer configuration, e.g. after " +
49+
"annotating a property to not be serialized if empty."
50+
) {
51+
private val ortSourcesDir by option(
52+
"--ort-sources-dir", "-i",
53+
help = "The directory containing the source code of ORT."
54+
).convert { it.expandTilde() }
55+
.file(mustExist = true, canBeFile = false, canBeDir = true, mustBeWritable = false, mustBeReadable = true)
56+
.convert { it.absoluteFile.normalize() }
57+
.required()
58+
59+
override fun run() {
60+
val candidateFiles = findCandidateFiles(ortSourcesDir)
61+
println("Found ${candidateFiles.size} candidate file(s).\n")
62+
63+
val candidateFilesByType = patchAll(candidateFiles)
64+
65+
candidateFilesByType.forEach { (type, files) ->
66+
files.forEach { file ->
67+
println("[$type] ${file.absolutePath}")
68+
}
69+
}
70+
71+
println("\nSummary\n")
72+
73+
candidateFilesByType.forEach { (type, files) ->
74+
println("$type: ${files.size}")
75+
}
76+
}
77+
}
78+
79+
// Fake values to replace the replace patterns with in order to make de-serialization work. The fake values must be
80+
// unique and no fake value must be a substring of another.
81+
private val REPLACE_PATTERN_REPLACEMENTS = listOf(
82+
"<REPLACE_JAVA>" to "33445501",
83+
"<REPLACE_OS>" to "33445502",
84+
"\"<REPLACE_PROCESSORS>\"" to "33445503",
85+
"\"<REPLACE_MAX_MEMORY>\"" to "33445504",
86+
"<REPLACE_DEFINITION_FILE_PATH>" to "33445505",
87+
"<REPLACE_ABSOLUTE_DEFINITION_FILE_PATH>" to "33445506",
88+
"<REPLACE_URL>" to "http://replace/url/non/processed",
89+
"<REPLACE_REVISION>" to "33445507",
90+
"<REPLACE_PATH>" to "replace/path",
91+
"<REPLACE_URL_PROCESSED>" to "http://replace/url/processed",
92+
)
93+
94+
private val TARGET_CLASSES = setOf(
95+
AnalyzerResult::class,
96+
AnalyzerRun::class,
97+
EvaluatorRun::class,
98+
OrtResult::class,
99+
ScannerRun::class,
100+
ScanResult::class,
101+
ProjectAnalyzerResult::class
102+
)
103+
104+
// Paths to nodes in the tree of JsonNodes, whose subtree shall not be changed. Explicitly ignoring these subtrees is
105+
// needed in order to avoid bringing in default values property values which are not present in the original file.
106+
private val IGNORE_PATHS_FOR_TARGET_CLASSES = mapOf(
107+
AnalyzerRun::class to listOf(
108+
"config",
109+
"environment"
110+
),
111+
EvaluatorRun::class to listOf(
112+
"config",
113+
"environment"
114+
),
115+
OrtResult::class to listOf(
116+
"analyzer/config",
117+
"analyzer/environment",
118+
"scanner/config",
119+
"scanner/environment",
120+
"evaluator/config",
121+
"evaluator/environment",
122+
"advisor/config",
123+
"advisor/environment"
124+
),
125+
ScannerRun::class to listOf(
126+
"config",
127+
"environment"
128+
)
129+
)
130+
131+
private fun findCandidateFiles(dir: File): List<File> =
132+
FileFormat.findFilesWithKnownExtensions(dir).filter {
133+
it.absolutePath.contains("/src/funTest/assets/") || it.absolutePath.contains("/src/test/assets/")
134+
}
135+
136+
private fun patchAll(candidateFiles: Collection<File>): Map<String, List<File>> =
137+
candidateFiles.groupByTo(sortedMapOf()) { it.patch() }
138+
139+
private fun File.patch(): String = TARGET_CLASSES.firstOrNull { patch(it) }?.simpleName ?: "Unknown"
140+
141+
private fun File.patch(kClass: KClass<out Any>): Boolean =
142+
runCatching {
143+
val content = readText().removeReplacePatterns()
144+
val mapper = FileFormat.forFile(this).mapper
145+
val value = mapper.readValue(content, kClass.java)
146+
var rewrittenContent = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(value)
147+
148+
val ignorePaths = IGNORE_PATHS_FOR_TARGET_CLASSES[kClass].orEmpty()
149+
150+
if (ignorePaths.isNotEmpty()) {
151+
val originalTree = mapper.readTree(content)
152+
val rewrittenTree = mapper.readTree(rewrittenContent)
153+
154+
ignorePaths.forEach { path ->
155+
rewrittenTree.replacePath(path, originalTree)
156+
rewrittenTree.replacePath(path, originalTree)
157+
}
158+
159+
rewrittenContent = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(rewrittenTree)
160+
}
161+
162+
writeText(rewrittenContent.recoverReplacePatterns())
163+
}.isSuccess
164+
165+
private fun JsonNode.replacePath(path: String, target: JsonNode) {
166+
val parentPath = path.substringBeforeLast("/")
167+
val key = path.substringAfterLast("/")
168+
169+
val parentNode = getNodeAtPath(parentPath) as? ObjectNode
170+
val targetNode = target.getNodeAtPath(path)
171+
172+
if (parentNode != null && targetNode != null) {
173+
parentNode.replace(key, targetNode)
174+
}
175+
}
176+
177+
private fun JsonNode.getNodeAtPath(path: String): JsonNode? {
178+
val keys = path.split("/").toMutableList()
179+
var result: JsonNode? = this
180+
181+
while (keys.isNotEmpty() && result != null) {
182+
val key = keys.removeFirst()
183+
result = result[key]
184+
}
185+
186+
return result
187+
}
188+
189+
private fun String.removeReplacePatterns(): String {
190+
var result = this
191+
192+
REPLACE_PATTERN_REPLACEMENTS.forEach { (pattern, replacement) ->
193+
result = result.replace(pattern, replacement)
194+
}
195+
196+
return result
197+
}
198+
199+
private fun String.recoverReplacePatterns(): String {
200+
var result = this
201+
202+
REPLACE_PATTERN_REPLACEMENTS.forEach { (pattern, replacement) ->
203+
result = result.replace(replacement, pattern)
204+
}
205+
206+
return result
207+
}

0 commit comments

Comments
 (0)