feat(model): Detect and exclude binary license files

Detect and exclude binary license files using Apache Tika. When a
non-text file is found during the license info creation process, a
warning is printed, and it is excluded from the final report.

This prevents the inclusion of binary files that previously caused the
reporter to enter an endless loop during report generation.

Signed-off-by: Julian Olderdissen <[email protected]>

Author: Juli0q

gradle/libs.versions.toml

@@ -67,6 +67,7 @@ slf4j = "2.0.17"
 springCore = "6.2.7"
 svnkit = "1.10.12"
 sw360Client = "17.0.1-m2"
+tika = "3.1.0"
 wagonHttp = "3.5.3"
 wiremock = "3.13.0"
 xmlutil = "0.91.1"
@@ -186,6 +187,7 @@ slf4j = { module = "org.slf4j:slf4j-api ", version.ref = "slf4j" }
 springCore = { module = "org.springframework:spring-core", version.ref = "springCore" }
 svnkit = { module = "com.tmatesoft.svnkit:svnkit", version.ref = "svnkit" }
 sw360Client = { module = "org.eclipse.sw360:client", version.ref = "sw360Client" }
+tika = { module = "org.apache.tika:tika-core", version.ref = "tika" }
 wagon-http = { module = "org.apache.maven.wagon:wagon-http", version.ref = "wagonHttp" }
 wiremock = { module = "org.wiremock:wiremock", version.ref = "wiremock" }
 xz = { module = "org.tukaani:xz", version.ref = "xz" }

model/build.gradle.kts

@@ -35,6 +35,7 @@ dependencies {
     api(libs.jackson.dataformat.yaml)
     api(libs.log4j.api)
 
+    implementation(libs.tika)
     implementation(libs.bundles.exposed)
     implementation(libs.bundles.hoplite)
     implementation(libs.hikari)

model/src/main/kotlin/utils/FileArchiver.kt

@@ -25,6 +25,7 @@ import kotlin.time.measureTime
 import kotlin.time.measureTimedValue
 
 import org.apache.logging.log4j.kotlin.logger
+import org.apache.tika.Tika
 
 import org.ossreviewtoolkit.model.KnownProvenance
 import org.ossreviewtoolkit.utils.common.FileMatcher
@@ -60,6 +61,13 @@ class FileArchiver(
         ignoreCase = true
     )
 
+    private val invalidTypes = listOf(
+        "application/octet-stream",
+        "application/x-executable",
+        "application/x-sharedlib",
+        "application/x-object"
+    )
+
     /**
      * Return whether an archive corresponding to [provenance] exists.
      */
@@ -77,6 +85,11 @@ class FileArchiver(
             directory.packZip(zipFile, overwrite = true) { file ->
                 val relativePath = file.relativeTo(directory).invariantSeparatorsPath
 
+                if (!file.isValidTextFile()) {
+                    logger.warn { "Not adding file '$relativePath' to archive because it is not a text file." }
+                    return@packZip false
+                }
+
                 matcher.matches(relativePath).also { result ->
                     logger.debug {
                         if (result) {
@@ -125,4 +138,9 @@ class FileArchiver(
             logger.error { "Failed to unarchive data for $provenance: ${it.collectMessages()}" }
         }.isSuccess
     }
+
+    private fun File.isValidTextFile(): Boolean {
+        val detected = Tika().detect(this)
+        return invalidTypes.none { detected.startsWith(it, ignoreCase = true) }
+    }
 }

model/src/test/kotlin/utils/FileArchiverTest.kt

@@ -22,9 +22,7 @@ package org.ossreviewtoolkit.model.utils
 import io.kotest.core.spec.style.StringSpec
 import io.kotest.core.test.TestCase
 import io.kotest.engine.spec.tempdir
-import io.kotest.matchers.file.aFile
-import io.kotest.matchers.file.exist
-import io.kotest.matchers.file.shouldContainNFiles
+import io.kotest.matchers.file.*
 import io.kotest.matchers.shouldBe
 import io.kotest.matchers.shouldNot
 
@@ -60,10 +58,11 @@ class FileArchiverTest : StringSpec() {
         storage = FileProvenanceFileStorage(LocalFileStorage(storageDir), FileArchiverConfiguration.ARCHIVE_FILENAME)
     }
 
-    private fun createFile(path: String) {
+    private fun createFile(path: String): File {
         val file = workingDir.resolve(path)
         file.parentFile.safeMkdirs()
         file.writeText(path)
+        return file
     }
 
     /**
@@ -162,5 +161,27 @@ class FileArchiverTest : StringSpec() {
             archiver.unarchive(targetDir, PROVENANCE) shouldBe true
             targetDir shouldContainNFiles 0
         }
+
+        "exclude basic binary license file" {
+            createFile("License").writeBytes(byteArrayOf(0xFF.toByte(), 0xD8.toByte()))
+
+            val archiver = FileArchiver.createDefault()
+            archiver.archive(workingDir, PROVENANCE)
+            val result = archiver.unarchive(targetDir, PROVENANCE)
+
+            result shouldBe true
+            targetDir shouldNot containFile("License")
+        }
+
+        "include utf8 file with japanese chars" {
+            createFile("License").writeText("ぁあぃいぅうぇえぉおかがきぎ")
+
+            val archiver = FileArchiver.createDefault()
+            archiver.archive(workingDir, PROVENANCE)
+            val result = archiver.unarchive(targetDir, PROVENANCE)
+
+            result shouldBe true
+            targetDir shouldNot containFile("License")
+        }
     }
 }