Improved built_and_test.yml (#1928)

nulls · web-flow · commit 88868beb5aa9 · 2023-03-03T16:11:18.000Z
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
@@ -40,7 +40,7 @@ jobs:
         with:
           path: ~/.konan
           key: ${{ runner.os }}-gradle-konan-${{ env.KOTLIN_VERSION }}
-      - name: Build all (excluding tests for save-orchestrator-common)
+      - name: Build all (excluding save-orchestrator-common, save-orchestrator and save-sandbox)
         uses: gradle/gradle-build-action@v2
         with:
           gradle-version: wrapper
@@ -50,19 +50,23 @@ jobs:
             -x detekt
             -x spotlessCheck
             -x :save-agent:linkDebugExecutableLinuxX64
-            -x :save-orchestrator-common:check
+            -x :save-orchestrator-common:build
+            -x :save-orchestrator:build
+            -x :save-sandbox:build
             -Pdetekt.multiplatform.disabled=true
             -PgprUser=${{ github.actor }}
             -PgprKey=${{ secrets.GITHUB_TOKEN }}
             --scan
             --build-cache
-      - name: Check save-orchestrator-common
+      - name: Build save-orchestrator-common, save-orchestrator and save-sandbox
         uses: gradle/gradle-build-action@v2
         with:
           gradle-version: wrapper
           gradle-home-cache-cleanup: true
           arguments: |
-            :save-orchestrator-common:check
+            :save-orchestrator-common:build
+            :save-orchestrator:build
+            :save-sandbox:build
             -x detekt
             -x spotlessCheck
             -x :save-agent:linkDebugExecutableLinuxX64
diff --git a/build.gradle.kts b/build.gradle.kts
@@ -3,6 +3,7 @@ import com.saveourtool.save.buildutils.*
 @Suppress("DSL_SCOPE_VIOLATION", "RUN_IN_SCRIPT")  // https://github.com/gradle/gradle/issues/22797
 plugins {
     id("com.saveourtool.save.buildutils.versioning-configuration")
+    id("com.saveourtool.save.buildutils.save-cli-download-configuration")
     id("com.saveourtool.save.buildutils.code-quality-convention")
     id("com.saveourtool.save.buildutils.publishing-configuration")
     alias(libs.plugins.talaiot.base)
diff --git a/gradle/plugins/src/main/kotlin/com/saveourtool/save/buildutils/VersioningConfiguration.kt b/gradle/plugins/src/main/kotlin/com/saveourtool/save/buildutils/VersioningConfiguration.kt
@@ -9,7 +9,9 @@ import org.ajoberstar.grgit.gradle.GrgitServicePlugin
 import org.ajoberstar.reckon.gradle.ReckonExtension
 import org.ajoberstar.reckon.gradle.ReckonPlugin
 import org.gradle.api.Project
+import org.gradle.api.provider.Provider
 import org.gradle.kotlin.dsl.*
+import java.util.*
 
 /**
  * @return path to the file with save-cli version for current build
@@ -50,6 +52,16 @@ fun Project.configureVersioning() {
     }
 }
 
+/**
+ * @return save-cli version for current build
+ */
+@Suppress("CUSTOM_GETTERS_SETTERS")
+internal fun Project.readSaveCliVersion(): Provider<String> = rootProject.tasks.named("getSaveCliVersion")
+    .map { getSaveCliVersionTask ->
+        val file = file(getSaveCliVersionTask.outputs.files.singleFile)
+        Properties().apply { load(file.reader()) }["version"] as String
+    }
+
 /**
  * @return true if this string denotes a snapshot version
  */
diff --git a/gradle/plugins/src/main/kotlin/com/saveourtool/save/buildutils/save-cli-configuration.gradle.kts b/gradle/plugins/src/main/kotlin/com/saveourtool/save/buildutils/save-cli-configuration.gradle.kts
@@ -5,77 +5,63 @@
 package com.saveourtool.save.buildutils
 
 import de.undercouch.gradle.tasks.download.Download
-import org.gradle.api.tasks.TaskProvider
 import org.gradle.kotlin.dsl.*
 import java.io.File
-import java.util.*
 
 plugins {
     kotlin("jvm")
     id("de.undercouch.download")
 }
 
-/**
- * @return version of save-cli from properties file
- */
-fun Project.readSaveCliVersion(): String {
-    val file = file(pathToSaveCliVersion)
-    return Properties().apply { load(file.reader()) }["version"] as String
-}
-
-/**
- * @return save-cli file path to copy
- */
-fun Project.getSaveCliPath(): String {
-    val saveCliVersion = readSaveCliVersion()
-    val saveCliPath = findProperty("saveCliPath")?.takeIf { saveCliVersion.isSnapshot() } as String?
-        ?: "https://github.com/saveourtool/save-cli/releases/download/v$saveCliVersion"
-    return "$saveCliPath/save-$saveCliVersion-linuxX64.kexe"
-}
-
 @Suppress("GENERIC_VARIABLE_WRONG_DECLARATION")
-val downloadSaveCliTaskProvider: TaskProvider<Download> = tasks.register<Download>("downloadSaveCli") {
-    dependsOn(":getSaveCliVersion")
-
-    src { getSaveCliPath() }
-    dest { "$buildDir/download/${File(getSaveCliPath()).name}" }
+val copySaveCliTaskProvider = tasks.register<Copy>("copySaveCli") {
+    val saveCliFile = rootProject.tasks.named<Download>("downloadSaveCli")
+        .map { downloadTask ->
+            downloadTask.dest
+        }
+    val outputDir = "$buildDir/download"
+    inputs.file(saveCliFile)
+    outputs.dir(outputDir)
 
-    overwrite(false)
+    from(saveCliFile.map { it.parentFile })
+    into(outputDir)
 }
 
 dependencies {
     add("runtimeOnly",
-        files(layout.buildDirectory.dir("$buildDir/download")).apply {
-            builtBy(downloadSaveCliTaskProvider)
-        }
+        files(layout.buildDirectory.dir(
+            copySaveCliTaskProvider.map { task ->
+                task.outputs.files.singleFile.absolutePath
+            }
+        ))
     )
 }
 
-val generateVersionFileTaskProvider = tasks.register("generateVersionFile") {
-    val versionsFile = File("$buildDir/generated/src/generated/Versions.kt")
+val generateSaveCliVersionFileTaskProvider = tasks.register("generateSaveCliVersionFile") {
+    val saveCliVersion = readSaveCliVersion()
+    val outputDir = File("$buildDir/generated/src")
+    val versionFile = outputDir.resolve("generated/SaveCliVersion.kt")
 
-    dependsOn(rootProject.tasks.named("getSaveCliVersion"))
-    inputs.file(pathToSaveCliVersion)
-    outputs.file(versionsFile)
+    inputs.property("save-cli version", saveCliVersion)
+    outputs.dir(outputDir)
 
     doFirst {
-        val saveCliVersion = readSaveCliVersion()
-        versionsFile.parentFile.mkdirs()
-        versionsFile.writeText(
+        versionFile.parentFile.mkdirs()
+        versionFile.writeText(
             """
             package generated
 
-            internal const val SAVE_CORE_VERSION = "$saveCliVersion"
+            internal const val SAVE_CORE_VERSION = "${saveCliVersion.get()}"
 
             """.trimIndent()
         )
     }
 }
 
 kotlin.sourceSets.getByName("main") {
-    kotlin.srcDir("$buildDir/generated/src")
-}
-
-tasks.withType<org.jetbrains.kotlin.gradle.tasks.KotlinCompile>().forEach {
-    it.dependsOn(generateVersionFileTaskProvider)
+    kotlin.srcDir(
+        generateSaveCliVersionFileTaskProvider.map {
+            it.outputs.files.singleFile
+        }
+    )
 }
diff --git a/gradle/plugins/src/main/kotlin/com/saveourtool/save/buildutils/save-cli-download-configuration.gradle.kts b/gradle/plugins/src/main/kotlin/com/saveourtool/save/buildutils/save-cli-download-configuration.gradle.kts
@@ -0,0 +1,25 @@
+/**
+ * Configuration task to download save-cli once per a project
+ */
+
+package com.saveourtool.save.buildutils
+
+import de.undercouch.gradle.tasks.download.Download
+import org.gradle.kotlin.dsl.*
+
+plugins {
+    id("de.undercouch.download")
+}
+
+tasks.register<Download>("downloadSaveCli") {
+    val saveCliVersion = readSaveCliVersion()
+    val saveCliFileName = saveCliVersion.map { "save-$it-linuxX64.kexe" }
+    val saveCliPath = saveCliVersion.zip(saveCliFileName) { version, fileName ->
+        findProperty("saveCliPath")?.takeIf { version.isSnapshot() } as String?
+            ?: "https://github.com/saveourtool/save-cli/releases/download/v$version/$fileName"
+    }
+    src { saveCliPath }
+    dest { saveCliFileName.map { fileName -> "$buildDir/download/$fileName" } }
+
+    overwrite(false)
+}
diff --git a/gradle/plugins/src/main/kotlin/com/saveourtool/save/buildutils/save-cloud-version-file-configuration.gradle.kts b/gradle/plugins/src/main/kotlin/com/saveourtool/save/buildutils/save-cloud-version-file-configuration.gradle.kts
@@ -8,13 +8,13 @@ import java.io.File
 
 tasks.register("generateSaveCloudVersionFile") {
     val outputDir = File("$buildDir/generated/src")
-    val versionsFile = outputDir.resolve("generated/SaveCloudVersion.kt")
+    val versionFile = outputDir.resolve("generated/SaveCloudVersion.kt")
     inputs.property("project version", version.toString())
-    outputs.dir("$buildDir/generated/src")
+    outputs.dir(outputDir)
 
     doFirst {
-        versionsFile.parentFile.mkdirs()
-        versionsFile.writeText(
+        versionFile.parentFile.mkdirs()
+        versionFile.writeText(
             """
             package generated
 
diff --git a/save-orchestrator-common/src/test/kotlin/com/saveourtool/save/orchestrator/docker/DockerContainerManagerTest.kt b/save-orchestrator-common/src/test/kotlin/com/saveourtool/save/orchestrator/docker/DockerContainerManagerTest.kt
@@ -9,13 +9,9 @@ import com.saveourtool.save.orchestrator.runner.ContainerRunnerException
 import com.saveourtool.save.orchestrator.service.OrchestratorAgentService
 import com.saveourtool.save.orchestrator.utils.DockerClientTestConfiguration
 import com.saveourtool.save.orchestrator.utils.silentlyCleanupContainer
-import com.saveourtool.save.orchestrator.utils.silentlyExec
 import com.saveourtool.save.utils.error
 import com.saveourtool.save.utils.getLogger
 import org.junit.jupiter.api.*
-import org.junit.jupiter.api.condition.DisabledOnOs
-import org.junit.jupiter.api.condition.EnabledOnOs
-import org.junit.jupiter.api.condition.OS
 import org.slf4j.Logger
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.boot.test.context.SpringBootTest
@@ -61,7 +57,6 @@ class DockerContainerManagerTest {
             .first {
                 it.repoTags?.contains("ghcr.io/saveourtool/save-base:eclipse-temurin-11") == true
             }
-        dockerClient.createVolumeCmd().withName("test-volume").exec()
     }
 
     @Test
@@ -113,7 +108,6 @@ class DockerContainerManagerTest {
         if (::testContainerId.isInitialized) {
             dockerClient.silentlyCleanupContainer(testContainerId)
         }
-        dockerClient.removeVolumeCmd("test-volume").silentlyExec()
     }
 
     companion object {
diff --git a/save-orchestrator-common/src/test/kotlin/com/saveourtool/save/orchestrator/service/ContainerServiceTest.kt b/save-orchestrator-common/src/test/kotlin/com/saveourtool/save/orchestrator/service/ContainerServiceTest.kt
@@ -2,23 +2,20 @@ package com.saveourtool.save.orchestrator.service
 
 import com.github.dockerjava.api.DockerClient
 import com.github.dockerjava.api.async.ResultCallback
-import com.github.dockerjava.api.command.PullImageResultCallback
 import com.github.dockerjava.api.model.*
 import com.saveourtool.save.entities.Execution
 import com.saveourtool.save.entities.Project
 import com.saveourtool.save.execution.ExecutionStatus
-import com.saveourtool.save.orchestrator.createTgzStream
 import com.saveourtool.save.orchestrator.docker.DockerContainerRunner
 import com.saveourtool.save.orchestrator.runner.ContainerRunnerException
 import com.saveourtool.save.orchestrator.utils.DockerClientTestConfiguration
+import com.saveourtool.save.orchestrator.utils.MockserverExtension
+import com.saveourtool.save.orchestrator.utils.MockserverExtension.Companion.MOCKSERVER_MOCK_URL
 
 import com.saveourtool.save.orchestrator.utils.silentlyCleanupContainer
-import com.saveourtool.save.orchestrator.utils.silentlyExec
 import com.saveourtool.save.utils.error
 import org.junit.jupiter.api.*
-import org.junit.jupiter.api.condition.DisabledOnOs
-import org.junit.jupiter.api.condition.EnabledOnOs
-import org.junit.jupiter.api.condition.OS
+import org.junit.jupiter.api.extension.ExtendWith
 import org.junit.jupiter.api.io.TempDir
 import org.mockito.kotlin.*
 import org.slf4j.LoggerFactory
@@ -29,7 +26,6 @@ import org.springframework.context.annotation.Import
 import org.springframework.http.ResponseEntity
 import org.springframework.test.context.ActiveProfiles
 import reactor.kotlin.core.publisher.toMono
-import java.net.ServerSocket
 import java.net.URL
 import java.nio.file.Path
 import kotlin.io.path.*
@@ -44,19 +40,12 @@ import kotlin.random.Random
     ContainerService::class,
     AgentService::class,
 )
+@ExtendWith(MockserverExtension::class)
 class ContainerServiceTest {
     @Autowired private lateinit var dockerClient: DockerClient
     @Autowired private lateinit var containerService: ContainerService
     @MockBean private lateinit var orchestratorAgentService: OrchestratorAgentService
     private lateinit var testContainerId: String
-    private lateinit var mockserverContainerId: String
-    private val mockserverPort: Int = ServerSocket(0).localPort
-    private val mockserverImageName = "mockserver/mockserver"
-    private val mockserverImageVersion = "5.15.0"
-    private val mockserverImageFullName = "$mockserverImageName:$mockserverImageVersion"
-    private val mockserverVolumeName = "mockserver-config"
-    private val mockserverConfigPath = "/config"
-    private val mockUrl = "/some-path-do-download-save-agent"
 
     init {
         if (System.getProperty("os.name").lowercase().contains("win")) {
@@ -68,57 +57,6 @@ class ContainerServiceTest {
     fun setUp(@TempDir tmpDir: Path) {
         whenever(orchestratorAgentService.updateExecutionStatus(any(), any(), anyOrNull()))
             .thenReturn(ResponseEntity.ok().build<Void>().toMono())
-
-        val configFile = (tmpDir / "initializerJson.json").createFile()
-            .also {
-                it.writeText(
-                    """
-                        [
-                          {
-                            "httpRequest": {
-                              "path": "$mockUrl"
-                            },
-                            "httpResponse": {
-                              "body": "!#/bin/bash\n echo \"sleep\"\n sleep 5000"
-                            }
-                          }
-                       ]
-                    """.trimIndent()
-                )
-            }
-        dockerClient.pullImageCmd(mockserverImageName)
-            .withTag(mockserverImageVersion)
-            .exec(PullImageResultCallback())
-            .awaitCompletion()
-        dockerClient.createVolumeCmd()
-            .withName(mockserverVolumeName)
-            .exec()
-        mockserverContainerId = dockerClient.createContainerCmd(mockserverImageFullName)
-            .withHostName("mockserver")
-            .withHostConfig(
-                HostConfig.newHostConfig()
-                    .withBinds(Bind(mockserverVolumeName, Volume(mockserverConfigPath)))
-                    .withPortBindings(
-                        Ports(
-                            ExposedPort.tcp(1080),
-                            Ports.Binding.bindPort(mockserverPort),
-                        )
-                    )
-            )
-            .withExposedPorts(ExposedPort.tcp(mockserverPort))
-            .withEnv("MOCKSERVER_INITIALIZATION_JSON_PATH=$mockserverConfigPath/${configFile.fileName}")
-            .exec()
-            .id
-            .also { containerId ->
-                createTgzStream(configFile.toFile()).use { out ->
-                    dockerClient.copyArchiveToContainerCmd(containerId)
-                        .withTarInputStream(out.toByteArray().inputStream())
-                        .withRemotePath(mockserverConfigPath)
-                        .exec()
-                }
-                dockerClient.startContainerCmd(containerId).exec()
-            }
-
     }
 
     @Test
@@ -134,7 +72,7 @@ class ContainerServiceTest {
         }
         val configuration = containerService.prepareConfiguration(
             testExecution.toRunRequest(
-                saveAgentUrl = URL("http://host.docker.internal:$mockserverPort$mockUrl"),
+                saveAgentUrl = URL("http://host.docker.internal:${MockserverExtension.getMockserverExposedPort()}$MOCKSERVER_MOCK_URL"),
             )
         )
         // start container and query backend
@@ -180,11 +118,6 @@ class ContainerServiceTest {
         if (::testContainerId.isInitialized) {
             dockerClient.silentlyCleanupContainer(testContainerId)
         }
-        if (::mockserverContainerId.isInitialized) {
-            dockerClient.silentlyCleanupContainer(mockserverContainerId)
-        }
-        dockerClient.removeVolumeCmd(mockserverVolumeName).silentlyExec()
-        dockerClient.removeImageCmd(mockserverImageFullName).silentlyExec()
     }
 
     companion object {
diff --git a/save-orchestrator-common/src/test/kotlin/com/saveourtool/save/orchestrator/utils/DockerClientTestConfiguration.kt b/save-orchestrator-common/src/test/kotlin/com/saveourtool/save/orchestrator/utils/DockerClientTestConfiguration.kt
@@ -14,7 +14,7 @@ class DockerClientTestConfiguration {
     /**
      * @return [DockerClient]
      */
-    @Bean
+    @Bean(destroyMethod = "")
     @Profile("docker-test")
     fun dockerClient(): DockerClient = DockerClientFactory.lazyClient()
 }
diff --git a/save-orchestrator-common/src/test/kotlin/com/saveourtool/save/orchestrator/utils/MockserverExtension.kt b/save-orchestrator-common/src/test/kotlin/com/saveourtool/save/orchestrator/utils/MockserverExtension.kt
diff --git a/save-orchestrator/build.gradle.kts b/save-orchestrator/build.gradle.kts
diff --git a/save-sandbox/build.gradle.kts b/save-sandbox/build.gradle.kts

Original file line number	Diff line number	Diff line change
`@@ -14,7 +14,7 @@ class DockerClientTestConfiguration {`
`14`	`14`	`/**`
`15`	`15`	`* @return [DockerClient]`
`16`	`16`	`*/`
`17`		`- @Bean`
	`17`	`+ @Bean(destroyMethod = "")`
`18`	`18`	`@Profile("docker-test")`
`19`	`19`	`fun dockerClient(): DockerClient = DockerClientFactory.lazyClient()`
`20`	`20`	`}`