From e1f206212cd825049bfe876c07e41751a5ab1723 Mon Sep 17 00:00:00 2001 From: Marc Nuri Date: Wed, 22 Mar 2023 19:53:02 +0100 Subject: [PATCH 1/4] refactor: use Fabric8 base extension --- it/pom.xml | 9 ++- .../jupiter/api/extension/BaseExtension.java | 64 ------------------- .../api/extension/GradleExtension.java | 1 + .../api/extension/HasKubernetesClient.java | 32 ---------- .../api/extension/RegistryExtension.java | 5 +- 5 files changed, 9 insertions(+), 102 deletions(-) delete mode 100644 it/src/main/java/org/eclipse/jkube/integrationtests/jupiter/api/extension/BaseExtension.java delete mode 100644 it/src/main/java/org/eclipse/jkube/integrationtests/jupiter/api/extension/HasKubernetesClient.java diff --git a/it/pom.xml b/it/pom.xml index ecd27dc1..be7203dd 100644 --- a/it/pom.xml +++ b/it/pom.xml @@ -44,6 +44,10 @@ commons-codec commons-codec + + io.fabric8 + kubernetes-junit-jupiter + io.fabric8 openshift-client @@ -85,11 +89,6 @@ maven-invoker test - - io.fabric8 - kubernetes-junit-jupiter - test - diff --git a/it/src/main/java/org/eclipse/jkube/integrationtests/jupiter/api/extension/BaseExtension.java b/it/src/main/java/org/eclipse/jkube/integrationtests/jupiter/api/extension/BaseExtension.java deleted file mode 100644 index 77bcb02b..00000000 --- a/it/src/main/java/org/eclipse/jkube/integrationtests/jupiter/api/extension/BaseExtension.java +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Copyright (c) 2019 Red Hat, Inc. - * This program and the accompanying materials are made - * available under the terms of the Eclipse Public License 2.0 - * which is available at: - * - * https://www.eclipse.org/legal/epl-2.0/ - * - * SPDX-License-Identifier: EPL-2.0 - * - * Contributors: - * Red Hat, Inc. - initial API and implementation - */ -package org.eclipse.jkube.integrationtests.jupiter.api.extension; - -import org.junit.jupiter.api.extension.ExtensionContext; - -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -public interface BaseExtension { - - default ExtensionContext.Namespace getNamespace(ExtensionContext context) { - return ExtensionContext.Namespace.create(context.getRequiredTestClass()); - } - - default ExtensionContext.Store getStore(ExtensionContext context) { - return context.getRoot().getStore(getNamespace(context)); - } - - default Field[] extractFields(ExtensionContext context, Class clazz, Predicate... predicates) { - final var fields = new ArrayList<>(); - var testClass = context.getTestClass().orElse(Object.class); - do { - fields.addAll(extractFields(testClass, clazz, predicates)); - testClass = testClass.getSuperclass(); - } while (testClass != Object.class); - return fields.toArray(new Field[0]); - } - - private static List extractFields(Class classWhereFieldIs, Class fieldType, Predicate... predicates) { - if (classWhereFieldIs != null && classWhereFieldIs != Object.class) { - var fieldStream = Arrays.stream(classWhereFieldIs.getDeclaredFields()) - .filter(f -> fieldType.isAssignableFrom(f.getType())); - for (Predicate p : predicates) { - fieldStream = fieldStream.filter(p); - } - return fieldStream.collect(Collectors.toList()); - } - return Collections.emptyList(); - } - - default void setFieldValue(Field field, Object entity, Object value) throws IllegalAccessException { - final boolean isAccessible = field.isAccessible(); - field.setAccessible(true); - field.set(entity, value); - field.setAccessible(isAccessible); - } -} diff --git a/it/src/main/java/org/eclipse/jkube/integrationtests/jupiter/api/extension/GradleExtension.java b/it/src/main/java/org/eclipse/jkube/integrationtests/jupiter/api/extension/GradleExtension.java index f81e55a8..f636005f 100644 --- a/it/src/main/java/org/eclipse/jkube/integrationtests/jupiter/api/extension/GradleExtension.java +++ b/it/src/main/java/org/eclipse/jkube/integrationtests/jupiter/api/extension/GradleExtension.java @@ -13,6 +13,7 @@ */ package org.eclipse.jkube.integrationtests.jupiter.api.extension; +import io.fabric8.junit.jupiter.BaseExtension; import org.eclipse.jkube.integrationtests.cli.CliUtils; import org.eclipse.jkube.integrationtests.gradle.JKubeGradleRunner; import org.eclipse.jkube.integrationtests.jupiter.api.Gradle; diff --git a/it/src/main/java/org/eclipse/jkube/integrationtests/jupiter/api/extension/HasKubernetesClient.java b/it/src/main/java/org/eclipse/jkube/integrationtests/jupiter/api/extension/HasKubernetesClient.java deleted file mode 100644 index 2a98a36c..00000000 --- a/it/src/main/java/org/eclipse/jkube/integrationtests/jupiter/api/extension/HasKubernetesClient.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright (c) 2019 Red Hat, Inc. - * This program and the accompanying materials are made - * available under the terms of the Eclipse Public License 2.0 - * which is available at: - * - * https://www.eclipse.org/legal/epl-2.0/ - * - * SPDX-License-Identifier: EPL-2.0 - * - * Contributors: - * Red Hat, Inc. - initial API and implementation - */ -package org.eclipse.jkube.integrationtests.jupiter.api.extension; - -import io.fabric8.kubernetes.client.KubernetesClient; -import io.fabric8.kubernetes.client.KubernetesClientBuilder; -import org.junit.jupiter.api.extension.ExtensionContext; - -public interface HasKubernetesClient extends BaseExtension { - - default KubernetesClient getClient(ExtensionContext context) { - synchronized (context.getRoot()) { - var client = getStore(context).get(KubernetesClient.class, KubernetesClient.class); - if (client == null) { - client = new KubernetesClientBuilder().build(); - getStore(context).put(KubernetesClient.class, client); - } - return client; - } - } -} diff --git a/it/src/main/java/org/eclipse/jkube/integrationtests/jupiter/api/extension/RegistryExtension.java b/it/src/main/java/org/eclipse/jkube/integrationtests/jupiter/api/extension/RegistryExtension.java index a2d6e7f6..e83da08b 100644 --- a/it/src/main/java/org/eclipse/jkube/integrationtests/jupiter/api/extension/RegistryExtension.java +++ b/it/src/main/java/org/eclipse/jkube/integrationtests/jupiter/api/extension/RegistryExtension.java @@ -13,6 +13,9 @@ */ package org.eclipse.jkube.integrationtests.jupiter.api.extension; +import io.fabric8.junit.jupiter.HasKubernetesClient; +import io.fabric8.kubernetes.api.model.IntOrString; +import io.fabric8.kubernetes.api.model.ServiceBuilder; import org.eclipse.jkube.integrationtests.cli.CliUtils; import org.eclipse.jkube.integrationtests.jupiter.api.DockerRegistry; import org.eclipse.jkube.integrationtests.jupiter.api.DockerRegistryHost; @@ -31,7 +34,7 @@ import static org.eclipse.jkube.integrationtests.cli.CliUtils.isWindows; import static org.hamcrest.MatcherAssert.assertThat; -public class RegistryExtension implements BaseExtension, BeforeAllCallback, BeforeEachCallback, AfterAllCallback { +public class RegistryExtension implements HasKubernetesClient, BeforeAllCallback, BeforeEachCallback, AfterAllCallback { private static final Logger log = LoggerFactory.getLogger(RegistryExtension.class); From ad1ee2da46bcf1b2a405ba233256217b9e3ed840 Mon Sep 17 00:00:00 2001 From: Marc Nuri Date: Wed, 22 Mar 2023 19:53:27 +0100 Subject: [PATCH 2/4] feat: deploy Docker registry on Kubernetes --- .github/workflows/e2e-tests.yml | 5 +- README.md | 10 +++ .../api/extension/RegistryExtension.java | 64 +++++++++++++++---- 3 files changed, 65 insertions(+), 14 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 741029e8..2198c248 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -101,7 +101,7 @@ jobs: minikube version: v1.25.2 kubernetes version: ${{ matrix.kubernetes }} github token: ${{ secrets.GITHUB_TOKEN }} - start args: --force + start args: '--force --insecure-registry "0.0.0.0/0"' - name: Harden Runner uses: step-security/harden-runner@8e0b4153524c217e5dce81df0986478e95a7004a with: @@ -139,7 +139,8 @@ jobs: us-east4-docker.pkg.dev:443 - name: Install and Run Integration Tests run: | - JKUBE_VERSION=$(./mvnw -q -f 'jkube/pom.xml' -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec) \ + eval $(minikube docker-env) \ + && JKUBE_VERSION=$(./mvnw -q -f 'jkube/pom.xml' -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec) \ && ./mvnw -B -PKubernetes,${{ matrix.suite }} clean verify -Djkube.version="$JKUBE_VERSION" - name: Save reports as artifact if: always() diff --git a/README.md b/README.md index 0f877584..ce0ae64b 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,16 @@ This project hosts Integration test suites for https://github.com/eclipse/jkube. +## Running locally + +### Minikube + +To be able to run the full suite in Minikube, you need to enable insecure registries (k8s:push tests). + +```shell +minikube start --insecure-registry "0.0.0.0/0" +``` + ## Test Structure In order to be able to run the tests in a CI environment without hogging the resources diff --git a/it/src/main/java/org/eclipse/jkube/integrationtests/jupiter/api/extension/RegistryExtension.java b/it/src/main/java/org/eclipse/jkube/integrationtests/jupiter/api/extension/RegistryExtension.java index e83da08b..5cb556a4 100644 --- a/it/src/main/java/org/eclipse/jkube/integrationtests/jupiter/api/extension/RegistryExtension.java +++ b/it/src/main/java/org/eclipse/jkube/integrationtests/jupiter/api/extension/RegistryExtension.java @@ -19,7 +19,6 @@ import org.eclipse.jkube.integrationtests.cli.CliUtils; import org.eclipse.jkube.integrationtests.jupiter.api.DockerRegistry; import org.eclipse.jkube.integrationtests.jupiter.api.DockerRegistryHost; -import org.hamcrest.Matchers; import org.junit.jupiter.api.extension.AfterAllCallback; import org.junit.jupiter.api.extension.BeforeAllCallback; import org.junit.jupiter.api.extension.BeforeEachCallback; @@ -30,9 +29,12 @@ import java.io.File; import java.io.IOException; import java.lang.reflect.Field; +import java.util.UUID; +import java.util.stream.Stream; import static org.eclipse.jkube.integrationtests.cli.CliUtils.isWindows; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.notNullValue; public class RegistryExtension implements HasKubernetesClient, BeforeAllCallback, BeforeEachCallback, AfterAllCallback { @@ -43,22 +45,23 @@ public void beforeAll(ExtensionContext context) throws Exception { final var annotation = context.getRequiredTestClass().getAnnotation(DockerRegistry.class); CliUtils.runCommand("docker rm -f " + getName(annotation)); log.debug(() -> "Starting Docker Registry Extension"); - final CliUtils.CliResult dockerRegistry; + final RegistryInfo dockerRegistry; if (isWindows()) { dockerRegistry = startWindowsDockerRegistry(annotation); } else { - dockerRegistry = startRegularDockerRegistry(annotation); + dockerRegistry = startKubernetesDockerRegistry(context); } - assertThat(dockerRegistry.getOutput(), dockerRegistry.getExitCode(), Matchers.equalTo(0)); + assertThat(dockerRegistry.assertionContext, dockerRegistry.host, notNullValue()); + getStore(context).put(RegistryInfo.class, dockerRegistry); log.debug(() -> "Docker Registry started successfully"); } @Override public void beforeEach(ExtensionContext context) throws Exception { - final var annotation = context.getRequiredTestClass().getAnnotation(DockerRegistry.class); + final var registryInfo = getStore(context).get(RegistryInfo.class, RegistryInfo.class); for (Field f : context.getRequiredTestClass().getDeclaredFields()) { if (f.isAnnotationPresent(DockerRegistryHost.class) && f.getType() == String.class) { - setFieldValue(f, context.getRequiredTestInstance(), getDockerHost() + ":" + annotation.port()); + setFieldValue(f, context.getRequiredTestInstance(), registryInfo.host + ":" + registryInfo.port); } } } @@ -67,22 +70,45 @@ public void beforeEach(ExtensionContext context) throws Exception { public void afterAll(ExtensionContext context) throws Exception { log.debug(() -> "Closing Docker Registry"); CliUtils.runCommand("docker stop " + getName(context.getRequiredTestClass().getAnnotation(DockerRegistry.class))); + if (!isWindows()) { + final var registryInfo = getStore(context).get(RegistryInfo.class, RegistryInfo.class); + final var client = getClient(context); + Stream.of( + client.pods().withName(registryInfo.name), + client.services().withName(registryInfo.name) + ).forEach(r -> r.withGracePeriod(0L).delete()); + } } - private static CliUtils.CliResult startRegularDockerRegistry(DockerRegistry dockerRegistry) throws IOException, InterruptedException { - log.debug(() -> "Starting standard Docker Registry"); - return CliUtils.runCommand("docker run --rm -d -p " + dockerRegistry.port() +":5000 --name " + - getName(dockerRegistry) + " registry:2"); + private RegistryInfo startKubernetesDockerRegistry(ExtensionContext context) throws IOException, InterruptedException { + final var name = "registry" + UUID.randomUUID().toString().replace("-", ""); + final var client = getClient(context); + final var ip = CliUtils.runCommand("minikube ip").getOutput().trim(); + final var dockerRegistry = client.run().withName(name).withImage("registry:2") + .withNewRunConfig() + .addToLabels("app", "docker-registry").addToLabels("group", "jkube-it").done(); + final var service = client.services().resource(new ServiceBuilder() + .withNewMetadata().withName(name) + .addToLabels("app", "docker-registry").addToLabels("group", "jkube-it").endMetadata() + .withNewSpec().withType("NodePort").withSelector(dockerRegistry.getMetadata().getLabels()) + .addNewPort().withName("http").withPort(5000).withTargetPort(new IntOrString(5000)).endPort().endSpec() + .build()) + .serverSideApply(); // Unsupported in K8s 1.12 + return new RegistryInfo(name, ip, service.getSpec().getPorts().get(0).getNodePort(), null); } - private static CliUtils.CliResult startWindowsDockerRegistry(DockerRegistry dockerRegistry) throws IOException, InterruptedException { + private static RegistryInfo startWindowsDockerRegistry(DockerRegistry dockerRegistry) throws IOException, InterruptedException { log.debug(() -> "Starting Windows specific Docker Registry"); final var registry = new File("C:\\registry"); if (!registry.exists() && !registry.mkdirs()) { throw new IllegalStateException("Directory C:\\registry cannot be created"); } - return CliUtils.runCommand("docker run --rm -d -p " + dockerRegistry.port() +":5000 --name " + + final var result = CliUtils.runCommand("docker run --rm -d -p " + dockerRegistry.port() +":5000 --name " + getName(dockerRegistry) + " -v C:\\registry:C:\\registry marcnuri/docker-registry-windows:ltsc2022"); + if (result.getExitCode() != 0) { + return new RegistryInfo(null, null, -1, result.getOutput()); + } + return new RegistryInfo("windows-docker-registry", getDockerHost(), dockerRegistry.port(), result.getOutput()); } private static String getName(DockerRegistry dockerRegistry) { @@ -98,4 +124,18 @@ private static String getDockerHost() { .replaceAll(":\\d+$", ""); } } + + private static final class RegistryInfo { + private final String name; + private final String host; + private final int port; + private final String assertionContext; + + public RegistryInfo(String name, String host, int port, String assertionContext) { + this.name = name; + this.host = host; + this.port = port; + this.assertionContext = assertionContext; + } + } } From 0ea98052b61be33846b5d49e6a9356d14753b5ff Mon Sep 17 00:00:00 2001 From: Marc Nuri Date: Wed, 22 Mar 2023 11:35:42 +0100 Subject: [PATCH 3/4] feat: improve exception handling and debug experience --- .../assertions/InvocationResultAssertion.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/it/src/test/java/org/eclipse/jkube/integrationtests/assertions/InvocationResultAssertion.java b/it/src/test/java/org/eclipse/jkube/integrationtests/assertions/InvocationResultAssertion.java index a6b940a6..131fe999 100644 --- a/it/src/test/java/org/eclipse/jkube/integrationtests/assertions/InvocationResultAssertion.java +++ b/it/src/test/java/org/eclipse/jkube/integrationtests/assertions/InvocationResultAssertion.java @@ -14,6 +14,9 @@ package org.eclipse.jkube.integrationtests.assertions; import org.apache.maven.shared.invoker.InvocationResult; +import org.eclipse.jkube.integrationtests.maven.MavenInvocationResult; + +import java.util.Arrays; public class InvocationResultAssertion { @@ -26,6 +29,10 @@ public static void assertInvocation(InvocationResult invocationResult) { if (invocationResult.getExecutionException() != null) { message.append("\n").append(invocationResult.getExecutionException().getMessage()); } + if (invocationResult instanceof MavenInvocationResult){ + Arrays.stream(((MavenInvocationResult) invocationResult).getStdOut().split("\r?\n")) + .forEach(line -> message.append("\n > ").append(line)); + } throw new AssertionError(message.toString()); } } From 3dce1e07a7319715f11b9d33ef90079bcdf87f03 Mon Sep 17 00:00:00 2001 From: Marc Nuri Date: Thu, 23 Mar 2023 06:23:12 +0100 Subject: [PATCH 4/4] ci: remove checks for Minikube legacy Makes things too complicated, users shouldn't be there anyway. --- .github/workflows/e2e-tests.yml | 80 +-------------------------------- 1 file changed, 2 insertions(+), 78 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 2198c248..03262baf 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -149,82 +149,6 @@ jobs: name: Test reports (Minikube ${{ matrix.kubernetes }}-${{ matrix.suite }}) path: ./it/target/jkube-test-report.txt - minikube-legacy: - name: K8S - needs: build-jkube - runs-on: ubuntu-20.04 - strategy: - fail-fast: false - matrix: - suite: ['quarkus','quarkus-native','springboot','webapp','other','dockerfile'] - steps: - - name: Checkout - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f - - name: Setup Java 11 - uses: actions/setup-java@5ffc13f4174014e2d4d4572b3d74c3fa61aeb2c2 - with: - java-version: '11' - distribution: 'temurin' - - name: Cache configuration - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 - with: - path: | - ~/.m2/repository - ./jkube - key: cache-it-${{ github.run_id }} - - name: Setup Minikube-Kubernetes - uses: manusa/actions-setup-minikube@1e9b544a8e64993bad1693f0aebeb108b459edd0 - with: - minikube version: v1.25.2 - kubernetes version: v1.12.10 - github token: ${{ secrets.GITHUB_TOKEN }} - start args: --force - - name: Harden Runner - uses: step-security/harden-runner@8e0b4153524c217e5dce81df0986478e95a7004a - with: - egress-policy: block - allowed-endpoints: > - api.github.com:443 - auth.docker.io:443 - azure.archive.ubuntu.com:80 - cdn03.quay.io:443 - gcr.io:443 - github.com:443 - jcenter.bintray.com:443 - k8s.gcr.io:443 - maven.repository.redhat.com:443 - md-hdd-51w5snc21ccf.z49.blob.storage.azure.net:443 - md-hdd-bfh3mwcdlxsh.z21.blob.storage.azure.net:443 - md-hdd-dxgvrxd2cnjf.z22.blob.storage.azure.net:443 - objects.githubusercontent.com:443 - packages.microsoft.com:443 - plugins-artifacts.gradle.org:443 - plugins.gradle.org:443 - ppa.launchpadcontent.net:443 - ppa.launchpad.net:80 - production.cloudflare.docker.com:443 - quay.io:443 - registry-1.docker.io:443 - registry.access.redhat.com:443 - registry.k8s.io:443 - repo1.maven.org:443 - repo.maven.apache.org:443 - repository.jboss.org:443 - storage.googleapis.com:443 - us-south1-docker.pkg.dev:443 - us-west2-docker.pkg.dev:443 - us-east4-docker.pkg.dev:443 - - name: Install and Run Integration Tests - run: | - JKUBE_VERSION=$(./mvnw -q -f 'jkube/pom.xml' -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec) \ - && ./mvnw -B -PKubernetes,${{ matrix.suite }} clean verify -Djkube.version="$JKUBE_VERSION" - - name: Save reports as artifact - if: always() - uses: actions/upload-artifact@65d862660abb392b8c4a3d1195a2108db131dd05 - with: - name: Test reports (Minikube ${{ matrix.kubernetes }}-${{ matrix.suite }}) - path: ./it/target/jkube-test-report.txt - openshift: name: OpenShift needs: build-jkube @@ -235,7 +159,7 @@ jobs: openshift: [v3.11.0,v3.10.0] suite: ['quarkus','springboot','webapp','other'] steps: -# This seems to cause problems with OpenShift Setup Action +# This seems to cause problems with OpenShift Setup Action # - name: Harden Runner # uses: step-security/harden-runner@8e0b4153524c217e5dce81df0986478e95a7004a # with: @@ -276,7 +200,7 @@ jobs: ./jkube key: cache-it-${{ github.run_id }} - name: Check Docker Status - run: systemctl status docker.service + run: systemctl status docker.service - name: Setup OpenShift uses: manusa/actions-setup-openshift@4fbc3dc0710ea1f9ff4c7e1e6196dafac8e78a28 with: