From a3bfefc6287ad5a41361f42cf3951b277bc74204 Mon Sep 17 00:00:00 2001 From: csviri Date: Mon, 10 Apr 2023 13:46:42 +0200 Subject: [PATCH 1/3] improvement: use etcd health check --- .../jenvtest/KubeAPIServer.java | 4 +- .../jenvtest/process/EtcdProcess.java | 5 + .../process/KubeAPIServerProcess.java | 120 +------------- .../process/ProcessReadinessChecker.java | 148 ++++++++++++++++++ 4 files changed, 156 insertions(+), 121 deletions(-) create mode 100644 core/src/main/java/io/javaoperatorsdk/jenvtest/process/ProcessReadinessChecker.java diff --git a/core/src/main/java/io/javaoperatorsdk/jenvtest/KubeAPIServer.java b/core/src/main/java/io/javaoperatorsdk/jenvtest/KubeAPIServer.java index 0d39ee7..4bc0b05 100644 --- a/core/src/main/java/io/javaoperatorsdk/jenvtest/KubeAPIServer.java +++ b/core/src/main/java/io/javaoperatorsdk/jenvtest/KubeAPIServer.java @@ -14,8 +14,6 @@ public class KubeAPIServer implements UnexpectedProcessStopHandler { private static final Logger log = LoggerFactory.getLogger(KubeAPIServer.class); - public static final int STARTUP_TIMEOUT = 10_000; - private final KubeAPIServerConfig config; private final BinaryManager binaryManager; private final CertManager certManager; @@ -46,7 +44,7 @@ public void start() { if (config.isUpdateKubeConfig()) { kubeConfig.updateKubeConfig(apiServerPort); } - kubeApiServerProcess.waitUntilDefaultNamespaceCreated(); + kubeApiServerProcess.waitUntilReady(); log.debug("API Server ready to use"); } diff --git a/core/src/main/java/io/javaoperatorsdk/jenvtest/process/EtcdProcess.java b/core/src/main/java/io/javaoperatorsdk/jenvtest/process/EtcdProcess.java index 9f35528..4bc7de3 100644 --- a/core/src/main/java/io/javaoperatorsdk/jenvtest/process/EtcdProcess.java +++ b/core/src/main/java/io/javaoperatorsdk/jenvtest/process/EtcdProcess.java @@ -67,12 +67,17 @@ public int startEtcd() { return null; }); log.debug("etcd started on port: {}", port); + waitUntilEtcdHealthy(port); return port; } catch (IOException e) { throw new JenvtestException(e); } } + private void waitUntilEtcdHealthy(int port) { + new ProcessReadinessChecker(port, "health", "etcd", false).waitUntilReady(); + } + public void cleanEtcdData() { try { FileUtils.deleteDirectory(tempDataDir); diff --git a/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java b/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java index 52f2244..65f6c80 100644 --- a/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java +++ b/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java @@ -2,24 +2,9 @@ import java.io.File; import java.io.IOException; -import java.net.ConnectException; -import java.net.Socket; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.security.KeyManagementException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.CertificateException; -import java.security.cert.X509Certificate; -import java.time.LocalTime; -import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.List; -import javax.net.ssl.*; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,14 +12,11 @@ import io.javaoperatorsdk.jenvtest.binary.BinaryManager; import io.javaoperatorsdk.jenvtest.cert.CertManager; -import static io.javaoperatorsdk.jenvtest.KubeAPIServer.STARTUP_TIMEOUT; - public class KubeAPIServerProcess { private static final Logger log = LoggerFactory.getLogger(KubeAPIServerProcess.class); private static final Logger apiLog = LoggerFactory.getLogger(KubeAPIServerProcess.class .getName() + ".APIServerProcessLogs"); - public static final int POLLING_INTERVAL = 150; private final CertManager certManager; private final BinaryManager binaryManager; @@ -102,106 +84,8 @@ private List createCommand(File apiServerBinary, int apiServerPort, int return command; } - public void waitUntilDefaultNamespaceCreated() { - try { - var client = getHttpClient(); - var request = getHttpRequest(); - var startedAt = LocalTime.now(); - while (true) { - if (ready(client, request)) { - return; - } - if (LocalTime.now().isAfter(startedAt.plus(STARTUP_TIMEOUT, ChronoUnit.MILLIS))) { - throw new JenvtestException("API Server did not start properly"); - } - Thread.sleep(POLLING_INTERVAL); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new JenvtestException(e); - } - } - - private boolean ready(HttpClient client, HttpRequest request) { - try { - var response = client.send(request, HttpResponse.BodyHandlers.ofString()); - log.debug("Ready Response message:{} code: {} Api Server Port: {}", response.body(), - response.statusCode(), - apiServerPort); - return response.statusCode() == 200; - } catch (ConnectException e) { - // still want to retry - log.warn("Cannot connect to the server", e); - return false; - } catch (IOException e) { - throw new JenvtestException(e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new JenvtestException(e); - } - } - - private HttpRequest getHttpRequest() { - try { - return HttpRequest.newBuilder() - .uri(new URI("https://127.0.0.1:" + apiServerPort + "/readyz")) - .GET() - .build(); - } catch (URISyntaxException e) { - throw new JenvtestException(e); - } - } - - private static HttpClient getHttpClient() { - try { - var sslContext = SSLContext.getInstance("TLS"); - sslContext.init( - null, - new TrustManager[] { - new X509ExtendedTrustManager() { - @Override - public void checkClientTrusted(X509Certificate[] chain, String authType, - Socket socket) throws CertificateException { - - } - - public X509Certificate[] getAcceptedIssuers() { - return null; - } - - public void checkClientTrusted( - final X509Certificate[] a_certificates, - final String a_auth_type) {} - - public void checkServerTrusted( - final X509Certificate[] a_certificates, - final String a_auth_type) {} - - - public void checkServerTrusted( - final X509Certificate[] a_certificates, - final String a_auth_type, - final Socket a_socket) {} - - @Override - public void checkClientTrusted(X509Certificate[] chain, String authType, - SSLEngine engine) throws CertificateException { - - } - - public void checkServerTrusted( - final X509Certificate[] a_certificates, - final String a_auth_type, - final SSLEngine a_engine) {} - } - }, - null); - return HttpClient.newBuilder() - .sslContext(sslContext) - .build(); - } catch (NoSuchAlgorithmException | KeyManagementException e) { - throw new JenvtestException(e); - } + public void waitUntilReady() { + new ProcessReadinessChecker(apiServerPort, "readyz", "Kube API Server", true).waitUntilReady(); } public void stopApiServer() { diff --git a/core/src/main/java/io/javaoperatorsdk/jenvtest/process/ProcessReadinessChecker.java b/core/src/main/java/io/javaoperatorsdk/jenvtest/process/ProcessReadinessChecker.java new file mode 100644 index 0000000..6d1db51 --- /dev/null +++ b/core/src/main/java/io/javaoperatorsdk/jenvtest/process/ProcessReadinessChecker.java @@ -0,0 +1,148 @@ +package io.javaoperatorsdk.jenvtest.process; + +import java.io.IOException; +import java.net.ConnectException; +import java.net.Socket; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.time.LocalTime; +import java.time.temporal.ChronoUnit; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLEngine; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509ExtendedTrustManager; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.javaoperatorsdk.jenvtest.JenvtestException; + +public class ProcessReadinessChecker { + private static final Logger log = LoggerFactory.getLogger(ProcessReadinessChecker.class); + + public static final int STARTUP_TIMEOUT = 10_000; + public static final int POLLING_INTERVAL = 150; + + private final int port; + private final String readyCheckPath; + private final String processName; + private final boolean useTLS; + + + public ProcessReadinessChecker(int port, String readyCheckPath, String processName, boolean useTLS) { + this.port = port; + this.readyCheckPath = readyCheckPath; + this.processName = processName; + this.useTLS = useTLS; + } + + public void waitUntilReady() { + try { + var client = getHttpClient(); + var request = getHttpRequest(); + var startedAt = LocalTime.now(); + while (true) { + if (ready(client, request)) { + return; + } + if (LocalTime.now().isAfter(startedAt.plus(STARTUP_TIMEOUT, ChronoUnit.MILLIS))) { + throw new JenvtestException(processName + " did not start properly"); + } + Thread.sleep(POLLING_INTERVAL); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new JenvtestException(e); + } + } + + private boolean ready(HttpClient client, HttpRequest request) { + try { + var response = client.send(request, HttpResponse.BodyHandlers.ofString()); + log.debug("Ready Response message:{} code: {} for {} on Port: {}", response.body(), + response.statusCode(), processName, + port); + return response.statusCode() == 200; + } catch (ConnectException e) { + // still want to retry + log.warn("Cannot connect to the server", e); + return false; + } catch (IOException e) { + throw new JenvtestException(e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new JenvtestException(e); + } + } + + private HttpRequest getHttpRequest() { + try { + return HttpRequest.newBuilder() + .uri(new URI((useTLS ? "https" : "http") + "://127.0.0.1:" + port + "/" + readyCheckPath)) + .GET() + .build(); + } catch (URISyntaxException e) { + throw new JenvtestException(e); + } + } + + private static HttpClient getHttpClient() { + try { + var sslContext = SSLContext.getInstance("TLS"); + sslContext.init( + null, + new TrustManager[] { + new X509ExtendedTrustManager() { + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType, + Socket socket) throws CertificateException { + + } + + public X509Certificate[] getAcceptedIssuers() { + return null; + } + + public void checkClientTrusted( + final X509Certificate[] a_certificates, + final String a_auth_type) {} + + public void checkServerTrusted( + final X509Certificate[] a_certificates, + final String a_auth_type) {} + + + public void checkServerTrusted( + final X509Certificate[] a_certificates, + final String a_auth_type, + final Socket a_socket) {} + + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType, + SSLEngine engine) throws CertificateException { + + } + + public void checkServerTrusted( + final X509Certificate[] a_certificates, + final String a_auth_type, + final SSLEngine a_engine) {} + } + }, + null); + return HttpClient.newBuilder() + .sslContext(sslContext) + .build(); + } catch (NoSuchAlgorithmException | KeyManagementException e) { + throw new JenvtestException(e); + } + } +} From 294da2cb85e625ad5324945b8e5e17f086681d81 Mon Sep 17 00:00:00 2001 From: csviri Date: Mon, 10 Apr 2023 13:49:52 +0200 Subject: [PATCH 2/3] format --- .../jenvtest/process/ProcessReadinessChecker.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/io/javaoperatorsdk/jenvtest/process/ProcessReadinessChecker.java b/core/src/main/java/io/javaoperatorsdk/jenvtest/process/ProcessReadinessChecker.java index 6d1db51..d7b1ccd 100644 --- a/core/src/main/java/io/javaoperatorsdk/jenvtest/process/ProcessReadinessChecker.java +++ b/core/src/main/java/io/javaoperatorsdk/jenvtest/process/ProcessReadinessChecker.java @@ -37,7 +37,8 @@ public class ProcessReadinessChecker { private final boolean useTLS; - public ProcessReadinessChecker(int port, String readyCheckPath, String processName, boolean useTLS) { + public ProcessReadinessChecker(int port, String readyCheckPath, String processName, + boolean useTLS) { this.port = port; this.readyCheckPath = readyCheckPath; this.processName = processName; From a4637ba60857a84c7903caa12aff61b1a68ce033 Mon Sep 17 00:00:00 2001 From: csviri Date: Mon, 10 Apr 2023 15:09:13 +0200 Subject: [PATCH 3/3] configs --- .../jenvtest/KubeAPIServer.java | 3 ++- .../jenvtest/KubeAPIServerConfig.java | 13 +++++++++++- .../jenvtest/KubeAPIServerConfigBuilder.java | 20 ++++++++++++++++++- .../jenvtest/process/EtcdProcess.java | 8 ++++++-- .../jenvtest/sample/KubeApiServerTest.java | 12 +++++++++++ 5 files changed, 51 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/io/javaoperatorsdk/jenvtest/KubeAPIServer.java b/core/src/main/java/io/javaoperatorsdk/jenvtest/KubeAPIServer.java index 4bc0b05..94fd335 100644 --- a/core/src/main/java/io/javaoperatorsdk/jenvtest/KubeAPIServer.java +++ b/core/src/main/java/io/javaoperatorsdk/jenvtest/KubeAPIServer.java @@ -30,7 +30,8 @@ public KubeAPIServer(KubeAPIServerConfig config) { this.binaryManager = new BinaryManager(config); this.certManager = new CertManager(config.getJenvtestDir()); this.kubeConfig = new KubeConfig(certManager, binaryManager); - this.etcdProcess = new EtcdProcess(binaryManager, this); + this.etcdProcess = new EtcdProcess(binaryManager, this, + config.isWaitForEtcdHealthCheckOnStartup()); this.kubeApiServerProcess = new KubeAPIServerProcess(certManager, binaryManager, this, config); } diff --git a/core/src/main/java/io/javaoperatorsdk/jenvtest/KubeAPIServerConfig.java b/core/src/main/java/io/javaoperatorsdk/jenvtest/KubeAPIServerConfig.java index 0ff463c..379b727 100644 --- a/core/src/main/java/io/javaoperatorsdk/jenvtest/KubeAPIServerConfig.java +++ b/core/src/main/java/io/javaoperatorsdk/jenvtest/KubeAPIServerConfig.java @@ -42,13 +42,20 @@ public class KubeAPIServerConfig { */ private final boolean updateKubeConfig; + /** + * This is mostly not needed, and increases startup time. + */ + private final boolean waitForEtcdHealthCheckOnStartup; + KubeAPIServerConfig(String jenvtestDir, String apiServerVersion, boolean offlineMode, - List apiServerFlags, boolean updateKubeConfig) { + List apiServerFlags, boolean updateKubeConfig, + boolean waitForEtcdHealthCheckOnStartup) { this.jenvtestDir = jenvtestDir; this.apiServerVersion = apiServerVersion; this.offlineMode = offlineMode; this.apiServerFlags = apiServerFlags; this.updateKubeConfig = updateKubeConfig; + this.waitForEtcdHealthCheckOnStartup = waitForEtcdHealthCheckOnStartup; } public String getJenvtestDir() { @@ -70,4 +77,8 @@ public List getApiServerFlags() { public boolean isUpdateKubeConfig() { return updateKubeConfig; } + + public boolean isWaitForEtcdHealthCheckOnStartup() { + return waitForEtcdHealthCheckOnStartup; + } } diff --git a/core/src/main/java/io/javaoperatorsdk/jenvtest/KubeAPIServerConfigBuilder.java b/core/src/main/java/io/javaoperatorsdk/jenvtest/KubeAPIServerConfigBuilder.java index 44d45f0..c423354 100644 --- a/core/src/main/java/io/javaoperatorsdk/jenvtest/KubeAPIServerConfigBuilder.java +++ b/core/src/main/java/io/javaoperatorsdk/jenvtest/KubeAPIServerConfigBuilder.java @@ -10,6 +10,8 @@ public final class KubeAPIServerConfigBuilder { public static final String JENVTEST_DOWNLOAD_BINARIES = "JENVTEST_OFFLINE_MODE"; public static final String JENVTEST_DIR_ENV_VAR = "JENVTEST_DIR"; public static final String JENVTEST_API_SERVER_VERSION_ENV_VAR = "JENVTEST_API_SERVER_VERSION"; + public static final String JENVTEST_WAIT_FOR_ETCD_HEALTH_CHECK = + "JENVTEST_WAIT_FOR_ETCD_HEALTH_CHECK"; public static final String DIRECTORY_NAME = ".jenvtest"; @@ -18,6 +20,7 @@ public final class KubeAPIServerConfigBuilder { private Boolean offlineMode; private boolean updateKubeConfig = false; private final List apiServerFlags = new ArrayList<>(0); + private Boolean waitForEtcdHealthCheckOnStartup; public KubeAPIServerConfigBuilder() {} @@ -63,8 +66,17 @@ public KubeAPIServerConfig build() { this.apiServerVersion = apiServerVersionEnvVar; } } + if (waitForEtcdHealthCheckOnStartup == null) { + var waitForEtcdHealthCheckOnStartup = System.getenv(JENVTEST_API_SERVER_VERSION_ENV_VAR); + if (waitForEtcdHealthCheckOnStartup != null) { + this.waitForEtcdHealthCheckOnStartup = + Boolean.parseBoolean(waitForEtcdHealthCheckOnStartup); + } else { + this.waitForEtcdHealthCheckOnStartup = false; + } + } return new KubeAPIServerConfig(jenvtestDir, apiServerVersion, offlineMode, apiServerFlags, - updateKubeConfig); + updateKubeConfig, waitForEtcdHealthCheckOnStartup); } public KubeAPIServerConfigBuilder withUpdateKubeConfig(boolean updateKubeConfig) { @@ -90,6 +102,12 @@ public KubeAPIServerConfigBuilder withApiServerFlag(String key) { return this; } + public KubeAPIServerConfigBuilder withWaitForEtcdHealthCheckOnStartup( + boolean waitForEtcdHealthCheckOnStartup) { + this.waitForEtcdHealthCheckOnStartup = waitForEtcdHealthCheckOnStartup; + return this; + } + private void checkKeyPrefix(String key) { if (!key.startsWith("--")) { throw new JenvtestException( diff --git a/core/src/main/java/io/javaoperatorsdk/jenvtest/process/EtcdProcess.java b/core/src/main/java/io/javaoperatorsdk/jenvtest/process/EtcdProcess.java index 4bc7de3..23b7b56 100644 --- a/core/src/main/java/io/javaoperatorsdk/jenvtest/process/EtcdProcess.java +++ b/core/src/main/java/io/javaoperatorsdk/jenvtest/process/EtcdProcess.java @@ -23,13 +23,15 @@ public class EtcdProcess { private volatile Process etcdProcess; private volatile boolean stopped = false; private final UnexpectedProcessStopHandler processStopHandler; + private final boolean waitForHealthCheck; private File tempWalDir; private File tempDataDir; public EtcdProcess(BinaryManager binaryManager, - UnexpectedProcessStopHandler processStopHandler) { + UnexpectedProcessStopHandler processStopHandler, boolean waitForHealthCheck) { this.binaryManager = binaryManager; this.processStopHandler = processStopHandler; + this.waitForHealthCheck = waitForHealthCheck; } public int startEtcd() { @@ -67,7 +69,9 @@ public int startEtcd() { return null; }); log.debug("etcd started on port: {}", port); - waitUntilEtcdHealthy(port); + if (waitForHealthCheck) { + waitUntilEtcdHealthy(port); + } return port; } catch (IOException e) { throw new JenvtestException(e); diff --git a/core/src/test/java/io/javaoperatorsdk/jenvtest/sample/KubeApiServerTest.java b/core/src/test/java/io/javaoperatorsdk/jenvtest/sample/KubeApiServerTest.java index 3e99c50..b6189d3 100644 --- a/core/src/test/java/io/javaoperatorsdk/jenvtest/sample/KubeApiServerTest.java +++ b/core/src/test/java/io/javaoperatorsdk/jenvtest/sample/KubeApiServerTest.java @@ -38,6 +38,18 @@ void usingKubeConfigFileToInitClient() { TestUtils.simpleTest(client); } + @Test + void canWaitForEtcdHealthCheckOnStartup() { + var kubeApi = new KubeAPIServer(KubeAPIServerConfigBuilder.anAPIServerConfig() + .withWaitForEtcdHealthCheckOnStartup(true) + .build()); + kubeApi.start(); + + var client = createClient(kubeApi.getKubeConfigYaml()); + TestUtils.simpleTest(client); + + kubeApi.stop(); + } @Test void usingWildcardVersion() {