Skip to content
This repository was archived by the owner on Apr 10, 2024. It is now read-only.

improvement: use etcd health check #78

Merged
merged 3 commits into from
Apr 10, 2023
Merged

improvement: use etcd health check #78

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a3bfefc
improvement: use etcd health check
csviri Apr 10, 2023
294da2c
format
csviri Apr 10, 2023
a4637ba
configs
csviri Apr 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions core/src/main/java/io/javaoperatorsdk/jenvtest/KubeAPIServer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,6 @@ public class KubeAPIServer implements UnexpectedProcessStopHandler {

private static final Logger log = LoggerFactory.getLogger(KubeAPIServer.class);

public static final int STARTUP_TIMEOUT = 10_000;

private final KubeAPIServerConfig config;
private final BinaryManager binaryManager;
private final CertManager certManager;
Expand All @@ -32,7 +30,8 @@ public KubeAPIServer(KubeAPIServerConfig config) {
this.binaryManager = new BinaryManager(config);
this.certManager = new CertManager(config.getJenvtestDir());
this.kubeConfig = new KubeConfig(certManager, binaryManager);
this.etcdProcess = new EtcdProcess(binaryManager, this);
this.etcdProcess = new EtcdProcess(binaryManager, this,
config.isWaitForEtcdHealthCheckOnStartup());
this.kubeApiServerProcess =
new KubeAPIServerProcess(certManager, binaryManager, this, config);
}
Expand All @@ -46,7 +45,7 @@ public void start() {
if (config.isUpdateKubeConfig()) {
kubeConfig.updateKubeConfig(apiServerPort);
}
kubeApiServerProcess.waitUntilDefaultNamespaceCreated();
kubeApiServerProcess.waitUntilReady();
log.debug("API Server ready to use");
}

Expand Down
13 changes: 12 additions & 1 deletion core/src/main/java/io/javaoperatorsdk/jenvtest/KubeAPIServerConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,13 +42,20 @@ public class KubeAPIServerConfig {
*/
private final boolean updateKubeConfig;

/**
* This is mostly not needed, and increases startup time.
*/
private final boolean waitForEtcdHealthCheckOnStartup;

KubeAPIServerConfig(String jenvtestDir, String apiServerVersion, boolean offlineMode,
List<String> apiServerFlags, boolean updateKubeConfig) {
List<String> apiServerFlags, boolean updateKubeConfig,
boolean waitForEtcdHealthCheckOnStartup) {
this.jenvtestDir = jenvtestDir;
this.apiServerVersion = apiServerVersion;
this.offlineMode = offlineMode;
this.apiServerFlags = apiServerFlags;
this.updateKubeConfig = updateKubeConfig;
this.waitForEtcdHealthCheckOnStartup = waitForEtcdHealthCheckOnStartup;
}

public String getJenvtestDir() {
Expand All @@ -70,4 +77,8 @@ public List<String> getApiServerFlags() {
public boolean isUpdateKubeConfig() {
return updateKubeConfig;
}

public boolean isWaitForEtcdHealthCheckOnStartup() {
return waitForEtcdHealthCheckOnStartup;
}
}
20 changes: 19 additions & 1 deletion core/src/main/java/io/javaoperatorsdk/jenvtest/KubeAPIServerConfigBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ public final class KubeAPIServerConfigBuilder {
public static final String JENVTEST_DOWNLOAD_BINARIES = "JENVTEST_OFFLINE_MODE";
public static final String JENVTEST_DIR_ENV_VAR = "JENVTEST_DIR";
public static final String JENVTEST_API_SERVER_VERSION_ENV_VAR = "JENVTEST_API_SERVER_VERSION";
public static final String JENVTEST_WAIT_FOR_ETCD_HEALTH_CHECK =
"JENVTEST_WAIT_FOR_ETCD_HEALTH_CHECK";

public static final String DIRECTORY_NAME = ".jenvtest";

Expand All @@ -18,6 +20,7 @@ public final class KubeAPIServerConfigBuilder {
private Boolean offlineMode;
private boolean updateKubeConfig = false;
private final List<String> apiServerFlags = new ArrayList<>(0);
private Boolean waitForEtcdHealthCheckOnStartup;

public KubeAPIServerConfigBuilder() {}

Expand Down Expand Up @@ -63,8 +66,17 @@ public KubeAPIServerConfig build() {
this.apiServerVersion = apiServerVersionEnvVar;
}
}
if (waitForEtcdHealthCheckOnStartup == null) {
var waitForEtcdHealthCheckOnStartup = System.getenv(JENVTEST_API_SERVER_VERSION_ENV_VAR);
if (waitForEtcdHealthCheckOnStartup != null) {
this.waitForEtcdHealthCheckOnStartup =
Boolean.parseBoolean(waitForEtcdHealthCheckOnStartup);
} else {
this.waitForEtcdHealthCheckOnStartup = false;
}
}
return new KubeAPIServerConfig(jenvtestDir, apiServerVersion, offlineMode, apiServerFlags,
updateKubeConfig);
updateKubeConfig, waitForEtcdHealthCheckOnStartup);
}

public KubeAPIServerConfigBuilder withUpdateKubeConfig(boolean updateKubeConfig) {
Expand All @@ -90,6 +102,12 @@ public KubeAPIServerConfigBuilder withApiServerFlag(String key) {
return this;
}

public KubeAPIServerConfigBuilder withWaitForEtcdHealthCheckOnStartup(
boolean waitForEtcdHealthCheckOnStartup) {
this.waitForEtcdHealthCheckOnStartup = waitForEtcdHealthCheckOnStartup;
return this;
}

private void checkKeyPrefix(String key) {
if (!key.startsWith("--")) {
throw new JenvtestException(
Expand Down
11 changes: 10 additions & 1 deletion core/src/main/java/io/javaoperatorsdk/jenvtest/process/EtcdProcess.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,15 @@ public class EtcdProcess {
private volatile Process etcdProcess;
private volatile boolean stopped = false;
private final UnexpectedProcessStopHandler processStopHandler;
private final boolean waitForHealthCheck;
private File tempWalDir;
private File tempDataDir;

public EtcdProcess(BinaryManager binaryManager,
UnexpectedProcessStopHandler processStopHandler) {
UnexpectedProcessStopHandler processStopHandler, boolean waitForHealthCheck) {
this.binaryManager = binaryManager;
this.processStopHandler = processStopHandler;
this.waitForHealthCheck = waitForHealthCheck;
}

public int startEtcd() {
Expand Down Expand Up @@ -67,12 +69,19 @@ public int startEtcd() {
return null;
});
log.debug("etcd started on port: {}", port);
if (waitForHealthCheck) {
waitUntilEtcdHealthy(port);
}
return port;
} catch (IOException e) {
throw new JenvtestException(e);
}
}

private void waitUntilEtcdHealthy(int port) {
new ProcessReadinessChecker(port, "health", "etcd", false).waitUntilReady();
}

public void cleanEtcdData() {
try {
FileUtils.deleteDirectory(tempDataDir);
Expand Down
120 changes: 2 additions & 118 deletions core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,39 +2,21 @@

import java.io.File;
import java.io.IOException;
import java.net.ConnectException;
import java.net.Socket;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.time.LocalTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.List;

import javax.net.ssl.*;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import io.javaoperatorsdk.jenvtest.*;
import io.javaoperatorsdk.jenvtest.binary.BinaryManager;
import io.javaoperatorsdk.jenvtest.cert.CertManager;

import static io.javaoperatorsdk.jenvtest.KubeAPIServer.STARTUP_TIMEOUT;

public class KubeAPIServerProcess {

private static final Logger log = LoggerFactory.getLogger(KubeAPIServerProcess.class);
private static final Logger apiLog = LoggerFactory.getLogger(KubeAPIServerProcess.class
.getName() + ".APIServerProcessLogs");
public static final int POLLING_INTERVAL = 150;

private final CertManager certManager;
private final BinaryManager binaryManager;
Expand Down Expand Up @@ -102,106 +84,8 @@ private List<String> createCommand(File apiServerBinary, int apiServerPort, int
return command;
}

public void waitUntilDefaultNamespaceCreated() {
try {
var client = getHttpClient();
var request = getHttpRequest();
var startedAt = LocalTime.now();
while (true) {
if (ready(client, request)) {
return;
}
if (LocalTime.now().isAfter(startedAt.plus(STARTUP_TIMEOUT, ChronoUnit.MILLIS))) {
throw new JenvtestException("API Server did not start properly");
}
Thread.sleep(POLLING_INTERVAL);
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new JenvtestException(e);
}
}

private boolean ready(HttpClient client, HttpRequest request) {
try {
var response = client.send(request, HttpResponse.BodyHandlers.ofString());
log.debug("Ready Response message:{} code: {} Api Server Port: {}", response.body(),
response.statusCode(),
apiServerPort);
return response.statusCode() == 200;
} catch (ConnectException e) {
// still want to retry
log.warn("Cannot connect to the server", e);
return false;
} catch (IOException e) {
throw new JenvtestException(e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new JenvtestException(e);
}
}

private HttpRequest getHttpRequest() {
try {
return HttpRequest.newBuilder()
.uri(new URI("https://127.0.0.1:" + apiServerPort + "/readyz"))
.GET()
.build();
} catch (URISyntaxException e) {
throw new JenvtestException(e);
}
}

private static HttpClient getHttpClient() {
try {
var sslContext = SSLContext.getInstance("TLS");
sslContext.init(
null,
new TrustManager[] {
new X509ExtendedTrustManager() {
@Override
public void checkClientTrusted(X509Certificate[] chain, String authType,
Socket socket) throws CertificateException {

}

public X509Certificate[] getAcceptedIssuers() {
return null;
}

public void checkClientTrusted(
final X509Certificate[] a_certificates,
final String a_auth_type) {}

public void checkServerTrusted(
final X509Certificate[] a_certificates,
final String a_auth_type) {}


public void checkServerTrusted(
final X509Certificate[] a_certificates,
final String a_auth_type,
final Socket a_socket) {}

@Override
public void checkClientTrusted(X509Certificate[] chain, String authType,
SSLEngine engine) throws CertificateException {

}

public void checkServerTrusted(
final X509Certificate[] a_certificates,
final String a_auth_type,
final SSLEngine a_engine) {}
}
},
null);
return HttpClient.newBuilder()
.sslContext(sslContext)
.build();
} catch (NoSuchAlgorithmException | KeyManagementException e) {
throw new JenvtestException(e);
}
public void waitUntilReady() {
new ProcessReadinessChecker(apiServerPort, "readyz", "Kube API Server", true).waitUntilReady();
}

public void stopApiServer() {
Expand Down
Loading