buildscripts: xDS Kubernetes Interop tests buildscript WIP

buildscripts/kokoro/xds-k8s-install-test-driver.sh

@@ -0,0 +1,189 @@
+#!/usr/bin/env bash
+# TODO(sergiitk): move to grpc/grpc when implementing support of other languages
+set -eo pipefail
+
+# Constants
+readonly PYTHON_BIN="python3.6"
+# Test driver
+readonly TEST_DRIVER_REPO_NAME="grpc"
+readonly TEST_DRIVER_REPO_URL="https://github.com/grpc/grpc.git"
+readonly TEST_DRIVER_BRANCH="${TEST_DRIVER_BRANCH:-master}"
+readonly TEST_DRIVER_PATH="tools/run_tests/xds_k8s_test_driver"
+readonly TEST_DRIVER_PROTOS_PATH="src/proto/grpc/testing"
+
+run_safe() {
+  # Run command end report its exit code.
+  # Don't terminate the script if the code is negative.
+  local exit_code=-1
+  "$@" || exit_code=$?
+  echo "Exit code: ${exit_code}"
+}
+
+parse_src_repo_git_info() {
+  local src_dir="${SRC_DIR:?SRC_DIR must be set}"
+  readonly GIT_ORIGIN_URL=$(git -C "${src_dir}" remote get-url origin)
+  readonly GIT_COMMIT_SHORT=$(git -C "${src_dir}" rev-parse --short HEAD)
+}
+
+gcloud_gcr_list_matching_tags() {
+  gcloud container images list-tags --format="table[box](tags,digest,timestamp.date())" --filter="tags:$2" "$1"
+}
+
+gcloud_update() {
+  echo "Update gcloud components:"
+  gcloud -q components update
+}
+
+gcloud_get_cluster_credentials() {
+  gcloud container clusters get-credentials "${GKE_CLUSTER_NAME}" --zone "${GKE_CLUSTER_ZONE}"
+  readonly KUBE_CONTEXT="$(kubectl config current-context)"
+}
+
+test_driver_get_source() {
+  if [[ -d "${TEST_DRIVER_REPO_DIR}" ]]; then
+    echo "Found driver directory: ${TEST_DRIVER_REPO_DIR}"
+  else
+    echo "Cloning driver to ${TEST_DRIVER_REPO_URL} branch ${TEST_DRIVER_BRANCH} to ${TEST_DRIVER_REPO_DIR}"
+    git clone -b "${TEST_DRIVER_BRANCH}" --depth=1 "${TEST_DRIVER_REPO_URL}" "${TEST_DRIVER_REPO_DIR}"
+  fi
+}
+
+test_driver_pip_install() {
+  echo "Install python dependencies"
+  cd "${TEST_DRIVER_FULL_DIR}"
+
+  # Create and activate virtual environment already using one
+  if [[ -z "${VIRTUAL_ENV}" ]]; then
+    local venv_dir="${TEST_DRIVER_FULL_DIR}/venv"
+    if [[ -d "${venv_dir}" ]]; then
+      echo "Found python virtual environment directory: ${venv_dir}"
+    else
+      echo "Creating python virtual environment: ${venv_dir}"
+      "${PYTHON_BIN} -m venv ${venv_dir}"
+    fi
+    source "${venv_dir}/bin/activate"
+  fi
+
+  pip install -r requirements.txt
+  echo "Installed Python packages:"
+  pip list
+}
+
+test_driver_compile_protos() {
+  declare -a protos
+  protos=(
+    "${TEST_DRIVER_PROTOS_PATH}/test.proto"
+    "${TEST_DRIVER_PROTOS_PATH}/messages.proto"
+    "${TEST_DRIVER_PROTOS_PATH}/empty.proto"
+  )
+  echo "Generate python code from grpc.testing protos: ${protos[*]}"
+  cd "${TEST_DRIVER_REPO_DIR}"
+  python3 -m grpc_tools.protoc \
+    --proto_path=. \
+    --python_out="${TEST_DRIVER_FULL_DIR}" \
+    --grpc_python_out="${TEST_DRIVER_FULL_DIR}" \
+    "${protos[@]}"
+  local protos_out_dir="${TEST_DRIVER_FULL_DIR}/${TEST_DRIVER_PROTOS_PATH}"
+  echo "Generated files ${protos_out_dir}:"
+  ls -Fl "${protos_out_dir}"
+}
+
+test_driver_install() {
+  readonly TEST_DRIVER_REPO_DIR="${1:?Usage kokoro_init TEST_DRIVER_REPO_DIR}"
+  readonly TEST_DRIVER_FULL_DIR="${TEST_DRIVER_REPO_DIR}/${TEST_DRIVER_PATH}"
+  # Test driver installation:
+  # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#installation
+  test_driver_get_source
+  test_driver_pip_install
+  test_driver_compile_protos
+}
+
+kokoro_print_version() {
+  echo "Kokoro VM version:"
+  if [[ -f /VERSION ]]; then
+    cat /VERSION
+  fi
+  run_safe lsb_release -a
+}
+
+kokoro_write_sponge_properties() {
+  # CSV format: "property_name","property_value"
+  # Bump TESTS_FORMAT_VERSION when reported test name changed enough to when it
+  # makes more sense to discard previous test results from a testgrid board.
+  # Use GIT_ORIGIN_URL to exclude test runs executed against repo forks from
+  # testgrid reports.
+  cat >"${KOKORO_ARTIFACTS_DIR}/custom_sponge_config.csv" <<EOF
+TESTS_FORMAT_VERSION,2
+TESTGRID_EXCLUDE,0
+GIT_ORIGIN_URL,${GIT_ORIGIN_URL:?GIT_ORIGIN_URL must be set}
+GIT_COMMIT_SHORT,${GIT_COMMIT_SHORT:?GIT_COMMIT_SHORT must be set}
+EOF
+  echo "Sponge properties:"
+  cat "${KOKORO_ARTIFACTS_DIR}/custom_sponge_config.csv"
+}
+
+kokoro_export_secrets() {
+  readonly PRIVATE_API_KEY=$(cat "${KOKORO_KEYSTORE_DIR}/73836_grpc_xds_interop_tests_gcp_alpha_apis_key")
+  export PRIVATE_API_KEY
+}
+
+kokoro_setup_python_virtual_environment() {
+  # Kokoro provides pyenv, so use it instead `python -m venv`
+  echo "Setup pyenv environment"
+  eval "$(pyenv init -)"
+  eval "$(pyenv virtualenv-init -)"
+  echo "Activating python virtual environment"
+  pyenv virtualenv 3.6.1 k8s_xds_test_runner
+  pyenv local k8s_xds_test_runner
+  pyenv activate k8s_xds_test_runner
+}
+
+kokoro_init() {
+  local src_repository_name="${1:?Usage kokoro_init GITHUB_REPOSITORY_NAME}"
+  # Capture Kokoro VM version info in the log.
+  kokoro_print_version
+
+  # Kokoro clones repo to ${KOKORO_ARTIFACTS_DIR}/github/${GITHUB_REPOSITORY}
+  local github_root="${KOKORO_ARTIFACTS_DIR}/github"
+  readonly SRC_DIR="${github_root}/${src_repository_name}"
+  local test_driver_repo_dir="${github_root}/${TEST_DRIVER_REPO_NAME}"
+  parse_src_repo_git_info SRC_DIR
+  # Report extra information about the job via sponge properties
+  kokoro_write_sponge_properties
+
+  # Turn off command trace print before exporting secrets.
+  local debug_on=0
+  if [[ $- =~ x ]]; then
+    debug_on=1
+    set +x
+  fi
+  kokoro_export_secrets
+  kokoro_setup_python_virtual_environment
+  # Re-enable debug output after secrets exported and noisy pyenv activated
+  ((debug_on == 0)) || set -x
+
+  # gcloud requires python, so this should be executed after pyenv setup
+  gcloud_update
+  gcloud_get_cluster_credentials
+  test_driver_install "${test_driver_repo_dir}"
+  readonly TEST_DRIVER_CONFIG="config/grpc-testing.cfg"
+  # Test artifacts dir: xml reports, logs, etc.
+  local artifacts_dir="${KOKORO_ARTIFACTS_DIR}/artifacts"
+  # Folders after $ARTIFACTS_DIR reported as target name
+  readonly TEST_XML_OUTPUT_DIR="${artifacts_dir}/${KOKORO_JOB_NAME}"
+  mkdir -p "${artifacts_dir}" "${TEST_XML_OUTPUT_DIR}"
+}
+
+local_init() {
+  local script_dir="${1:?Usage: local_init SCRIPT_DIR}"
+  readonly SRC_DIR="$(git -C "${script_dir}" rev-parse --show-toplevel)"
+  parse_src_repo_git_info SRC_DIR
+  readonly KUBE_CONTEXT="${KUBE_CONTEXT:-$(kubectl config current-context)}"
+  local test_driver_repo_dir
+  test_driver_repo_dir="${TEST_DRIVER_REPO_DIR:-$(mktemp -d)/${TEST_DRIVER_REPO_NAME}}"
+  test_driver_install "${test_driver_repo_dir}"
+  readonly TEST_DRIVER_CONFIG="config/local-dev.cfg"
+  # Test out
+  readonly TEST_XML_OUTPUT_DIR="${TEST_DRIVER_FULL_DIR}/out"
+  mkdir -p "${TEST_XML_OUTPUT_DIR}"
+}

buildscripts/kokoro/xds-k8s.cfg

@@ -3,9 +3,20 @@
 # Location of the continuous shell script in repository.
 build_file: "grpc-java/buildscripts/kokoro/xds-k8s.sh"
 timeout_mins: 90
+
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 73836
+      keyname: "grpc_xds_interop_tests_gcp_alpha_apis_key"
+    }
+  }
+}
+
 action {
   define_artifacts {
-    regex: "artifacts/*sponge_log.xml"
-    regex: "artifacts/*sponge_log.log"
+    regex: "artifacts/**/*sponge_log.xml"
+    regex: "artifacts/**/*sponge_log.log"
+    strip_prefix: "artifacts"
   }
 }

buildscripts/kokoro/xds-k8s.sh

@@ -1,4 +1,88 @@
-#!/bin/bash
+#!/usr/bin/env bash
+set -eo pipefail
 
-# A placeholder for xDS interop tests executed on GKE
-echo "Coming soon"
+# Constants
+readonly GITHUB_REPOSITORY_NAME="grpc-java"
+readonly SCRIPT_DIR=$(dirname "$0")
+# GKE Cluster
+readonly GKE_CLUSTER_NAME="interop-test-psm-sec1-us-central1"
+readonly GKE_CLUSTER_ZONE="us-central1-a"
+## xDS test server/client Docker images
+readonly IMAGE_NAME="gcr.io/grpc-testing/xds-k8s-test-java"
+readonly IMAGE_BUILD_SKIP="${IMAGE_BUILD_SKIP:-1}"
+
+build_java_test_app() {
+  echo "Building Java test app"
+  cd "${SRC_DIR}"
+  ./gradlew --no-daemon grpc-interop-testing:installDist -x test -PskipCodegen=true -PskipAndroid=true --console=plain
+  # Test-run binaries
+  run_safe "${TEST_APP_BUILD_APP_DIR}/bin/xds-test-client" --help
+  run_safe "${TEST_APP_BUILD_APP_DIR}/bin/xds-test-server" --help
+}
+
+build_test_app_docker_images() {
+  local docker_dir="${SRC_DIR}/buildscripts/xds-k8s"
+  echo "Building Java test app Docker Images"
+  # Copy Docker files and log properties to the build dir
+  cp -rv "${docker_dir}/"*.Dockerfile "${TEST_APP_BUILD_INSTALL_DIR}"
+  cp -rv "${docker_dir}/"*.properties "${TEST_APP_BUILD_INSTALL_DIR}"
+  # Run Google Cloud Build
+  gcloud builds submit "${TEST_APP_BUILD_INSTALL_DIR}" \
+    --config "${docker_dir}/cloudbuild.yaml" \
+    --substitutions "_IMAGE_NAME=${IMAGE_NAME},_SERVER_IMAGE_TAG=${SERVER_IMAGE_TAG},_CLIENT_IMAGE_TAG=${CLIENT_IMAGE_TAG}"
+  # TODO(sergiitk): extra "cosmetic" tags for versioned branches
+}
+
+build_docker_images_if_needed() {
+  # Check if images already exist
+  server_tags=$(gcloud_gcr_list_matching_tags "${IMAGE_NAME}" "${SERVER_IMAGE_TAG}")
+  printf "Server image: %s:%s\n%s\n" "${IMAGE_NAME}" "${SERVER_IMAGE_TAG}" "${server_tags:-Server image not found}"
+  client_tags=$(gcloud_gcr_list_matching_tags "${IMAGE_NAME}" "${CLIENT_IMAGE_TAG}")
+  printf "Client image: %s:%s\n%s\n" "${IMAGE_NAME}" "${CLIENT_IMAGE_TAG}" "${client_tags:-Client image not found}"
+
+  # Build if one of images is missing or IMAGE_BUILD_SKIP=0
+  if [[ "${IMAGE_BUILD_SKIP}" == "0" || -z "${server_tags}" && -z "${client_tags}" ]]; then
+    readonly TEST_APP_BUILD_INSTALL_DIR="${SRC_DIR}/interop-testing/build/install"
+    readonly TEST_APP_BUILD_APP_DIR="${TEST_APP_BUILD_OUT_DIR}/grpc-interop-testing"
+    build_java_test_app
+    build_test_app_docker_images
+  else
+    echo "Skipping Java test app build"
+  fi
+}
+
+run_test() {
+  # Test driver usage:
+  # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#basic-usage
+  local test_name="${1:?Usage: run_test test_name}"
+  cd "${TEST_DRIVER_FULL_DIR}"
+  set -x
+  python -m "tests.${test_name}" \
+    --flagfile="${TEST_DRIVER_CONFIG}" \
+    --kube_context="${KUBE_CONTEXT}" \
+    --server_image="${IMAGE_NAME}:${SERVER_IMAGE_TAG}" \
+    --client_image="${IMAGE_NAME}:${CLIENT_IMAGE_TAG}" \
+    --xml_output_file="${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml" \
+    --force_cleanup
+  set +x
+}
+
+main() {
+  # Intentional: source from the same dir as xds-k8s.sh
+  # shellcheck disable=SC1090
+  source "${SCRIPT_DIR}/xds-k8s-install-test-driver.sh"
+  set -x
+  if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then
+    kokoro_init "${GITHUB_REPOSITORY_NAME}"
+  else
+    local_init "${SCRIPT_DIR}"
+  fi
+  readonly SERVER_IMAGE_TAG="server-${GIT_COMMIT_SHORT}"
+  readonly CLIENT_IMAGE_TAG="client-${GIT_COMMIT_SHORT}"
+  build_docker_images_if_needed
+  # Run tests
+  run_test baseline_test
+  run_test security_test
+}
+
+main "$@"

buildscripts/xds-k8s/cloudbuild.yaml

@@ -0,0 +1,23 @@
+steps:
+- name: 'gcr.io/cloud-builders/docker'
+  args:
+    - 'build'
+    - '--tag=${_IMAGE_NAME}:${_SERVER_IMAGE_TAG}'
+    - '--file=test-server.Dockerfile'
+    - '.'
+
+- name: 'gcr.io/cloud-builders/docker'
+  args:
+    - 'build'
+    - '--tag=${_IMAGE_NAME}:${_CLIENT_IMAGE_TAG}'
+    - '--file=test-client.Dockerfile'
+    - '.'
+
+substitutions:
+  _IMAGE_NAME: gcr.io/grpc-testing/xds-k8s-test-java
+  _SERVER_IMAGE_TAG: server-dev
+  _CLIENT_IMAGE_TAG: client-dev
+
+images:
+  - '${_IMAGE_NAME}:${_SERVER_IMAGE_TAG}'
+  - '${_IMAGE_NAME}:${_CLIENT_IMAGE_TAG}'

buildscripts/xds-k8s/logging-debug.properties

@@ -0,0 +1,7 @@
+handlers=java.util.logging.ConsoleHandler
+io.grpc.ChannelLogger.level=FINEST
+io.grpc.level=FINEST
+io.netty.level=FINEST
+java.util.logging.ConsoleHandler.level=ALL
+java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter
+java.util.logging.SimpleFormatter.format=[%1$tF %1$tT.%1$tL %1tZ] [%4$-7s] [%3$s] %5$s %6$s %n

buildscripts/xds-k8s/logging-json.properties

@@ -0,0 +1,8 @@
+handlers=java.util.logging.ConsoleHandler
+java.util.logging.ConsoleHandler.level=FINEST
+java.util.logging.ConsoleHandler.formatter=io.github.devatherock.json.formatter.JSONFormatter
+io.github.devatherock.json.formatter.JSONFormatter.key_timestamp=time
+io.github.devatherock.json.formatter.JSONFormatter.key_log_level=severity
+io.github.devatherock.json.formatter.JSONFormatter.use_slf4j_level_names=true
+io.grpc.ChannelLogger.level=FINEST
+io.grpc.xds.level=FINEST

buildscripts/xds-k8s/logging.properties

@@ -0,0 +1,6 @@
+handlers=java.util.logging.ConsoleHandler
+io.grpc.ChannelLogger.level=FINEST
+io.grpc.xds.level=FINEST
+java.util.logging.ConsoleHandler.level=FINEST
+java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter
+java.util.logging.SimpleFormatter.format=[%1$tF %1$tT.%1$tL %1tZ] [%4$-7s] [%3$s] %5$s %6$s %n

buildscripts/xds-k8s/test-client.Dockerfile

@@ -0,0 +1,15 @@
+# Build runtime image.
+FROM openjdk:11.0.9.1-jdk
+
+ENV APP_DIR=/usr/src/app
+WORKDIR $APP_DIR
+
+# Install the app
+COPY grpc-interop-testing/ $APP_DIR/
+
+# Copy all logging profiles, use json logging by default
+COPY logging*.properties $APP_DIR/
+ENV JAVA_OPTS="-Djava.util.logging.config.file=$APP_DIR/logging-json.properties"
+
+# Client
+ENTRYPOINT ["bin/xds-test-client"]

buildscripts/xds-k8s/test-server.Dockerfile

@@ -0,0 +1,15 @@
+# Build runtime image.
+FROM openjdk:11.0.9.1-jdk
+
+ENV APP_DIR=/usr/src/app
+WORKDIR $APP_DIR
+
+# Install the app
+COPY grpc-interop-testing/ $APP_DIR/
+
+# Copy all logging profiles, use json logging by default
+COPY logging*.properties $APP_DIR/
+ENV JAVA_OPTS="-Djava.util.logging.config.file=$APP_DIR/logging-json.properties"
+
+# Server
+ENTRYPOINT ["bin/xds-test-server"]

interop-testing/build.gradle

@@ -32,6 +32,13 @@ dependencies {
             libraries.truth
     censusGrpcMetricDependency 'implementation'
     compileOnly libraries.javax_annotation
+    // TODO(sergiitk): replace with com.google.cloud:google-cloud-logging
+    // Used instead of google-cloud-logging because it's failing
+    // due to a circular dependency on grpc.
+    // https://cloud.google.com/logging/docs/setup/java#the_javautillogging_handler
+    // Error example: "java.util.logging.ErrorManager: 1"
+    // Latest failing version com.google.cloud:google-cloud-logging:2.1.2
+    runtimeOnly group: 'io.github.devatherock', name: 'jul-jsonformatter', version: '1.1.0'
     runtimeOnly libraries.opencensus_impl,
             libraries.netty_tcnative,
             project(':grpc-grpclb')