From 5183135e045cd7d73e8927f3bb6e1f06d6627b15 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 27 Mar 2020 15:05:25 +0000 Subject: [PATCH 1/2] update to centralized travis configuration --- .ci/Dockerfile | 23 +++++++++++++++++ .ci/Dockerfile.elasticsearch | 21 ++++++++++++++++ .ci/docker-compose.override.yml | 32 ++++++++++++++++++++++++ {ci => .ci}/docker-run.sh | 7 ++++-- {ci => .ci}/docker-setup.sh | 12 +++++---- {ci => .ci}/elasticsearch-run.sh | 0 {ci => .ci}/logstash-run.sh | 25 ++++++++++--------- .travis.yml | 17 ++++--------- ci/Dockerfile.elasticsearch | 21 ---------------- ci/Dockerfile.logstash | 25 ------------------- ci/docker-compose.yml | 42 -------------------------------- 11 files changed, 106 insertions(+), 119 deletions(-) create mode 100644 .ci/Dockerfile create mode 100644 .ci/Dockerfile.elasticsearch create mode 100644 .ci/docker-compose.override.yml rename {ci => .ci}/docker-run.sh (51%) rename {ci => .ci}/docker-setup.sh (90%) rename {ci => .ci}/elasticsearch-run.sh (100%) rename {ci => .ci}/logstash-run.sh (62%) delete mode 100644 ci/Dockerfile.elasticsearch delete mode 100644 ci/Dockerfile.logstash delete mode 100644 ci/docker-compose.yml diff --git a/.ci/Dockerfile b/.ci/Dockerfile new file mode 100644 index 000000000..9f1c9455a --- /dev/null +++ b/.ci/Dockerfile @@ -0,0 +1,23 @@ +ARG ELASTIC_STACK_VERSION +# TODO: refactor this to be implicitly resolved by logstash-plugins/.ci/Dockerfile +ARG DISTRIBUTION_SUFFIX +FROM docker.elastic.co/logstash/logstash$DISTRIBUTION_SUFFIX:$ELASTIC_STACK_VERSION +USER logstash +COPY --chown=logstash:logstash Gemfile /usr/share/plugins/plugin/Gemfile +COPY --chown=logstash:logstash *.gemspec VERSION* version* /usr/share/plugins/plugin/ +RUN cp /usr/share/logstash/logstash-core/versions-gem-copy.yml /usr/share/logstash/versions.yml +ENV PATH="${PATH}:/usr/share/logstash/vendor/jruby/bin" +ENV LOGSTASH_SOURCE="1" +ENV ELASTIC_STACK_VERSION=$ELASTIC_STACK_VERSION +# DISTRIBUTION="default" (by default) or "oss" +ARG DISTRIBUTION +ENV DISTRIBUTION=$DISTRIBUTION +# INTEGRATION="true" while integration testing (false-y by default) +ARG INTEGRATION +ENV INTEGRATION=$INTEGRATION +RUN gem install bundler -v '< 2' +WORKDIR /usr/share/plugins/plugin +RUN bundle install --with test ci +COPY --chown=logstash:logstash . /usr/share/plugins/plugin +RUN bundle exec rake vendor +RUN .ci/setup.sh diff --git a/.ci/Dockerfile.elasticsearch b/.ci/Dockerfile.elasticsearch new file mode 100644 index 000000000..95cbc4714 --- /dev/null +++ b/.ci/Dockerfile.elasticsearch @@ -0,0 +1,21 @@ +ARG ELASTIC_STACK_VERSION +ARG DISTRIBUTION_SUFFIX +FROM docker.elastic.co/elasticsearch/elasticsearch$DISTRIBUTION_SUFFIX:$ELASTIC_STACK_VERSION + +ARG plugin_path=/usr/share/plugins/plugin +ARG es_path=/usr/share/elasticsearch +ARG es_yml=$es_path/config/elasticsearch.yml +ARG SECURE_INTEGRATION + +RUN rm -f $es_path/config/scripts + +COPY --chown=elasticsearch:elasticsearch spec/fixtures/scripts/groovy/* $es_path/config/scripts/ +COPY --chown=elasticsearch:elasticsearch spec/fixtures/test_certs/* $es_path/config/test_certs/ +COPY --chown=elasticsearch:elasticsearch .ci/elasticsearch-run.sh $es_path/ + +RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then echo "xpack.security.http.ssl.enabled: $SECURE_INTEGRATION" >> $es_yml; fi +RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then echo "xpack.security.http.ssl.key: $es_path/config/test_certs/test.key" >> $es_yml; fi +RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then echo "xpack.security.http.ssl.certificate: $es_path/config/test_certs/test.crt" >> $es_yml; fi +RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then echo "xpack.security.http.ssl.certificate_authorities: [ '$es_path/config/test_certs/ca.crt' ]" >> $es_yml; fi +RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then $es_path/bin/elasticsearch-users useradd simpleuser -p abc123 -r superuser; fi +RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then $es_path/bin/elasticsearch-users useradd 'f@ncyuser' -p 'ab%12#' -r superuser; fi diff --git a/.ci/docker-compose.override.yml b/.ci/docker-compose.override.yml new file mode 100644 index 000000000..42832109b --- /dev/null +++ b/.ci/docker-compose.override.yml @@ -0,0 +1,32 @@ +version: '3' + +services: + + logstash: + command: /usr/share/plugins/plugin/.ci/logstash-run.sh + build: + args: + - ELASTIC_STACK_VERSION=$ELASTIC_STACK_VERSION + - DISTRIBUTION=${DISTRIBUTION:-default} + - DISTRIBUTION_SUFFIX=${DISTRIBUTION_SUFFIX} + environment: + - DISTRIBUTION=${DISTRIBUTION:-default} + - DISTRIBUTION_SUFFIX=${DISTRIBUTION_SUFFIX} + - INTEGRATION=${INTEGRATION:-false} + - SECURE_INTEGRATION=${SECURE_INTEGRATION:-false} + + elasticsearch: + build: + context: ../ + dockerfile: .ci/Dockerfile.elasticsearch + args: + - ELASTIC_STACK_VERSION=$ELASTIC_STACK_VERSION + - INTEGRATION=${INTEGRATION:-false} + - SECURE_INTEGRATION=${SECURE_INTEGRATION:-false} + - DISTRIBUTION_SUFFIX=${DISTRIBUTION_SUFFIX} + command: /usr/share/elasticsearch/elasticsearch-run.sh + tty: true + ports: + - "9200:9200" + user: elasticsearch + diff --git a/ci/docker-run.sh b/.ci/docker-run.sh similarity index 51% rename from ci/docker-run.sh rename to .ci/docker-run.sh index 3a5ef9f41..02f0c42c4 100755 --- a/ci/docker-run.sh +++ b/.ci/docker-run.sh @@ -2,8 +2,11 @@ # This is intended to be run inside the docker container as the command of the docker-compose. set -ex + +cd .ci + if [ "$INTEGRATION" == "true" ]; then - docker-compose -f ci/docker-compose.yml up --exit-code-from logstash + docker-compose up --exit-code-from logstash else - docker-compose -f ci/docker-compose.yml up --exit-code-from logstash logstash + docker-compose up --exit-code-from logstash logstash fi diff --git a/ci/docker-setup.sh b/.ci/docker-setup.sh similarity index 90% rename from ci/docker-setup.sh rename to .ci/docker-setup.sh index 4230e9183..99e346aea 100755 --- a/ci/docker-setup.sh +++ b/.ci/docker-setup.sh @@ -1,6 +1,6 @@ #!/bin/bash -# This is intended to be run the plugin's root directory. `ci/docker-test.sh` +# This is intended to be run the plugin's root directory. `.ci/docker-setup.sh` # Ensure you have Docker installed locally and set the ELASTIC_STACK_VERSION environment variable. set -e @@ -67,12 +67,14 @@ if [ "$ELASTIC_STACK_VERSION" ]; then rm Gemfile.lock fi + cd .ci + if [ "$INTEGRATION" == "true" ]; then - docker-compose -f ci/docker-compose.yml down - docker-compose -f ci/docker-compose.yml build + docker-compose down + docker-compose build else - docker-compose -f ci/docker-compose.yml down - docker-compose -f ci/docker-compose.yml build logstash + docker-compose down + docker-compose build logstash fi else echo "Please set the ELASTIC_STACK_VERSION environment variable" diff --git a/ci/elasticsearch-run.sh b/.ci/elasticsearch-run.sh similarity index 100% rename from ci/elasticsearch-run.sh rename to .ci/elasticsearch-run.sh diff --git a/ci/logstash-run.sh b/.ci/logstash-run.sh similarity index 62% rename from ci/logstash-run.sh rename to .ci/logstash-run.sh index 72e23c49d..ce7be08e4 100755 --- a/ci/logstash-run.sh +++ b/.ci/logstash-run.sh @@ -3,21 +3,20 @@ set -ex export PATH=$BUILD_DIR/gradle/bin:$PATH +if [[ "$SECURE_INTEGRATION" == "true" ]]; then + ES_URL="https://elasticsearch:9200 -k" +else + ES_URL="http://elasticsearch:9200" +fi + wait_for_es() { - echo "Waiting for elasticsearch to respond..." - es_url="http://elasticsearch:9200" - if [[ "$SECURE_INTEGRATION" == "true" ]]; then - es_url="https://elasticsearch:9200 -k" - fi count=120 - while ! curl --silent $es_url && [[ $count -ne 0 ]]; do + while ! curl -s $ES_URL >/dev/null && [[ $count -ne 0 ]]; do count=$(( $count - 1 )) - [[ $count -eq 0 ]] && return 1 + [[ $count -eq 0 ]] && exit 1 sleep 1 done - echo "Elasticsearch is Up !" - - return 0 + echo $(curl -s $ES_URL | python2 -c "import sys, json; print json.load(sys.stdin)['version']['number']") } if [[ "$INTEGRATION" != "true" ]]; then @@ -35,6 +34,8 @@ else elif [[ "$DISTRIBUTION" == "default" ]]; then extra_tag_args="$extra_tag_args --tag ~distribution:oss --tag distribution:xpack" fi - wait_for_es - bundle exec rspec -fd $extra_tag_args --tag update_tests:painless --tag update_tests:groovy --tag es_version:$ELASTIC_STACK_VERSION spec/integration + echo "Waiting for elasticsearch to respond..." + ES_VERSION=$(wait_for_es) + echo "Elasticsearch $ES_VERSION is Up!" + bundle exec rspec -fd $extra_tag_args --tag update_tests:painless --tag update_tests:groovy --tag es_version:$ES_VERSION spec/integration fi diff --git a/.travis.yml b/.travis.yml index 68ae198e0..d453c9e64 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,19 +1,12 @@ ---- -sudo: required -services: docker -addons: - apt: - packages: - - docker-ce +import: +- logstash-plugins/.ci:travis/travis.yml@1.x + env: - - INTEGRATION=false ELASTIC_STACK_VERSION=5.x - - INTEGRATION=false ELASTIC_STACK_VERSION=6.x - - INTEGRATION=false ELASTIC_STACK_VERSION=7.x + - DISTRIBUTION=default INTEGRATION=false ELASTIC_STACK_VERSION=6.x + - DISTRIBUTION=default INTEGRATION=false ELASTIC_STACK_VERSION=7.x - DISTRIBUTION=default INTEGRATION=true ELASTIC_STACK_VERSION=6.x - DISTRIBUTION=default INTEGRATION=true ELASTIC_STACK_VERSION=7.x - DISTRIBUTION=default INTEGRATION=true ELASTIC_STACK_VERSION=7.x SNAPSHOT=true - DISTRIBUTION=default INTEGRATION=true ELASTIC_STACK_VERSION=8.x SNAPSHOT=true - DISTRIBUTION=default SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x - DISTRIBUTION=oss INTEGRATION=true ELASTIC_STACK_VERSION=7.x -install: ci/docker-setup.sh -script: ci/docker-run.sh diff --git a/ci/Dockerfile.elasticsearch b/ci/Dockerfile.elasticsearch deleted file mode 100644 index 894185947..000000000 --- a/ci/Dockerfile.elasticsearch +++ /dev/null @@ -1,21 +0,0 @@ -ARG elastic_stack_version -ARG distribution_suffix -FROM docker.elastic.co/elasticsearch/elasticsearch$distribution_suffix:$elastic_stack_version - -ARG plugin_path=/usr/share/plugins/this -ARG es_path=/usr/share/elasticsearch -ARG es_yml=$es_path/config/elasticsearch.yml -ARG secure_integration - -RUN rm -f $es_path/config/scripts - -COPY --chown=elasticsearch:elasticsearch spec/fixtures/scripts/groovy/* $es_path/config/scripts/ -COPY --chown=elasticsearch:elasticsearch spec/fixtures/test_certs/* $es_path/config/test_certs/ -COPY --chown=elasticsearch:elasticsearch ci/elasticsearch-run.sh $es_path/ - -RUN if [ "$secure_integration" = "true" ] ; then echo "xpack.security.http.ssl.enabled: $secure_integration" >> $es_yml; fi -RUN if [ "$secure_integration" = "true" ] ; then echo "xpack.security.http.ssl.key: $es_path/config/test_certs/test.key" >> $es_yml; fi -RUN if [ "$secure_integration" = "true" ] ; then echo "xpack.security.http.ssl.certificate: $es_path/config/test_certs/test.crt" >> $es_yml; fi -RUN if [ "$secure_integration" = "true" ] ; then echo "xpack.security.http.ssl.certificate_authorities: [ '$es_path/config/test_certs/ca.crt' ]" >> $es_yml; fi -RUN if [ "$secure_integration" = "true" ] ; then $es_path/bin/elasticsearch-users useradd simpleuser -p abc123 -r superuser; fi -RUN if [ "$secure_integration" = "true" ] ; then $es_path/bin/elasticsearch-users useradd 'f@ncyuser' -p 'ab%12#' -r superuser; fi diff --git a/ci/Dockerfile.logstash b/ci/Dockerfile.logstash deleted file mode 100644 index d62a36930..000000000 --- a/ci/Dockerfile.logstash +++ /dev/null @@ -1,25 +0,0 @@ -ARG elastic_stack_version -ARG distribution_suffix -FROM docker.elastic.co/logstash/logstash$distribution_suffix:$elastic_stack_version -USER root -RUN yum install -y openssl -USER logstash -ENV JARS_SKIP="true" -ENV LOGSTASH_SOURCE=1 -ENV LS_JAVA_OPTS="-Xmx256m -Xms256m" -ENV PATH="${PATH}:/usr/share/logstash/vendor/jruby/bin" -COPY --chown=logstash:logstash Gemfile /usr/share/plugins/this/Gemfile -COPY --chown=logstash:logstash *.gemspec /usr/share/plugins/this/ -RUN cp /usr/share/logstash/logstash-core/versions-gem-copy.yml /usr/share/logstash/versions.yml -RUN gem install bundler -v '< 2' -WORKDIR /usr/share/plugins/this -RUN bundle install --path=vendor/bundler -COPY --chown=logstash:logstash . /usr/share/plugins/this -ARG elastic_stack_version -ARG distribution -ARG integration -ARG secure_integration -ENV INTEGRATION $integration -ENV SECURE_INTEGRATION $secure_integration -ENV DISTRIBUTION $distribution -ENV ELASTIC_STACK_VERSION $elastic_stack_version diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml deleted file mode 100644 index 2c5b397e3..000000000 --- a/ci/docker-compose.yml +++ /dev/null @@ -1,42 +0,0 @@ -version: '3' - -# run tests: cd ci; docker-compose up --build --force-recreate -# manual: cd ci; docker-compose run logstash bash -services: - - logstash: - build: - context: ../ - dockerfile: ci/Dockerfile.logstash - args: - - elastic_stack_version=$ELASTIC_STACK_VERSION - - distribution=${DISTRIBUTION:-default} - - distribution_suffix=${DISTRIBUTION_SUFFIX} - - integration=${INTEGRATION:-false} - - secure_integration=${SECURE_INTEGRATION:-false} - command: /usr/share/plugins/this/ci/logstash-run.sh - environment: - - SPEC_OPTS - tty: true - #volumes: - # - ./:/usr/share/plugins/this - - elasticsearch: - build: - context: ../ - dockerfile: ci/Dockerfile.elasticsearch - args: - - elastic_stack_version=$ELASTIC_STACK_VERSION - - distribution=${DISTRIBUTION:-default} - - distribution_suffix=${DISTRIBUTION_SUFFIX} - - integration=${INTEGRATION:-false} - - secure_integration=${SECURE_INTEGRATION:-false} - command: /usr/share/elasticsearch/elasticsearch-run.sh - tty: true - ports: - - "9200:9200" - user: elasticsearch - - #volumes: - # - ./:/usr/share/plugins/this - From 4bc2e532c87ce703cce21848c4218e2276aebc49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Duarte?= Date: Mon, 30 Mar 2020 10:55:36 +0100 Subject: [PATCH 2/2] Update .ci/logstash-run.sh Co-Authored-By: Karol Bucek --- .ci/logstash-run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/logstash-run.sh b/.ci/logstash-run.sh index ce7be08e4..39d0a3dd1 100755 --- a/.ci/logstash-run.sh +++ b/.ci/logstash-run.sh @@ -16,7 +16,7 @@ wait_for_es() { [[ $count -eq 0 ]] && exit 1 sleep 1 done - echo $(curl -s $ES_URL | python2 -c "import sys, json; print json.load(sys.stdin)['version']['number']") + echo $(curl -s $ES_URL | python -c "import sys, json; print(json.load(sys.stdin)['version']['number'])") } if [[ "$INTEGRATION" != "true" ]]; then