From 1e64cd6620483e5c8e8cb84ead38e1e93afe4957 Mon Sep 17 00:00:00 2001 From: asm0dey Date: Tue, 1 Oct 2019 00:12:35 +0300 Subject: [PATCH 1/5] ci: verify Ansible roles with ansible-lint Fix #479 --- .gitignore | 5 ++ .travis.yml | 1 + src/main/scripts/ci/check-build-and-verify.sh | 47 +++++++++++++------ 3 files changed, 38 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 0198b08dce..d978a9a06e 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,8 @@ docker/mysql_backup_mystamps.sql.gz # maven-wrapper .mvn/wrapper/maven-wrapper.jar + +# @asm0dey uses molecule for role testing which generates .pyc files +# and also creates molecule directory +*.pyc +molecule/ diff --git a/.travis.yml b/.travis.yml index fc0df0fdfd..a8b1ab02a6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,7 @@ before_script: if [ "$TRAVIS_BRANCH" = 'prod' -a "$TRAVIS_PULL_REQUEST" = 'false' ]; then pip install --user ansible==2.1.1.0; fi; + pip install --user ansible-lint; if [ "$TRAVIS_PULL_REQUEST" != 'false' ]; then gem install danger nokogiri --no-ri --no-rdoc; fi; diff --git a/src/main/scripts/ci/check-build-and-verify.sh b/src/main/scripts/ci/check-build-and-verify.sh index f30475247f..a28c7c701f 100755 --- a/src/main/scripts/ci/check-build-and-verify.sh +++ b/src/main/scripts/ci/check-build-and-verify.sh @@ -32,6 +32,7 @@ TEST_STATUS= CODENARC_STATUS= SPOTBUGS_STATUS= VERIFY_STATUS= +ANSIBLE_LINT_STATUS= DANGER_STATUS=skip if [ "${SPRING_PROFILES_ACTIVE:-}" = 'travis' ] && [ "${TRAVIS_PULL_REQUEST:-false}" != 'false' ]; then @@ -66,6 +67,7 @@ if [ "$RUN_ONLY_INTEGRATION_TESTS" = 'no' ]; then AFFECTS_GROOVY_FILES="$(echo "$MODIFIED_FILES" | grep -q '\.groovy$' || echo 'no')" AFFECTS_PROPERTIES="$(echo "$MODIFIED_FILES" | grep -q '\.properties$' || echo 'no')" AFFECTS_LICENSE_HEADER="$(echo "$MODIFIED_FILES" | grep -q 'license_header\.txt$' || echo 'no')" + AFFECTS_PLAYBOOKS="$(echo "$MODIFIED_FILES" | grep -Eq '(vagrant|deploy|bootstrap|/roles/.+)\.yml$' || echo 'no')" if [ "$AFFECTS_POM_XML" = 'no' ]; then POM_STATUS=skip @@ -97,7 +99,12 @@ if [ "$RUN_ONLY_INTEGRATION_TESTS" = 'no' ]; then fi [ "$AFFECTS_ROBOT_FILES" != 'no' ] || RFLINT_STATUS=skip [ "$AFFECTS_SHELL_FILES" != 'no' ] || SHELLCHECK_STATUS=skip + + if [ "$AFFECTS_PLAYBOOKS" = 'no' ]; then + ANSIBLE_LINT_STATUS=skip + fi fi + echo 'INFO: Some checks could be skipped' else echo "INFO: Couldn't determine list of modified files." @@ -218,6 +225,15 @@ if [ "$RUN_ONLY_INTEGRATION_TESTS" = 'no' ]; then >spotbugs.log 2>&1 || SPOTBUGS_STATUS=fail fi print_status "$SPOTBUGS_STATUS" 'Run SpotBugs' + + if [ "$ANSIBLE_LINT_STATUS" != 'skip' ]; then + ansible-lint \ + vagrant/provisioning/vagrant.yml \ + vagrant/provisioning/bootstrap.yml \ + src/main/scripts/ci/ansible/deploy.yml \ + >ansible_lint.log 2>&1 || ANSIBLE_LINT_STATUS=fail + fi + print_status "$ANSIBLE_LINT_STATUS" 'Run Ansible Lint' fi mvn --batch-mode --activate-profiles frontend,native2ascii verify -Denforcer.skip=true -DskipUnitTests=true \ @@ -234,19 +250,20 @@ fi print_status "$DANGER_STATUS" 'Run danger' if [ "$RUN_ONLY_INTEGRATION_TESTS" = 'no' ]; then - [ "$CS_STATUS" = 'skip' ] || print_log cs.log 'Run CheckStyle' - [ "$PMD_STATUS" = 'skip' ] || print_log pmd.log 'Run PMD' - [ "$LICENSE_STATUS" = 'skip' ] || print_log license.log 'Check license headers' - [ "$POM_STATUS" = 'skip' ] || print_log pom.log 'Check sorting of pom.xml' - [ "$BOOTLINT_STATUS" = 'skip' ] || print_log bootlint.log 'Run bootlint' - [ "$RFLINT_STATUS" = 'skip' ] || print_log rflint.log 'Run robot framework lint' - [ "$SHELLCHECK_STATUS" = 'skip' ] || print_log shellcheck.log 'Run shellcheck' - [ "$JASMINE_STATUS" = 'skip' ] || print_log jasmine.log 'Run JavaScript unit tests' - [ "$HTML_STATUS" = 'skip' ] || print_log validator.log 'Run html5validator' - [ "$ENFORCER_STATUS" = 'skip' ] || print_log enforcer.log 'Run maven-enforcer-plugin' - [ "$TEST_STATUS" = 'skip' ] || print_log test.log 'Run unit tests' - [ "$CODENARC_STATUS" = 'skip' ] || print_log codenarc.log 'Run CodeNarc' - [ "$SPOTBUGS_STATUS" = 'skip' ] || print_log spotbugs.log 'Run SpotBugs' + [ "$CS_STATUS" = 'skip' ] || print_log cs.log 'Run CheckStyle' + [ "$PMD_STATUS" = 'skip' ] || print_log pmd.log 'Run PMD' + [ "$LICENSE_STATUS" = 'skip' ] || print_log license.log 'Check license headers' + [ "$POM_STATUS" = 'skip' ] || print_log pom.log 'Check sorting of pom.xml' + [ "$BOOTLINT_STATUS" = 'skip' ] || print_log bootlint.log 'Run bootlint' + [ "$RFLINT_STATUS" = 'skip' ] || print_log rflint.log 'Run robot framework lint' + [ "$SHELLCHECK_STATUS" = 'skip' ] || print_log shellcheck.log 'Run shellcheck' + [ "$JASMINE_STATUS" = 'skip' ] || print_log jasmine.log 'Run JavaScript unit tests' + [ "$HTML_STATUS" = 'skip' ] || print_log validator.log 'Run html5validator' + [ "$ENFORCER_STATUS" = 'skip' ] || print_log enforcer.log 'Run maven-enforcer-plugin' + [ "$TEST_STATUS" = 'skip' ] || print_log test.log 'Run unit tests' + [ "$CODENARC_STATUS" = 'skip' ] || print_log codenarc.log 'Run CodeNarc' + [ "$SPOTBUGS_STATUS" = 'skip' ] || print_log spotbugs.log 'Run SpotBugs' + [ "$ANSIBLE_LINT_STATUS" = 'skip' ] || print_log ansible_lint.log 'Run Ansible Lint' fi print_log verify.log 'Run integration tests' @@ -255,8 +272,8 @@ if [ "$DANGER_STATUS" != 'skip' ]; then print_log danger.log 'Run danger' fi -rm -f cs.log pmd.log license.log pom.log bootlint.log rflint.log shellcheck.log jasmine.log validator.log enforcer.log test.log codenarc.log spotbugs.log verify-raw.log verify.log danger.log +rm -f cs.log pmd.log license.log pom.log bootlint.log rflint.log shellcheck.log jasmine.log validator.log enforcer.log test.log codenarc.log spotbugs.log verify-raw.log verify.log danger.log ansible_lint.log -if echo "$CS_STATUS$PMD_STATUS$LICENSE_STATUS$POM_STATUS$BOOTLINT_STATUS$RFLINT_STATUS$SHELLCHECK_STATUS$JASMINE_STATUS$HTML_STATUS$ENFORCER_STATUS$TEST_STATUS$CODENARC_STATUS$SPOTBUGS_STATUS$VERIFY_STATUS$DANGER_STATUS" | grep -Fqs 'fail'; then +if echo "$CS_STATUS$PMD_STATUS$LICENSE_STATUS$POM_STATUS$BOOTLINT_STATUS$RFLINT_STATUS$SHELLCHECK_STATUS$JASMINE_STATUS$HTML_STATUS$ENFORCER_STATUS$TEST_STATUS$CODENARC_STATUS$SPOTBUGS_STATUS$VERIFY_STATUS$DANGER_STATUS$ANSIBLE_LINT_STATUS" | grep -Fqs 'fail'; then exit 1 fi From c98d18c1901b6a11100f119c7f5a0c64ad17b46c Mon Sep 17 00:00:00 2001 From: asm0dey Date: Sun, 20 Oct 2019 19:52:59 +0300 Subject: [PATCH 2/5] ci: fixes E602 Don't compare to empty string Should work regardless of https://github.com/ansible/ansible-lint/issues/457 because `monitorid` and `apikey` are fields of uptimerobot object, not plain variables --- src/main/scripts/ci/ansible/deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scripts/ci/ansible/deploy.yml b/src/main/scripts/ci/ansible/deploy.yml index 347451cdd5..ef7cf72ff0 100644 --- a/src/main/scripts/ci/ansible/deploy.yml +++ b/src/main/scripts/ci/ansible/deploy.yml @@ -31,7 +31,7 @@ monitorid: "{{ uptimerobot.monitorid }}" apikey: "{{ uptimerobot.apikey }}" state: paused - when: uptimerobot is defined and uptimerobot.monitorid != '' and uptimerobot.apikey != '' + when: uptimerobot is defined and uptimerobot.monitorid and uptimerobot.apikey # we can't use service module here because our sudoers allows to execute only exact commands - name: Stopping service @@ -57,5 +57,5 @@ monitorid: "{{ uptimerobot.monitorid }}" apikey: "{{ uptimerobot.apikey }}" state: started - when: uptimerobot is defined and uptimerobot.monitorid != '' and uptimerobot.apikey != '' + when: uptimerobot is defined and uptimerobot.monitorid and uptimerobot.apikey From a49f1aa125af8a21df6b6478a427628f1744d5b4 Mon Sep 17 00:00:00 2001 From: asm0dey Date: Sun, 20 Oct 2019 19:58:17 +0300 Subject: [PATCH 3/5] ci: Fixes E301 Commands should not change things if nothing needs doing This fix is intended to allow playbook to manually start and stop mystamps service. Potentially this is not intended behaviour and should be migrated to systemd task and handler. But current behaviour is well-tested and as such should be presereved at least for now. --- src/main/scripts/ci/ansible/deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scripts/ci/ansible/deploy.yml b/src/main/scripts/ci/ansible/deploy.yml index ef7cf72ff0..d8ebd2af67 100644 --- a/src/main/scripts/ci/ansible/deploy.yml +++ b/src/main/scripts/ci/ansible/deploy.yml @@ -34,7 +34,7 @@ when: uptimerobot is defined and uptimerobot.monitorid and uptimerobot.apikey # we can't use service module here because our sudoers allows to execute only exact commands - - name: Stopping service + - name: Stopping service # noqa 301 ignoring this because we're always stopping service before deploy raw: sudo systemctl stop mystamps @@ -48,7 +48,7 @@ backup: yes # we can't use service module here because our sudoers allows to execute only exact commands - - name: Starting service + - name: Starting service # noqa 301 ignoring this because we're always starting service after deploy raw: sudo systemctl start mystamps From aed2c3574c852b5271b2b7129f4f3cd2809331cf Mon Sep 17 00:00:00 2001 From: asm0dey Date: Sun, 20 Oct 2019 20:00:28 +0300 Subject: [PATCH 4/5] ci: removes E303 Using command rather than module After migration to systemd module (and newer ansible) this should be fixed of course. But currently this is the only supported way to reload systemd daemon --- vagrant/provisioning/roles/mystamps-app/handlers/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vagrant/provisioning/roles/mystamps-app/handlers/main.yml b/vagrant/provisioning/roles/mystamps-app/handlers/main.yml index 9795b9319b..136a111002 100644 --- a/vagrant/provisioning/roles/mystamps-app/handlers/main.yml +++ b/vagrant/provisioning/roles/mystamps-app/handlers/main.yml @@ -2,7 +2,7 @@ # @todo #399 mystamps-app: use systemd module # (requires ansible 2.2+) -- name: Reloading systemd service +- name: Reloading systemd service # noqa 303 command: systemctl daemon-reload From 24370972341d61e325ccfa18acfc7260cfcfe0db Mon Sep 17 00:00:00 2001 From: asm0dey Date: Wed, 23 Oct 2019 22:23:59 +0300 Subject: [PATCH 5/5] ci: install robotframework-lint and html5validator in one command Signed-off-by: asm0dey --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a8b1ab02a6..a22fcc34a7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,8 +16,7 @@ before_script: mkdir -p /tmp/uploads /tmp/preview; cp src/main/resources/test/test.png /tmp/uploads/1.png; npm install -g bootlint; - pip install --user html5validator; - pip install --user robotframework-lint; + pip install --user html5validator robotframework-lint; if [ "$TRAVIS_BRANCH" = 'prod' -a "$TRAVIS_PULL_REQUEST" = 'false' ]; then pip install --user ansible==2.1.1.0; fi;