From 46e433448e0b68c2dfb1cbfac5dd9bad4ebf6217 Mon Sep 17 00:00:00 2001 From: Yuriy Gerasimov Date: Fri, 22 Nov 2019 08:22:32 -0800 Subject: [PATCH] Diffy visual regression testing integration. --- .ci/test/diffy-visual-regression/README.md | 7 ++ .ci/test/diffy-visual-regression/run | 125 +++++++++++++++++++++ .circleci/config.yml | 29 +++++ README.md | 13 ++- 4 files changed, 171 insertions(+), 3 deletions(-) create mode 100644 .ci/test/diffy-visual-regression/README.md create mode 100755 .ci/test/diffy-visual-regression/run diff --git a/.ci/test/diffy-visual-regression/README.md b/.ci/test/diffy-visual-regression/README.md new file mode 100644 index 000000000..b699e54c8 --- /dev/null +++ b/.ci/test/diffy-visual-regression/README.md @@ -0,0 +1,7 @@ +# Diffy Integration + +You need to set up two environmental variables `DIFFY_API_KEY` and `DIFFY_PROJECT_ID` in CircleCI for integration to work. + +There is a terminus command [https://github.com/DiffyWebsite/diffy-build-tools-plugin](https://github.com/DiffyWebsite/diffy-build-tools-plugin) available to help you out. + +Read more about this set up at [https://diffy.website/documentation/pantheon-build-tools](https://diffy.website/documentation/pantheon-build-tools). diff --git a/.ci/test/diffy-visual-regression/run b/.ci/test/diffy-visual-regression/run new file mode 100755 index 000000000..5a4201ae8 --- /dev/null +++ b/.ci/test/diffy-visual-regression/run @@ -0,0 +1,125 @@ +#!/bin/bash + +# +# This script runs visual regression tests using Diffy (https://diffy.website). +# To configure integration check https://github.com/DiffyWebsite/diffy-build-tools-plugin. +# Tests are run if composer.lock has changed +# but composer.json has not (e.g. dependency updates) +# OR if the last commit message contains [vr] +# + +# Variables +BUILD_DIR=$(pwd) +GITHUB_API_URL="https://api.github.com/repos/$CI_PROJECT_USERNAME/$CI_PROJECT_REPONAME" + +if [ -z "$DIFFY_API_KEY" ] +then + echo "Diffy integration is not configured. Add DIFFY_API_KEY to CircleCI variables." + exit 1; +fi + +if [ -z "$DIFFY_PROJECT_ID" ] +then + echo "Diffy integration is not configured. Add DIFFY_PROJECT_ID to CircleCI variables." + exit 1; +fi + +GIT_FILE_MODIFIED() +{ + # Stash list of changed files + GIT_FILES_CHANGED="$(git diff origin/master --name-only)" + + while read -r changedFile; do + if [[ "${changedFile}" == "$1" ]] + then + return 0; + fi + done <<< "$GIT_FILES_CHANGED" + + return 1; +} + +LAST_GIT_COMMIT_MESSAGE=$(git log -1 --pretty=%B) + +if [[ ${LAST_GIT_COMMIT_MESSAGE} == *"--skip-vr"* ]] +then + echo -e "\nVisual regression tests skipped because the last commit contains --skip-vr" + exit 0 +fi + +# Always run visual tests if "[vr]" is in the last commit message +if [[ ${LAST_GIT_COMMIT_MESSAGE} != *"[vr]"* ]] +then + + # Skip visual tests if there hasn't been a modification to composer.lock + if ! GIT_FILE_MODIFIED 'composer.lock' + then + echo -e "\nSkipping visual regression tests since composer.lock has NOT changed" + exit 0 + fi + + # Skip visual tests if has been a modification to composer.json + if GIT_FILE_MODIFIED 'composer.json' + then + echo -e "\nSkipping visual regression tests since composer.json HAS changed" + exit 0 + fi + +else + echo -e "\nRunning visual regression tests because the latest commit message demands it" +fi + +# Ping the multidev environment to wake it from sleep +echo -e "\nPinging the ${TERMINUS_ENV} multidev environment to wake it from sleep..." +curl -I "$MULTIDEV_SITE_URL" >/dev/null + +# Ping the live environment to wake it from sleep +echo -e "\nPinging the dev environment to wake it from sleep..." +curl -I "$DEV_SITE_URL" >/dev/null + +# Get an access token from Diffy. +echo -e "\nGetting an access token from Diffy" +TOKEN=`curl -X POST "https://app.diffy.website/api/auth/key" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"key\":\"$DIFFY_API_KEY\"}" | php -r 'echo json_decode(file_get_contents("php://stdin"))->token;'` + +# Trigger a job to compare environments. +echo -e "\nCompare a build ${MULTIDEV_SITE_URL} with DEV environment..." +DIFF_ID=`curl -X POST "https://app.diffy.website/api/projects/$DIFFY_PROJECT_ID/compare" -H "Accept: application/json" -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" -d "{\"env1\":\"dev\",\"env2\":\"custom\",\"env2Url\":\"${MULTIDEV_SITE_URL}\"}"` +sleep 30 + +DIFF_COMPLETED='' +while [ -z "$DIFF_COMPLETED" ] +do + echo -e "\nChecking status of the diff $DIFF_ID" + # State 4 or 8 means that diff completed. 8 means that Diffy creates an archive but we already can treat it as completed. + DIFF_INFO=`curl -X GET "https://app.diffy.website/api/diffs/$DIFF_ID" -H "Accept: application/json" -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json"` + echo -e "\n Result of the check: $DIFF_INFO" + DIFF_COMPLETED=`echo $DIFF_INFO | grep '"state":8\|"state":4'` + echo -e "\n Diff completed? $DIFF_COMPLETED" + sleep 5 +done + +CHANGES=`echo $DIFF_COMPLETED | php -r 'echo json_decode(file_get_contents("php://stdin"))->result;'` +echo "Diff result: $CHANGES" +if [ "$CHANGES" -eq "0" ] +then + # visual regression passed + echo -e "\n\nVisual regression test passed! No changes found." + PR_MESSAGE="Diffy visual regression test passed! No changes found." +else + # visual regression failed + echo -e "\nVisual regression test failed!" + REPORT_LINK="https://app.diffy.website/#/diffs/$DIFF_ID" + PR_MESSAGE="Diffy visual regression test failed! $REPORT_LINK" +fi + +# Post the image back to the pull request on GitHub +if [[ -n ${CI_REPOSITORY_URL} && ${CI_REPOSITORY_URL} == *"github"* ]] +then + echo -e "\nPosting visual regression results back to PR #$PR_NUMBER " + curl -s -i -u "$CI_PROJECT_USERNAME:$GITHUB_TOKEN" -d "{\"body\": \"$PR_MESSAGE\"}" $GITHUB_API_URL/issues/$PR_NUMBER/comments > /dev/null +fi + +if [ "$CHANGES" != "0" ] +then + exit 1 +fi diff --git a/.circleci/config.yml b/.circleci/config.yml index 7dcbb6fbd..8aad62191 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -189,6 +189,27 @@ jobs: path: /tmp/artifacts destination: artifacts + diffy_visual_regression_test: + <<: *defaults + docker: + - image: circleci/php:7.3.9 + steps: + - checkout + + - attach_workspace: + at: /tmp/workspace + + - run: cp /tmp/workspace/bash_env.txt $BASH_ENV + - run: echo "export CI_BUILD_URL='${CIRCLE_BUILD_URL}'" >> $BASH_ENV + - run: echo "export CI_NODE_INDEX='${CIRCLE_NODE_INDEX}'" >> $BASH_ENV + - run: echo "export CI_REPOSITORY_URL='${CIRCLE_REPOSITORY_URL}'" >> $BASH_ENV + - run: echo "export ARTIFACTS_DIR_URL='${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/artifacts'" >> $BASH_ENV + - run: source $BASH_ENV + + - run: + name: diffy visual regression test + command: ./.ci/test/diffy-visual-regression/run + visual_regression_test: <<: *defaults docker: @@ -244,6 +265,14 @@ workflows: requires: - static_tests - build_php + - diffy_visual_regression_test: + requires: + - configure_env_vars + - deploy_to_pantheon + filters: + branches: + ignore: + - master - visual_regression_test: requires: - configure_env_vars diff --git a/README.md b/README.md index e19a16de4..529641d58 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Example Drops 8 Composer [![CircleCI](https://circleci.com/gh/pantheon-systems/example-drops-8-composer.svg?style=shield)](https://circleci.com/gh/pantheon-systems/example-drops-8-composer) -[![Pantheon example-drops-8-composer](https://img.shields.io/badge/dashboard-drops_8-yellow.svg)](https://dashboard.pantheon.io/sites/c401fd14-f745-4e51-9af2-f30b45146a0c#dev/code) +[![Pantheon example-drops-8-composer](https://img.shields.io/badge/dashboard-drops_8-yellow.svg)](https://dashboard.pantheon.io/sites/c401fd14-f745-4e51-9af2-f30b45146a0c#dev/code) [![Dev Site example-drops-8-composer](https://img.shields.io/badge/site-drops_8-blue.svg)](http://dev-example-drops-8-composer.pantheonsite.io/) This repository is a reference implementation and start state for a modern Drupal 8 workflow utilizing [Composer](https://getcomposer.org/), Continuous Integration (CI), Automated Testing, and Pantheon. Even though this is a good starting point, you will need to customize and maintain the CI/testing set up for your projects. @@ -29,7 +29,7 @@ One of the directories moved to the git root is `/config`. This directory holds ### `composer.json` This project uses Composer to manage third-party PHP dependencies. -The `require` section of `composer.json` should be used for any dependencies your web project needs, even those that might only be used on non-Live environments. All dependencies in `require` will be pushed to Pantheon. +The `require` section of `composer.json` should be used for any dependencies your web project needs, even those that might only be used on non-Live environments. All dependencies in `require` will be pushed to Pantheon. The `require-dev` section should be used for dependencies that are not a part of the web application but are necesarry to build or test the project. Some example are `php_codesniffer` and `phpunit`. Dev dependencies will not be deployed to Pantheon. @@ -68,12 +68,19 @@ Static tests analyze code without executing it. It is good at detecting syntax e - `tests/unit/bootstrap.php` Bootstraps the Composer autoloader - `tests/unit/TestAssert.php` An example Unit test. Project specific test files will need to be created in `tests/unit`. -**Visual Regression Testing** `.ci/test/visual-regression` +**BackstopJS Visual Regression Testing** `.ci/test/visual-regression` Visual regression testing uses a headless browser to take screenshots of web pages and compare them for visual differences. - `.ci/test/visual-regression/run` Runs [BackstopJS](https://github.com/garris/BackstopJS) visual regression testing. - `.ci/test/visual-regression/backstopConfig.js` The [BackstopJS](https://github.com/garris/BackstopJS) configuration file. Setting here will need to be updated for your project. For example, the `pathsToTest` variable determines the URLs to test. +**[Diffy Visual Regression Testing](https://diffy.website)** `.ci/test/diffy-visual-regression` +Visual regression testing uses a cloud service Diffy to take screenshots of web pages and compare them for visual differences. +All project settings are stored in Diffy's UI. Read more about [Diffy features](https://diffy.website/features). + +- `.ci/test/diffy-visual-regression/run` Runs [Diffy](https://diffy.website) visual regression testing. + + **Behat Testing** `.ci/test/behat` and `tests/behat` [Behat](http://behat.org/en/latest/) is an acceptance/end-to-end testing framework written in PHP. It faciliates testing the fully built Drupal site on Pantheon infrastucture. [The Drupal Behat Extension](https://www.drupal.org/project/drupalextension) is used to help with integrating Behat and Drupal.