Merge pull request #12165 from rabbitmq/run-selenium-on-cluster

Run Selenium suites against a cluster of nodes

Author: michaelklishin

.github/workflows/test-selenium.yaml

@@ -85,19 +85,36 @@ jobs:
         cd ${SELENIUM_DIR}
         docker build -t mocha-test --target test .
 
-    - name: Run Suites
+    - name: Run full ui suites on a standalone rabbitmq server
       run: |
-        RABBITMQ_DOCKER_IMAGE=bazel/packaging/docker-image:rabbitmq-amd64 ${SELENIUM_DIR}/run-suites.sh
+        RABBITMQ_DOCKER_IMAGE=bazel/packaging/docker-image:rabbitmq-amd64 \
+            ${SELENIUM_DIR}/run-suites.sh
+        mkdir -p /tmp/full-suite
+        mv /tmp/selenium/* /tmp/full-suite
+        mkdir -p /tmp/full-suite/logs
+        mv ${SELENIUM_DIR}/logs/* /tmp/full-suite/logs
+        mkdir -p /tmp/full-suite/screens
+        mv ${SELENIUM_DIR}/screens/* /tmp/full-suite/screens
+
+    - name: Run short ui suite on a 3-node rabbitmq cluster
+      run: |
+        RABBITMQ_DOCKER_IMAGE=bazel/packaging/docker-image:rabbitmq-amd64 \
+            ADDON_PROFILES=cluster ${SELENIUM_DIR}/run-suites.sh short-suite-management-ui
+        mkdir -p /tmp/short-suite
+        mv /tmp/selenium/* /tmp/short-suite
+        mkdir -p /tmp/short-suite/logs
+        mv ${SELENIUM_DIR}/logs/* /tmp/short-suite/logs
+        mkdir -p /tmp/short-suite/screens
+        mv ${SELENIUM_DIR}/screens/* /tmp/short-suite/screens
 
     - name: Upload Test Artifacts
       if: always()
       uses: actions/[email protected]
       with:
         name: test-artifacts-${{ matrix.browser }}-${{ matrix.erlang_version }}
         path: |
-          logs/*
-          screens/*
-          /tmp/selenium/*
+          /tmp/full-suite
+          /tmp/short-suite
 
   summary-selenium:
     needs:

deps/rabbitmq_management/selenium/bin/components/rabbitmq

@@ -10,14 +10,42 @@ init_rabbitmq() {
   [[ -z "${OAUTH_SERVER_CONFIG_DIR}" ]] || print "> OAUTH_SERVER_CONFIG_DIR: ${OAUTH_SERVER_CONFIG_DIR}"
 
 }
+
 start_rabbitmq() {
   if [[ "$PROFILES" == *"docker"* ]]; then
-    start_docker_rabbitmq
+      if [[ "$PROFILES" == *"cluster"* ]]; then
+          start_docker_cluster_rabbitmq
+      else
+          start_docker_rabbitmq
+      fi
   else
     start_local_rabbitmq
   fi
 }
-
+stop_rabbitmq() {
+  if [[ "$PROFILES" == *"docker"* ]]; then
+      if [[ "$PROFILES" == *"cluster"* ]]; then
+          docker compose -f $CONF_DIR/rabbitmq/compose.yml kill
+      else
+          kill_container_if_exist "$component"
+      fi
+  else
+      stop_local_rabbitmq
+  fi
+}
+stop_local_rabbitmq() {
+    RABBITMQ_SERVER_ROOT=$(realpath $TEST_DIR/../../../../)
+    gmake --directory=${RABBITMQ_SERVER_ROOT} stop-node
+}
+save_logs_rabbitmq() {
+    if [[ "$PROFILES" == *"docker"* ]]; then
+        if [[ "$PROFILES" == *"cluster"* ]]; then
+            docker compose -f $CONF_DIR/rabbitmq/compose.yml logs > $LOGS/rabbitmq.log
+        else
+            save_container_logs "rabbitmq"
+        fi
+    fi
+}
 start_local_rabbitmq() {
   begin "Starting rabbitmq ..."
 
@@ -31,22 +59,76 @@ start_local_rabbitmq() {
 
   print "> EFFECTIVE RABBITMQ_CONFIG_FILE: /tmp$MOUNT_RABBITMQ_CONF"
   ${BIN_DIR}/gen-advanced-config ${RABBITMQ_CONFIG_DIR} $ENV_FILE /tmp$MOUNT_ADVANCED_CONFIG
+  cp ${RABBITMQ_CONFIG_DIR}/enabled_plugins /tmp/etc/rabbitmq/
   RESULT=$?
   if [ $RESULT -eq 0 ]; then
     print "> EFFECTIVE RABBITMQ_CONFIG_FILE: /tmp$MOUNT_ADVANCED_CONFIG"
   	gmake --directory=${RABBITMQ_SERVER_ROOT} run-broker \
-  		RABBITMQ_ENABLED_PLUGINS_FILE=${RABBITMQ_CONFIG_DIR}/enabled_plugins \
+  		RABBITMQ_ENABLED_PLUGINS_FILE=/tmp/etc/rabbitmq/enabled_plugins \
   		RABBITMQ_CONFIG_FILE=/tmp$MOUNT_RABBITMQ_CONF \
       RABBITMQ_ADVANCED_CONFIG_FILE=/tmp$MOUNT_ADVANCED_CONFIG
   else
     gmake --directory=${RABBITMQ_SERVER_ROOT} run-broker \
-  		RABBITMQ_ENABLED_PLUGINS_FILE=${RABBITMQ_CONFIG_DIR}/enabled_plugins \
+  		RABBITMQ_ENABLED_PLUGINS_FILE=/tmp/etc/rabbitmq/enabled_plugins \
   		RABBITMQ_CONFIG_FILE=/tmp$MOUNT_RABBITMQ_CONF
   fi
   print "> RABBITMQ_TEST_DIR: ${RABBITMQ_CONFIG_DIR}"
 
 
 }
+start_docker_cluster_rabbitmq() {
+    begin "Starting rabbitmq cluster in docker ..."
+    init_rabbitmq
+    kill_container_if_exist rabbitmq
+    kill_container_if_exist rabbitmq1
+    kill_container_if_exist rabbitmq2
+
+    mkdir -p $CONF_DIR/rabbitmq
+    MOUNT_RABBITMQ_CONF="/etc/rabbitmq/rabbitmq.conf"
+    MOUNT_ADVANCED_CONFIG="/etc/rabbitmq/advanced.config"
+
+    RABBITMQ_TEST_DIR="/var/rabbitmq" ${BIN_DIR}/gen-rabbitmq-conf ${RABBITMQ_CONFIG_DIR} $ENV_FILE $CONF_DIR/rabbitmq/rabbitmq.conf
+    print "> EFFECTIVE RABBITMQ_CONFIG_FILE: $CONF_DIR/rabbitmq/rabbitmq.conf"
+    ${BIN_DIR}/gen-advanced-config ${RABBITMQ_CONFIG_DIR} $ENV_FILE /$CONF_DIR/rabbitmq/advanced.config
+    RESULT=$?
+    if [ $RESULT -eq 0 ]; then
+        if [ -s $RESULT ]; then
+            print "> EFFECTIVE ADVANCED_CONFIG_FILE: $CONF_DIR/rabbitmq/advanced.config"
+        else
+            rm $CONF_DIR/rabbitmq/advanced.config
+        fi
+    fi
+    mkdir -p $CONF_DIR/rabbitmq/conf.d/
+    cp ${RABBITMQ_CONFIG_DIR}/logging.conf $CONF_DIR/rabbitmq/conf.d/
+    if [ -f ${RABBITMQ_CONFIG_DIR}/enabled_plugins ]; then
+        cp ${RABBITMQ_CONFIG_DIR}/enabled_plugins $CONF_DIR/rabbitmq
+    fi
+    if [ -d ${RABBITMQ_CONFIG_DIR}/certs ]; then
+        cp -r ${RABBITMQ_CONFIG_DIR}/certs $CONF_DIR/rabbitmq
+    fi
+    if [ -d ${RABBITMQ_CONFIG_DIR}/imports ]; then
+        cp -r ${RABBITMQ_CONFIG_DIR}/imports $CONF_DIR/rabbitmq
+    fi
+
+    cat > $CONF_DIR/rabbitmq/image_compose.yml <<EOF
+x-rabbitmq_image: &rabbitmq_image
+    image: ${RABBITMQ_DOCKER_IMAGE}
+    volumes:
+    - $CONF_DIR/rabbitmq/:/etc/rabbitmq
+    - $CONF_DIR/rabbitmq/:/var/rabbitmq
+    - ${TEST_DIR}:/config
+EOF
+    cat $CONF_DIR/rabbitmq/image_compose.yml > $CONF_DIR/rabbitmq/compose.yml
+    cat ${BIN_DIR}/components/../rabbit-compose.yml >> $CONF_DIR/rabbitmq/compose.yml
+
+    docker compose -f $CONF_DIR/rabbitmq/compose.yml up -d
+
+    wait_for_message rabbitmq "Server startup complete"
+    wait_for_message rabbitmq1 "Server startup complete"
+    wait_for_message rabbitmq2 "Server startup complete"
+    end "RabbitMQ cluster ready"
+}
+
 start_docker_rabbitmq() {
   begin "Starting rabbitmq in docker ..."
 

deps/rabbitmq_management/selenium/bin/components/selenium

@@ -1,11 +1,11 @@
 #!/usr/bin/env bash
 
-arch=$(uname -i)
+arch=$(uname -a)
 if [[ $arch == arm* ]]; then
   SELENIUM_DOCKER_IMAGE=selenium/standalone-chrome:123.0
 else
   SELENIUM_DOCKER_IMAGE=seleniarm/standalone-chromium:123.0
-fi 
+fi
 
 start_selenium() {
   begin "Starting selenium  ..."

deps/rabbitmq_management/selenium/bin/gen-advanced-config

@@ -28,5 +28,6 @@ then
 fi
 if [ "$FOUND_TEMPLATES_COUNT" -lt 1 ]
 then
+  rm $FINAL_CONFIG_FILE
   exit -1
 fi

deps/rabbitmq_management/selenium/bin/rabbit-compose.yml

@@ -0,0 +1,49 @@
+
+
+# https://docs.docker.com/compose/compose-file/#networks
+networks:
+  rabbitmq_net:
+      name: rabbitmq_net
+      external: true
+
+services:
+  rmq0: &rabbitmq
+    # https://hub.docker.com/r/pivotalrabbitmq/rabbitmq-prometheus/tags
+    << : *rabbitmq_image
+    networks:
+      - "rabbitmq_net"
+    ports:
+      - "5672:5672"
+      - "15672:15672"
+      - "15692:15692"
+    # https://unix.stackexchange.com/questions/71940/killing-tcp-connection-in-linux
+    # https://en.wikipedia.org/wiki/Tcpkill
+    # https://www.digitalocean.com/community/tutorials/iptables-essentials-common-firewall-rules-and-commands#block-an-ip-address
+    cap_add:
+      - ALL
+    hostname: rabbitmq
+    container_name: rabbitmq
+    environment:
+      RABBITMQ_ERLANG_COOKIE: rmq0
+
+    # we want to simulate hitting thresholds
+    ulimits:
+      nofile:
+        soft: "2000"
+        hard: "2000"
+  rmq1:
+    << : *rabbitmq
+    container_name: rabbitmq1
+    hostname: rabbitmq1
+    ports:
+      - "5677:5672"
+      - "15677:15672"
+      - "15697:15692"
+  rmq2:
+    << : *rabbitmq
+    hostname: rabbitmq2
+    container_name: rabbitmq2
+    ports:
+      - "5678:5672"
+      - "15678:15672"
+      - "15698:15692"

deps/rabbitmq_management/selenium/bin/suite_template

@@ -32,6 +32,8 @@ SCREENS=${SELENIUM_ROOT_FOLDER}/screens/${SUITE}
 CONF_DIR=/tmp/selenium/${SUITE}
 ENV_FILE=$CONF_DIR/.env
 
+rm -rf $CONF_DIR
+
 for f in $SCRIPT/components/*; do
   if [[ ! "$f" == *README.md ]]
   then
@@ -54,6 +56,9 @@ parse_arguments() {
     elif [[ "$1" == "stop-others" ]]
       then
       echo "stop-others"
+    elif [[ "$1" == "stop-rabbitmq" ]]
+      then
+      echo "stop-rabbitmq"
     elif [[ "$1" == "test" ]]
       then
       echo "test $2"
@@ -107,7 +112,10 @@ init_suite() {
   print "> TEST_CASES_DIR: ${TEST_CASES_DIR} "
   print "> TEST_CONFIG_DIR: ${TEST_CONFIG_DIR} "
   print "> DOCKER_NETWORK: ${DOCKER_NETWORK} "
-  print "> PROFILES: ${PROFILES} "
+  print "> initial PROFILES: ${PROFILES} "
+  print "> (+) ADDON_PROFILES: ${ADDON_PROFILES} "
+  PROFILES="${PROFILES} ${ADDON_PROFILES}"
+  print "> (=) final PROFILES: ${PROFILES} "
   print "> ENV_FILE: ${ENV_FILE} "
   print "> COMMAND: ${COMMAND}"
   end "Initialized suite"
@@ -239,25 +247,68 @@ wait_for_url_docker() {
   done
 }
 
-
+test_on_cluster() {
+    IFS=', ' read -r -a array <<< "$RABBITMQ_CLUSTER_NODES"
+    begin "Running against all nodes in cluster $RABBITMQ_CLUSTER_NODES :"
+    for item in "${array[@]}"
+    do
+        RMQ_HOST_0=${RABBITMQ_HOST:-rabbitmq:15672}
+        RMQ_HOST=$(rewrite_rabbitmq_hostname ${item} $RMQ_HOST_0)
+        PUBLIC_RMQ_HOST_0=${PUBLIC_RABBITMQ_HOST:-$RMQ_HOST}
+        PUBLIC_RMQ_HOST=$(rewrite_rabbitmq_hostname ${item} $PUBLIC_RMQ_HOST_0)
+        RMQ_URL=$(calculate_rabbitmq_url $PUBLIC_RMQ_HOST)
+        RMQ_HOSTNAME=${item}
+        _test $RMQ_HOST \
+              $PUBLIC_RMQ_HOST \
+              $RMQ_URL \
+              $RMQ_HOSTNAME
+        TEST_RESULT=$?
+        if [ $TEST_RESULT -ne 0 ]; then
+            break
+        fi
+    done
+    end "Finishing running test ($TEST_RESULT)"
+}
+rewrite_rabbitmq_hostname() {
+    IFS=':' read -r -a array <<< "$2"
+    if [ "${array[0]}" == "rabbitmq" ]; then
+        echo "${2//rabbitmq/$1}"
+    else
+        echo "$2"
+    fi
+}
 test() {
+  if [[ "$PROFILES" == *"cluster"* && ! -z "$RABBITMQ_CLUSTER_NODES" ]]; then
+      test_on_cluster
+  else
+      RMQ_HOST=${RABBITMQ_HOST:-rabbitmq:15672}
+      PUBLIC_RMQ_HOST=${PUBLIC_RABBITMQ_HOST:-$RMQ_HOST}
+      _test $RABBITMQ_HOST \
+            $PUBLIC_RMQ_HOST \
+            $(calculate_rabbitmq_url $PUBLIC_RMQ_HOST) \
+            ${RABBITMQ_HOSTNAME:-rabbitmq}
+  fi
+}
+
+_test() {
+  RMQ_HOST=$1
+  PUBLIC_RMQ_HOST=$2
+  RMQ_URL=$3
+  RMQ_HOSTNAME=$4
+
   kill_container_if_exist mocha
-  begin "Running tests with env variables:"
+  begin "Running tests against $RMQ_HOSTNAME with these env variable:"
 
-  RABBITMQ_HOST=${RABBITMQ_HOST:-rabbitmq:15672}
-  PUBLIC_RABBITMQ_HOST=${PUBLIC_RABBITMQ_HOST:-$RABBITMQ_HOST}
-  RABBITMQ_URL=$(calculate_rabbitmq_url $PUBLIC_RABBITMQ_HOST)
-  RABBITMQ_HOSTNAME=${RABBITMQ_HOSTNAME:-rabbitmq}
   SELENIUM_TIMEOUT=${SELENIUM_TIMEOUT:-20000}
   SELENIUM_POLLING=${SELENIUM_POLLING:-500}
 
   print "> SELENIUM_TIMEOUT: ${SELENIUM_TIMEOUT}"
   print "> SELENIUM_POLLING: ${SELENIUM_POLLING}"
-  print "> RABBITMQ_HOST: ${RABBITMQ_HOST}"
-  print "> RABBITMQ_HOSTNAME: ${RABBITMQ_HOSTNAME}"
-  print "> PUBLIC_RABBITMQ_HOST: ${PUBLIC_RABBITMQ_HOST}"
+  print "> RABBITMQ_HOST: ${RMQ_HOST}"
+  print "> RABBITMQ_HOSTNAME: ${RMQ_HOSTNAME}"
+  print "> PUBLIC_RABBITMQ_HOST: ${PUBLIC_RMQ_HOST}"
   print "> RABBITMQ_PATH: ${RABBITMQ_PATH}"
-  print "> RABBITMQ_URL: ${RABBITMQ_URL}"
+  print "> RABBITMQ_URL: ${RMQ_URL}"
   print "> UAA_URL: ${UAA_URL}"
   print "> FAKEPORTAL_URL: ${FAKEPORTAL_URL}"
   mocha_test_tag=($(md5sum $SELENIUM_ROOT_FOLDER/package.json))
@@ -270,8 +321,8 @@ test() {
     --rm \
     --name mocha \
     --net ${DOCKER_NETWORK} \
-    --env RABBITMQ_URL=${RABBITMQ_URL} \
-    --env RABBITMQ_HOSTNAME=${RABBITMQ_HOSTNAME} \
+    --env RABBITMQ_URL=${RMQ_URL} \
+    --env RABBITMQ_HOSTNAME=${RMQ_HOSTNAME} \
     --env UAA_URL=${UAA_URL} \
     --env FAKE_PORTAL_URL=${FAKEPORTAL_URL} \
     --env RUN_LOCAL=false \
@@ -354,6 +405,9 @@ run_local_with() {
   elif [[ "$COMMAND" == "stop-others" ]]
   then
     teardown_local_others
+  elif [[ "$COMMAND" == "stop-rabbitmq" ]]
+  then
+    stop_local_rabbitmq
   elif [[ "$COMMAND" =~ test[[:space:]]*([^[:space:]]*) ]]
   then
     test_local ${BASH_REMATCH[1]}
@@ -466,13 +520,15 @@ start_components() {
     $start
   done
 }
+
 teardown_components() {
   begin "Tear down ..."
   for i in "${REQUIRED_COMPONENTS[@]}"
   do
     local component="$i"
+    stop="stop_$i"
+    type "$stop" &>/dev/null && $stop || kill_container_if_exist "$component"
     print "Tear down $component"
-    kill_container_if_exist "$component"
   done
   end "Finished teardown"
 }
@@ -481,8 +537,9 @@ save_components_logs() {
   for i in "${REQUIRED_COMPONENTS[@]}"
   do
     local component="$i"
+    save="save_logs_$i"
+    type "$save" &>/dev/null && $save || save_container_logs "$component"
     print "Saving logs for component $component"
-    save_container_logs "$component"
   done
   end "Finished saving logs"
 }