The shell CLI program to check and operate LambdaRedshiftLoader batches

cliBatchOperations.sh

@@ -0,0 +1,337 @@
+#!/bin/bash
+# Purpose: The shell CLI program to check and operate LambdaRedshiftLoader batches
+# Author: Andrey Suvorov
+# Created: 2021-09-14
+
+SCRIPT_NAME=$0
+
+usage(){
+    echo ""
+    echo "usage: $SCRIPT_NAME [-arpg] [-a account] [-r role] [-p profile] [-g region] [-u 1]"
+    echo "arguments: "
+    echo "  -a      aws account id"
+    echo "  -r      aws role"
+    echo "  -p      aws default profile"
+    echo "  -g      aws region"
+    echo "  -u      (1) show usage"
+    echo ""
+    exit 1
+}
+
+# get arguments
+while getopts ":a:r:p:g:u:" opt; do
+  case $opt in
+    a) AWS_ACCOUNT_ID="$OPTARG"
+      ;;
+    r) AWS_ROLE="$OPTARG"
+      ;;
+    p) AWS_PROFILE="$OPTARG"
+      ;;
+    g) AWS_REGION="$OPTARG"
+      ;;
+    u)
+      usage
+      ;;
+    :)
+      echo "$SCRIPT_NAME: missing value for -$OPTARG argument" >&2
+      usage
+      ;;
+    \?)
+      echo "$SCRIPT_NAME: invalid argument -$OPTARG provided" >&2
+      usage
+      ;;
+  esac
+done
+
+# set default arguments here:
+if [ "$AWS_ACCOUNT_ID" == "" ]; then AWS_ACCOUNT_ID="000000000000"; fi
+if [ "$AWS_ROLE" == "" ]; then AWS_ROLE="Developer"; fi
+if [ "$AWS_PROFILE" == "" ]; then AWS_PROFILE="default"; fi
+if [ "$AWS_REGION" == "" ]; then AWS_REGION="us-west-2"; fi
+
+menuHeader(){
+  clear
+  echo " "
+  echo "*****************************************************************************************"
+  echo "*                                      $1                                      *"
+  echo "*****************************************************************************************"
+}
+
+displayMsg(){
+  echo " "
+  echo "---------------------------------------- MESSAGE ----------------------------------------"
+  echo " $1"
+  echo "-----------------------------------------------------------------------------------------"
+  echo " "
+}
+
+getCredentials(){
+  local l_retryLogin
+  menuHeader " AWS Login "
+  displayMsg "AWS account: $AWS_ACCOUNT_ID\n AWS Region: $AWS_REGION\n AWS Role: $AWS_ROLE\n AWS Profile: $AWS_PROFILE"
+  while ! eiamcli getAWSTempCredentials -a "$AWS_ACCOUNT_ID" -r "$AWS_ROLE" -p "$AWS_PROFILE" ; do
+    if ! eiamcli login; then
+      displayMsg "Can't authenticate the user. Exiting program."
+      read -r -p "Would you like to try login again? (y/n): " l_retryLogin
+      if ! [ "$l_retryLogin" == 'y' ]; then
+        displayMsg "Exiting program"
+        exit 1
+      fi
+    else
+      eiamcli getAWSTempCredentials -a "$AWS_ACCOUNT_ID" -r "$AWS_ROLE" -p "$AWS_PROFILE"
+    fi
+  done
+}
+
+inputDate(){
+  local dt=""
+  while ! [[ "$dt" =~ ^[0-1][0-9]/[0-3][0-9]/[0-9]{4}$ && $(date -j -f %m/%d/%Y "$dt"  +%s) ]]; do
+    if ! [[ "$dt" == "" ]]; then displayMsg "Invalid date format for $1 : $dt" > /dev/tty; fi
+    read -r -p "Please enter $1 (MM/DD/YYYY): " dt
+  done
+  echo "$dt"
+}
+
+batchAction(){
+  local l_batchStatus=$1
+  local l_response=$2
+
+  batchCount=$(echo "$2" | jq '. | length')
+  action_menu=true
+  action_executed=""
+  menuHeader "Action Menu"
+
+  while [[ "$action_menu" == true ]]; do
+    if [[ "$action_executed" == "" ]]; then
+      displayMsg "Found $batchCount batches with '$l_batchStatus' status from $startDate to $endDate"
+    else
+      displayMsg "$action_executed batches command completed"
+      action_executed=""
+    fi
+    PS3="Please select the action from the menu above: "
+    select action in "${actions[@]}"; do
+      case $action in
+        "Display")
+          displayMsg "$action batches with $l_batchStatus status"
+          echo "$l_response" | jq
+          action_executed=$action
+          break
+          ;;
+        "Display with description")
+          displayMsg "$action batches with $l_batchStatus status"
+
+          for row in $(echo "$l_response" | jq -r '.[] | @base64'); do
+            _jq() {
+                  echo "${row}" | base64 --decode | jq -r "${1}"
+                  }
+
+            batchId=$(_jq '.batchId')
+            s3Prefix=$(_jq '.s3Prefix')
+
+            echo "==> Describe batch $counter of $batchCount (batchId: ${batchId} s3Prefix: ${s3Prefix})"
+
+            node describeBatch.js \
+                  --region "${AWS_REGION}" \
+                  --batchId "${batchId}" \
+                  --s3prefix "${s3Prefix}" | jq
+
+            echo ""
+
+            counter=$((counter+1))
+          done
+          action_executed=$action
+          break
+          ;;
+        "Reprocess")
+          displayMsg "$action batches with $l_batchStatus status"
+          counter=1
+          for row in $(echo "$l_response" | jq -r '.[] | @base64'); do
+            _jq() {
+                  echo "${row}" | base64 --decode | jq -r "${1}"
+                  }
+
+            batchId=$(_jq '.batchId')
+            s3Prefix=$(_jq '.s3Prefix')
+
+            echo "==> Reprocessing batch $counter of $batchCount (batchId: ${batchId} s3Prefix: ${s3Prefix})"
+
+            node reprocessBatch.js \
+                  --region "${AWS_REGION}" \
+                  --batchId "${batchId}" \
+                  --prefix "${s3Prefix}"
+
+            counter=$((counter+1))
+          done
+          action_executed=$action
+          break
+          ;;
+        "Unlock")
+          displayMsg "$action batches with $l_batchStatus status"
+          counter=1
+          for row in $(echo "$l_response" | jq -r '.[] | @base64'); do
+            _jq() {
+                  echo "${row}" | base64 --decode | jq -r "${1}"
+                  }
+
+            batchId=$(_jq '.batchId')
+            s3Prefix=$(_jq '.s3Prefix')
+
+            echo "==> Unlocking batch $counter of $batchCount (batchId: ${batchId} s3Prefix: ${s3Prefix})"
+
+            node unlockBatch.js \
+                  "${AWS_REGION}" \
+                  "${batchId}" \
+                  "${s3Prefix}"
+
+            counter=$((counter+1))
+          done
+          action_executed=$action
+          break
+          ;;
+        "Delete")
+          displayMsg "$action batches with $l_batchStatus status from $startDate to $endDate"
+
+          node deleteBatches.js \
+                --region "${AWS_REGION}" \
+                --startDate "${startDateUnix}" \
+                --endDate "${endDateUnix}" \
+                --batchStatus "$l_batchStatus" \
+                --dryRun false
+          action_executed=$action
+          break
+          ;;
+        "Main Menu")
+          clear
+          action_menu=false
+          break
+          ;;
+        *) displayMsg "Invalid option selected $REPLY"
+          ;;
+      esac
+    done
+  done
+}
+
+queryBatch(){
+  response=$(node queryBatches.js \
+                  --region "${AWS_REGION}" \
+                  --batchStatus "$1" \
+                  --startDate "${startDateUnix}" \
+                  --endDate "${endDateUnix}" \
+            )
+
+  echo "$response"
+}
+
+getCredentials
+
+startDate=$(date -v-7d '+%m/%d/%Y')
+startDateUnix=$(date -j -f %m/%d/%Y "$startDate"  +%s)
+endDate=$(date '+%m/%d/%Y')
+endDateUnix=$(date -j -f %m/%d/%Y "$endDate"  +%s)
+dates_set=false
+
+main_menu=true
+while [ "$main_menu" == true ]; do
+
+  menuHeader " Main Menu "
+
+  if ! [[ "$startDate" == "" && "$endDate" == "" ]]; then
+    if [[ "$dates_set" == false  ]]; then
+      read -r -p "Do you want to use selected dates to query batches? $startDate to $endDate (y/n): " defaultDates
+      if ! [[ $defaultDates =~ ^[Yy]$ ]]; then
+        startDateUnix=0
+        endDateUnix=0
+        while ! [[ "$startDateUnix" < "$endDateUnix" ]]; do
+          startDate=$(inputDate 'Start Date')
+          startDateUnix=$(date -j -f %m/%d/%Y "$startDate"  +%s)
+          endDate=$(inputDate 'End Date')
+          endDateUnix=$(date -j -f %m/%d/%Y "$endDate"  +%s)
+          if ! [[ "$startDateUnix" < "$endDateUnix" ]]; then
+            displayMsg "The Start Date $startDate can't be greater than End Date $endDate";
+          fi
+        done
+      fi
+      dates_set=true
+    fi
+  fi
+
+  menuHeader " Main Menu "
+  if ! [[ $outputMsg == "" ]]; then
+    displayMsg "$outputMsg"
+    outputMsg=""
+  else
+    displayMsg "Query dates set from Start Date $startDate to End Date $endDate\n Please use option 6 to change dates"
+  fi
+
+  PS3="What you would like to do? Please select from the menu: "
+  statuses=("Check Error batches" "Check Locked batches" "Check Open batches"
+            "Check Complete batches" "Check Other batches" "Change query dates" "Exit")
+  select status in "${statuses[@]}"; do
+    case $status in
+      "Check Error batches")
+        queryResult=$(queryBatch "error")
+        if ! [[ "$queryResult" == "[]" ]]; then
+          actions=("Display" "Display with description" "Reprocess" "Delete" "Main Menu")
+          batchAction "error" "$queryResult"
+        else
+          outputMsg="No batches with status 'error' found for specified dates $startDate - $endDate"
+        fi
+        break
+        ;;
+      "Check Locked batches")
+        queryResult=$(queryBatch "locked")
+        if ! [[ "$queryResult" == "[]" ]]; then
+          actions=("Display" "Display with description" "Unlock" "Reprocess" "Delete" "Main Menu")
+          batchAction "locked" "$queryResult"
+        else
+          outputMsg="No batches with status 'locked' found for specified dates $startDate - $endDate"
+        fi
+        break
+        ;;
+      "Check Open batches")
+        queryResult=$(queryBatch "open")
+        if ! [[ "$queryResult" == "[]" ]]; then
+          actions=("Display" "Display with description" "Main Menu")
+          batchAction "open" "$queryResult"
+        else
+          outputMsg="No batches with status 'open' found for specified dates $startDate - $endDate"
+        fi
+        break
+        ;;
+      "Check Complete batches")
+        queryResult=$(queryBatch "complete")
+        if ! [[ "$queryResult" == "[]" ]]; then
+          actions=("Display" "Display with description" "Delete" "Main Menu")
+          batchAction "complete" "$queryResult"
+        else
+          outputMsg="No batches with status 'complete' found for specified dates $startDate - $endDate"
+        fi
+        break
+        ;;
+      "Check Other batches")
+        read -r -p "Please type in batch status to query: " batchStatus
+        queryResult=$(queryBatch "$batchStatus")
+        if ! [[ "$queryResult" == "[]" ]]; then
+          actions=("Display" "Display with description" "Delete" "Main Menu")
+          batchAction "$batchStatus" "$queryResult"
+        else
+          outputMsg="No batches with status '$batchStatus' found for specified dates $startDate - $endDate"
+        fi
+        break
+        ;;
+      "Change query dates")
+        dates_set=false
+        clear
+        break
+        ;;
+      "Exit")
+        displayMsg "Exiting program"
+        exit
+        ;;
+      *)
+        displayMsg "Invalid option selected $REPLY"
+        ;;
+    esac
+  done
+done