diff --git a/cliBatchOperations.sh b/cliBatchOperations.sh new file mode 100644 index 0000000..63e04fd --- /dev/null +++ b/cliBatchOperations.sh @@ -0,0 +1,337 @@ +#!/bin/bash +# Purpose: The shell CLI program to check and operate LambdaRedshiftLoader batches +# Author: Andrey Suvorov +# Created: 2021-09-14 + +SCRIPT_NAME=$0 + +usage(){ + echo "" + echo "usage: $SCRIPT_NAME [-arpg] [-a account] [-r role] [-p profile] [-g region] [-u 1]" + echo "arguments: " + echo " -a aws account id" + echo " -r aws role" + echo " -p aws default profile" + echo " -g aws region" + echo " -u (1) show usage" + echo "" + exit 1 +} + +# get arguments +while getopts ":a:r:p:g:u:" opt; do + case $opt in + a) AWS_ACCOUNT_ID="$OPTARG" + ;; + r) AWS_ROLE="$OPTARG" + ;; + p) AWS_PROFILE="$OPTARG" + ;; + g) AWS_REGION="$OPTARG" + ;; + u) + usage + ;; + :) + echo "$SCRIPT_NAME: missing value for -$OPTARG argument" >&2 + usage + ;; + \?) + echo "$SCRIPT_NAME: invalid argument -$OPTARG provided" >&2 + usage + ;; + esac +done + +# set default arguments here: +if [ "$AWS_ACCOUNT_ID" == "" ]; then AWS_ACCOUNT_ID="000000000000"; fi +if [ "$AWS_ROLE" == "" ]; then AWS_ROLE="Developer"; fi +if [ "$AWS_PROFILE" == "" ]; then AWS_PROFILE="default"; fi +if [ "$AWS_REGION" == "" ]; then AWS_REGION="us-west-2"; fi + +menuHeader(){ + clear + echo " " + echo "*****************************************************************************************" + echo "* $1 *" + echo "*****************************************************************************************" +} + +displayMsg(){ + echo " " + echo "---------------------------------------- MESSAGE ----------------------------------------" + echo " $1" + echo "-----------------------------------------------------------------------------------------" + echo " " +} + +getCredentials(){ + local l_retryLogin + menuHeader " AWS Login " + displayMsg "AWS account: $AWS_ACCOUNT_ID\n AWS Region: $AWS_REGION\n AWS Role: $AWS_ROLE\n AWS Profile: $AWS_PROFILE" + while ! eiamcli getAWSTempCredentials -a "$AWS_ACCOUNT_ID" -r "$AWS_ROLE" -p "$AWS_PROFILE" ; do + if ! eiamcli login; then + displayMsg "Can't authenticate the user. Exiting program." + read -r -p "Would you like to try login again? (y/n): " l_retryLogin + if ! [ "$l_retryLogin" == 'y' ]; then + displayMsg "Exiting program" + exit 1 + fi + else + eiamcli getAWSTempCredentials -a "$AWS_ACCOUNT_ID" -r "$AWS_ROLE" -p "$AWS_PROFILE" + fi + done +} + +inputDate(){ + local dt="" + while ! [[ "$dt" =~ ^[0-1][0-9]/[0-3][0-9]/[0-9]{4}$ && $(date -j -f %m/%d/%Y "$dt" +%s) ]]; do + if ! [[ "$dt" == "" ]]; then displayMsg "Invalid date format for $1 : $dt" > /dev/tty; fi + read -r -p "Please enter $1 (MM/DD/YYYY): " dt + done + echo "$dt" +} + +batchAction(){ + local l_batchStatus=$1 + local l_response=$2 + + batchCount=$(echo "$2" | jq '. | length') + action_menu=true + action_executed="" + menuHeader "Action Menu" + + while [[ "$action_menu" == true ]]; do + if [[ "$action_executed" == "" ]]; then + displayMsg "Found $batchCount batches with '$l_batchStatus' status from $startDate to $endDate" + else + displayMsg "$action_executed batches command completed" + action_executed="" + fi + PS3="Please select the action from the menu above: " + select action in "${actions[@]}"; do + case $action in + "Display") + displayMsg "$action batches with $l_batchStatus status" + echo "$l_response" | jq + action_executed=$action + break + ;; + "Display with description") + displayMsg "$action batches with $l_batchStatus status" + + for row in $(echo "$l_response" | jq -r '.[] | @base64'); do + _jq() { + echo "${row}" | base64 --decode | jq -r "${1}" + } + + batchId=$(_jq '.batchId') + s3Prefix=$(_jq '.s3Prefix') + + echo "==> Describe batch $counter of $batchCount (batchId: ${batchId} s3Prefix: ${s3Prefix})" + + node describeBatch.js \ + --region "${AWS_REGION}" \ + --batchId "${batchId}" \ + --s3prefix "${s3Prefix}" | jq + + echo "" + + counter=$((counter+1)) + done + action_executed=$action + break + ;; + "Reprocess") + displayMsg "$action batches with $l_batchStatus status" + counter=1 + for row in $(echo "$l_response" | jq -r '.[] | @base64'); do + _jq() { + echo "${row}" | base64 --decode | jq -r "${1}" + } + + batchId=$(_jq '.batchId') + s3Prefix=$(_jq '.s3Prefix') + + echo "==> Reprocessing batch $counter of $batchCount (batchId: ${batchId} s3Prefix: ${s3Prefix})" + + node reprocessBatch.js \ + --region "${AWS_REGION}" \ + --batchId "${batchId}" \ + --prefix "${s3Prefix}" + + counter=$((counter+1)) + done + action_executed=$action + break + ;; + "Unlock") + displayMsg "$action batches with $l_batchStatus status" + counter=1 + for row in $(echo "$l_response" | jq -r '.[] | @base64'); do + _jq() { + echo "${row}" | base64 --decode | jq -r "${1}" + } + + batchId=$(_jq '.batchId') + s3Prefix=$(_jq '.s3Prefix') + + echo "==> Unlocking batch $counter of $batchCount (batchId: ${batchId} s3Prefix: ${s3Prefix})" + + node unlockBatch.js \ + "${AWS_REGION}" \ + "${batchId}" \ + "${s3Prefix}" + + counter=$((counter+1)) + done + action_executed=$action + break + ;; + "Delete") + displayMsg "$action batches with $l_batchStatus status from $startDate to $endDate" + + node deleteBatches.js \ + --region "${AWS_REGION}" \ + --startDate "${startDateUnix}" \ + --endDate "${endDateUnix}" \ + --batchStatus "$l_batchStatus" \ + --dryRun false + action_executed=$action + break + ;; + "Main Menu") + clear + action_menu=false + break + ;; + *) displayMsg "Invalid option selected $REPLY" + ;; + esac + done + done +} + +queryBatch(){ + response=$(node queryBatches.js \ + --region "${AWS_REGION}" \ + --batchStatus "$1" \ + --startDate "${startDateUnix}" \ + --endDate "${endDateUnix}" \ + ) + + echo "$response" +} + +getCredentials + +startDate=$(date -v-7d '+%m/%d/%Y') +startDateUnix=$(date -j -f %m/%d/%Y "$startDate" +%s) +endDate=$(date '+%m/%d/%Y') +endDateUnix=$(date -j -f %m/%d/%Y "$endDate" +%s) +dates_set=false + +main_menu=true +while [ "$main_menu" == true ]; do + + menuHeader " Main Menu " + + if ! [[ "$startDate" == "" && "$endDate" == "" ]]; then + if [[ "$dates_set" == false ]]; then + read -r -p "Do you want to use selected dates to query batches? $startDate to $endDate (y/n): " defaultDates + if ! [[ $defaultDates =~ ^[Yy]$ ]]; then + startDateUnix=0 + endDateUnix=0 + while ! [[ "$startDateUnix" < "$endDateUnix" ]]; do + startDate=$(inputDate 'Start Date') + startDateUnix=$(date -j -f %m/%d/%Y "$startDate" +%s) + endDate=$(inputDate 'End Date') + endDateUnix=$(date -j -f %m/%d/%Y "$endDate" +%s) + if ! [[ "$startDateUnix" < "$endDateUnix" ]]; then + displayMsg "The Start Date $startDate can't be greater than End Date $endDate"; + fi + done + fi + dates_set=true + fi + fi + + menuHeader " Main Menu " + if ! [[ $outputMsg == "" ]]; then + displayMsg "$outputMsg" + outputMsg="" + else + displayMsg "Query dates set from Start Date $startDate to End Date $endDate\n Please use option 6 to change dates" + fi + + PS3="What you would like to do? Please select from the menu: " + statuses=("Check Error batches" "Check Locked batches" "Check Open batches" + "Check Complete batches" "Check Other batches" "Change query dates" "Exit") + select status in "${statuses[@]}"; do + case $status in + "Check Error batches") + queryResult=$(queryBatch "error") + if ! [[ "$queryResult" == "[]" ]]; then + actions=("Display" "Display with description" "Reprocess" "Delete" "Main Menu") + batchAction "error" "$queryResult" + else + outputMsg="No batches with status 'error' found for specified dates $startDate - $endDate" + fi + break + ;; + "Check Locked batches") + queryResult=$(queryBatch "locked") + if ! [[ "$queryResult" == "[]" ]]; then + actions=("Display" "Display with description" "Unlock" "Reprocess" "Delete" "Main Menu") + batchAction "locked" "$queryResult" + else + outputMsg="No batches with status 'locked' found for specified dates $startDate - $endDate" + fi + break + ;; + "Check Open batches") + queryResult=$(queryBatch "open") + if ! [[ "$queryResult" == "[]" ]]; then + actions=("Display" "Display with description" "Main Menu") + batchAction "open" "$queryResult" + else + outputMsg="No batches with status 'open' found for specified dates $startDate - $endDate" + fi + break + ;; + "Check Complete batches") + queryResult=$(queryBatch "complete") + if ! [[ "$queryResult" == "[]" ]]; then + actions=("Display" "Display with description" "Delete" "Main Menu") + batchAction "complete" "$queryResult" + else + outputMsg="No batches with status 'complete' found for specified dates $startDate - $endDate" + fi + break + ;; + "Check Other batches") + read -r -p "Please type in batch status to query: " batchStatus + queryResult=$(queryBatch "$batchStatus") + if ! [[ "$queryResult" == "[]" ]]; then + actions=("Display" "Display with description" "Delete" "Main Menu") + batchAction "$batchStatus" "$queryResult" + else + outputMsg="No batches with status '$batchStatus' found for specified dates $startDate - $endDate" + fi + break + ;; + "Change query dates") + dates_set=false + clear + break + ;; + "Exit") + displayMsg "Exiting program" + exit + ;; + *) + displayMsg "Invalid option selected $REPLY" + ;; + esac + done +done