diff --git a/batchOperations.js b/batchOperations.js index cef83bc..91e6195 100644 --- a/batchOperations.js +++ b/batchOperations.js @@ -190,7 +190,9 @@ exports.doQuery = doQuery; * @param callback * @returns */ -function deleteBatch(s3Prefix, batchId, callback) { +function deleteBatch(setRegion, s3Prefix, batchId, callback) { + init(setRegion); + var deleteParams = { TableName: batchTable, Key: { @@ -236,7 +238,7 @@ function deleteBatches(setRegion, batchStatus, startDate, endDate, dryRun, callb async.map(data, function (batchItem, asyncCallback) { // pass the request through the function that deletes the // item from DynamoDB - deleteBatch(batchItem.s3Prefix, batchItem.batchId, function (err, data) { + deleteBatch(setRegion, batchItem.s3Prefix, batchItem.batchId, function (err, data) { if (err) { asyncCallback(err); } else { diff --git a/cliBatchOperations.sh b/cliBatchOperations.sh index 63e04fd..bc24285 100644 --- a/cliBatchOperations.sh +++ b/cliBatchOperations.sh @@ -2,6 +2,7 @@ # Purpose: The shell CLI program to check and operate LambdaRedshiftLoader batches # Author: Andrey Suvorov # Created: 2021-09-14 +# Updated: 2021-11-02 SCRIPT_NAME=$0 @@ -111,20 +112,24 @@ batchAction(){ PS3="Please select the action from the menu above: " select action in "${actions[@]}"; do case $action in - "Display") + "List") displayMsg "$action batches with $l_batchStatus status" echo "$l_response" | jq action_executed=$action break ;; - "Display with description") + "Describe") displayMsg "$action batches with $l_batchStatus status" + counter=1 for row in $(echo "$l_response" | jq -r '.[] | @base64'); do _jq() { echo "${row}" | base64 --decode | jq -r "${1}" } + # exit individual items action menu + if ! [[ "$action_executed" == "" ]]; then break; fi + batchId=$(_jq '.batchId') s3Prefix=$(_jq '.s3Prefix') @@ -138,6 +143,57 @@ batchAction(){ echo "" counter=$((counter+1)) + + # select an individual action for each batch in the query, one by one + PS3="Please select the batch action from the menu above: " + if [[ "$l_batchStatus" == "error" || "$l_batchStatus" == "locked" ]]; then + batch_actions=("Reprocess" "Unlock" "Delete" "Next" "Exit") + elif [[ "$l_batchStatus" == "complete" ]]; then + batch_actions=("Next" "Delete" "Exit") + elif [[ "$l_batchStatus" == "open" ]]; then + batch_actions=("Next" "Exit") + fi + + select batch_action in "${batch_actions[@]}"; do + case $batch_action in + "Reprocess") + echo "==> Reprocessing (batchId: ${batchId} s3Prefix: ${s3Prefix})" + node reprocessBatch.js \ + --region "${AWS_REGION}" \ + --batchId "${batchId}" \ + --prefix "${s3Prefix}" + + break + ;; + "Unlock") + echo "==> Unlocking (batchId: ${batchId} s3Prefix: ${s3Prefix})" + node unlockBatch.js \ + "${AWS_REGION}" \ + "${batchId}" \ + "${s3Prefix}" + break + ;; + "Delete") + echo "==> Deleting (batchId: ${batchId} s3Prefix: ${s3Prefix})" + node deleteBatch.js \ + --region "${AWS_REGION}" \ + --batchId "${batchId}" \ + --s3Prefix "${s3Prefix}" + break + ;; + "Next") + echo "==> Next batch" + break + ;; + "Exit") + clear + action_executed=$action + break + ;; + *) displayMsg "Invalid option selected $REPLY" + ;; + esac + done done action_executed=$action break @@ -223,115 +279,119 @@ queryBatch(){ echo "$response" } -getCredentials - -startDate=$(date -v-7d '+%m/%d/%Y') -startDateUnix=$(date -j -f %m/%d/%Y "$startDate" +%s) -endDate=$(date '+%m/%d/%Y') -endDateUnix=$(date -j -f %m/%d/%Y "$endDate" +%s) -dates_set=false - -main_menu=true -while [ "$main_menu" == true ]; do - - menuHeader " Main Menu " - - if ! [[ "$startDate" == "" && "$endDate" == "" ]]; then - if [[ "$dates_set" == false ]]; then - read -r -p "Do you want to use selected dates to query batches? $startDate to $endDate (y/n): " defaultDates - if ! [[ $defaultDates =~ ^[Yy]$ ]]; then - startDateUnix=0 - endDateUnix=0 - while ! [[ "$startDateUnix" < "$endDateUnix" ]]; do - startDate=$(inputDate 'Start Date') - startDateUnix=$(date -j -f %m/%d/%Y "$startDate" +%s) - endDate=$(inputDate 'End Date') - endDateUnix=$(date -j -f %m/%d/%Y "$endDate" +%s) - if ! [[ "$startDateUnix" < "$endDateUnix" ]]; then - displayMsg "The Start Date $startDate can't be greater than End Date $endDate"; - fi - done +main(){ + startDate=$(date -v-7d '+%m/%d/%Y') + startDateUnix=$(date -j -f %m/%d/%Y "$startDate" +%s) + endDate=$(date '+%m/%d/%Y') + endDateUnix=$(date -j -f %m/%d/%Y "$endDate" +%s) + dates_set=false + + getCredentials + + main_menu=true + while [ "$main_menu" == true ]; do + + menuHeader " Main Menu " + + if ! [[ "$startDate" == "" && "$endDate" == "" ]]; then + if [[ "$dates_set" == false ]]; then + read -r -p "Do you want to use selected dates to query batches? $startDate to $endDate (y/n): " defaultDates + if ! [[ $defaultDates =~ ^[Yy]$ ]]; then + startDateUnix=0 + endDateUnix=0 + while ! [[ "$startDateUnix" < "$endDateUnix" ]]; do + startDate=$(inputDate 'Start Date') + startDateUnix=$(date -j -f %m/%d/%Y "$startDate" +%s) + endDate=$(inputDate 'End Date') + endDateUnix=$(date -j -f %m/%d/%Y "$endDate" +%s) + if ! [[ "$startDateUnix" < "$endDateUnix" ]]; then + displayMsg "The Start Date $startDate can't be greater than End Date $endDate"; + fi + done + fi + dates_set=true fi - dates_set=true fi - fi - - menuHeader " Main Menu " - if ! [[ $outputMsg == "" ]]; then - displayMsg "$outputMsg" - outputMsg="" - else - displayMsg "Query dates set from Start Date $startDate to End Date $endDate\n Please use option 6 to change dates" - fi - - PS3="What you would like to do? Please select from the menu: " - statuses=("Check Error batches" "Check Locked batches" "Check Open batches" - "Check Complete batches" "Check Other batches" "Change query dates" "Exit") - select status in "${statuses[@]}"; do - case $status in - "Check Error batches") - queryResult=$(queryBatch "error") - if ! [[ "$queryResult" == "[]" ]]; then - actions=("Display" "Display with description" "Reprocess" "Delete" "Main Menu") - batchAction "error" "$queryResult" - else - outputMsg="No batches with status 'error' found for specified dates $startDate - $endDate" - fi - break - ;; - "Check Locked batches") - queryResult=$(queryBatch "locked") - if ! [[ "$queryResult" == "[]" ]]; then - actions=("Display" "Display with description" "Unlock" "Reprocess" "Delete" "Main Menu") - batchAction "locked" "$queryResult" - else - outputMsg="No batches with status 'locked' found for specified dates $startDate - $endDate" - fi - break - ;; - "Check Open batches") - queryResult=$(queryBatch "open") - if ! [[ "$queryResult" == "[]" ]]; then - actions=("Display" "Display with description" "Main Menu") - batchAction "open" "$queryResult" - else - outputMsg="No batches with status 'open' found for specified dates $startDate - $endDate" - fi - break - ;; - "Check Complete batches") - queryResult=$(queryBatch "complete") - if ! [[ "$queryResult" == "[]" ]]; then - actions=("Display" "Display with description" "Delete" "Main Menu") - batchAction "complete" "$queryResult" - else - outputMsg="No batches with status 'complete' found for specified dates $startDate - $endDate" - fi - break - ;; - "Check Other batches") - read -r -p "Please type in batch status to query: " batchStatus - queryResult=$(queryBatch "$batchStatus") - if ! [[ "$queryResult" == "[]" ]]; then - actions=("Display" "Display with description" "Delete" "Main Menu") - batchAction "$batchStatus" "$queryResult" - else - outputMsg="No batches with status '$batchStatus' found for specified dates $startDate - $endDate" - fi - break - ;; - "Change query dates") - dates_set=false - clear - break - ;; - "Exit") - displayMsg "Exiting program" - exit - ;; - *) - displayMsg "Invalid option selected $REPLY" - ;; - esac + + menuHeader " Main Menu " + if ! [[ $outputMsg == "" ]]; then + displayMsg "$outputMsg" + outputMsg="" + else + displayMsg "Query dates set from Start Date $startDate to End Date $endDate\n Please use option 6 to change dates" + fi + + PS3="What you would like to do? Please select from the menu: " + statuses=("Check Error batches" "Check Locked batches" "Check Open batches" + "Check Complete batches" "Check Other batches" "Change query dates" "Exit") + select status in "${statuses[@]}"; do + case $status in + "Check Error batches") + queryResult=$(queryBatch "error") + if ! [[ "$queryResult" == "[]" ]]; then + actions=("List" "Describe" "Reprocess" "Delete" "Main Menu") + batchAction "error" "$queryResult" + else + outputMsg="No batches with status 'error' found for specified dates $startDate - $endDate" + fi + break + ;; + "Check Locked batches") + queryResult=$(queryBatch "locked") + if ! [[ "$queryResult" == "[]" ]]; then + actions=("List" "Describe" "Unlock" "Reprocess" "Delete" "Main Menu") + batchAction "locked" "$queryResult" + else + outputMsg="No batches with status 'locked' found for specified dates $startDate - $endDate" + fi + break + ;; + "Check Open batches") + queryResult=$(queryBatch "open") + if ! [[ "$queryResult" == "[]" ]]; then + actions=("List" "Describe" "Main Menu") + batchAction "open" "$queryResult" + else + outputMsg="No batches with status 'open' found for specified dates $startDate - $endDate" + fi + break + ;; + "Check Complete batches") + queryResult=$(queryBatch "complete") + if ! [[ "$queryResult" == "[]" ]]; then + actions=("List" "Describe" "Delete" "Main Menu") + batchAction "complete" "$queryResult" + else + outputMsg="No batches with status 'complete' found for specified dates $startDate - $endDate" + fi + break + ;; + "Check Other batches") + read -r -p "Please type in batch status to query: " batchStatus + queryResult=$(queryBatch "$batchStatus") + if ! [[ "$queryResult" == "[]" ]]; then + actions=("List" "Describe" "Delete" "Main Menu") + batchAction "$batchStatus" "$queryResult" + else + outputMsg="No batches with status '$batchStatus' found for specified dates $startDate - $endDate" + fi + break + ;; + "Change query dates") + dates_set=false + clear + break + ;; + "Exit") + displayMsg "Exiting program" + exit + ;; + *) + displayMsg "Invalid option selected $REPLY" + ;; + esac + done done -done +} + +main \ No newline at end of file diff --git a/deleteBatch.js b/deleteBatch.js new file mode 100644 index 0000000..d8d4eb2 --- /dev/null +++ b/deleteBatch.js @@ -0,0 +1,29 @@ +/* + Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Amazon Software License (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at + + http://aws.amazon.com/asl/ + + or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +var args = require('minimist')(process.argv.slice(2)); +var batchOperations = require("./batchOperations"); + +var region = args.region +var s3Prefix = args.s3Prefix; +var batchId = args.batchId; + +if (!region || !s3Prefix || !batchId) { + console.log("You must provide an AWS Region Code, Batch ID, and configured Input Location"); + process.exit(-1); +} + +batchOperations.deleteBatch(region, s3Prefix, batchId, function(err, data) { + if (err) { + console.log("Error: " + err); + process.exit(-1); + } else { + console.log("OK: Deleted " + JSON.stringify(data)); + } +});