Skip to content

Commit 4514bfd

Browse files
IanMeyersIanMeyers
authored
Merge pull request #234 from andreys70/master
The shell CLI program to check and operate LambdaRedshiftLoader batches
2 parents caf79fa + 598321b commit 4514bfd

File tree

1 file changed

+337
-0
lines changed

1 file changed

+337
-0
lines changed

‎cliBatchOperations.sh

Lines changed: 337 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,337 @@
1+
#!/bin/bash
2+
# Purpose: The shell CLI program to check and operate LambdaRedshiftLoader batches
3+
# Author: Andrey Suvorov
4+
# Created: 2021-09-14
5+
6+
SCRIPT_NAME=$0
7+
8+
usage(){
9+
echo ""
10+
echo "usage: $SCRIPT_NAME [-arpg] [-a account] [-r role] [-p profile] [-g region] [-u 1]"
11+
echo "arguments: "
12+
echo " -a aws account id"
13+
echo " -r aws role"
14+
echo " -p aws default profile"
15+
echo " -g aws region"
16+
echo " -u (1) show usage"
17+
echo ""
18+
exit 1
19+
}
20+
21+
# get arguments
22+
while getopts ":a:r:p:g:u:" opt; do
23+
case $opt in
24+
a) AWS_ACCOUNT_ID="$OPTARG"
25+
;;
26+
r) AWS_ROLE="$OPTARG"
27+
;;
28+
p) AWS_PROFILE="$OPTARG"
29+
;;
30+
g) AWS_REGION="$OPTARG"
31+
;;
32+
u)
33+
usage
34+
;;
35+
:)
36+
echo "$SCRIPT_NAME: missing value for -$OPTARG argument" >&2
37+
usage
38+
;;
39+
\?)
40+
echo "$SCRIPT_NAME: invalid argument -$OPTARG provided" >&2
41+
usage
42+
;;
43+
esac
44+
done
45+
46+
# set default arguments here:
47+
if [ "$AWS_ACCOUNT_ID" == "" ]; then AWS_ACCOUNT_ID="000000000000"; fi
48+
if [ "$AWS_ROLE" == "" ]; then AWS_ROLE="Developer"; fi
49+
if [ "$AWS_PROFILE" == "" ]; then AWS_PROFILE="default"; fi
50+
if [ "$AWS_REGION" == "" ]; then AWS_REGION="us-west-2"; fi
51+
52+
menuHeader(){
53+
clear
54+
echo " "
55+
echo "*****************************************************************************************"
56+
echo "* $1 *"
57+
echo "*****************************************************************************************"
58+
}
59+
60+
displayMsg(){
61+
echo " "
62+
echo "---------------------------------------- MESSAGE ----------------------------------------"
63+
echo " $1"
64+
echo "-----------------------------------------------------------------------------------------"
65+
echo " "
66+
}
67+
68+
getCredentials(){
69+
local l_retryLogin
70+
menuHeader " AWS Login "
71+
displayMsg "AWS account: $AWS_ACCOUNT_ID\n AWS Region: $AWS_REGION\n AWS Role: $AWS_ROLE\n AWS Profile: $AWS_PROFILE"
72+
while ! eiamcli getAWSTempCredentials -a "$AWS_ACCOUNT_ID" -r "$AWS_ROLE" -p "$AWS_PROFILE" ; do
73+
if ! eiamcli login; then
74+
displayMsg "Can't authenticate the user. Exiting program."
75+
read -r -p "Would you like to try login again? (y/n): " l_retryLogin
76+
if ! [ "$l_retryLogin" == 'y' ]; then
77+
displayMsg "Exiting program"
78+
exit 1
79+
fi
80+
else
81+
eiamcli getAWSTempCredentials -a "$AWS_ACCOUNT_ID" -r "$AWS_ROLE" -p "$AWS_PROFILE"
82+
fi
83+
done
84+
}
85+
86+
inputDate(){
87+
local dt=""
88+
while ! [[ "$dt" =~ ^[0-1][0-9]/[0-3][0-9]/[0-9]{4}$ && $(date -j -f %m/%d/%Y "$dt" +%s) ]]; do
89+
if ! [[ "$dt" == "" ]]; then displayMsg "Invalid date format for $1 : $dt" > /dev/tty; fi
90+
read -r -p "Please enter $1 (MM/DD/YYYY): " dt
91+
done
92+
echo "$dt"
93+
}
94+
95+
batchAction(){
96+
local l_batchStatus=$1
97+
local l_response=$2
98+
99+
batchCount=$(echo "$2" | jq '. | length')
100+
action_menu=true
101+
action_executed=""
102+
menuHeader "Action Menu"
103+
104+
while [[ "$action_menu" == true ]]; do
105+
if [[ "$action_executed" == "" ]]; then
106+
displayMsg "Found $batchCount batches with '$l_batchStatus' status from $startDate to $endDate"
107+
else
108+
displayMsg "$action_executed batches command completed"
109+
action_executed=""
110+
fi
111+
PS3="Please select the action from the menu above: "
112+
select action in "${actions[@]}"; do
113+
case $action in
114+
"Display")
115+
displayMsg "$action batches with $l_batchStatus status"
116+
echo "$l_response" | jq
117+
action_executed=$action
118+
break
119+
;;
120+
"Display with description")
121+
displayMsg "$action batches with $l_batchStatus status"
122+
123+
for row in $(echo "$l_response" | jq -r '.[] | @base64'); do
124+
_jq() {
125+
echo "${row}" | base64 --decode | jq -r "${1}"
126+
}
127+
128+
batchId=$(_jq '.batchId')
129+
s3Prefix=$(_jq '.s3Prefix')
130+
131+
echo "==> Describe batch $counter of $batchCount (batchId: ${batchId} s3Prefix: ${s3Prefix})"
132+
133+
node describeBatch.js \
134+
--region "${AWS_REGION}" \
135+
--batchId "${batchId}" \
136+
--s3prefix "${s3Prefix}" | jq
137+
138+
echo ""
139+
140+
counter=$((counter+1))
141+
done
142+
action_executed=$action
143+
break
144+
;;
145+
"Reprocess")
146+
displayMsg "$action batches with $l_batchStatus status"
147+
counter=1
148+
for row in $(echo "$l_response" | jq -r '.[] | @base64'); do
149+
_jq() {
150+
echo "${row}" | base64 --decode | jq -r "${1}"
151+
}
152+
153+
batchId=$(_jq '.batchId')
154+
s3Prefix=$(_jq '.s3Prefix')
155+
156+
echo "==> Reprocessing batch $counter of $batchCount (batchId: ${batchId} s3Prefix: ${s3Prefix})"
157+
158+
node reprocessBatch.js \
159+
--region "${AWS_REGION}" \
160+
--batchId "${batchId}" \
161+
--prefix "${s3Prefix}"
162+
163+
counter=$((counter+1))
164+
done
165+
action_executed=$action
166+
break
167+
;;
168+
"Unlock")
169+
displayMsg "$action batches with $l_batchStatus status"
170+
counter=1
171+
for row in $(echo "$l_response" | jq -r '.[] | @base64'); do
172+
_jq() {
173+
echo "${row}" | base64 --decode | jq -r "${1}"
174+
}
175+
176+
batchId=$(_jq '.batchId')
177+
s3Prefix=$(_jq '.s3Prefix')
178+
179+
echo "==> Unlocking batch $counter of $batchCount (batchId: ${batchId} s3Prefix: ${s3Prefix})"
180+
181+
node unlockBatch.js \
182+
"${AWS_REGION}" \
183+
"${batchId}" \
184+
"${s3Prefix}"
185+
186+
counter=$((counter+1))
187+
done
188+
action_executed=$action
189+
break
190+
;;
191+
"Delete")
192+
displayMsg "$action batches with $l_batchStatus status from $startDate to $endDate"
193+
194+
node deleteBatches.js \
195+
--region "${AWS_REGION}" \
196+
--startDate "${startDateUnix}" \
197+
--endDate "${endDateUnix}" \
198+
--batchStatus "$l_batchStatus" \
199+
--dryRun false
200+
action_executed=$action
201+
break
202+
;;
203+
"Main Menu")
204+
clear
205+
action_menu=false
206+
break
207+
;;
208+
*) displayMsg "Invalid option selected $REPLY"
209+
;;
210+
esac
211+
done
212+
done
213+
}
214+
215+
queryBatch(){
216+
response=$(node queryBatches.js \
217+
--region "${AWS_REGION}" \
218+
--batchStatus "$1" \
219+
--startDate "${startDateUnix}" \
220+
--endDate "${endDateUnix}" \
221+
)
222+
223+
echo "$response"
224+
}
225+
226+
getCredentials
227+
228+
startDate=$(date -v-7d '+%m/%d/%Y')
229+
startDateUnix=$(date -j -f %m/%d/%Y "$startDate" +%s)
230+
endDate=$(date '+%m/%d/%Y')
231+
endDateUnix=$(date -j -f %m/%d/%Y "$endDate" +%s)
232+
dates_set=false
233+
234+
main_menu=true
235+
while [ "$main_menu" == true ]; do
236+
237+
menuHeader " Main Menu "
238+
239+
if ! [[ "$startDate" == "" && "$endDate" == "" ]]; then
240+
if [[ "$dates_set" == false ]]; then
241+
read -r -p "Do you want to use selected dates to query batches? $startDate to $endDate (y/n): " defaultDates
242+
if ! [[ $defaultDates =~ ^[Yy]$ ]]; then
243+
startDateUnix=0
244+
endDateUnix=0
245+
while ! [[ "$startDateUnix" < "$endDateUnix" ]]; do
246+
startDate=$(inputDate 'Start Date')
247+
startDateUnix=$(date -j -f %m/%d/%Y "$startDate" +%s)
248+
endDate=$(inputDate 'End Date')
249+
endDateUnix=$(date -j -f %m/%d/%Y "$endDate" +%s)
250+
if ! [[ "$startDateUnix" < "$endDateUnix" ]]; then
251+
displayMsg "The Start Date $startDate can't be greater than End Date $endDate";
252+
fi
253+
done
254+
fi
255+
dates_set=true
256+
fi
257+
fi
258+
259+
menuHeader " Main Menu "
260+
if ! [[ $outputMsg == "" ]]; then
261+
displayMsg "$outputMsg"
262+
outputMsg=""
263+
else
264+
displayMsg "Query dates set from Start Date $startDate to End Date $endDate\n Please use option 6 to change dates"
265+
fi
266+
267+
PS3="What you would like to do? Please select from the menu: "
268+
statuses=("Check Error batches" "Check Locked batches" "Check Open batches"
269+
"Check Complete batches" "Check Other batches" "Change query dates" "Exit")
270+
select status in "${statuses[@]}"; do
271+
case $status in
272+
"Check Error batches")
273+
queryResult=$(queryBatch "error")
274+
if ! [[ "$queryResult" == "[]" ]]; then
275+
actions=("Display" "Display with description" "Reprocess" "Delete" "Main Menu")
276+
batchAction "error" "$queryResult"
277+
else
278+
outputMsg="No batches with status 'error' found for specified dates $startDate - $endDate"
279+
fi
280+
break
281+
;;
282+
"Check Locked batches")
283+
queryResult=$(queryBatch "locked")
284+
if ! [[ "$queryResult" == "[]" ]]; then
285+
actions=("Display" "Display with description" "Unlock" "Reprocess" "Delete" "Main Menu")
286+
batchAction "locked" "$queryResult"
287+
else
288+
outputMsg="No batches with status 'locked' found for specified dates $startDate - $endDate"
289+
fi
290+
break
291+
;;
292+
"Check Open batches")
293+
queryResult=$(queryBatch "open")
294+
if ! [[ "$queryResult" == "[]" ]]; then
295+
actions=("Display" "Display with description" "Main Menu")
296+
batchAction "open" "$queryResult"
297+
else
298+
outputMsg="No batches with status 'open' found for specified dates $startDate - $endDate"
299+
fi
300+
break
301+
;;
302+
"Check Complete batches")
303+
queryResult=$(queryBatch "complete")
304+
if ! [[ "$queryResult" == "[]" ]]; then
305+
actions=("Display" "Display with description" "Delete" "Main Menu")
306+
batchAction "complete" "$queryResult"
307+
else
308+
outputMsg="No batches with status 'complete' found for specified dates $startDate - $endDate"
309+
fi
310+
break
311+
;;
312+
"Check Other batches")
313+
read -r -p "Please type in batch status to query: " batchStatus
314+
queryResult=$(queryBatch "$batchStatus")
315+
if ! [[ "$queryResult" == "[]" ]]; then
316+
actions=("Display" "Display with description" "Delete" "Main Menu")
317+
batchAction "$batchStatus" "$queryResult"
318+
else
319+
outputMsg="No batches with status '$batchStatus' found for specified dates $startDate - $endDate"
320+
fi
321+
break
322+
;;
323+
"Change query dates")
324+
dates_set=false
325+
clear
326+
break
327+
;;
328+
"Exit")
329+
displayMsg "Exiting program"
330+
exit
331+
;;
332+
*)
333+
displayMsg "Invalid option selected $REPLY"
334+
;;
335+
esac
336+
done
337+
done

0 commit comments

Comments
 (0)