Skip to content

Commit e5fd8b2

Browse files
author
Swapneil Singh
committed
Don't force environment variables to be set for CVE automations
1 parent 2360dd7 commit e5fd8b2

File tree

4 files changed

+332
-37
lines changed

4 files changed

+332
-37
lines changed

‎scripts/check_for_new_al_version.sh

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/bash
2+
3+
most_recent_al2=$(./scripts/most_recent_al2.sh)
4+
last_used_al2=$(grep "Amazon Linux base container image version" CHANGELOG.md | head -n 1 | grep -o ': .*$' | cut -c 3-)
5+
6+
# Return true if there's a newer version than ours
7+
if [ "$last_used_al2" != "$most_recent_al2" ]; then
8+
echo "true"
9+
else
10+
echo "false"
11+
fi

‎scripts/generate_changelog.sh

Lines changed: 134 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,8 @@
11
#!/bin/bash
22
set -xeuo pipefail
33

4-
# Initialize variables
5-
next_token=""
6-
all_tags=()
7-
8-
# Get all amazonlinux container image tags
9-
while true; do
10-
if [ -z "$next_token" ]; then
11-
response=$(curl -sSL \
12-
--header "Content-Type: application/json" \
13-
--request POST \
14-
--data '{"registryAliasName":"amazonlinux","repositoryName":"amazonlinux","maxResults":250}' \
15-
https://api.us-east-1.gallery.ecr.aws/describeImageTags)
16-
else
17-
response=$(curl -sSL \
18-
--header "Content-Type: application/json" \
19-
--request POST \
20-
--data "{\"registryAliasName\":\"amazonlinux\",\"repositoryName\":\"amazonlinux\",\"nextToken\":\"$next_token\",\"maxResults\":250}" \
21-
https://api.us-east-1.gallery.ecr.aws/describeImageTags)
22-
fi
23-
24-
# Extract tags and add them to the array
25-
tags=$(echo "$response" | jq -r '.imageTagDetails[].imageTag')
26-
all_tags+=($tags)
27-
28-
# Check if there's a next token
29-
next_token=$(echo "$response" | jq -r '.nextToken')
30-
if [[ "$next_token" == "null" ]]; then
31-
break
32-
fi
33-
done
34-
354
# Find the most recent AL2 tag
36-
most_recent_al2=$(printf '%s\n' "${all_tags[@]}" | grep '^2\.' | grep -v minimal | grep -v arm | grep -v amd | sort -V | tail -n 1)
5+
most_recent_al2=$(./scripts/most_recent_al2.sh)
376

387
# Read the JSON file
398
json_file="linux.version"
@@ -46,17 +15,145 @@ cloudwatch_plugin_version=$(echo "$json_content" | jq -r '.linux."cloudwatch-plu
4615
kinesis_plugin_version=$(echo "$json_content" | jq -r '.linux."kinesis-plugin"')
4716
firehose_plugin_version=$(echo "$json_content" | jq -r '.linux."firehose-plugin"')
4817

18+
changelog_commit=$(git log -n1 --pretty=format:%h CHANGELOG.md)
19+
changelog_commit_date="$(git show --no-patch --format=%ci $changelog_commit)"
20+
current_commit=$(git log -n1 --pretty=format:%h)
21+
22+
# Test Overrides
23+
24+
25+
upstream_changes=false
26+
newer_upstream_changes=""
27+
if [ "${FLUENT_BIT_DIRECTORY+false}" && "${#FLUENT_BIT_DIRECTORY}" -ge 1 ]; then
28+
newer_upstream_changes=$(cd $FLUENT_BIT_DIRECTORY; git log --pretty=format:%s --after "$changelog_commit_date")
29+
# If there are NOT newer changes
30+
if [ ${#newer_upstream_changes} -le 1 ]; then
31+
FLUENT_BIT_DIRECTORY=""
32+
fi
33+
fi
34+
35+
compared_string=""
36+
37+
add_changelog_changes=false
38+
flb_has_changes=false
39+
we_have_changes=false
40+
if [ "${FLUENT_BIT_DIRECTORY+false}" && "${#FLUENT_BIT_DIRECTORY}" -ge 1 ]; then
41+
flb_has_changes=true
42+
fi
43+
if [ "$changelog_commit" != "$current_commit" ]; then
44+
we_have_changes=true
45+
fi
46+
47+
if [ "$we_have_changes" = true ] || [ "$flb_has_changes" = true ]; then
48+
compared_string="
49+
Compared to the previous release, this release adds:"
50+
commit_names=$(git log --pretty=format:%s "$changelog_commit".."$current_commit")
51+
if [ "$flb_has_changes" = true ]; then
52+
# Get commits from the upstream as well
53+
# Add newline to separate from existing commits
54+
commit_names+="
55+
"
56+
# Add FLB commit names to the list considered for changelog updates
57+
commit_names+="$newer_upstream_changes"
58+
fi
59+
original_ifs=$IFS
60+
IFS="
61+
"
62+
for line in $commit_names
63+
do
64+
IFS=" "
65+
found_label=false
66+
for word in $line
67+
do
68+
# Look for an indication of what this commit is about
69+
# unless we already are at the stage of collecting
70+
# the name
71+
if [ "$found_label" = false ]; then
72+
# Factor out setting this to true if a
73+
# label is found, reducing boilerplate
74+
found_label=true
75+
if [ "$word" = "feature:" ]; then
76+
compared_string+="
77+
* Feature - "
78+
continue
79+
elif [ "$word" = "enhancement:" ]; then
80+
compared_string+="
81+
* Enhancement - "
82+
continue
83+
elif [ "$word" = "fix:" ] || [ "$word" = "bugfix:" ]; then
84+
compared_string+="
85+
* Fix - "
86+
continue
87+
elif [ "$word" = "Fix" ]; then
88+
compared_string+="
89+
* Fix - "
90+
# We don't continue for the enclosing elif,
91+
# i.e. we add "Fix" as the first word
92+
elif [ $(echo "$word" | grep "\(aws:\|out_cloudwatch_logs:\|out_s3:\|out_kinesis:\|filter_ecs:\|filter_kubernetes:\)") ]; then
93+
change_type="Enhancement"
94+
for inner_word in $line
95+
do
96+
if [ $(echo "$inner_word" | grep '\(fix\|Fix\|bugfix\)') ]; then
97+
change_type="Fix"
98+
break
99+
elif [ $(echo "$inner_word" | grep '\(add\|Add\)') ]; then
100+
change_type="Feature"
101+
break
102+
elif [ $(echo "$inner_word" | grep '\(allow\|support\)') ]; then
103+
# Enhancement is the default
104+
break
105+
fi
106+
done
107+
108+
compared_string+="
109+
* $change_type - "
110+
continue
111+
else
112+
# No label was found
113+
found_label=false
114+
fi
115+
fi
116+
if [ "$found_label" = false ]; then
117+
# If no label, skip this commit
118+
break 1
119+
else
120+
# Add each word aside from the prefix to the commit changelog description
121+
compared_string+=$word
122+
compared_string+=" "
123+
fi
124+
done
125+
IFS="
126+
"
127+
done
128+
# Append newline to separate from the next changelog entry
129+
compared_string+="
130+
"
131+
IFS=$original_ifs
132+
fi
133+
49134
# Generate the changelog entry
50-
cat << EOF
135+
136+
new_changelog="
51137
### $version
52138
This release includes:
53139
* Fluent Bit [$fluent_bit_version](https://github.com/fluent/fluent-bit/tree/v$fluent_bit_version)
54140
* Amazon CloudWatch Logs for Fluent Bit ${cloudwatch_plugin_version#v}
55141
* Amazon Kinesis Streams for Fluent Bit ${kinesis_plugin_version#v}
56142
* Amazon Kinesis Firehose for Fluent Bit ${firehose_plugin_version#v}
57143
* Amazon Linux base container image version: $most_recent_al2
144+
$compared_string"
145+
146+
if [ "${FLUENT_BIT_DIRECTORY+false}" ]; then
147+
148+
heads=$(head -n 1 CHANGELOG.md)
149+
tails=$(tail -n +3 CHANGELOG.md)
150+
151+
rm -f temp_changelog.txt
152+
echo "$heads" >> temp_changelog.txt
153+
echo "$new_changelog" >> temp_changelog.txt
154+
echo "$tails" >> temp_changelog.txt
155+
156+
mv -f temp_changelog.txt CHANGELOG.md
157+
rm -f temp_changelog.txt
58158

59-
Compared to the previous release, this release adds:
60-
* Fix - TODO blah blah [#TODO](https://github.com/amazon-contributing/upstream-to-fluent-bit/pull/TODO)
61-
* Enhancement - TODO blah blah [#TODO](https://github.com/aws/aws-for-fluent-bit/pull/TODO)
62-
EOF
159+
fi

‎scripts/most_recent_al2.sh

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#!/bin/bash
2+
3+
# Initialize variables
4+
next_token=""
5+
all_tags=()
6+
7+
# Get all amazonlinux container image tags
8+
while true; do
9+
if [ -z "$next_token" ]; then
10+
response=$(curl -sSL \
11+
--header "Content-Type: application/json" \
12+
--request POST \
13+
--data '{"registryAliasName":"amazonlinux","repositoryName":"amazonlinux","maxResults":250}' \
14+
https://api.us-east-1.gallery.ecr.aws/describeImageTags)
15+
else
16+
response=$(curl -sSL \
17+
--header "Content-Type: application/json" \
18+
--request POST \
19+
--data "{\"registryAliasName\":\"amazonlinux\",\"repositoryName\":\"amazonlinux\",\"nextToken\":\"$next_token\",\"maxResults\":250}" \
20+
https://api.us-east-1.gallery.ecr.aws/describeImageTags)
21+
fi
22+
23+
# Extract tags and add them to the array
24+
tags=$(echo "$response" | jq -r '.imageTagDetails[].imageTag')
25+
all_tags+=($tags)
26+
27+
# Check if there's a next token
28+
next_token=$(echo "$response" | jq -r '.nextToken')
29+
if [[ "$next_token" == "null" ]]; then
30+
break
31+
fi
32+
done
33+
34+
# Find the most recent AL2 tag
35+
most_recent_al2=$(printf '%s\n' "${all_tags[@]}" | grep '^2\.' | grep -v minimal | grep -v arm | grep -v amd | sort -V | tail -n 1)
36+
37+
echo $most_recent_al2

‎scripts/publish_cve_update.py

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
#!/usr/bin/env python3
2+
3+
from datetime import datetime
4+
import subprocess
5+
import os
6+
7+
# Check if this is an automated or manual execution
8+
automated = os.getenv("FLUENT_BIT_CVE_AUTOMATION")
9+
automated = isinstance(automated, string) and len(automated) > 0 and int(automated) > 0
10+
11+
git_cli_path = '/usr/bin/git'
12+
13+
def run_command(command_arr, timeout=None, tries=1, cwd=None, fail_on_error=False, env=None):
14+
i = 0
15+
output = None
16+
for i in range(tries):
17+
try:
18+
output = subprocess.run(command_arr,
19+
stdout=subprocess.PIPE,
20+
timeout=timeout,
21+
cwd=cwd,
22+
env=env).stdout.decode('utf-8')
23+
except subprocess.TimeoutExpired as ex:
24+
print(f"Command {command_arr} timed out after {ex.timeout} seconds.")
25+
print(f"Partial output (stdout): {ex.stdout}")
26+
print(f"Partial output (stderr): {ex.stderr}")
27+
except Exception as ex:
28+
print(f"Command {command_arr} had unexpected exception {ex}")
29+
30+
# Return output if the command succeeded
31+
if output is not None:
32+
return output
33+
34+
# Failure handling
35+
print(f"Failed to run command {command_arr}")
36+
if fail_on_error:
37+
print("Quitting...")
38+
quit()
39+
40+
41+
is_new_al_version = run_command(["./scripts/check_for_new_al_version.sh"], timeout=10, tries=3, fail_on_error=True)
42+
new_al2_version = None
43+
if is_new_al_version == 'false':
44+
print("Amazon Linux version is up to date.")
45+
quit()
46+
else:
47+
print("New Amazon Linux version is available.")
48+
new_al2_version = run_command(["./scripts/most_recent_al2.sh"], timeout=10, tries=3, fail_on_error=True)
49+
print("New AL2 version: {}".format(new_al2_version))
50+
51+
# Get upstream_to_fluent_bit
52+
max_clone_attempts = 3
53+
upstream_repo = "/tmp/fluent-bit/"
54+
upstream_uri = "https://github.com/amazon-contributing/upstream-to-fluent-bit.git"
55+
for i in range(max_clone_attempts):
56+
57+
run_command(["rm", "-rf", upstream_repo])
58+
run_command(["mkdir", upstream_repo])
59+
clone_output = run_command(["git", "clone", upstream_uri, upstream_repo],
60+
fail_on_error=False, timeout=30)
61+
if clone_output is not None:
62+
run_command(["git", "checkout", "1.9.10"], cwd=upstream_repo,
63+
fail_on_error=True)
64+
break
65+
else:
66+
print("Couldn't clone FLB upstream. Will ignore upstream commits.")
67+
upstream_repo = None
68+
69+
# Go to a new branch for this automation.
70+
numeric_date_string = datetime.now().strftime("%Y%m%d")
71+
branch_name = "patch-automation-{}".format(numeric_date_string)
72+
run_command([git_cli_path, "fetch".format(numeric_date_string)],
73+
fail_on_error=True, tries=2)
74+
run_command([git_cli_path, "switch", "-C", branch_name],
75+
fail_on_error=True)
76+
current_branch = run_command(["git", "rev-parse", "--abbrev-ref", "HEAD"],
77+
fail_on_error=True)
78+
current_branch = current_branch.strip()
79+
if current_branch != branch_name:
80+
print("failed to switch branch to {}".format(branch_name))
81+
print("current branch: {}".format(current_branch))
82+
quit()
83+
# Reset the branch to master in case it isn't already
84+
run_command([git_cli_path, "reset", "--hard", "mainline"])
85+
86+
# Derive the new version number
87+
old_version = run_command(["cat", "AWS_FOR_FLUENT_BIT_VERSION"], fail_on_error=True)
88+
semantic_version_components = old_version.split('.')
89+
version_num_count = len(semantic_version_components)
90+
if version_num_count == 3:
91+
semantic_version_components = semantic_version_components + [numeric_date_string]
92+
elif version_num_count == 4:
93+
semantic_version_components[3] = numeric_date_string
94+
else:
95+
print("Invalid semantic versioning for current version. Aborting...")
96+
quit()
97+
98+
new_version = ".".join(semantic_version_components)
99+
100+
# Update version file
101+
with open("AWS_FOR_FLUENT_BIT_VERSION", "w") as version_file:
102+
version_file.write(new_version)
103+
104+
# Update linux.version file
105+
with open("linux.version", 'r') as version_file:
106+
linux_version = version_file.readlines()
107+
108+
old_version = "{}".format(old_version)
109+
print("old aws-for-fluent-bit version number: {}".format(old_version))
110+
print("new aws-for-fluent-bit version number: {}".format(new_version))
111+
for linum in range(len(linux_version)):
112+
# print("linux.version line: {}".format(linux_version[linum]))
113+
if old_version in linux_version[linum]:
114+
linux_version[linum] = linux_version[linum].replace(old_version, new_version)
115+
break
116+
else:
117+
print("could not auto-locate version number in linux.version!!!")
118+
print("Guessing at line to modify based on file format...")
119+
linux_version[2] = ' "version": "{}",\n'.format(new_version)
120+
121+
with open("linux.version", 'w') as version_file:
122+
version_file.writelines(linux_version)
123+
124+
# Update changelog
125+
run_command(["./scripts/generate_changelog.sh"],
126+
# Provide fluent bit directory to the changelog script,
127+
# if we were able to clone it.
128+
env={"FLUENT_BIT_DIRECTORY": upstream_repo} if upstream_repo is not None else None,
129+
timeout=20, tries=3,
130+
fail_on_error=True)
131+
132+
# Commit changes
133+
run_command(["git", "commit", "-a", "--message", '"Release {}"'.format(new_version)],
134+
fail_on_error=True)
135+
136+
# Push changes to remote.
137+
# Might fail if there are unexpected commits, but we ignore that as expected
138+
core_push_command = ["git", "push"]
139+
if automated:
140+
core_push_command = core_push_command + ["--force"]
141+
push_output = run_command(core_push_command,
142+
fail_on_error=False)
143+
if push_output is None or push_output == "" or ("fatal: The current branch" in push_output and "has no upstream branch" in push_output):
144+
# Sometimes you need to set the pustream if it doesn't exist
145+
print("push may have failed due to missing set-upstream. Retrying")
146+
push_output = run_command(core_push_command + ["--set-upstream", "origin", branch_name],
147+
fail_on_error=False)
148+
149+
# Go back to mainline
150+
run_command(["git", "checkout", "mainline"], timeout=30)

0 commit comments

Comments
 (0)