diff --git a/.github/workflows/cpp-packaging.yml b/.github/workflows/cpp-packaging.yml index 13db3f4219..7b630174ce 100644 --- a/.github/workflows/cpp-packaging.yml +++ b/.github/workflows/cpp-packaging.yml @@ -782,6 +782,7 @@ jobs: - name: Use GitHub API to start workflow shell: bash run: | + pip install -r scripts/gha/requirements.txt if [[ -z ${USE_EXPANDED_MATRIX} ]]; then USE_EXPANDED_MATRIX=0 fi diff --git a/scripts/gha/github.py b/scripts/gha/github.py index 204c50f363..e61458a4bf 100644 --- a/scripts/gha/github.py +++ b/scripts/gha/github.py @@ -21,6 +21,7 @@ import requests import json import shutil +import re from absl import logging from requests.adapters import HTTPAdapter @@ -35,9 +36,24 @@ REPO = 'firebase-cpp-sdk' BASE_URL = 'https://api.github.com' -FIREBASE_URL = '%s/repos/%s/%s' % (BASE_URL, OWNER, REPO) +GITHUB_API_URL = '%s/repos/%s/%s' % (BASE_URL, OWNER, REPO) logging.set_verbosity(logging.INFO) + +def set_repo_url(repo): + match = re.match(r'https://github\.com/([^/]+)/([^/.]+)', repo) + if not match: + logging.info('Error, only pattern https://github.com/\{repo_owner\}/\{repo_name\} are allowed.') + return False + + (repo_owner, repo_name) = match.groups() + global OWNER, REPO, GITHUB_API_URL + OWNER = repo_owner + REPO = repo_name + GITHUB_API_URL = '%s/repos/%s/%s' % (BASE_URL, OWNER, REPO) + return True + + def requests_retry_session(retries=RETRIES, backoff_factor=BACKOFF, status_forcelist=RETRY_STATUS): @@ -54,7 +70,7 @@ def requests_retry_session(retries=RETRIES, def create_issue(token, title, label, body): """Create an issue: https://docs.github.com/en/rest/reference/issues#create-an-issue""" - url = f'{FIREBASE_URL}/issues' + url = f'{GITHUB_API_URL}/issues' headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'} data = {'title': title, 'labels': [label], 'body': body} with requests.post(url, headers=headers, data=json.dumps(data), timeout=TIMEOUT) as response: @@ -64,7 +80,7 @@ def create_issue(token, title, label, body): def get_issue_body(token, issue_number): """https://docs.github.com/en/rest/reference/issues#get-an-issue-comment""" - url = f'{FIREBASE_URL}/issues/{issue_number}' + url = f'{GITHUB_API_URL}/issues/{issue_number}' headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'} with requests_retry_session().get(url, headers=headers, timeout=TIMEOUT) as response: logging.info("get_issue_body: %s response: %s", url, response) @@ -73,7 +89,7 @@ def get_issue_body(token, issue_number): def update_issue(token, issue_number, data): """Update an issue: https://docs.github.com/en/rest/reference/issues#update-an-issue""" - url = f'{FIREBASE_URL}/issues/{issue_number}' + url = f'{GITHUB_API_URL}/issues/{issue_number}' headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'} with requests_retry_session().patch(url, headers=headers, data=json.dumps(data), timeout=TIMEOUT) as response: logging.info("update_issue: %s response: %s", url, response) @@ -102,7 +118,7 @@ def search_issues_by_label(label): def list_comments(token, issue_number): """https://docs.github.com/en/rest/reference/issues#list-issue-comments""" - url = f'{FIREBASE_URL}/issues/{issue_number}/comments' + url = f'{GITHUB_API_URL}/issues/{issue_number}/comments' headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'} with requests_retry_session().get(url, headers=headers, timeout=TIMEOUT) as response: logging.info("list_comments: %s response: %s", url, response) @@ -111,7 +127,7 @@ def list_comments(token, issue_number): def add_comment(token, issue_number, comment): """https://docs.github.com/en/rest/reference/issues#create-an-issue-comment""" - url = f'{FIREBASE_URL}/issues/{issue_number}/comments' + url = f'{GITHUB_API_URL}/issues/{issue_number}/comments' headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'} data = {'body': comment} with requests.post(url, headers=headers, data=json.dumps(data), timeout=TIMEOUT) as response: @@ -120,7 +136,7 @@ def add_comment(token, issue_number, comment): def update_comment(token, comment_id, comment): """https://docs.github.com/en/rest/reference/issues#update-an-issue-comment""" - url = f'{FIREBASE_URL}/issues/comments/{comment_id}' + url = f'{GITHUB_API_URL}/issues/comments/{comment_id}' headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'} data = {'body': comment} with requests_retry_session().patch(url, headers=headers, data=json.dumps(data), timeout=TIMEOUT) as response: @@ -129,7 +145,7 @@ def update_comment(token, comment_id, comment): def delete_comment(token, comment_id): """https://docs.github.com/en/rest/reference/issues#delete-an-issue-comment""" - url = f'{FIREBASE_URL}/issues/comments/{comment_id}' + url = f'{GITHUB_API_URL}/issues/comments/{comment_id}' headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'} with requests.delete(url, headers=headers, timeout=TIMEOUT) as response: logging.info("delete_comment: %s response: %s", url, response) @@ -137,7 +153,7 @@ def delete_comment(token, comment_id): def add_label(token, issue_number, label): """https://docs.github.com/en/rest/reference/issues#add-labels-to-an-issue""" - url = f'{FIREBASE_URL}/issues/{issue_number}/labels' + url = f'{GITHUB_API_URL}/issues/{issue_number}/labels' headers={} headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'} data = [label] @@ -147,7 +163,7 @@ def add_label(token, issue_number, label): def delete_label(token, issue_number, label): """https://docs.github.com/en/rest/reference/issues#delete-a-label""" - url = f'{FIREBASE_URL}/issues/{issue_number}/labels/{label}' + url = f'{GITHUB_API_URL}/issues/{issue_number}/labels/{label}' headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'} with requests.delete(url, headers=headers, timeout=TIMEOUT) as response: logging.info("delete_label: %s response: %s", url, response) @@ -155,7 +171,7 @@ def delete_label(token, issue_number, label): def list_artifacts(token, run_id): """https://docs.github.com/en/rest/reference/actions#list-workflow-run-artifacts""" - url = f'{FIREBASE_URL}/actions/runs/{run_id}/artifacts' + url = f'{GITHUB_API_URL}/actions/runs/{run_id}/artifacts' headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'} with requests_retry_session().get(url, headers=headers, timeout=TIMEOUT) as response: logging.info("list_artifacts: %s response: %s", url, response) @@ -164,7 +180,7 @@ def list_artifacts(token, run_id): def download_artifact(token, artifact_id, output_path): """https://docs.github.com/en/rest/reference/actions#download-an-artifact""" - url = f'{FIREBASE_URL}/actions/artifacts/{artifact_id}/zip' + url = f'{GITHUB_API_URL}/actions/artifacts/{artifact_id}/zip' headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'} with requests.get(url, headers=headers, stream=True, timeout=TIMEOUT) as response: logging.info("download_artifact: %s response: %s", url, response) @@ -174,7 +190,7 @@ def download_artifact(token, artifact_id, output_path): def dismiss_review(token, pull_number, review_id, message): """https://docs.github.com/en/rest/reference/pulls#dismiss-a-review-for-a-pull-request""" - url = f'{FIREBASE_URL}/pulls/{pull_number}/reviews/{review_id}/dismissals' + url = f'{GITHUB_API_URL}/pulls/{pull_number}/reviews/{review_id}/dismissals' headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'} data = {'message': message} with requests_retry_session().put(url, headers=headers, data=json.dumps(data), @@ -182,9 +198,10 @@ def dismiss_review(token, pull_number, review_id, message): logging.info("dismiss_review: %s response: %s", url, response) return response.json() + def get_reviews(token, pull_number): """https://docs.github.com/en/rest/reference/pulls#list-reviews-for-a-pull-request""" - url = f'{FIREBASE_URL}/pulls/{pull_number}/reviews' + url = f'{GITHUB_API_URL}/pulls/{pull_number}/reviews' headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'} page = 1 per_page = 100 @@ -203,19 +220,32 @@ def get_reviews(token, pull_number): return results -def workflow_dispatch(token, workflow_id, ref, inputs): +def create_workflow_dispatch(token, workflow_id, ref, inputs): """https://docs.github.com/en/rest/reference/actions#create-a-workflow-dispatch-event""" - url = f'{FIREBASE_URL}/actions/workflows/{workflow_id}/dispatches' + url = f'{GITHUB_API_URL}/actions/workflows/{workflow_id}/dispatches' headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'} data = {'ref': ref, 'inputs': inputs} with requests.post(url, headers=headers, data=json.dumps(data), stream=True, timeout=TIMEOUT) as response: - logging.info("workflow_dispatch: %s response: %s", url, response) + logging.info("create_workflow_dispatch: %s response: %s", url, response) + # Response Status: 204 No Content + return True if response.status_code == 204 else False + + +def list_workflows(token, workflow_id, branch): + """https://docs.github.com/en/rest/reference/actions#list-workflow-runs-for-a-repository""" + url = f'{GITHUB_API_URL}/actions/workflows/{workflow_id}/runs' + headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'} + data = {'event': 'workflow_dispatch', 'branch': branch} + with requests.get(url, headers=headers, data=json.dumps(data), + stream=True, timeout=TIMEOUT) as response: + logging.info("list_workflows: %s response: %s", url, response) + return response.json() def create_pull_request(token, head, base, title, body, maintainer_can_modify): """https://docs.github.com/en/rest/reference/pulls#create-a-pull-request""" - url = f'{FIREBASE_URL}/pulls' + url = f'{GITHUB_API_URL}/pulls' headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'} data = {'head': head, 'base': base, 'title': title, 'body': body, 'maintainer_can_modify': maintainer_can_modify} @@ -224,9 +254,10 @@ def create_pull_request(token, head, base, title, body, maintainer_can_modify): logging.info("create_pull_request: %s response: %s", head, response) return True if response.status_code == 201 else False + def list_pull_requests(token, state, head, base): """https://docs.github.com/en/rest/reference/pulls#list-pull-requests""" - url = f'{FIREBASE_URL}/pulls' + url = f'{GITHUB_API_URL}/pulls' headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'} page = 1 per_page = 100 diff --git a/scripts/gha/trigger_workflow.py b/scripts/gha/trigger_workflow.py index f44312ced3..f1726f2339 100644 --- a/scripts/gha/trigger_workflow.py +++ b/scripts/gha/trigger_workflow.py @@ -29,70 +29,43 @@ """ import argparse -import json -import os -import re import subprocess import time import urllib.parse +import github def main(): args = parse_cmdline_args() - if args.repo is None: - args.repo=subprocess.check_output(['git', 'config', '--get', 'remote.origin.url']).decode('utf-8').rstrip('\n').lower() - print('autodetected repo: %s' % args.repo) if args.branch is None: - args.branch=subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).decode('utf-8').rstrip('\n') - print('autodetected branch: %s' % args.branch) - if not args.repo.startswith('https://github.com/'): - print('Error, only https://github.com/ repositories are allowed.') + args.branch=subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).decode('utf-8').rstrip('\n') + print('autodetected branch: %s' % args.branch) + if args.repo: # else use default firebase/firebase-cpp-sdk repo + if not github.set_repo_url(args.repo): exit(2) - (repo_owner, repo_name) = re.match(r'https://github\.com/([^/]+)/([^/.]+)', args.repo).groups() + else: + print('set repo url to: %s' % github.GITHUB_API_URL) - # POST /repos/{owner}/{repo}/actions/workflows/{workflow_id}/dispatches - request_url = 'https://api.github.com/repos/%s/%s/actions/workflows/%s/dispatches' % (repo_owner, repo_name, args.workflow) json_params = {} for param in args.param: - json_params[param[0]] = param[1] - json_text = '{"ref":%s,"inputs":%s}' % (json.dumps(args.branch), json.dumps(json_params)) + json_params[param[0]] = param[1] if args.verbose or args.dryrun: - print('request_url: %s' % request_url) - print('request_body: %s' % json_text) + print(f'request_url: {github.GITHUB_API_URL}/actions/workflows/{args.workflow}/dispatches') + print(f'request_body: ref: {args.branch}, inputs: {json_params}') if args.dryrun: return(0) print('Sending request to GitHub API...') - run_output = subprocess.check_output([args.curl, - '-s', '-o', '-', '-w', '\nHTTP status %{http_code}\n', - '-X', 'POST', - '-H', 'Accept: application/vnd.github.v3+json', - '-H', 'Authorization: token %s' % args.token, - request_url, '-d', json_text] - + ([] if not args.verbose else ['-v'])).decode('utf-8').rstrip('\n') - if args.verbose: - print(run_output) - if not re.search('HTTP status 2[0-9][0-9]$', run_output): - if not args.verbose: - print(run_output) - # Super quick and dirty way to get the message text since the appended status code means that - # the contents are not valid JSON. - error_message = re.search(r'"message": "([^"]+)"', run_output).group(1) - print('%sFailed to trigger workflow %s: %s' % ( - '::error ::' if args.in_github_action else '', args.workflow, error_message)) + if not github.create_workflow_dispatch(args.token, args.workflow, args.branch, json_params): + print('%sFailed to trigger workflow %s' % ( + '::error ::' if args.in_github_action else '', args.workflow)) return(-1) print('Success!') time.sleep(args.sleep) # Give a few seconds for the job to become queued. # Unfortunately, the GitHub REST API doesn't return the new workflow's run ID. # Query the list of workflows to find the one we just added. - request_url = 'https://api.github.com/repos/%s/%s/actions/workflows/%s/runs?event=workflow_dispatch&branch=%s' % (repo_owner, repo_name, args.workflow, args.branch) - run_output = subprocess.check_output([args.curl, - '-s', '-X', 'GET', - '-H', 'Accept: application/vnd.github.v3+json', - '-H', 'Authorization: token %s' % args.token, - request_url]).decode('utf-8').rstrip('\n') + workflows = github.list_workflows(args.token, args.workflow, args.branch) run_id = 0 - workflows = json.loads(run_output) if "workflow_runs" in workflows: branch_sha = subprocess.check_output(['git', 'rev-parse', args.branch]).decode('utf-8').rstrip('\n') for workflow in workflows['workflow_runs']: @@ -108,8 +81,8 @@ def main(): workflow_url = 'https://github.com/firebase/firebase-cpp-sdk/actions/runs/%s' % (run_id) else: # Couldn't get a run ID, use a generic URL. - workflow_url = 'https://github.com/%s/%s/actions/workflows/%s?query=%s+%s' % ( - repo_owner, repo_name, args.workflow, + workflow_url = '/%s/actions/workflows/%s?query=%s+%s' % ( + github.GITHUB_API_URL, args.workflow, urllib.parse.quote('event:workflow_dispatch', safe=''), urllib.parse.quote('branch:'+args.branch, safe='')) print('%sStarted workflow %s: %s' % ('::warning ::' if args.in_github_action else '',