From ba59579141916d25a9d371ca945bc5e28289ce4e Mon Sep 17 00:00:00 2001 From: averikitsch Date: Mon, 15 Apr 2019 18:49:07 -0700 Subject: [PATCH 1/7] Add task with authentication sample --- appengine/flexible/tasks/Dockerfile | 17 ++++ .../tasks/create_http_task_with_token.py | 80 +++++++++++++++++++ .../tasks/create_http_task_with_token_test.py | 28 +++++++ appengine/flexible/tasks/requirements.txt | 2 +- 4 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 appengine/flexible/tasks/Dockerfile create mode 100644 appengine/flexible/tasks/create_http_task_with_token.py create mode 100644 appengine/flexible/tasks/create_http_task_with_token_test.py diff --git a/appengine/flexible/tasks/Dockerfile b/appengine/flexible/tasks/Dockerfile new file mode 100644 index 00000000000..64f160d8253 --- /dev/null +++ b/appengine/flexible/tasks/Dockerfile @@ -0,0 +1,17 @@ +# Use the official Python image. +# https://hub.docker.com/_/python +FROM python:3.7 + +# Copy local code to the container image. +ENV APP_HOME /app +WORKDIR $APP_HOME +COPY . . + +# Install production dependencies. +RUN pip install Flask gunicorn + +# Run the web service on container startup. Here we use the gunicorn +# webserver, with one worker process and 8 threads. +# For environments with multiple CPU cores, increase the number of workers +# to be equal to the cores available. +CMD exec gunicorn --bind :$PORT --workers 1 --threads 8 main:app diff --git a/appengine/flexible/tasks/create_http_task_with_token.py b/appengine/flexible/tasks/create_http_task_with_token.py new file mode 100644 index 00000000000..42188b550ad --- /dev/null +++ b/appengine/flexible/tasks/create_http_task_with_token.py @@ -0,0 +1,80 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import argparse +import datetime + + +def create_http_task(project, + queue, + location, + url, + payload=None, + in_seconds=None): + # [START cloud_tasks_create_http_task_with_token] + """Create a task for a given queue with an arbitrary payload.""" + + from google.cloud import tasks_v2beta3 + from google.protobuf import timestamp_pb2 + + # Create a client. + client = tasks_v2beta3.CloudTasksClient() + + # TODO(developer): Uncomment these lines and replace with your values. + # project = 'my-project-id' + # queue = 'my-appengine-queue' + # location = 'us-central1' + # url = 'https://.appspot.com/example_task_handler' + # payload = 'hello' + + # Construct the fully qualified queue name. + parent = client.queue_path(project, location, queue) + + # Construct the request body. + task = { + 'http_request': { # Specify the type of request. + 'http_method': 'POST', + 'url': url, # The full url path that the task will be sent to. + 'oidc_token': { + 'service_account_email': + 'client_id@project_id.iam.gserviceaccount.com' + } + } + } + if payload is not None: + # The API expects a payload of type bytes. + converted_payload = payload.encode() + + # Add the payload to the request. + task['http_request']['body'] = converted_payload + + if in_seconds is not None: + # Convert "seconds from now" into an rfc3339 datetime string. + d = datetime.datetime.utcnow() + datetime.timedelta(seconds=in_seconds) + + # Create Timestamp protobuf. + timestamp = timestamp_pb2.Timestamp() + timestamp.FromDatetime(d) + + # Add the timestamp to the tasks. + task['schedule_time'] = timestamp + + # Use the client to build and send the task. + response = client.create_task(parent, task) + + print('Created task {}'.format(response.name)) + return response +# [END cloud_tasks_create_http_task_with_token] diff --git a/appengine/flexible/tasks/create_http_task_with_token_test.py b/appengine/flexible/tasks/create_http_task_with_token_test.py new file mode 100644 index 00000000000..4d51fd04683 --- /dev/null +++ b/appengine/flexible/tasks/create_http_task_with_token_test.py @@ -0,0 +1,28 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import create_http_task_with_token + +TEST_PROJECT_ID = os.getenv('GCLOUD_PROJECT') +TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') +TEST_QUEUE_NAME = os.getenv('TEST_QUEUE_NAME', 'my-appengine-queue') + + +def test_create_http_task_with_token(): + url = 'https://example.com/example_task_handler' + result = create_http_task_with_token.create_http_task( + TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION, url) + assert TEST_QUEUE_NAME in result.name diff --git a/appengine/flexible/tasks/requirements.txt b/appengine/flexible/tasks/requirements.txt index 9aadd8a7747..fe50a5aa3b2 100644 --- a/appengine/flexible/tasks/requirements.txt +++ b/appengine/flexible/tasks/requirements.txt @@ -1,3 +1,3 @@ Flask==1.0.2 gunicorn==19.9.0 -google-cloud-tasks==0.6.0 +google-cloud-tasks==0.7.0 From a832385646d5921c45303d88cb358ba7c0318d9e Mon Sep 17 00:00:00 2001 From: averikitsch Date: Mon, 15 Apr 2019 18:59:45 -0700 Subject: [PATCH 2/7] Fix linting --- appengine/flexible/tasks/create_http_task_with_token.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/appengine/flexible/tasks/create_http_task_with_token.py b/appengine/flexible/tasks/create_http_task_with_token.py index 42188b550ad..5642233e627 100644 --- a/appengine/flexible/tasks/create_http_task_with_token.py +++ b/appengine/flexible/tasks/create_http_task_with_token.py @@ -14,10 +14,8 @@ from __future__ import print_function -import argparse import datetime - def create_http_task(project, queue, location, From a5554ef495c7474114be9324bbdef64d79a1354c Mon Sep 17 00:00:00 2001 From: averikitsch Date: Mon, 15 Apr 2019 18:59:45 -0700 Subject: [PATCH 3/7] Fix linting --- appengine/flexible/tasks/create_http_task_with_token.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/appengine/flexible/tasks/create_http_task_with_token.py b/appengine/flexible/tasks/create_http_task_with_token.py index 42188b550ad..5642233e627 100644 --- a/appengine/flexible/tasks/create_http_task_with_token.py +++ b/appengine/flexible/tasks/create_http_task_with_token.py @@ -14,10 +14,8 @@ from __future__ import print_function -import argparse import datetime - def create_http_task(project, queue, location, From 4c5d6b4e07b95158ee67c0cfaba9b3d2853d6f22 Mon Sep 17 00:00:00 2001 From: averikitsch Date: Tue, 16 Apr 2019 09:14:16 -0700 Subject: [PATCH 4/7] Fix spacing --- appengine/flexible/tasks/create_http_task_with_token.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/appengine/flexible/tasks/create_http_task_with_token.py b/appengine/flexible/tasks/create_http_task_with_token.py index 5642233e627..2c4f851e02a 100644 --- a/appengine/flexible/tasks/create_http_task_with_token.py +++ b/appengine/flexible/tasks/create_http_task_with_token.py @@ -16,6 +16,7 @@ import datetime + def create_http_task(project, queue, location, @@ -52,6 +53,7 @@ def create_http_task(project, } } } + if payload is not None: # The API expects a payload of type bytes. converted_payload = payload.encode() From 0e69eef12dd89e307983016b7db38d9887a97fe7 Mon Sep 17 00:00:00 2001 From: averikitsch Date: Tue, 16 Apr 2019 09:35:32 -0700 Subject: [PATCH 5/7] Update tests with service account --- .kokoro/system_tests.sh | 1 + appengine/flexible/tasks/create_http_task_with_token.py | 6 +++--- .../flexible/tasks/create_http_task_with_token_test.py | 8 ++++++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.kokoro/system_tests.sh b/.kokoro/system_tests.sh index 80b9a3173e5..aae4cda6707 100755 --- a/.kokoro/system_tests.sh +++ b/.kokoro/system_tests.sh @@ -25,6 +25,7 @@ SECRETS_PASSWORD=$(cat "${KOKORO_GFILE_DIR}/secrets-password.txt") source ./testing/test-env.sh export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/testing/service-account.json export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json +source ${KOKORO_GFILE_DIR}/tasks-service-account.sh # Run Cloud SQL proxy, if required if [ -n "${CLOUD_SQL_PROXY}" ]; then diff --git a/appengine/flexible/tasks/create_http_task_with_token.py b/appengine/flexible/tasks/create_http_task_with_token.py index 2c4f851e02a..28494928953 100644 --- a/appengine/flexible/tasks/create_http_task_with_token.py +++ b/appengine/flexible/tasks/create_http_task_with_token.py @@ -21,6 +21,7 @@ def create_http_task(project, queue, location, url, + service_account_email, payload=None, in_seconds=None): # [START cloud_tasks_create_http_task_with_token] @@ -48,12 +49,11 @@ def create_http_task(project, 'http_method': 'POST', 'url': url, # The full url path that the task will be sent to. 'oidc_token': { - 'service_account_email': - 'client_id@project_id.iam.gserviceaccount.com' + 'service_account_email': service_account_email } } } - + if payload is not None: # The API expects a payload of type bytes. converted_payload = payload.encode() diff --git a/appengine/flexible/tasks/create_http_task_with_token_test.py b/appengine/flexible/tasks/create_http_task_with_token_test.py index 4d51fd04683..aa33b672adb 100644 --- a/appengine/flexible/tasks/create_http_task_with_token_test.py +++ b/appengine/flexible/tasks/create_http_task_with_token_test.py @@ -19,10 +19,14 @@ TEST_PROJECT_ID = os.getenv('GCLOUD_PROJECT') TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') TEST_QUEUE_NAME = os.getenv('TEST_QUEUE_NAME', 'my-appengine-queue') +TEST_SERVICE_ACCOUNT = os.getenv('CLOUD_RUN_INVOKER_SERVICE_ACCOUNT') def test_create_http_task_with_token(): url = 'https://example.com/example_task_handler' - result = create_http_task_with_token.create_http_task( - TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION, url) + result = create_http_task_with_token.create_http_task(TEST_PROJECT_ID, + TEST_QUEUE_NAME, + TEST_LOCATION, + url, + TEST_SERVICE_ACCOUNT) assert TEST_QUEUE_NAME in result.name From 3fb01f6cbff20e83c9ba2fefe2edc5c8c50224dc Mon Sep 17 00:00:00 2001 From: averikitsch Date: Thu, 18 Apr 2019 11:23:27 -0700 Subject: [PATCH 6/7] Move samples and update READMEs --- .kokoro/system_tests.sh | 1 - appengine/flexible/tasks/README.md | 21 ------------ tasks/README.md | 34 +++---------------- .../create_http_task_with_token.py | 2 +- .../create_http_task_with_token_test.py | 3 +- 5 files changed, 8 insertions(+), 53 deletions(-) rename {appengine/flexible/tasks => tasks}/create_http_task_with_token.py (97%) rename {appengine/flexible/tasks => tasks}/create_http_task_with_token_test.py (93%) diff --git a/.kokoro/system_tests.sh b/.kokoro/system_tests.sh index aae4cda6707..80b9a3173e5 100755 --- a/.kokoro/system_tests.sh +++ b/.kokoro/system_tests.sh @@ -25,7 +25,6 @@ SECRETS_PASSWORD=$(cat "${KOKORO_GFILE_DIR}/secrets-password.txt") source ./testing/test-env.sh export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/testing/service-account.json export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json -source ${KOKORO_GFILE_DIR}/tasks-service-account.sh # Run Cloud SQL proxy, if required if [ -n "${CLOUD_SQL_PROXY}" ]; then diff --git a/appengine/flexible/tasks/README.md b/appengine/flexible/tasks/README.md index 303fbfcfcaf..c2888ba647b 100644 --- a/appengine/flexible/tasks/README.md +++ b/appengine/flexible/tasks/README.md @@ -101,24 +101,3 @@ endpoint, with a payload specified: ``` python create_app_engine_queue_task.py --project=$PROJECT_ID --queue=$QUEUE_ID --location=$LOCATION_ID --payload=hello ``` - -### Using HTTP Push Queues - -Set an environment variable for the endpoint to your task handler. This is an -example url to send requests to the App Engine task handler: -``` -export URL=https://.appspot.com/example_task_handler -``` - -Running the sample will create a task and send the task to the specific URL -endpoint, with a payload specified: - -``` -python create_http_task.py --project=$PROJECT_ID --queue=$QUEUE_ID --location=$LOCATION_ID --url=$URL --payload=hello -``` - -Now view that the payload was received and verify the payload: - -``` -gcloud app logs read -``` diff --git a/tasks/README.md b/tasks/README.md index 61ac422cb2e..00503ccd392 100644 --- a/tasks/README.md +++ b/tasks/README.md @@ -12,8 +12,11 @@ App Engine queues push tasks to an App Engine HTTP target. This directory contains both the App Engine app to deploy, as well as the snippets to run locally to push tasks to it, which could also be called on App Engine. -`create_app_engine_queue_task.py` is a simple command-line program to create -tasks to be pushed to the App Engine app. +`create_http_task.py` is a simple command-line program to create +tasks to be pushed to an URL endpoint. + +`create_http_task_with_token.py` is a simple command-line program to create +tasks to be pushed to an URL endpoint with authorization header. `main.py` is the main App Engine app. This app serves as an endpoint to receive App Engine task attempts. @@ -41,33 +44,6 @@ gcloud beta tasks queues create-app-engine-queue my-appengine-queue Note: A newly created queue will route to the default App Engine service and version unless configured to do otherwise. -## Deploying the App Engine App - -Deploy the App Engine app with gcloud: - -* To deploy to the Standard environment: - ``` - gcloud app deploy app.yaml - ``` -* To deploy to the Flexible environment: - ``` - gcloud app deploy app.flexible.yaml - ``` - -Verify the index page is serving: - -``` -gcloud app browse -``` - -The App Engine app serves as a target for the push requests. It has an -endpoint `/example_task_handler` that reads the payload (i.e., the request body) -of the HTTP POST request and logs it. The log output can be viewed with: - -``` -gcloud app logs read -``` - ## Run the Sample Using the Command Line Set environment variables: diff --git a/appengine/flexible/tasks/create_http_task_with_token.py b/tasks/create_http_task_with_token.py similarity index 97% rename from appengine/flexible/tasks/create_http_task_with_token.py rename to tasks/create_http_task_with_token.py index 28494928953..1b79a9b3fc7 100644 --- a/appengine/flexible/tasks/create_http_task_with_token.py +++ b/tasks/create_http_task_with_token.py @@ -37,7 +37,7 @@ def create_http_task(project, # project = 'my-project-id' # queue = 'my-appengine-queue' # location = 'us-central1' - # url = 'https://.appspot.com/example_task_handler' + # url = 'https://example.com/example_task_handler' # payload = 'hello' # Construct the fully qualified queue name. diff --git a/appengine/flexible/tasks/create_http_task_with_token_test.py b/tasks/create_http_task_with_token_test.py similarity index 93% rename from appengine/flexible/tasks/create_http_task_with_token_test.py rename to tasks/create_http_task_with_token_test.py index aa33b672adb..32cc4aa348b 100644 --- a/appengine/flexible/tasks/create_http_task_with_token_test.py +++ b/tasks/create_http_task_with_token_test.py @@ -19,7 +19,8 @@ TEST_PROJECT_ID = os.getenv('GCLOUD_PROJECT') TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') TEST_QUEUE_NAME = os.getenv('TEST_QUEUE_NAME', 'my-appengine-queue') -TEST_SERVICE_ACCOUNT = os.getenv('CLOUD_RUN_INVOKER_SERVICE_ACCOUNT') +TEST_SERVICE_ACCOUNT = 'test-run-invoker@python-docs-samples-tests.iam.gserviceaccount.com' + def test_create_http_task_with_token(): From ddb643fa548bd360bd61d6cc55506ab457e1c2e4 Mon Sep 17 00:00:00 2001 From: averikitsch Date: Thu, 18 Apr 2019 11:37:21 -0700 Subject: [PATCH 7/7] Update version and linting --- tasks/create_http_task_with_token_test.py | 4 ++-- tasks/requirements.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tasks/create_http_task_with_token_test.py b/tasks/create_http_task_with_token_test.py index 32cc4aa348b..5b98c566e9e 100644 --- a/tasks/create_http_task_with_token_test.py +++ b/tasks/create_http_task_with_token_test.py @@ -19,8 +19,8 @@ TEST_PROJECT_ID = os.getenv('GCLOUD_PROJECT') TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') TEST_QUEUE_NAME = os.getenv('TEST_QUEUE_NAME', 'my-appengine-queue') -TEST_SERVICE_ACCOUNT = 'test-run-invoker@python-docs-samples-tests.iam.gserviceaccount.com' - +TEST_SERVICE_ACCOUNT = ( + 'test-run-invoker@python-docs-samples-tests.iam.gserviceaccount.com') def test_create_http_task_with_token(): diff --git a/tasks/requirements.txt b/tasks/requirements.txt index 9aadd8a7747..fe50a5aa3b2 100644 --- a/tasks/requirements.txt +++ b/tasks/requirements.txt @@ -1,3 +1,3 @@ Flask==1.0.2 gunicorn==19.9.0 -google-cloud-tasks==0.6.0 +google-cloud-tasks==0.7.0