diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 45ede645eb7..00000000000 --- a/.coveragerc +++ /dev/null @@ -1,16 +0,0 @@ -[run] -include = - appengine/* - bigquery/* - blog/* - cloud_logging/* - compute/* - datastore/* - managed_vms/* - monitoring/* - speech/* - storage/* -[report] -exclude_lines = - pragma: NO COVER - if __name__ == '__main__': diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000000..90b740e9aa3 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,14 @@ +# Code owners file. +# This file controls who is tagged for review for any given pull request. + +# Alix Hamilton is the primary maintainer of the BigQuery samples. +bigquery/* @alixhami + +# Tim Swast is the primary maintainer of the BigQuery Data Transfer samples. +bigquery/transfer/* @tswast + +# Tim Swast is the primary maintainer of the Composer samples. +composer/* @tswast + +# Alix Hamilton is the primary maintainer of the Jupyter notebook samples +notebooks/* @alixhami diff --git a/.gitignore b/.gitignore index 176a48ea6e7..08a370acb39 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ *.pyc .coverage .tox +.pytest_cache +.ipynb_checkpoints +.executed_notebooks coverage.xml python-docs-samples.json service-account.json @@ -19,3 +22,6 @@ secrets.tar junit.xml credentials.dat .nox +.vscode/ +*sponge_log.xml +.DS_store diff --git a/.kokoro/common.cfg b/.kokoro/common.cfg new file mode 100644 index 00000000000..10f74078791 --- /dev/null +++ b/.kokoro/common.cfg @@ -0,0 +1,25 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download trampoline resources. These will be in ${KOKORO_GFILE_DIR} +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# All builds use the trampoline script to run in docker. +build_file: "python-docs-samples/.kokoro/trampoline.sh" + +# Use the Python worker docker image. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python@sha256:e11a459d01e5dcd3613fda35c7c94edfecfe911ed79c078580ff59de300b1938" +} + +# Specify project ID +env_vars: { + key: "GCP_PROJECT" + value: "python-docs-samples" +} + +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} diff --git a/.kokoro/presubmit_tests_appengine.cfg b/.kokoro/presubmit_tests_appengine.cfg new file mode 100644 index 00000000000..7fa4e15f74f --- /dev/null +++ b/.kokoro/presubmit_tests_appengine.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "gae" +} diff --git a/.kokoro/presubmit_tests_appengine_flexible.cfg b/.kokoro/presubmit_tests_appengine_flexible.cfg new file mode 100644 index 00000000000..493ec4209c0 --- /dev/null +++ b/.kokoro/presubmit_tests_appengine_flexible.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "py36 and flexible" +} diff --git a/.kokoro/presubmit_tests_auth.cfg b/.kokoro/presubmit_tests_auth.cfg new file mode 100644 index 00000000000..5b17f588026 --- /dev/null +++ b/.kokoro/presubmit_tests_auth.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "auth and py36 and not appengine" +} diff --git a/.kokoro/presubmit_tests_bigquery.cfg b/.kokoro/presubmit_tests_bigquery.cfg new file mode 100644 index 00000000000..b7ff3c59ff4 --- /dev/null +++ b/.kokoro/presubmit_tests_bigquery.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "bigquery and py36 and not appengine" +} diff --git a/.kokoro/presubmit_tests_bigtable.cfg b/.kokoro/presubmit_tests_bigtable.cfg new file mode 100644 index 00000000000..e701c9ac3bf --- /dev/null +++ b/.kokoro/presubmit_tests_bigtable.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "bigtable and py36 and not appengine" +} diff --git a/.kokoro/presubmit_tests_composer.cfg b/.kokoro/presubmit_tests_composer.cfg new file mode 100644 index 00000000000..d48442b0d7f --- /dev/null +++ b/.kokoro/presubmit_tests_composer.cfg @@ -0,0 +1,23 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "composer and py36 and not appengine" +} + +# Explicitly use choose unicode dependency to let the apache-airflow package +# installation continue. See: +# https://github.com/apache/incubator-airflow/pull/3660 +env_vars: { + key: "SLUGIFY_USES_TEXT_UNIDECODE" + value: "yes" +} diff --git a/.kokoro/presubmit_tests_compute.cfg b/.kokoro/presubmit_tests_compute.cfg new file mode 100644 index 00000000000..dfba68f75a1 --- /dev/null +++ b/.kokoro/presubmit_tests_compute.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "compute and py36 and not appengine" +} diff --git a/.kokoro/presubmit_tests_container_engine.cfg b/.kokoro/presubmit_tests_container_engine.cfg new file mode 100644 index 00000000000..b6b3ee4b344 --- /dev/null +++ b/.kokoro/presubmit_tests_container_engine.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "container_engine and py36 and not appengine" +} diff --git a/.kokoro/presubmit_tests_dataproc.cfg b/.kokoro/presubmit_tests_dataproc.cfg new file mode 100644 index 00000000000..9d035309b21 --- /dev/null +++ b/.kokoro/presubmit_tests_dataproc.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "dataproc and py36 and not appengine" +} diff --git a/.kokoro/presubmit_tests_datastore.cfg b/.kokoro/presubmit_tests_datastore.cfg new file mode 100644 index 00000000000..7a950e7f6d0 --- /dev/null +++ b/.kokoro/presubmit_tests_datastore.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "datastore and py36 and not appengine" +} diff --git a/.kokoro/presubmit_tests_dlp.cfg b/.kokoro/presubmit_tests_dlp.cfg new file mode 100644 index 00000000000..f5b60121400 --- /dev/null +++ b/.kokoro/presubmit_tests_dlp.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "dlp and py36 and not appengine" +} diff --git a/.kokoro/presubmit_tests_dns.cfg b/.kokoro/presubmit_tests_dns.cfg new file mode 100644 index 00000000000..053c25270d3 --- /dev/null +++ b/.kokoro/presubmit_tests_dns.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "dns and py36 and not appengine" +} diff --git a/.kokoro/presubmit_tests_endpoints.cfg b/.kokoro/presubmit_tests_endpoints.cfg new file mode 100644 index 00000000000..59476530bf6 --- /dev/null +++ b/.kokoro/presubmit_tests_endpoints.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "endpoints and py36 and not appengine" +} diff --git a/.kokoro/presubmit_tests_error_reporting.cfg b/.kokoro/presubmit_tests_error_reporting.cfg new file mode 100644 index 00000000000..1006a421e2d --- /dev/null +++ b/.kokoro/presubmit_tests_error_reporting.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "error_reporting and py36 and not appengine" +} diff --git a/.kokoro/presubmit_tests_functions.cfg b/.kokoro/presubmit_tests_functions.cfg new file mode 100644 index 00000000000..1b37bf7db23 --- /dev/null +++ b/.kokoro/presubmit_tests_functions.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "functions and (py36 or lint) and not venv" +} diff --git a/.kokoro/presubmit_tests_functions_sql.cfg b/.kokoro/presubmit_tests_functions_sql.cfg new file mode 100644 index 00000000000..a4fb2912164 --- /dev/null +++ b/.kokoro/presubmit_tests_functions_sql.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "functions and sql and py36 and not venv" +} diff --git a/.kokoro/presubmit_tests_iap.cfg b/.kokoro/presubmit_tests_iap.cfg new file mode 100644 index 00000000000..a0d1856348a --- /dev/null +++ b/.kokoro/presubmit_tests_iap.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "iap and py36 and not appengine" +} diff --git a/.kokoro/presubmit_tests_iot.cfg b/.kokoro/presubmit_tests_iot.cfg new file mode 100644 index 00000000000..3a8cee6e48f --- /dev/null +++ b/.kokoro/presubmit_tests_iot.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "iot and py36 and not appengine" +} diff --git a/.kokoro/presubmit_tests_kms.cfg b/.kokoro/presubmit_tests_kms.cfg new file mode 100644 index 00000000000..00cbe0dbba1 --- /dev/null +++ b/.kokoro/presubmit_tests_kms.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "kms and py36 and not appengine" +} diff --git a/.kokoro/presubmit_tests_language.cfg b/.kokoro/presubmit_tests_language.cfg new file mode 100644 index 00000000000..95330fd6d5f --- /dev/null +++ b/.kokoro/presubmit_tests_language.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "language and py36 and not appengine" +} diff --git a/.kokoro/presubmit_tests_logging.cfg b/.kokoro/presubmit_tests_logging.cfg new file mode 100644 index 00000000000..71e63e7993e --- /dev/null +++ b/.kokoro/presubmit_tests_logging.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "logging and py36 and not appengine" +} diff --git a/.kokoro/presubmit_tests_ml_engine.cfg b/.kokoro/presubmit_tests_ml_engine.cfg new file mode 100644 index 00000000000..47070d66d10 --- /dev/null +++ b/.kokoro/presubmit_tests_ml_engine.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "ml_engine and py36 and not appengine" +} diff --git a/.kokoro/presubmit_tests_monitoring.cfg b/.kokoro/presubmit_tests_monitoring.cfg new file mode 100644 index 00000000000..95f2cd2c32a --- /dev/null +++ b/.kokoro/presubmit_tests_monitoring.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "monitoring and py36 and not appengine" +} diff --git a/.kokoro/presubmit_tests_pubsub.cfg b/.kokoro/presubmit_tests_pubsub.cfg new file mode 100644 index 00000000000..7ccd9b6f676 --- /dev/null +++ b/.kokoro/presubmit_tests_pubsub.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "pubsub and py36 and not appengine" +} diff --git a/.kokoro/presubmit_tests_spanner.cfg b/.kokoro/presubmit_tests_spanner.cfg new file mode 100644 index 00000000000..456271c5bb2 --- /dev/null +++ b/.kokoro/presubmit_tests_spanner.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "spanner and py36 and not appengine" +} diff --git a/.kokoro/presubmit_tests_speech.cfg b/.kokoro/presubmit_tests_speech.cfg new file mode 100644 index 00000000000..a12337e0443 --- /dev/null +++ b/.kokoro/presubmit_tests_speech.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "speech and py36 and not appengine" +} diff --git a/.kokoro/presubmit_tests_storage.cfg b/.kokoro/presubmit_tests_storage.cfg new file mode 100644 index 00000000000..300aceb2202 --- /dev/null +++ b/.kokoro/presubmit_tests_storage.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "storage and py36 and not appengine" +} diff --git a/.kokoro/presubmit_tests_tasks.cfg b/.kokoro/presubmit_tests_tasks.cfg new file mode 100644 index 00000000000..5cd413436ea --- /dev/null +++ b/.kokoro/presubmit_tests_tasks.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "tasks and py36 and not appengine" +} diff --git a/.kokoro/presubmit_tests_trace.cfg b/.kokoro/presubmit_tests_trace.cfg new file mode 100644 index 00000000000..8438f4af281 --- /dev/null +++ b/.kokoro/presubmit_tests_trace.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "trace and py36 and not appengine" +} diff --git a/.kokoro/presubmit_tests_translate.cfg b/.kokoro/presubmit_tests_translate.cfg new file mode 100644 index 00000000000..059bcdc1594 --- /dev/null +++ b/.kokoro/presubmit_tests_translate.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "translate and py36 and not appengine" +} diff --git a/.kokoro/presubmit_tests_video.cfg b/.kokoro/presubmit_tests_video.cfg new file mode 100644 index 00000000000..917491b523a --- /dev/null +++ b/.kokoro/presubmit_tests_video.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "video and py36 and not appengine" +} diff --git a/.kokoro/presubmit_tests_vision.cfg b/.kokoro/presubmit_tests_vision.cfg new file mode 100644 index 00000000000..677b768a266 --- /dev/null +++ b/.kokoro/presubmit_tests_vision.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "vision and py36 and not appengine" +} diff --git a/.kokoro/system_tests.sh b/.kokoro/system_tests.sh new file mode 100755 index 00000000000..80b9a3173e5 --- /dev/null +++ b/.kokoro/system_tests.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +cd github/python-docs-samples + +# Unencrypt and extract secrets +SECRETS_PASSWORD=$(cat "${KOKORO_GFILE_DIR}/secrets-password.txt") +./scripts/decrypt-secrets.sh "${SECRETS_PASSWORD}" + +source ./testing/test-env.sh +export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/testing/service-account.json +export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json + +# Run Cloud SQL proxy, if required +if [ -n "${CLOUD_SQL_PROXY}" ]; then + cloud_sql_proxy -instances="${MYSQL_INSTANCE}"=tcp:3306 & + cloud_sql_proxy -instances="${POSTGRES_INSTANCE}"=tcp:5432 & +fi + +# Run tests +nox -k "${NOX_SESSION}" || ret_code=$? + +if [ -n "${CLOUD_SQL_PROXY}" ]; then + killall cloud_sql_proxy || true +fi + +# Workaround for Kokoro permissions issue: delete secrets +rm testing/{test-env.sh,client-secrets.json,service-account.json} + +exit ${ret_code} diff --git a/.kokoro/system_tests_appengine.cfg b/.kokoro/system_tests_appengine.cfg new file mode 100644 index 00000000000..7fa4e15f74f --- /dev/null +++ b/.kokoro/system_tests_appengine.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "gae" +} diff --git a/.kokoro/system_tests_appengine_flexible.cfg b/.kokoro/system_tests_appengine_flexible.cfg new file mode 100644 index 00000000000..493ec4209c0 --- /dev/null +++ b/.kokoro/system_tests_appengine_flexible.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "py36 and flexible" +} diff --git a/.kokoro/system_tests_auth.cfg b/.kokoro/system_tests_auth.cfg new file mode 100644 index 00000000000..5b17f588026 --- /dev/null +++ b/.kokoro/system_tests_auth.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "auth and py36 and not appengine" +} diff --git a/.kokoro/system_tests_bigquery.cfg b/.kokoro/system_tests_bigquery.cfg new file mode 100644 index 00000000000..b7ff3c59ff4 --- /dev/null +++ b/.kokoro/system_tests_bigquery.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "bigquery and py36 and not appengine" +} diff --git a/.kokoro/system_tests_bigtable.cfg b/.kokoro/system_tests_bigtable.cfg new file mode 100644 index 00000000000..e701c9ac3bf --- /dev/null +++ b/.kokoro/system_tests_bigtable.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "bigtable and py36 and not appengine" +} diff --git a/.kokoro/system_tests_composer.cfg b/.kokoro/system_tests_composer.cfg new file mode 100644 index 00000000000..8a8b5726a61 --- /dev/null +++ b/.kokoro/system_tests_composer.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "composer and py27 and not appengine" +} diff --git a/.kokoro/system_tests_compute.cfg b/.kokoro/system_tests_compute.cfg new file mode 100644 index 00000000000..dfba68f75a1 --- /dev/null +++ b/.kokoro/system_tests_compute.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "compute and py36 and not appengine" +} diff --git a/.kokoro/system_tests_container_engine.cfg b/.kokoro/system_tests_container_engine.cfg new file mode 100644 index 00000000000..b6b3ee4b344 --- /dev/null +++ b/.kokoro/system_tests_container_engine.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "container_engine and py36 and not appengine" +} diff --git a/.kokoro/system_tests_dataproc.cfg b/.kokoro/system_tests_dataproc.cfg new file mode 100644 index 00000000000..9d035309b21 --- /dev/null +++ b/.kokoro/system_tests_dataproc.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "dataproc and py36 and not appengine" +} diff --git a/.kokoro/system_tests_datastore.cfg b/.kokoro/system_tests_datastore.cfg new file mode 100644 index 00000000000..7a950e7f6d0 --- /dev/null +++ b/.kokoro/system_tests_datastore.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "datastore and py36 and not appengine" +} diff --git a/.kokoro/system_tests_dlp.cfg b/.kokoro/system_tests_dlp.cfg new file mode 100644 index 00000000000..f5b60121400 --- /dev/null +++ b/.kokoro/system_tests_dlp.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "dlp and py36 and not appengine" +} diff --git a/.kokoro/system_tests_dns.cfg b/.kokoro/system_tests_dns.cfg new file mode 100644 index 00000000000..053c25270d3 --- /dev/null +++ b/.kokoro/system_tests_dns.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "dns and py36 and not appengine" +} diff --git a/.kokoro/system_tests_endpoints.cfg b/.kokoro/system_tests_endpoints.cfg new file mode 100644 index 00000000000..59476530bf6 --- /dev/null +++ b/.kokoro/system_tests_endpoints.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "endpoints and py36 and not appengine" +} diff --git a/.kokoro/system_tests_error_reporting.cfg b/.kokoro/system_tests_error_reporting.cfg new file mode 100644 index 00000000000..1006a421e2d --- /dev/null +++ b/.kokoro/system_tests_error_reporting.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "error_reporting and py36 and not appengine" +} diff --git a/.kokoro/system_tests_iap.cfg b/.kokoro/system_tests_iap.cfg new file mode 100644 index 00000000000..a0d1856348a --- /dev/null +++ b/.kokoro/system_tests_iap.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "iap and py36 and not appengine" +} diff --git a/.kokoro/system_tests_iot.cfg b/.kokoro/system_tests_iot.cfg new file mode 100644 index 00000000000..3a8cee6e48f --- /dev/null +++ b/.kokoro/system_tests_iot.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "iot and py36 and not appengine" +} diff --git a/.kokoro/system_tests_kms.cfg b/.kokoro/system_tests_kms.cfg new file mode 100644 index 00000000000..00cbe0dbba1 --- /dev/null +++ b/.kokoro/system_tests_kms.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "kms and py36 and not appengine" +} diff --git a/.kokoro/system_tests_language.cfg b/.kokoro/system_tests_language.cfg new file mode 100644 index 00000000000..95330fd6d5f --- /dev/null +++ b/.kokoro/system_tests_language.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "language and py36 and not appengine" +} diff --git a/.kokoro/system_tests_logging.cfg b/.kokoro/system_tests_logging.cfg new file mode 100644 index 00000000000..71e63e7993e --- /dev/null +++ b/.kokoro/system_tests_logging.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "logging and py36 and not appengine" +} diff --git a/.kokoro/system_tests_ml_engine.cfg b/.kokoro/system_tests_ml_engine.cfg new file mode 100644 index 00000000000..47070d66d10 --- /dev/null +++ b/.kokoro/system_tests_ml_engine.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "ml_engine and py36 and not appengine" +} diff --git a/.kokoro/system_tests_monitoring.cfg b/.kokoro/system_tests_monitoring.cfg new file mode 100644 index 00000000000..95f2cd2c32a --- /dev/null +++ b/.kokoro/system_tests_monitoring.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "monitoring and py36 and not appengine" +} diff --git a/.kokoro/system_tests_pubsub.cfg b/.kokoro/system_tests_pubsub.cfg new file mode 100644 index 00000000000..7ccd9b6f676 --- /dev/null +++ b/.kokoro/system_tests_pubsub.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "pubsub and py36 and not appengine" +} diff --git a/.kokoro/system_tests_spanner.cfg b/.kokoro/system_tests_spanner.cfg new file mode 100644 index 00000000000..456271c5bb2 --- /dev/null +++ b/.kokoro/system_tests_spanner.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "spanner and py36 and not appengine" +} diff --git a/.kokoro/system_tests_speech.cfg b/.kokoro/system_tests_speech.cfg new file mode 100644 index 00000000000..a12337e0443 --- /dev/null +++ b/.kokoro/system_tests_speech.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "speech and py36 and not appengine" +} diff --git a/.kokoro/system_tests_storage.cfg b/.kokoro/system_tests_storage.cfg new file mode 100644 index 00000000000..300aceb2202 --- /dev/null +++ b/.kokoro/system_tests_storage.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "storage and py36 and not appengine" +} diff --git a/.kokoro/system_tests_tasks.cfg b/.kokoro/system_tests_tasks.cfg new file mode 100644 index 00000000000..5cd413436ea --- /dev/null +++ b/.kokoro/system_tests_tasks.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "tasks and py36 and not appengine" +} diff --git a/.kokoro/system_tests_trace.cfg b/.kokoro/system_tests_trace.cfg new file mode 100644 index 00000000000..8438f4af281 --- /dev/null +++ b/.kokoro/system_tests_trace.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "trace and py36 and not appengine" +} diff --git a/.kokoro/system_tests_translate.cfg b/.kokoro/system_tests_translate.cfg new file mode 100644 index 00000000000..059bcdc1594 --- /dev/null +++ b/.kokoro/system_tests_translate.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "translate and py36 and not appengine" +} diff --git a/.kokoro/system_tests_video.cfg b/.kokoro/system_tests_video.cfg new file mode 100644 index 00000000000..917491b523a --- /dev/null +++ b/.kokoro/system_tests_video.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "video and py36 and not appengine" +} diff --git a/.kokoro/system_tests_vision.cfg b/.kokoro/system_tests_vision.cfg new file mode 100644 index 00000000000..677b768a266 --- /dev/null +++ b/.kokoro/system_tests_vision.cfg @@ -0,0 +1,15 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download secrets from Cloud Storage. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-docs-samples/.kokoro/system_tests.sh" +} + +env_vars: { + key: "NOX_SESSION" + value: "vision and py36 and not appengine" +} diff --git a/.kokoro/trampoline.sh b/.kokoro/trampoline.sh new file mode 100755 index 00000000000..fef7c24de02 --- /dev/null +++ b/.kokoro/trampoline.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +# Always run the cleanup script, regardless of the success of bouncing into +# the container. + +function cleanup() { + chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh + ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh + echo "cleanup"; +} +trap cleanup EXIT + +python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" diff --git a/.travis.yml b/.travis.yml index 8b43097f2c0..e518ee8d4d7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,29 +1,35 @@ sudo: false language: python +python: +- "3.6" services: - memcached - mysql +- redis-server branches: only: - master cache: directories: - - $HOME/.cache + - "$HOME/.cache" env: global: - - PATH=${PATH}:${HOME}/gcloud/google-cloud-sdk/bin - - GOOGLE_APPLICATION_CREDENTIALS=${TRAVIS_BUILD_DIR}/testing/resources/service-account.json - - GOOGLE_CLIENT_SECRETS=${TRAVIS_BUILD_DIR}/testing/resources/client-secrets.json - - GAE_ROOT=${HOME}/.cache/ - - secure: Orp9Et2TIwCG/Hf59aa0NUDF1pNcwcS4TFulXX175918cFREOzf/cNZNg+Ui585ZRFjbifZdc858tVuCVd8XlxQPXQgp7bwB7nXs3lby3LYg4+HD83Gaz7KOWxRLWVor6IVn8OxeCzwl6fJkdmffsTTO9csC4yZ7izHr+u7hiO4= -before_install: -- pip install --upgrade pip wheel virtualenv -- openssl aes-256-cbc -k "$secrets_password" -in secrets.tar.enc -out secrets.tar -d -- tar xvf secrets.tar + # Explicitly use choose unicode dependency to let the apache-airflow + # package installation continue. See: + # https://github.com/apache/incubator-airflow/pull/3660 + - SLUGIFY_USES_TEXT_UNIDECODE=yes + - secure: fsBH64/WqTe7lRcn4awZU7q6+euS/LHgMq2Ee2ubaoxUei2GbK5jBgnGHxOKVL5sZ4KNfTc7d6NR5BB1ZouYr2v4q1ip7Il9kFG4g5qV4cIXzHusXkrjvIzQLupNpcD9JJZr1fmYh4AqXRs2kP/nZqb7xB6Jm/O+h+aeC1bhhBg= +addons: + apt: + sources: + - deadsnakes + packages: + - portaudio19-dev + - python3.6 + - python3.6-dev install: -- pip install nox-automation coverage +- pip install --upgrade pip wheel virtualenv +- pip install --upgrade nox +- pip install --upgrade git+https://github.com/dhermes/ci-diff-helper.git script: -- source ${TRAVIS_BUILD_DIR}/testing/resources/test-env.sh -- nox --stop-on-first-error -s lint travis -after_script: -- coverage report +- "./scripts/travis.sh" diff --git a/AUTHORING_GUIDE.md b/AUTHORING_GUIDE.md new file mode 100644 index 00000000000..b0f445fc403 --- /dev/null +++ b/AUTHORING_GUIDE.md @@ -0,0 +1,491 @@ +# Python Sample Authoring Guide + +We're happy you want to write a Python sample! Like a lot of Pythonistas, we're +opinationed and fussy. This guide intends to be a reference for the format and +style expected of samples that live in +[python-docs-samples](https://github.com/GoogleCloudPlatform/python-docs-samples). + +## Canonical sample + +The [Cloud Storage Sample](https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/storage/cloud-client/snippets.py) +is a great example of what we expect from samples. It's a great sample to copy +and start from. + +## The basics + +No matter what, all samples must: + +1. Have a license header. +1. Pass lint. +1. Be either a web application or a runnable console application. +1. Have a `requirements.txt` containing all of its third-party dependencies. +1. Work in Python 2.7, 3.5, & 3.6. App Engine Standard is exempt as it + only supports 2.7. Our default version is currently Python 3.5. +1. Have tests. +1. Declare all dependencies in a `requirements.txt`. All requirements must + be pinned. + +## Style & linting + +We follow [pep8](https://www.python.org/dev/peps/pep-0008/) and the +*external* [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html) +we verify this with [flake8](https://pypi.python.org/pypi/flake8). In general: + +1. 4 spaces. +1. `CamelCase` only for classes, `snake_case` elsewhere. +1. `_` for private variables, members, functions, and methods only. Samples + should generally have very few private items. +1. `CAP_WITH_UNDERSCORES` for constants. +1. `mixedCase` is only acceptable when interface with code that doesn't follow + our style guide. +1. 79-character line limit. +1. Imports should be in three sections separated by a blank line - standard + library, third-party, and package-local. Sample will have very few + package-local imports. + +See [Automated tools](#automated-tools) for information on how to run the lint +checker. + +Beyond PEP8, there are several idioms and style nits we prefer. + +1. Use single quotes (`'`) except for docstrings (which use `"""`). +1. Typically import modules over members, for example + `from gcloud import datastore` + instead of `from gcloud.datastore import Client`. Although you should use + your best judgment, for example + `from oauth2client.contrib.flask_util import UserOAuth2` + and `from oauth2client.client import GoogleCredentials` are both totally + fine. +1. Never alias imports unless there is a name collision. +1. Use `.format()` over `%` for string interpolation. +1. Generally put a blank line above control statements that start new indented + blocks, for example: + + ```python + # Good + do_stuff() + + if other_stuff(): + more_stuff() + + # Not so good + do_stuff() + if other_stuff(): + more_stuff() + ``` + + This rule can be relaxed for counter or accumulation variables used in loops. +1. Don't use parenthesis on multiple return values (`return one, two`) or in + destructuring assignment (`one, two = some_function()`). +2. Prefer not to do hanging indents if possible. If you break at the first + logical grouping, it shouldn't be necessary. For example: + + ```python + # Good + do_some_stuff_please( + a_parameter, another_parameter, wow_so_many_parameters, + much_parameter, very_function) + + # Not so good + do_some_stuff_please(a_parameter, another_parameter, wow_so_many_parameters, + much_parameter, very_function) + ``` + + Similarly with strings and other such things: + + ```python + long_string = ( + 'Alice was beginning to get very tired of sitting by her sister on ' + 'the bank, and of having nothing to do: once or twice she had peeped ' + 'into the book her sister was reading, but it had no pictures or ...' + ) + ``` +1. Use descriptive variables names in comprehensions, for example: + + ```python + # Good + blogs = [blog for blog in bob_laws_law_blog] + + # Not so good + blogs = [x for x in bob_laws_law_blog] + ``` + +## The sample format + +In general our sample format follows ideas borrowed from +[Literate Programming](https://en.wikipedia.org/wiki/Literate_programming). +Notably, your sample program should self-contained, readable from top to bottom, +and should be fairly self-documenting. Prefer descriptive names. Use comments +and docstrings only as needed to further explain. Always introduce functions and +variables before they are used. Prefer less indirection. Prefer imperative +programming as it is easier to understand. + +### Shebang + +If, and only if, your sample application is a command-line application then +include a shebang as the first line. Separate the shebang from the rest of +the application with a blank line. The shebang should always be: + +```python +#!/usr/bin/env python +``` + +Don't include shebangs in web applications or test files. + +### License header + +All samples should start with the following (modulo shebang line): + +``` +# Copyright 2017 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +``` + +### Module-level docstring + +All samples should contain a module-level docstring. For command-line +applications, this docstring will be used as the summary when `-h` is passed. +The docstring should be succinct and should avoid repeating information +available in readmes or documentation. + +Here's a simple docstring for a command-line application with straightforward +usage: + +``` +This application demonstrates how to perform basic operations on blobs +(objects) in a Google Cloud Storage bucket. + +For more information, see the README.md under /storage and the documentation +at https://cloud.google.com/storage/docs. +``` + +Here's a docstring from a command-line application that requires a little +bit more explanation: + +```python +"""This application demonstrates how to upload and download encrypted blobs +(objects) in Google Cloud Storage. + +Use `generate-encryption-key` to generate an example key: + + python encryption.py generate-encryption-key + +Then use the key to upload and download files encrypted with a custom key. + +For more information, see the README.md under /storage and the documentation +at https://cloud.google.com/storage/docs/encryption. +""" +``` + +Finally, here's a docstring from a web application: + +```python +"""Google Cloud Endpoints sample application. + +Demonstrates how to create a simple echo API as well as how to use the +various authentication methods available. + +For more information, see the README.md under /appengine/flexible and the +documentation at https://cloud.google.com/appengine/docs/flexible. +""" +``` + +### Functions & classes + +Very few samples will require authoring classes. Prefer functions whenever +possible. See [this video](https://www.youtube.com/watch?v=o9pEzgHorH0) for +some insight into why classes aren't as necessary as you might think in Python. +Classes also introduce cognitive load. If you do write a class in a sample be +prepared to justify its existence during code review. + +Always prefer descriptive function names even if they are long. +For example `upload_file`, `upload_encrypted_file`, and `list_resource_records`. +Similarly, prefer long and descriptive parameter names. For example +`source_file_name`, `dns_zone_name`, and `base64_encryption_key`. + +Here's an example of a top-level function in a command-line application: + +```python +def list_blobs(bucket_name): + """Lists all the blobs in the bucket.""" + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + + blobs = bucket.list_blobs() + + for blob in blobs: + print(blob.name) +``` + +Notice the simple docstring and descriptive argument name (`bucket_name` +implying a string instead of just `bucket` which could imply a class instance). + +This particular function is intended to be the "top of the stack" - the function +executed when the command-line sample is run by the user. As such, notice that +it prints the blobs instead of returning. In general top of the stack +functions in command-line applications should print, but use your best +judgment. + +Here's an example of a more complicated top-level function in a command-line +application: + +```python +def download_encrypted_blob( + bucket_name, source_blob_name, destination_file_name, + base64_encryption_key): + """Downloads a previously-encrypted blob from Google Cloud Storage. + + The encryption key provided must be the same key provided when uploading + the blob. + """ + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + blob = bucket.blob(source_blob_name) + + # Encryption key must be an AES256 key represented as a bytestring with + # 32 bytes. Since it's passed in as a base64 encoded string, it needs + # to be decoded. + encryption_key = base64.b64decode(base64_encryption_key) + + blob.download_to_filename( + destination_file_name, encryption_key=encryption_key) + + print('Blob {} downloaded to {}.'.format( + source_blob_name, + destination_file_name)) +``` + +Note the verbose parameter names and the extended description that helps the +user form context. If there were more parameters or if the parameters had +complex context, then it might make sense to expand the docstring to include an +`Args` section such as: + +``` +Args: + bucket_name: The name of the cloud storage bucket. + source_blob_name: The name of the blob in the bucket to download. + destination_file_name: The blob will be downloaded to this path. + base64_encryption_key: A base64-encoded RSA256 encryption key. Must be the + same key used to encrypt the file. +``` + +Generally, however, it's rarely necessary to exhaustively document the +parameters this way. Lean towards unsurprising arguments with descriptive names, +as having to resort to this kind of docstring might be extremely accurate but +it comes at the cost of high redundancy, signal-to-noise ratio, and increased +cognitive load. + +Finally, if absolutely necessary feel free to document the type for the +parameters, for example: + +``` +Args: + credentials (google.oauth2.credentials.Credentials): Credentials authorized + for the current user. +``` + +If documenting primitive types, be sure to note if they have a particular set +of constraints, for example `A base64-encoded string` or `Must be between 0 and +10`. + +### Request handlers + +In general these follow the same rules as top-level functions. +Here's a sample function from a web application: + +```python +@app.route('/pubsub/push', methods=['POST']) +def pubsub_push(): + """Receives push notifications from Cloud Pub/Sub.""" + # Verify the token - if it's not the same token used when creating the + # notification channel then this request did not come from Pub/Sub. + if (request.args.get('token', '') != + current_app.config['PUBSUB_VERIFICATION_TOKEN']): + return 'Invalid request', 400 + + envelope = json.loads(request.data.decode('utf-8')) + payload = base64.b64decode(envelope['message']['data']) + + MESSAGES.append(payload) + + # Returning any 2xx status indicates successful receipt of the message. + return 'OK', 200 +``` + +Note the name of the function matches the URL route. The docstring is kept +simple because reading the function body reveals how the parameters are +used. + +Use `print` or `logging.info` in request handlers to print useful information +as needed. + +### Argparse section + +For command-line samples, you'll need an argparse section to handle +parsing command-line arguments and executing the sample functions. This +section lives within the `if __name__ == '__main__'` clause: + +```python +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) +``` + +Note the use of `__doc__` (the module-level docstring) as the description for +the parser. This helps you not repeat yourself and gives users useful +information when invoking the program. We also use `RawDescriptionHelpFormatter` +to prevent argparse from re-formatting the docstring. + +Command-line arguments should generally have a 1-to-1 match to function +arguments. For example: + +```python +parser.add_argument('source_file_name') +parser.add_argument('destination_blob_name') +``` + +Again, descriptive names prevent you from having to exhaustively describe +every parameter. + +Some samples demonstrate multiple functions. You should use *subparsers* to +handle this, for example: + +```python +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('bucket_name', help='Your cloud storage bucket.') + + subparsers = parser.add_subparsers(dest='command') + + subparsers.add_parser('list', help=list_blobs.__doc__) + + upload_parser = subparsers.add_parser('upload', help=upload_blob.__doc__) + upload_parser.add_argument('source_file_name') + upload_parser.add_argument('destination_blob_name') + + args = parser.parse_args() + + if args.command == 'list': + list_blobs(args.bucket_name) + elif args.command == 'upload': + upload_blob( + args.bucket_name, + args.source_file_name, + args.destination_blob_name) +``` + +### Local server + +For web application samples using Flask that don't run on App Engine Standard, +the `if __name__ == '__main__'` clause should handle starting the development +server: + +```python +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) +``` +## Writing tests + +* Use [pytest](https://docs.pytest.org/en/latest/)-style tests and plain + asserts. Don't use `unittest`-style tests or `assertX` mthods. +* All tests in this repository are **system tests**. This means they hit real + services and should use little to no mocking. +* Tests should avoid doing very strict assertions. The exact output format + from an API call can change, but as long as sample still works assertions + should pass. +* Tests will run against Python 2.7 and 3. The only exception is App Engine + standard- these samples are only be tested against Python 2.7. +* Samples that use App Engine Standard should use the App Engine testbed for + system testing. See existing App Engine tests for how to use this. + +## Running tests and automated tools + +### Installing interpreters + +You need python 2.7 and 3.6, and the dev packages for each. + +For example, to install with apt you'd use: +`apt-get install python2.7 python2.7-dev python3.6 python3.6-dev` + +### Using nox + +The testing of `python-docs-samples` is managed by +[nox](https://nox.readthedocs.io). Nox allows us to run a variety of tests, +including the linter, Python 2.7, Python 3, App Engine, and automatic README +generation. + +To use nox, install it globally with `pip`: + + $ pip install nox + +Nox automatically discovers all samples in the repository and generates three +types of sessions for *each* sample in this repository: + +1. A test sessions (`gae`, `py27` and `py35`) for running the system tests + against a specific Python version. +2. `lint` sessions for running the style linter . +3. `readmegen` sessions for regenerating READMEs. + +Because nox generates all of these sessions, it's often useful to filter down +by just the sample you're working on. For example, if you just want to see +which sessions are available for storage samples: + + $ nox -k storage -l + * gae(sample='./appengine/standard/storage/api-client') + * gae(sample='./appengine/standard/storage/appengine-client') + * lint(sample='./appengine/flexible/storage') + * lint(sample='./appengine/standard/storage/api-client') + * lint(sample='./appengine/standard/storage/appengine-client') + * lint(sample='./storage/api') + * lint(sample='./storage/cloud-client') + * lint(sample='./storage/transfer_service') + * py27(sample='./appengine/flexible/storage') + * py27(sample='./storage/api') + * py27(sample='./storage/cloud-client') + * py35(sample='./appengine/flexible/storage') + * py35(sample='./storage/api') + * py35(sample='./storage/cloud-client') + * readmegen(sample='./storage/api') + * readmegen(sample='./storage/cloud-client') + * readmegen(sample='./storage/transfer_service') + +Now you can use nox to run a specific session, for example, if you want to lint +the storage cloud-client samples: + + $ nox -s "lint(sample='./storage/cloud-client')" + +### Test environment setup + +Because all the tests here are system tests, you'll need to have a Google +Cloud project with billing enabled. Once you have this configured, you'll +need to set environment variables for the tests to be able to use your project +and its resources. See `testing/test-env.tmpl.sh` for a list of all environment +variables used by all tests. Not every test needs all of these variables. + +#### Google Cloud Storage resources + +Certain samples require integration with Google Cloud Storage (GCS), +most commonly for APIs that read files from GCS. To run the tests for +these samples, configure your GCS bucket name via the `CLOUD_STORAGE_BUCKET` +environment variable. + +The resources required by tests can usually be found in the `./resources` +folder inside the sample directory. You can upload these resources to your +own bucket to run the tests, e.g. using `gsutil`: +`gsutil cp ./resources/* gs://$CLOUD_STORAGE_BUCKET/` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cd004ed073d..36e7ffd5bd0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,11 +9,9 @@ Please fill out either the individual or corporate Contributor License Agreement (CLA). * If you are an individual writing original source code and you're sure you - own the intellectual property, then you'll need to sign an [individual CLA] - (https://developers.google.com/open-source/cla/individual). + own the intellectual property, then you'll need to sign an [individual CLA](https://developers.google.com/open-source/cla/individual). * If you work for a company that wants to allow you to contribute your work, - then you'll need to sign a [corporate CLA] - (https://developers.google.com/open-source/cla/corporate). + then you'll need to sign a [corporate CLA](https://developers.google.com/open-source/cla/corporate). Follow either of the two links above to access the appropriate CLA and instructions for how to sign and return it. Once we receive it, we'll @@ -27,13 +25,14 @@ be able to accept your pull requests. Contributor License Agreement (see details above). 1. Fork the desired repo, develop and test your code changes. 1. Ensure that your code adheres to the existing style in the sample to which - you are contributing. Refer to the - [Google Cloud Platform Samples Style Guide] - (https://github.com/GoogleCloudPlatform/Template/wiki/style.html) for the - recommended coding standards for this organization. + you are contributing. 1. Ensure that your code has an appropriate set of unit tests which all pass. 1. Submit a pull request. -## Testing +## Setting up a development environment -See [TESTING.md](TESTING.md). +* [Mac development environment guide](MAC_SETUP.md) + +## Authoring, testing, and contributing samples + +See [AUTHORING_GUIDE.md](AUTHORING_GUIDE.md). diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md new file mode 100644 index 00000000000..95b687c0151 --- /dev/null +++ b/ISSUE_TEMPLATE.md @@ -0,0 +1,12 @@ +## In which file did you encounter the issue? + + + +### Did you change the file? If so, how? + + + +## Describe the issue + + diff --git a/MAC_SETUP.md b/MAC_SETUP.md new file mode 100644 index 00000000000..c72b2151840 --- /dev/null +++ b/MAC_SETUP.md @@ -0,0 +1,113 @@ +# Setting up a Mac development environment with pyenv and pyenv-virtualenv + +In this guide, you'll set up a local Python development environment with +multiple Python versions, managed by [pyenv](https://github.com/pyenv/pyenv). + +This guide differs from the [Google Cloud Python development +instructions](https://cloud.google.com/python/setup) because developers of +samples and libraries need to be able to use multiple versions of Python to +test their code. + +## Before you begin + +1. [Optional] Install [homebrew](https://brew.sh/). + +## Installing pyenv and pyenv-virtualenv + +1. Install [pyenv](https://github.com/pyenv/pyenv). + + I (tswast@) use [homebrew](https://brew.sh/) to install it. + + ``` + brew update + brew install pyenv + ``` + +1. Install the [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv) + plugin. + + ``` + brew install pyenv-virtualenv + ``` + +1. Append the following to your `~/.bashrc`: + + ``` + eval "$(pyenv init -)" + eval "$(pyenv virtualenv-init -)" + ``` + + Note that this also works with ZSH. + +1. Reload your shell. + + ``` + source ~/.bashrc + ``` + +1. Verify that you are now using the pyenv Python shim. + + ``` + $ which python + /Users/tswast/.pyenv/shims/python + ``` + +## Installing multiple Python versions + + +1. See the available Python versions with + + ``` + pyenv install --list + ``` + + The Python versions are at the top of the long list. If the Python + version you want isn't listed, you may need to upgrade your pyenv with + homebrew. + + ``` + brew update + brew upgrade pyenv + ``` + +1. Compile the necessary Python versions with pyenv. Use the latest release + of the versions you wish to test against. + + As of August 8, 2018 my (tswast@) Python versions are: + + * 2.7.15 (latest 2.7.x release) + * 3.5.4 (latest 3.5.x release) + * 3.6.4 (latest 3.6.x release) + * 3.7.0 (latest 3.7.x release) + +## Using pyenv and pyenv-virtualenv to manage your Python versions + +1. Change to the desired source directory. + + ``` + cd ~/src/python-docs-samples + ``` + +1. Create a virtualenv using `pyenv virtualenv`. + + ``` + pyenv virtualenv 3.6.4 python-docs-samples + ``` + + This creates a virtualenv folder within `~/.pyenv/versions/`. + +1. Set the local Python version(s) with `pyenv local` + + ``` + # pyenv local [name of virtualenv] [list of python versions to use] + pyenv local python-docs-samples 3.6.4 3.7.0 3.5.4 2.7.15 + ``` + +1. Now, when you `cd` into the source directory or a subdirectory within it, + pyenv will make your virtualenv the default Python. Since you specified + more than one version, it will also add binaries like `python36` and + `python27` to your PATH, which nox uses when picking Python interpreters. + +1. Add `.python-version` to your [global gitignore + file](https://help.github.com/articles/ignoring-files/#create-a-global-gitignore), + so it wont be committed into the repository. diff --git a/README.md b/README.md index 935b2b78926..76f46ce2f83 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ ## Google Cloud Platform Python Samples +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=./README.md + This repository holds the samples used in the python documentation on [cloud.google.com](https://cloud.google.com). [![Build Status](https://travis-ci.org/GoogleCloudPlatform/python-docs-samples.svg)](https://travis-ci.org/GoogleCloudPlatform/python-docs-samples) diff --git a/TESTING.md b/TESTING.md deleted file mode 100644 index fc04d1fdb59..00000000000 --- a/TESTING.md +++ /dev/null @@ -1,84 +0,0 @@ -# Testing - -The tests in this repository are system tests and run against live services, therefore, it takes a bit of configuration to run all of the tests locally. - -Before you can run tests locally you must have: - -* The latest [nox](https://nox.readthedocs.org/en/latest/), - [pip](https://pypi.python.org/pypi/pip), and [gcp-python-repo-tools](https://pypi.python.org/pypi/gcp-python-repo-tools) installed. - - $ sudo pip install --upgrade nox-automation pip gcp-python-repo-tools - -* The [Google Cloud SDK](https://cloud.google.com/sdk/) installed. You - can do so with the following command: - - $ curl https://sdk.cloud.google.com | bash - -## Preparing a project for testing - -Most tests require you to have an active, billing-enabled project on the -[Google Cloud Console](https://console.cloud.google.com). - -### Creating resources - -Some resources need to be created in a project ahead of time before testing. We have a script that can create everything needed: - - gcloud config set project - scripts/prepare-testing-project.sh - -The script will also instruct you to follow a URL to enable APIs. You will need to do that. - -### Getting a service account key - -From the Cloud Console, create a new Service Account and download its json key. Place this file in `testing/resources/service-account.json`. - -## Environment variables - -* Copy `testing/resources/test-env.tmpl.sh` to `testing/resources/test-env.sh`, and updated it with your configuration. -* Run `source testing/resources/test-env.sh`. -* Run `export GOOGLE_APPLICATION_CREDENTIALS=testing/resources/service-account.json`. - -If you want to run the Google App Engine tests, you will need: - -* The App Engine Python SDK. You can also download it programatically with `gcp-python-repo-tools`: - - $ gcp-python-repo-tools download-appengine-sdk - -* Set the `GAE_PYTHONPATH` variable: - - $ export GAE_PYTHONPATH= - -### Test environments - -We use [nox](https://nox.readthedocs.org/en/latest/) to configure -multiple python sessions: - -* ``tests`` contains tests for samples that run in a normal Python 2.7 or 3.4 - environment. This is everything outside of the ``appengine`` directory. It's - parameterized to run all the tests using the 2.7 and 3.4 interpreters. -* ``gae`` contains tests for samples that run only in Google App Engine. This is - (mostly) everything in the ``appengine`` directory. -* ``lint`` just runs the linter. - -To see a list of the available sessions: - - nox -l - -To run tests for a particular session, with a particular parameter, invoke nox -with the ``-s`` flag: - - nox -s "tests(interpreter='python2.7')" - -To run one particular session or provide additional parameters to ``py.test``, -invoke nox like this: - - nox -s tests -- storage/api - -### Adding new tests -When adding a new top-level directory, be sure to edit ``.coveragerc`` to -include it in coverage reporting. - -To add new tests that require Google App Engine, please place them in -the ``appengine`` directory if possible. If you place them elsewhere, -you will need to modify ``nox.py`` to make the environments -appropriately run or ignore your test. diff --git a/appengine/README.md b/appengine/README.md deleted file mode 100644 index d95a5e6d8fe..00000000000 --- a/appengine/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# Google App Engine Samples - -This section contains samples for [Google Cloud Storage](https://cloud.google.com/storage). Most of these samples have associated documentation that is linked -within the docstring of the sample itself. - -## Running the samples locally - -1. Download the [Google App Engine Python SDK](https://cloud.google.com/appengine/downloads) for your platform. -2. Many samples require extra libraries to be installed. If there is a `requirements.txt`, you will need to install the dependencies with [`pip`](pip.readthedocs.org). - - pip install -t lib -r requirements.txt - -3. Use `dev_appserver.py` to run the sample: - - dev_appserver.py app.yaml - -4. Visit `http://localhost:8080` to view your application. - -Some samples may require additional setup. Refer to individual sample READMEs. - -## Deploying the samples - -1. Download the [Google App Engine Python SDK](https://cloud.google.com/appengine/downloads) for your platform. -2. Many samples require extra libraries to be installed. If there is a `requirements.txt`, you will need to install the dependencies with [`pip`](pip.readthedocs.org). - - pip install -t lib -r requirements.txt - -3. Use `appcfg.py` to deploy the sample, you will need to specify your Project ID and a version number: - - appcfg.py update -A your-app-id -V your-version app.yaml - -4. Visit `https://your-app-id.appost.com` to view your application. - -## Additional resources - -For more information on App Engine: - -> https://cloud.google.com/appengine - -For more information on Python on App Engine: - -> https://cloud.google.com/appengine/docs/python diff --git a/appengine/angular/README.md b/appengine/angular/README.md deleted file mode 100644 index 4e3c3539b85..00000000000 --- a/appengine/angular/README.md +++ /dev/null @@ -1,5 +0,0 @@ -## App Engine & Angular JS - -A simple [AngularJS](http://angularjs.org/) CRUD application for [Google App Engine](https://appengine.google.com/). - -Refer to the [App Engine Samples README](../README.md) for information on how to run and deploy this sample. diff --git a/appengine/angular/app.yaml b/appengine/angular/app.yaml deleted file mode 100644 index 56ae94266f9..00000000000 --- a/appengine/angular/app.yaml +++ /dev/null @@ -1,21 +0,0 @@ -application: your-app-id -version: 1 -runtime: python27 -threadsafe: true -api_version: 1 - -handlers: -- url: /favicon\.ico - static_files: favicon.ico - upload: favicon\.ico - -- url: /rest/.* - script: main.APP - -- url: (.*)/ - static_files: app\1/index.html - upload: app - -- url: (.*) - static_files: app\1 - upload: app diff --git a/appengine/angular/main.py b/appengine/angular/main.py deleted file mode 100644 index 3380d304a1a..00000000000 --- a/appengine/angular/main.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2013 Google, Inc -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json - -import model - -import webapp2 - - -def AsDict(guest): - return {'id': guest.key.id(), 'first': guest.first, 'last': guest.last} - - -class RestHandler(webapp2.RequestHandler): - - def dispatch(self): - # time.sleep(1) - super(RestHandler, self).dispatch() - - def SendJson(self, r): - self.response.headers['content-type'] = 'text/plain' - self.response.write(json.dumps(r)) - - -class QueryHandler(RestHandler): - - def get(self): - guests = model.AllGuests() - r = [AsDict(guest) for guest in guests] - self.SendJson(r) - - -class UpdateHandler(RestHandler): - - def post(self): - r = json.loads(self.request.body) - guest = model.UpdateGuest(r['id'], r['first'], r['last']) - r = AsDict(guest) - self.SendJson(r) - - -class InsertHandler(RestHandler): - - def post(self): - r = json.loads(self.request.body) - guest = model.InsertGuest(r['first'], r['last']) - r = AsDict(guest) - self.SendJson(r) - - -class DeleteHandler(RestHandler): - - def post(self): - r = json.loads(self.request.body) - model.DeleteGuest(r['id']) - - -APP = webapp2.WSGIApplication([ - ('/rest/query', QueryHandler), - ('/rest/insert', InsertHandler), - ('/rest/delete', DeleteHandler), - ('/rest/update', UpdateHandler), -], debug=True) diff --git a/appengine/app_identity/signing/main.py b/appengine/app_identity/signing/main.py deleted file mode 100644 index 7e325d8315c..00000000000 --- a/appengine/app_identity/signing/main.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Sample Google App Engine application that demonstrates usage of the app -identity API. - -For more information about App Engine, see README.md under /appengine. -""" - -# [START all] - -import base64 - -from Crypto.Hash import SHA256 -from Crypto.PublicKey import RSA -from Crypto.Signature import PKCS1_v1_5 -from Crypto.Util.asn1 import DerSequence -from google.appengine.api import app_identity -import webapp2 - - -def verify_signature(data, signature, x509_certificate): - """Verifies a signature using the given x.509 public key certificate.""" - - # PyCrypto 2.6 doesn't support x.509 certificates directly, so we'll need - # to extract the public key from it manually. - # This code is based on https://github.com/google/oauth2client/blob/master - # /oauth2client/_pycrypto_crypt.py - pem_lines = x509_certificate.replace(b' ', b'').split() - cert_der = base64.urlsafe_b64decode(b''.join(pem_lines[1:-1])) - cert_seq = DerSequence() - cert_seq.decode(cert_der) - tbs_seq = DerSequence() - tbs_seq.decode(cert_seq[0]) - public_key = RSA.importKey(tbs_seq[6]) - - signer = PKCS1_v1_5.new(public_key) - digest = SHA256.new(data) - - return signer.verify(digest, signature) - - -def verify_signed_by_app(data, signature): - """Checks the signature and data against all currently valid certificates - for the application.""" - public_certificates = app_identity.get_public_certificates() - - for cert in public_certificates: - if verify_signature(data, signature, cert.x509_certificate_pem): - return True - - return False - - -class MainPage(webapp2.RequestHandler): - def get(self): - message = 'Hello, world!' - signing_key_name, signature = app_identity.sign_blob(message) - verified = verify_signed_by_app(message, signature) - - self.response.content_type = 'text/plain' - self.response.write('Message: {}\n'.format(message)) - self.response.write( - 'Signature: {}\n'.format(base64.b64encode(signature))) - self.response.write('Verified: {}\n'.format(verified)) - - -app = webapp2.WSGIApplication([ - ('/', MainPage) -], debug=True) - -# [END all] diff --git a/appengine/app_identity/signing/main_test.py b/appengine/app_identity/signing/main_test.py deleted file mode 100644 index 2c2fdbc7bc8..00000000000 --- a/appengine/app_identity/signing/main_test.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import main -import webtest - - -def test_app(testbed): - app = webtest.TestApp(main.app) - response = app.get('/') - assert response.status_int == 200 - assert 'Verified: True' in response.text diff --git a/appengine/bigquery/README.md b/appengine/bigquery/README.md deleted file mode 100644 index 0dc4b7cdb56..00000000000 --- a/appengine/bigquery/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# Google App Engine accessing BigQuery using OAuth2 - -This sample demonstrates [authenticating to BigQuery in App Engine using OAuth2](https://cloud.google.com/bigquery/authentication). - - -These samples are used on the following documentation pages: - -> -* https://cloud.google.com/bigquery/authentication -* https://cloud.google.com/monitoring/api/authentication - - - -Refer to the [App Engine Samples README](../README.md) for information on how to run and deploy this sample. - -## Setup - -1. You'll need a client id for your project. Follow [these instructions](https://cloud.google.com/bigquery/authentication#clientsecrets). Once you've downloaded the client's json secret copy it into the sample directory and rename it to `client_secrets.json`. - -2. Update `main.py` and replace `` with your project's ID. diff --git a/appengine/bigquery/app.yaml b/appengine/bigquery/app.yaml deleted file mode 100644 index 6fde9bebf2f..00000000000 --- a/appengine/bigquery/app.yaml +++ /dev/null @@ -1,9 +0,0 @@ -application: cloud-samples-tests -version: 1 -runtime: python27 -api_version: 1 -threadsafe: yes - -handlers: -- url: .* - script: main.app diff --git a/appengine/bigquery/client_secrets.json b/appengine/bigquery/client_secrets.json deleted file mode 100644 index 767caf645e6..00000000000 --- a/appengine/bigquery/client_secrets.json +++ /dev/null @@ -1,3 +0,0 @@ -{"web":{ - "client_id":"NOTE: this is just a placeholder for unit tests. See the README for what to replace this file with.", - "auth_uri":"TODO","token_uri":"TODO","auth_provider_x509_cert_url":"TODO","client_email":"","client_x509_cert_url":"","client_secret":"TODO","redirect_uris":["TODO","TODO"],"javascript_origins":["TODO","TODO"]}} diff --git a/appengine/bigquery/main.py b/appengine/bigquery/main.py deleted file mode 100644 index 1c035272ce7..00000000000 --- a/appengine/bigquery/main.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START all] - -""" -Sample App Engine application that demonstrates authentication to BigQuery -using User OAuth2 as opposed to OAuth2 Service Accounts. - -For more information, see README.md. -""" - -import json -import os - -from googleapiclient.discovery import build -from oauth2client.appengine import OAuth2DecoratorFromClientSecrets -import webapp2 - - -# The project id whose datasets you'd like to list -PROJECTID = '' - -# Create the method decorator for oauth. -decorator = OAuth2DecoratorFromClientSecrets( - os.path.join(os.path.dirname(__file__), 'client_secrets.json'), - scope='https://www.googleapis.com/auth/bigquery') - -# Create the bigquery api client -service = build('bigquery', 'v2') - - -class MainPage(webapp2.RequestHandler): - - # oauth_required ensures that the user goes through the OAuth2 - # authorization flow before reaching this handler. - @decorator.oauth_required - def get(self): - # This is an httplib2.Http instance that is signed with the user's - # credentials. This allows you to access the BigQuery API on behalf - # of the user. - http = decorator.http() - - response = service.datasets().list(projectId=PROJECTID).execute(http) - - self.response.out.write('

Datasets.list raw response:

') - self.response.out.write('
%s
' % - json.dumps(response, sort_keys=True, indent=4, - separators=(',', ': '))) - - -app = webapp2.WSGIApplication([ - ('/', MainPage), - # Create the endpoint to receive oauth flow callbacks - (decorator.callback_path, decorator.callback_handler()) -], debug=True) -# [END all] diff --git a/appengine/bigquery/main_test.py b/appengine/bigquery/main_test.py deleted file mode 100644 index 77e192b2dac..00000000000 --- a/appengine/bigquery/main_test.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from googleapiclient.http import HttpMock -import main -import mock -import pytest -import webtest - - -@pytest.fixture -def app(cloud_config, testbed): - main.PROJECTID = cloud_config.project - return webtest.TestApp(main.app) - - -def test_anonymous(app): - response = app.get('/') - - # Should redirect to login - assert response.status_int == 302 - assert re.search(r'.*accounts.*Login.*', response.headers['Location']) - - -def test_loggedin(app, login): - login() - - response = app.get('/') - - # Should redirect to oauth2 - assert response.status_int == 302 - assert re.search(r'.*oauth2.*', response.headers['Location']) - - -def test_oauthed(resource, app, login): - login() - - mock_http = HttpMock( - resource('datasets-list.json'), - {'status': '200'}) - - with mock.patch.object(main.decorator, 'http', return_value=mock_http): - with mock.patch.object( - main.decorator, 'has_credentials', return_value=True): - response = app.get('/') - - # Should make the api call - assert response.status_int == 200 - assert re.search( - re.compile(r'.*datasets.*datasetReference.*etag.*', re.DOTALL), - response.body) diff --git a/appengine/bigquery/requirements.txt b/appengine/bigquery/requirements.txt deleted file mode 100644 index c3b2784ce87..00000000000 --- a/appengine/bigquery/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -google-api-python-client==1.5.0 diff --git a/appengine/bigquery/resources/datasets-list.json b/appengine/bigquery/resources/datasets-list.json deleted file mode 100644 index c9a3877c4db..00000000000 --- a/appengine/bigquery/resources/datasets-list.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "datasets": [ - { - "datasetReference": { - "datasetId": "test_dataset_java", - "projectId": "cloud-samples-tests" - }, - "id": "cloud-samples-tests:test_dataset_java", - "kind": "bigquery#dataset" - } - ], - "etag": "\"ZduQht1tG1odVP6IPm66xfuN2eI/HmGRlylAN_zCB6N4JDeX_XDO0R0\"", - "kind": "bigquery#datasetList" -} diff --git a/appengine/blobstore/README.md b/appengine/blobstore/README.md deleted file mode 100644 index d708d72cbf1..00000000000 --- a/appengine/blobstore/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# App Engine Blobstore Sample - - -These samples are used on the following documentation page: - -> https://cloud.google.com/appengine/docs/python/blobstore/ - - - --> diff --git a/appengine/blobstore/main.py b/appengine/blobstore/main.py deleted file mode 100644 index 841cbf1701d..00000000000 --- a/appengine/blobstore/main.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Sample application that demonstrates how to use the App Engine Blobstore API. - -For more information, see README.md. -""" - -# [START all] -from google.appengine.api import users -from google.appengine.ext import blobstore -from google.appengine.ext import ndb -from google.appengine.ext.webapp import blobstore_handlers -import webapp2 - - -# This datastore model keeps track of which users uploaded which photos. -class UserPhoto(ndb.Model): - user = ndb.StringProperty() - blob_key = ndb.BlobKeyProperty() - - -class PhotoUploadFormHandler(webapp2.RequestHandler): - def get(self): - # [START upload_url] - upload_url = blobstore.create_upload_url('/upload_photo') - # [END upload_url] - # [START upload_form] - # To upload files to the blobstore, the request method must be "POST" - # and enctype must be set to "multipart/form-data". - self.response.out.write(""" - -
- Upload File:
- -
-""".format(upload_url)) - # [END upload_form] - - -# [START upload_handler] -class PhotoUploadHandler(blobstore_handlers.BlobstoreUploadHandler): - def post(self): - try: - upload = self.get_uploads()[0] - user_photo = UserPhoto( - user=users.get_current_user().user_id(), - blob_key=upload.key()) - user_photo.put() - - self.redirect('/view_photo/%s' % upload.key()) - - except: - self.error(500) -# [END upload_handler] - - -# [START download_handler] -class ViewPhotoHandler(blobstore_handlers.BlobstoreDownloadHandler): - def get(self, photo_key): - if not blobstore.get(photo_key): - self.error(404) - else: - self.send_blob(photo_key) -# [END download_handler] - - -app = webapp2.WSGIApplication([ - ('/', PhotoUploadFormHandler), - ('/upload_photo', PhotoUploadHandler), - ('/view_photo/([^/]+)?', ViewPhotoHandler), -], debug=True) -# [END all] diff --git a/appengine/blobstore/main_test.py b/appengine/blobstore/main_test.py deleted file mode 100644 index 2b62074dae4..00000000000 --- a/appengine/blobstore/main_test.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import main -import webtest - - -def test_app(testbed, login): - app = webtest.TestApp(main.app) - - login() - response = app.get('/') - - assert '/_ah/upload' in response diff --git a/appengine/cloudsql/README.md b/appengine/cloudsql/README.md deleted file mode 100644 index a87d4f1e013..00000000000 --- a/appengine/cloudsql/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Using Cloud SQL from Google App Engine - -This is an example program showing how to use the native MySQL connections from Google App Engine to [Google Cloud SQL](https://cloud.google.com/sql). - -Refer to the [App Engine Samples README](../README.md) for information on how to run and deploy this sample. - -## Setup - -1. You will need to create a [Cloud SQL instance](https://cloud.google.com/sql/docs/create-instance). - -2. Edit the `CLOUDSQL_INSTANCE` and `CLOUDSQL_PROJECT` values in `main.py`. - -3. To run locally, you will need to be running a local instance of MySQL. You may need to update the connection code in `main.py` with the appropriate local username and password. diff --git a/appengine/cloudsql/app.yaml b/appengine/cloudsql/app.yaml deleted file mode 100644 index 58843564e0e..00000000000 --- a/appengine/cloudsql/app.yaml +++ /dev/null @@ -1,11 +0,0 @@ -runtime: python27 -api_version: 1 -threadsafe: yes - -handlers: -- url: / - script: main.app - -libraries: -- name: MySQLdb - version: "latest" diff --git a/appengine/cloudsql/main.py b/appengine/cloudsql/main.py deleted file mode 100644 index 1f70a0470f6..00000000000 --- a/appengine/cloudsql/main.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2013 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Sample App Engine application demonstrating how to connect to Google Cloud SQL -using App Engine's native unix socket. - -For more information, see the README.md. -""" - -import os - -import MySQLdb -import webapp2 - - -CLOUDSQL_PROJECT = '' -CLOUDSQL_INSTANCE = '' - - -class MainPage(webapp2.RequestHandler): - def get(self): - self.response.headers['Content-Type'] = 'text/plain' - - # When running on Google App Engine, use the special unix socket - # to connect to Cloud SQL. - if os.getenv('SERVER_SOFTWARE', '').startswith('Google App Engine/'): - db = MySQLdb.connect( - unix_socket='/cloudsql/{}:{}'.format( - CLOUDSQL_PROJECT, - CLOUDSQL_INSTANCE), - user='root') - # When running locally, you can either connect to a local running - # MySQL instance, or connect to your Cloud SQL instance over TCP. - else: - db = MySQLdb.connect(host='localhost', user='root') - - cursor = db.cursor() - cursor.execute('SHOW VARIABLES') - - for r in cursor.fetchall(): - self.response.write('{}\n'.format(r)) - - -app = webapp2.WSGIApplication([ - ('/', MainPage), -], debug=True) diff --git a/appengine/cloudsql/main_test.py b/appengine/cloudsql/main_test.py deleted file mode 100644 index f2debea7a96..00000000000 --- a/appengine/cloudsql/main_test.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import re - -import main -import pytest -import webtest - - -@pytest.mark.skipif( - not os.path.exists('/var/run/mysqld/mysqld.sock'), - reason='MySQL server not available.') -def test_app(): - app = webtest.TestApp(main.app) - response = app.get('/') - - assert response.status_int == 200 - assert re.search( - re.compile(r'.*version.*', re.DOTALL), - response.body) diff --git a/appengine/conftest.py b/appengine/conftest.py deleted file mode 100644 index 4fd85dcc906..00000000000 --- a/appengine/conftest.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Import py.test hooks and fixtures for App Engine -from gcp.testing.appengine import ( - login, - pytest_configure, - pytest_runtest_call, - run_tasks, - testbed) - -(login) -(pytest_configure) -(pytest_runtest_call) -(run_tasks) -(testbed) diff --git a/managed_vms/.gitignore b/appengine/flexible/.gitignore similarity index 100% rename from managed_vms/.gitignore rename to appengine/flexible/.gitignore diff --git a/appengine/flexible/README.md b/appengine/flexible/README.md new file mode 100644 index 00000000000..c2182a219f1 --- /dev/null +++ b/appengine/flexible/README.md @@ -0,0 +1,73 @@ +## Google App Engine Flexible Environment Python Samples + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/flexible/README.md + +These are samples for using Python on Google App Engine Flexible Environment. These samples are typically referenced from the [docs](https://cloud.google.com/appengine/docs). + +See our other [Google Cloud Platform github repos](https://github.com/GoogleCloudPlatform) for sample applications and +scaffolding for other frameworks and use cases. + +## Run Locally + +Some samples have specific instructions. If there is a README in the sample folder, pleaese refer to it for any additional steps required to run the sample. + +In general, the samples typically require: + +1. Install the [Google Cloud SDK](https://cloud.google.com/sdk/), including the [gcloud tool](https://cloud.google.com/sdk/gcloud/), and [gcloud app component](https://cloud.google.com/sdk/gcloud-app). + +2. Setup the gcloud tool. This provides authentication to Google Cloud APIs and services. + + ``` + gcloud init + ``` + +3. Clone this repo. + + ``` + git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + cd python-docs-samples/appengine/flexible + ``` + +4. Open a sample folder, create a virtualenv, install dependencies, and run the sample: + + ``` + cd hello-world + virtualenv env + source env/bin/activate + pip install -r requirements.txt + python main.py + ``` + +5. Visit the application at [http://localhost:8080](http://localhost:8080). + + +## Deploying + +Some samples in this repositories may have special deployment instructions. Refer to the readme in the sample directory. + +1. Use the [Google Developers Console](https://console.developer.google.com) to create a project/app id. (App id and project id are identical) + +2. Setup the gcloud tool, if you haven't already. + + ``` + gcloud init + ``` + +3. Use gcloud to deploy your app. + + ``` + gcloud app deploy + ``` + +4. Congratulations! Your application is now live at `your-app-id.appspot.com` + +## Contributing changes + +* See [CONTRIBUTING.md](../CONTRIBUTING.md) + +## Licensing + +* See [LICENSE](../LICENSE) diff --git a/appengine/flexible/analytics/README.md b/appengine/flexible/analytics/README.md new file mode 100644 index 00000000000..2646e144ed0 --- /dev/null +++ b/appengine/flexible/analytics/README.md @@ -0,0 +1,25 @@ +# Google Analytics Measurement Protocol sample for Google App Engine Flexible + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/flexible/analytics/README.md + +This sample demonstrates how to use the [Google Analytics Measurement Protocol](https://developers.google.com/analytics/devguides/collection/protocol/v1/) (or any other SQL server) on [Google App Engine Flexible Environment](https://cloud.google.com/appengine). + +## Setup + +Before you can run or deploy the sample, you will need to do the following: + +1. Create a Google Analytics Property and obtain the Tracking ID. + +2. Update the environment variables in in ``app.yaml`` with your Tracking ID. + +## Running locally + +Refer to the [top-level README](../README.md) for instructions on running and deploying. + +You will need to set the following environment variables via your shell before running the sample: + + $ export GA_TRACKING_ID=[your Tracking ID] + $ python main.py diff --git a/appengine/flexible/analytics/app.yaml b/appengine/flexible/analytics/app.yaml new file mode 100644 index 00000000000..e4d6df6b018 --- /dev/null +++ b/appengine/flexible/analytics/app.yaml @@ -0,0 +1,11 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 + +#[START gae_flex_analytics_env_variables] +env_variables: + GA_TRACKING_ID: your-tracking-id +#[END gae_flex_analytics_env_variables] diff --git a/appengine/flexible/analytics/main.py b/appengine/flexible/analytics/main.py new file mode 100644 index 00000000000..034f66f8222 --- /dev/null +++ b/appengine/flexible/analytics/main.py @@ -0,0 +1,74 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START gae_flex_analytics_track_event] +import logging +import os + +from flask import Flask +import requests + + +app = Flask(__name__) + + +# Environment variables are defined in app.yaml. +GA_TRACKING_ID = os.environ['GA_TRACKING_ID'] + + +def track_event(category, action, label=None, value=0): + data = { + 'v': '1', # API Version. + 'tid': GA_TRACKING_ID, # Tracking ID / Property ID. + # Anonymous Client Identifier. Ideally, this should be a UUID that + # is associated with particular user, device, or browser instance. + 'cid': '555', + 't': 'event', # Event hit type. + 'ec': category, # Event category. + 'ea': action, # Event action. + 'el': label, # Event label. + 'ev': value, # Event value, must be an integer + } + + response = requests.post( + 'https://www.google-analytics.com/collect', data=data) + + # If the request fails, this will raise a RequestException. Depending + # on your application's needs, this may be a non-error and can be caught + # by the caller. + response.raise_for_status() + + +@app.route('/') +def track_example(): + track_event( + category='Example', + action='test action') + return 'Event tracked.' + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
{}
+ See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) +# [END gae_flex_analytics_track_event] diff --git a/appengine/flexible/analytics/main_test.py b/appengine/flexible/analytics/main_test.py new file mode 100644 index 00000000000..490e43ea967 --- /dev/null +++ b/appengine/flexible/analytics/main_test.py @@ -0,0 +1,47 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +import pytest +import responses + + +@pytest.fixture +def app(monkeypatch): + monkeypatch.setenv('GA_TRACKING_ID', '1234') + + import main + + main.app.testing = True + return main.app.test_client() + + +@responses.activate +def test_tracking(app): + responses.add( + responses.POST, + re.compile(r'.*'), + body='{}', + content_type='application/json') + + r = app.get('/') + + assert r.status_code == 200 + assert 'Event tracked' in r.data.decode('utf-8') + + assert len(responses.calls) == 1 + request_body = responses.calls[0].request.body + assert 'tid=1234' in request_body + assert 'ea=test+action' in request_body diff --git a/appengine/flexible/analytics/requirements.txt b/appengine/flexible/analytics/requirements.txt new file mode 100644 index 00000000000..23558464899 --- /dev/null +++ b/appengine/flexible/analytics/requirements.txt @@ -0,0 +1,3 @@ +Flask==1.0.2 +gunicorn==19.9.0 +requests[security]==2.21.0 diff --git a/appengine/flexible/cloudsql/README.md b/appengine/flexible/cloudsql/README.md new file mode 100644 index 00000000000..947ff96ddb1 --- /dev/null +++ b/appengine/flexible/cloudsql/README.md @@ -0,0 +1,57 @@ +# Python Google Cloud SQL sample for Google App Engine Flexible + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/flexible/cloudsql/README.md + +This sample demonstrates how to use [Google Cloud SQL](https://cloud.google.com/sql/) (or any other SQL server) on [Google App Engine Flexible](https://cloud.google.com/appengine). + +## Setup + +Before you can run or deploy the sample, you will need to do the following: + +1. Create a [Second Generation Cloud SQL](https://cloud.google.com/sql/docs/create-instance) instance. You can do this from the [Cloud Console](https://console.developers.google.com) or via the [Cloud SDK](https://cloud.google.com/sdk). To create it via the SDK use the following command: + + $ gcloud sql instances create YOUR_INSTANCE_NAME \ + --activation-policy=ALWAYS \ + --tier=db-n1-standard-1 + +1. Set the root password on your Cloud SQL instance: + + $ gcloud sql instances set-root-password YOUR_INSTANCE_NAME --password YOUR_INSTANCE_ROOT_PASSWORD + +1. Create a [Service Account](https://cloud.google.com/sql/docs/external#createServiceAccount) for your project. You'll use this service account to connect to your Cloud SQL instance locally. + +1. Download the [Cloud SQL Proxy](https://cloud.google.com/sql/docs/sql-proxy). + +1. Run the proxy to allow connecting to your instance from your machine. + + $ cloud_sql_proxy \ + -dir /tmp/cloudsql \ + -instances=YOUR_PROJECT_ID:us-central1:YOUR_INSTANCE_NAME=tcp:3306 \ + -credential_file=PATH_TO_YOUR_SERVICE_ACCOUNT_JSON + +1. Use the MySQL command line tools (or a management tool of your choice) to create a [new user](https://cloud.google.com/sql/docs/create-user) and [database](https://cloud.google.com/sql/docs/create-database) for your application: + + $ mysql -h 127.0.0.1 -u root -p + mysql> create database YOUR_DATABASE; + mysql> create user 'YOUR_USER'@'%' identified by 'PASSWORD'; + mysql> grant all on YOUR_DATABASE.* to 'YOUR_USER'@'%'; + +1. Set the connection string environment variable. This allows the app to connect to your Cloud SQL instance through the proxy: + + export SQLALCHEMY_DATABASE_URI=mysql+pymysql://USER:PASSWORD@127.0.0.1/YOUR_DATABASE + +1. Run ``create_tables.py`` to ensure that the database is properly configured and to create the tables needed for the sample. + +1. Update the connection string in ``app.yaml`` with your configuration values. These values are used when the application is deployed. + +## Running locally + +Refer to the [top-level README](../README.md) for instructions on running and deploying. + +It's recommended to follow the instructions above to run the Cloud SQL proxy. You will need to set the following environment variables via your shell before running the sample: + + $ export SQLALCHEMY_DATABASE_URI=[your connection string] + $ python main.py diff --git a/appengine/flexible/cloudsql/app.yaml b/appengine/flexible/cloudsql/app.yaml new file mode 100644 index 00000000000..4d53bd6aa01 --- /dev/null +++ b/appengine/flexible/cloudsql/app.yaml @@ -0,0 +1,21 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 + +#[START gae_flex_mysql_env] +env_variables: + # Replace user, password, database, and instance connection name with the values obtained + # when configuring your Cloud SQL instance. + SQLALCHEMY_DATABASE_URI: >- + mysql+pymysql://USER:PASSWORD@/DATABASE?unix_socket=/cloudsql/INSTANCE_CONNECTION_NAME +#[END gae_flex_mysql_env] + +#[START gae_flex_mysql_settings] +# Replace project and instance with the values obtained when configuring your +# Cloud SQL instance. +beta_settings: + cloud_sql_instances: INSTANCE_CONNECTION_NAME +#[END gae_flex_mysql_settings] diff --git a/appengine/flexible/cloudsql/create_tables.py b/appengine/flexible/cloudsql/create_tables.py new file mode 100755 index 00000000000..acb8c563f12 --- /dev/null +++ b/appengine/flexible/cloudsql/create_tables.py @@ -0,0 +1,22 @@ +#! /usr/bin/env python +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from main import db + + +if __name__ == '__main__': + print('Creating all database tables...') + db.create_all() + print('Done!') diff --git a/appengine/flexible/cloudsql/main.py b/appengine/flexible/cloudsql/main.py new file mode 100644 index 00000000000..832cb4f246a --- /dev/null +++ b/appengine/flexible/cloudsql/main.py @@ -0,0 +1,97 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import logging +import os +import socket + +from flask import Flask, request +from flask_sqlalchemy import SQLAlchemy +import sqlalchemy + + +app = Flask(__name__) + + +def is_ipv6(addr): + """Checks if a given address is an IPv6 address.""" + try: + socket.inet_pton(socket.AF_INET6, addr) + return True + except socket.error: + return False + + +# [START gae_flex_mysql_app] +# Environment variables are defined in app.yaml. +app.config['SQLALCHEMY_DATABASE_URI'] = os.environ['SQLALCHEMY_DATABASE_URI'] +app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False + +db = SQLAlchemy(app) + + +class Visit(db.Model): + id = db.Column(db.Integer, primary_key=True) + timestamp = db.Column(db.DateTime()) + user_ip = db.Column(db.String(46)) + + def __init__(self, timestamp, user_ip): + self.timestamp = timestamp + self.user_ip = user_ip + + +@app.route('/') +def index(): + user_ip = request.remote_addr + + # Keep only the first two octets of the IP address. + if is_ipv6(user_ip): + user_ip = ':'.join(user_ip.split(':')[:2]) + else: + user_ip = '.'.join(user_ip.split('.')[:2]) + + visit = Visit( + user_ip=user_ip, + timestamp=datetime.datetime.utcnow() + ) + + db.session.add(visit) + db.session.commit() + + visits = Visit.query.order_by(sqlalchemy.desc(Visit.timestamp)).limit(10) + + results = [ + 'Time: {} Addr: {}'.format(x.timestamp, x.user_ip) + for x in visits] + + output = 'Last 10 visits:\n{}'.format('\n'.join(results)) + + return output, 200, {'Content-Type': 'text/plain; charset=utf-8'} +# [END gae_flex_mysql_app] + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
{}
+ See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/managed_vms/cloudsql/main_test.py b/appengine/flexible/cloudsql/main_test.py similarity index 100% rename from managed_vms/cloudsql/main_test.py rename to appengine/flexible/cloudsql/main_test.py diff --git a/appengine/flexible/cloudsql/requirements.txt b/appengine/flexible/cloudsql/requirements.txt new file mode 100644 index 00000000000..cf2681656a0 --- /dev/null +++ b/appengine/flexible/cloudsql/requirements.txt @@ -0,0 +1,4 @@ +Flask==1.0.2 +Flask-SQLAlchemy==2.3.2 +gunicorn==19.9.0 +PyMySQL==0.9.3 diff --git a/appengine/flexible/cloudsql_postgresql/app.yaml b/appengine/flexible/cloudsql_postgresql/app.yaml new file mode 100644 index 00000000000..dee7b123b98 --- /dev/null +++ b/appengine/flexible/cloudsql_postgresql/app.yaml @@ -0,0 +1,21 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 + +#[START gae_flex_postgres_env] +env_variables: + # Replace user, password, database, and instance connection name with the values obtained + # when configuring your Cloud SQL instance. + SQLALCHEMY_DATABASE_URI: >- + postgresql+psycopg2://USER:PASSWORD@/DATABASE?host=/cloudsql/INSTANCE_CONNECTION_NAME +#[END gae_flex_postgres_env] + +#[START gae_flex_postgres_settings] +# Replace project and instance with the values obtained when configuring your +# Cloud SQL instance. +beta_settings: + cloud_sql_instances: INSTANCE_CONNECTION_NAME +#[END gae_flex_postgres_settings] diff --git a/appengine/flexible/cloudsql_postgresql/create_tables.py b/appengine/flexible/cloudsql_postgresql/create_tables.py new file mode 100755 index 00000000000..acb8c563f12 --- /dev/null +++ b/appengine/flexible/cloudsql_postgresql/create_tables.py @@ -0,0 +1,22 @@ +#! /usr/bin/env python +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from main import db + + +if __name__ == '__main__': + print('Creating all database tables...') + db.create_all() + print('Done!') diff --git a/appengine/flexible/cloudsql_postgresql/main.py b/appengine/flexible/cloudsql_postgresql/main.py new file mode 100644 index 00000000000..5ecc0622591 --- /dev/null +++ b/appengine/flexible/cloudsql_postgresql/main.py @@ -0,0 +1,104 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This sample shows how to connect to PostgreSQL running on Cloud SQL. + +See the documentation for details on how to setup and use this sample: + https://cloud.google.com/appengine/docs/flexible/python\ + /using-cloud-sql-postgres +""" + +import datetime +import logging +import os +import socket + +from flask import Flask, request +from flask_sqlalchemy import SQLAlchemy +import sqlalchemy + + +app = Flask(__name__) + + +def is_ipv6(addr): + """Checks if a given address is an IPv6 address.""" + try: + socket.inet_pton(socket.AF_INET6, addr) + return True + except socket.error: + return False + + +# [START gae_flex_postgres_app] +# Environment variables are defined in app.yaml. +app.config['SQLALCHEMY_DATABASE_URI'] = os.environ['SQLALCHEMY_DATABASE_URI'] +app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False + +db = SQLAlchemy(app) + + +class Visit(db.Model): + id = db.Column(db.Integer, primary_key=True) + timestamp = db.Column(db.DateTime()) + user_ip = db.Column(db.String(46)) + + def __init__(self, timestamp, user_ip): + self.timestamp = timestamp + self.user_ip = user_ip + + +@app.route('/') +def index(): + user_ip = request.remote_addr + + # Keep only the first two octets of the IP address. + if is_ipv6(user_ip): + user_ip = ':'.join(user_ip.split(':')[:2]) + else: + user_ip = '.'.join(user_ip.split('.')[:2]) + + visit = Visit( + user_ip=user_ip, + timestamp=datetime.datetime.utcnow() + ) + + db.session.add(visit) + db.session.commit() + + visits = Visit.query.order_by(sqlalchemy.desc(Visit.timestamp)).limit(10) + + results = [ + 'Time: {} Addr: {}'.format(x.timestamp, x.user_ip) + for x in visits] + + output = 'Last 10 visits:\n{}'.format('\n'.join(results)) + + return output, 200, {'Content-Type': 'text/plain; charset=utf-8'} +# [END gae_flex_postgres_app] + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
{}
+ See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/appengine/flexible/cloudsql_postgresql/main_test.py b/appengine/flexible/cloudsql_postgresql/main_test.py new file mode 100644 index 00000000000..49032fc0336 --- /dev/null +++ b/appengine/flexible/cloudsql_postgresql/main_test.py @@ -0,0 +1,26 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import main + + +def test_index(): + main.db.create_all() + + main.app.testing = True + client = main.app.test_client() + + r = client.get('/', environ_base={'REMOTE_ADDR': '127.0.0.1'}) + assert r.status_code == 200 + assert '127.0' in r.data.decode('utf-8') diff --git a/appengine/flexible/cloudsql_postgresql/requirements.txt b/appengine/flexible/cloudsql_postgresql/requirements.txt new file mode 100644 index 00000000000..93fefb961ba --- /dev/null +++ b/appengine/flexible/cloudsql_postgresql/requirements.txt @@ -0,0 +1,4 @@ +Flask==1.0.2 +Flask-SQLAlchemy==2.3.2 +gunicorn==19.9.0 +psycopg2==2.7.7 diff --git a/appengine/flexible/datastore/README.md b/appengine/flexible/datastore/README.md new file mode 100644 index 00000000000..3a3a0b9e4bc --- /dev/null +++ b/appengine/flexible/datastore/README.md @@ -0,0 +1,24 @@ +# Python Google Cloud Datastore sample for Google App Engine Flexible Environment + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/flexible/datastore/README.md + +This sample demonstrates how to use [Google Cloud Datastore](https://cloud.google.com/datastore/) on [Google App Engine Flexible Environment](https://cloud.google.com/appengine). + +## Setup + +Before you can run or deploy the sample, you will need to enable the Cloud Datastore API in the [Google Developers Console](https://console.developers.google.com/project/_/apiui/apiview/datastore/overview). + +## Running locally + +Refer to the [top-level README](../README.md) for instructions on running and deploying. + +When running locally, you can use the [Google Cloud SDK](https://cloud.google.com/sdk) to provide authentication to use Google Cloud APIs: + + $ gcloud init + +Starting your application: + + $ python main.py diff --git a/appengine/flexible/datastore/app.yaml b/appengine/flexible/datastore/app.yaml new file mode 100644 index 00000000000..e5ac514e8b6 --- /dev/null +++ b/appengine/flexible/datastore/app.yaml @@ -0,0 +1,6 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 diff --git a/appengine/flexible/datastore/main.py b/appengine/flexible/datastore/main.py new file mode 100644 index 00000000000..65e628b1a57 --- /dev/null +++ b/appengine/flexible/datastore/main.py @@ -0,0 +1,80 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import logging +import socket + +from flask import Flask, request +from google.cloud import datastore + + +app = Flask(__name__) + + +def is_ipv6(addr): + """Checks if a given address is an IPv6 address.""" + try: + socket.inet_pton(socket.AF_INET6, addr) + return True + except socket.error: + return False + + +# [START gae_flex_datastore_app] +@app.route('/') +def index(): + ds = datastore.Client() + + user_ip = request.remote_addr + + # Keep only the first two octets of the IP address. + if is_ipv6(user_ip): + user_ip = ':'.join(user_ip.split(':')[:2]) + else: + user_ip = '.'.join(user_ip.split('.')[:2]) + + entity = datastore.Entity(key=ds.key('visit')) + entity.update({ + 'user_ip': user_ip, + 'timestamp': datetime.datetime.utcnow() + }) + + ds.put(entity) + + query = ds.query(kind='visit', order=('-timestamp',)) + + results = [ + 'Time: {timestamp} Addr: {user_ip}'.format(**x) + for x in query.fetch(limit=10)] + + output = 'Last 10 visits:\n{}'.format('\n'.join(results)) + + return output, 200, {'Content-Type': 'text/plain; charset=utf-8'} +# [END gae_flex_datastore_app] + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
{}
+ See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/managed_vms/datastore/main_test.py b/appengine/flexible/datastore/main_test.py similarity index 100% rename from managed_vms/datastore/main_test.py rename to appengine/flexible/datastore/main_test.py diff --git a/appengine/flexible/datastore/requirements.txt b/appengine/flexible/datastore/requirements.txt new file mode 100644 index 00000000000..bdd19c09fe6 --- /dev/null +++ b/appengine/flexible/datastore/requirements.txt @@ -0,0 +1,3 @@ +Flask==1.0.2 +google-cloud-datastore==1.7.3 +gunicorn==19.9.0 diff --git a/appengine/flexible/disk/app.yaml b/appengine/flexible/disk/app.yaml new file mode 100644 index 00000000000..e5ac514e8b6 --- /dev/null +++ b/appengine/flexible/disk/app.yaml @@ -0,0 +1,6 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 diff --git a/appengine/flexible/disk/main.py b/appengine/flexible/disk/main.py new file mode 100644 index 00000000000..188eb841bb5 --- /dev/null +++ b/appengine/flexible/disk/main.py @@ -0,0 +1,74 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import socket + +from flask import Flask, request + + +app = Flask(__name__) + + +def is_ipv6(addr): + """Checks if a given address is an IPv6 address.""" + try: + socket.inet_pton(socket.AF_INET6, addr) + return True + except socket.error: + return False + + +# [START example] +@app.route('/') +def index(): + instance_id = os.environ.get('GAE_INSTANCE', '1') + + user_ip = request.remote_addr + + # Keep only the first two octets of the IP address. + if is_ipv6(user_ip): + user_ip = ':'.join(user_ip.split(':')[:2]) + else: + user_ip = '.'.join(user_ip.split('.')[:2]) + + with open('/tmp/seen.txt', 'a') as f: + f.write('{}\n'.format(user_ip)) + + with open('/tmp/seen.txt', 'r') as f: + seen = f.read() + + output = """ +Instance: {} +Seen: +{}""".format(instance_id, seen) + + return output, 200, {'Content-Type': 'text/plain; charset=utf-8'} +# [END example] + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
{}
+ See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/managed_vms/disk/main_test.py b/appengine/flexible/disk/main_test.py similarity index 100% rename from managed_vms/disk/main_test.py rename to appengine/flexible/disk/main_test.py diff --git a/appengine/flexible/disk/requirements.txt b/appengine/flexible/disk/requirements.txt new file mode 100644 index 00000000000..a34d076bacf --- /dev/null +++ b/appengine/flexible/disk/requirements.txt @@ -0,0 +1,2 @@ +Flask==1.0.2 +gunicorn==19.9.0 diff --git a/appengine/flexible/django_cloudsql/README.md b/appengine/flexible/django_cloudsql/README.md new file mode 100644 index 00000000000..3a723b5f9f1 --- /dev/null +++ b/appengine/flexible/django_cloudsql/README.md @@ -0,0 +1,25 @@ +# Getting started with Django on Google Cloud Platform on App Engine Flexible + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/flexible/django_cloudsql/README.md + +This repository is an example of how to run a [Django](https://www.djangoproject.com/) +app on Google App Engine Flexible Environment. It uses the +[Writing your first Django app](https://docs.djangoproject.com/en/1.9/intro/tutorial01/) as the +example app to deploy. + + +# Tutorial +See our [Running Django in the App Engine Flexible Environment](https://cloud.google.com/python/django/flexible-environment) tutorial for instructions for setting up and deploying this sample application. + + +## Contributing changes + +* See [CONTRIBUTING.md](CONTRIBUTING.md) + + +## Licensing + +* See [LICENSE](LICENSE) diff --git a/appengine/flexible/django_cloudsql/app.yaml b/appengine/flexible/django_cloudsql/app.yaml new file mode 100644 index 00000000000..96c7a992b2d --- /dev/null +++ b/appengine/flexible/django_cloudsql/app.yaml @@ -0,0 +1,11 @@ +# [START runtime] +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT mysite.wsgi + +beta_settings: + cloud_sql_instances: + +runtime_config: + python_version: 3 +# [END runtime] diff --git a/managed_vms/django_cloudsql/manage.py b/appengine/flexible/django_cloudsql/manage.py similarity index 100% rename from managed_vms/django_cloudsql/manage.py rename to appengine/flexible/django_cloudsql/manage.py diff --git a/managed_vms/django_cloudsql/mysite/__init__.py b/appengine/flexible/django_cloudsql/mysite/__init__.py similarity index 100% rename from managed_vms/django_cloudsql/mysite/__init__.py rename to appengine/flexible/django_cloudsql/mysite/__init__.py diff --git a/appengine/flexible/django_cloudsql/mysite/settings.py b/appengine/flexible/django_cloudsql/mysite/settings.py new file mode 100644 index 00000000000..d3f5cf133b2 --- /dev/null +++ b/appengine/flexible/django_cloudsql/mysite/settings.py @@ -0,0 +1,127 @@ +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +import os + +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'pf-@jxtojga)z+4s*uwbgjrq$aep62-thd0q7f&o77xtpka!_m' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +# SECURITY WARNING: App Engine's security features ensure that it is safe to +# have ALLOWED_HOSTS = ['*'] when the app is deployed. If you deploy a Django +# app not on App Engine, make sure to set an appropriate host here. +# See https://docs.djangoproject.com/en/1.10/ref/settings/ +ALLOWED_HOSTS = ['*'] + +# Application definition + +INSTALLED_APPS = ( + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'polls' +) + +MIDDLEWARE = ( + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +) + +ROOT_URLCONF = 'mysite.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'mysite.wsgi.application' + +# Database +# https://docs.djangoproject.com/en/1.8/ref/settings/#databases + +# [START dbconfig] +DATABASES = { + 'default': { + # If you are using Cloud SQL for MySQL rather than PostgreSQL, set + # 'ENGINE': 'django.db.backends.mysql' instead of the following. + 'ENGINE': 'django.db.backends.postgresql', + 'NAME': 'polls', + 'USER': '', + 'PASSWORD': '', + # For MySQL, set 'PORT': '3306' instead of the following. Any Cloud + # SQL Proxy instances running locally must also be set to tcp:3306. + 'PORT': '5432', + } +} +# In the flexible environment, you connect to CloudSQL using a unix socket. +# Locally, you can use the CloudSQL proxy to proxy a localhost connection +# to the instance +DATABASES['default']['HOST'] = '/cloudsql/' +if os.getenv('GAE_INSTANCE'): + pass +else: + DATABASES['default']['HOST'] = '127.0.0.1' +# [END dbconfig] + +# Internationalization +# https://docs.djangoproject.com/en/1.8/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/1.8/howto/static-files/ + +# [START staticurl] +# Fill in your cloud bucket and switch which one of the following 2 lines +# is commented to serve static content from GCS +# STATIC_URL = 'https://storage.googleapis.com//static/' +STATIC_URL = '/static/' +# [END staticurl] + +STATIC_ROOT = 'static/' diff --git a/appengine/flexible/django_cloudsql/mysite/urls.py b/appengine/flexible/django_cloudsql/mysite/urls.py new file mode 100644 index 00000000000..302e8e9e675 --- /dev/null +++ b/appengine/flexible/django_cloudsql/mysite/urls.py @@ -0,0 +1,28 @@ +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from django.conf import settings +from django.conf.urls import include, url +from django.contrib import admin +from django.contrib.staticfiles.urls import staticfiles_urlpatterns + +urlpatterns = [url(r'^', include('polls.urls')), + url(r'^admin/', admin.site.urls)] + + +# This enables static files to be served from the Gunicorn server +# In Production, serve static files from Google Cloud Storage or an alternative +# CDN +if settings.DEBUG: + urlpatterns += staticfiles_urlpatterns() diff --git a/managed_vms/django_cloudsql/mysite/wsgi.py b/appengine/flexible/django_cloudsql/mysite/wsgi.py similarity index 100% rename from managed_vms/django_cloudsql/mysite/wsgi.py rename to appengine/flexible/django_cloudsql/mysite/wsgi.py diff --git a/managed_vms/django_cloudsql/polls/__init__.py b/appengine/flexible/django_cloudsql/polls/__init__.py similarity index 100% rename from managed_vms/django_cloudsql/polls/__init__.py rename to appengine/flexible/django_cloudsql/polls/__init__.py diff --git a/managed_vms/django_cloudsql/polls/admin.py b/appengine/flexible/django_cloudsql/polls/admin.py similarity index 100% rename from managed_vms/django_cloudsql/polls/admin.py rename to appengine/flexible/django_cloudsql/polls/admin.py diff --git a/managed_vms/django_cloudsql/polls/apps.py b/appengine/flexible/django_cloudsql/polls/apps.py similarity index 100% rename from managed_vms/django_cloudsql/polls/apps.py rename to appengine/flexible/django_cloudsql/polls/apps.py diff --git a/managed_vms/django_cloudsql/polls/models.py b/appengine/flexible/django_cloudsql/polls/models.py similarity index 100% rename from managed_vms/django_cloudsql/polls/models.py rename to appengine/flexible/django_cloudsql/polls/models.py diff --git a/managed_vms/django_cloudsql/polls/tests.py b/appengine/flexible/django_cloudsql/polls/tests.py similarity index 100% rename from managed_vms/django_cloudsql/polls/tests.py rename to appengine/flexible/django_cloudsql/polls/tests.py diff --git a/managed_vms/django_cloudsql/polls/urls.py b/appengine/flexible/django_cloudsql/polls/urls.py similarity index 100% rename from managed_vms/django_cloudsql/polls/urls.py rename to appengine/flexible/django_cloudsql/polls/urls.py diff --git a/managed_vms/django_cloudsql/polls/views.py b/appengine/flexible/django_cloudsql/polls/views.py similarity index 100% rename from managed_vms/django_cloudsql/polls/views.py rename to appengine/flexible/django_cloudsql/polls/views.py diff --git a/appengine/flexible/django_cloudsql/requirements.txt b/appengine/flexible/django_cloudsql/requirements.txt new file mode 100644 index 00000000000..c19a8e2bed0 --- /dev/null +++ b/appengine/flexible/django_cloudsql/requirements.txt @@ -0,0 +1,5 @@ +Django==2.1.5 +mysqlclient==1.4.1 +wheel==0.32.3 +gunicorn==19.9.0 +psycopg2==2.7.7 diff --git a/managed_vms/extending_runtime/.dockerignore b/appengine/flexible/extending_runtime/.dockerignore similarity index 100% rename from managed_vms/extending_runtime/.dockerignore rename to appengine/flexible/extending_runtime/.dockerignore diff --git a/managed_vms/extending_runtime/Dockerfile b/appengine/flexible/extending_runtime/Dockerfile similarity index 100% rename from managed_vms/extending_runtime/Dockerfile rename to appengine/flexible/extending_runtime/Dockerfile diff --git a/appengine/flexible/extending_runtime/app.yaml b/appengine/flexible/extending_runtime/app.yaml new file mode 100644 index 00000000000..ce2a124359b --- /dev/null +++ b/appengine/flexible/extending_runtime/app.yaml @@ -0,0 +1,2 @@ +runtime: custom +env: flex diff --git a/appengine/flexible/extending_runtime/main.py b/appengine/flexible/extending_runtime/main.py new file mode 100644 index 00000000000..4cb173f7a68 --- /dev/null +++ b/appengine/flexible/extending_runtime/main.py @@ -0,0 +1,46 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START app] +import logging +import subprocess + +from flask import Flask + + +app = Flask(__name__) + + +# [START example] +@app.route('/') +def fortune(): + output = subprocess.check_output('/usr/games/fortune') + return output, 200, {'Content-Type': 'text/plain; charset=utf-8'} +# [END example] + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
{}
+ See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See CMD in Dockerfile. + app.run(host='127.0.0.1', port=8080, debug=True) +# [END app] diff --git a/appengine/flexible/extending_runtime/main_test.py b/appengine/flexible/extending_runtime/main_test.py new file mode 100644 index 00000000000..7c6fe2e1f10 --- /dev/null +++ b/appengine/flexible/extending_runtime/main_test.py @@ -0,0 +1,31 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytest + +import main + + +@pytest.mark.skipif( + not os.path.exists('/usr/games/fortune'), + reason='Fortune executable is not installed.') +def test_index(): + main.app.testing = True + client = main.app.test_client() + + r = client.get('/') + assert r.status_code == 200 + assert len(r.data) diff --git a/appengine/flexible/extending_runtime/requirements.txt b/appengine/flexible/extending_runtime/requirements.txt new file mode 100644 index 00000000000..a34d076bacf --- /dev/null +++ b/appengine/flexible/extending_runtime/requirements.txt @@ -0,0 +1,2 @@ +Flask==1.0.2 +gunicorn==19.9.0 diff --git a/appengine/flexible/hello_world/app.yaml b/appengine/flexible/hello_world/app.yaml new file mode 100644 index 00000000000..e6461a14157 --- /dev/null +++ b/appengine/flexible/hello_world/app.yaml @@ -0,0 +1,17 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 + +# This sample incurs costs to run on the App Engine flexible environment. +# The settings below are to reduce costs during testing and are not appropriate +# for production use. For more information, see: +# https://cloud.google.com/appengine/docs/flexible/python/configuring-your-app-with-app-yaml +manual_scaling: + instances: 1 +resources: + cpu: 1 + memory_gb: 0.5 + disk_size_gb: 10 diff --git a/appengine/flexible/hello_world/main.py b/appengine/flexible/hello_world/main.py new file mode 100644 index 00000000000..967eb1a5097 --- /dev/null +++ b/appengine/flexible/hello_world/main.py @@ -0,0 +1,43 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START gae_flex_quickstart] +import logging + +from flask import Flask + + +app = Flask(__name__) + + +@app.route('/') +def hello(): + """Return a friendly HTTP greeting.""" + return 'Hello World!' + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
{}
+ See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) +# [END gae_flex_quickstart] diff --git a/managed_vms/hello_world/main_test.py b/appengine/flexible/hello_world/main_test.py similarity index 100% rename from managed_vms/hello_world/main_test.py rename to appengine/flexible/hello_world/main_test.py diff --git a/appengine/flexible/hello_world/requirements.txt b/appengine/flexible/hello_world/requirements.txt new file mode 100644 index 00000000000..a34d076bacf --- /dev/null +++ b/appengine/flexible/hello_world/requirements.txt @@ -0,0 +1,2 @@ +Flask==1.0.2 +gunicorn==19.9.0 diff --git a/managed_vms/hello_world_django/.gitignore b/appengine/flexible/hello_world_django/.gitignore similarity index 100% rename from managed_vms/hello_world_django/.gitignore rename to appengine/flexible/hello_world_django/.gitignore diff --git a/appengine/flexible/hello_world_django/README.md b/appengine/flexible/hello_world_django/README.md new file mode 100644 index 00000000000..e62558ef64e --- /dev/null +++ b/appengine/flexible/hello_world_django/README.md @@ -0,0 +1,63 @@ +# Django sample for Google App Engine Flexible Environment + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/flexible/hello_world_django/README.md + +This is a basic hello world [Django](https://www.djangoproject.com/) example +for [Google App Engine Flexible Environment](https://cloud.google.com/appengine). + +## Running locally + +You can run locally using django's `manage.py`: + + $ python manage.py runserver + +## Deployment & how the application runs on Google App Engine. + +Follow the standard deployment instructions in +[the top-level README](../README.md). Google App Engine runs the application +using [gunicorn](http://gunicorn.org/) as defined by `entrypoint` in +[`app.yaml`](app.yaml). You can use a different WSGI container if you want, as +long as it listens for web traffic on port `$PORT` and is declared in +[`requirements.txt`](requirements.txt). + +## How this was created + +This project was created using standard Django commands: + + $ virtualenv env + $ source env/bin/activate + $ pip install django gunicorn + $ pip freeze > requirements.txt + $ django-admin startproject project_name + $ python manage.py startapp helloworld + +Then, we added a simple view in `hellworld.views`, added the app to +`project_name.settings.INSTALLED_APPS`, and finally added a URL rule to +`project_name.urls`. + +In order to deploy to Google App Engine, we created a simple +[`app.yaml`](app.yaml). + +## Database notice + +This sample project uses Django's default sqlite database. This isn't suitable +for production as your application can run multiple instances and each will +have a different sqlite database. Additionally, instance disks are ephemeral, +so data will not survive restarts. + +For production applications running on Google Cloud Platform, you have +the following options: + +* Use [Cloud SQL](https://cloud.google.com/sql), a fully-managed MySQL database. + There is a [Flask CloudSQL](../cloudsql) sample that should be straightfoward + to adapt to Django. +* Use any database of your choice hosted on + [Google Compute Engine](https://cloud.google.com/compute). The + [Cloud Launcher](https://cloud.google.com/launcher/) can be used to easily + deploy common databases. +* Use third-party database services, or services hosted by other providers, + provided you have configured access. + diff --git a/appengine/flexible/hello_world_django/app.yaml b/appengine/flexible/hello_world_django/app.yaml new file mode 100644 index 00000000000..ea714d595d5 --- /dev/null +++ b/appengine/flexible/hello_world_django/app.yaml @@ -0,0 +1,6 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT project_name.wsgi + +runtime_config: + python_version: 3 diff --git a/managed_vms/hello_world_django/helloworld/__init__.py b/appengine/flexible/hello_world_django/helloworld/__init__.py similarity index 100% rename from managed_vms/hello_world_django/helloworld/__init__.py rename to appengine/flexible/hello_world_django/helloworld/__init__.py diff --git a/managed_vms/hello_world_django/helloworld/views.py b/appengine/flexible/hello_world_django/helloworld/views.py similarity index 100% rename from managed_vms/hello_world_django/helloworld/views.py rename to appengine/flexible/hello_world_django/helloworld/views.py diff --git a/managed_vms/hello_world_django/manage.py b/appengine/flexible/hello_world_django/manage.py similarity index 100% rename from managed_vms/hello_world_django/manage.py rename to appengine/flexible/hello_world_django/manage.py diff --git a/managed_vms/hello_world_django/project_name/__init__.py b/appengine/flexible/hello_world_django/project_name/__init__.py similarity index 100% rename from managed_vms/hello_world_django/project_name/__init__.py rename to appengine/flexible/hello_world_django/project_name/__init__.py diff --git a/appengine/flexible/hello_world_django/project_name/settings.py b/appengine/flexible/hello_world_django/project_name/settings.py new file mode 100644 index 00000000000..af069259e59 --- /dev/null +++ b/appengine/flexible/hello_world_django/project_name/settings.py @@ -0,0 +1,116 @@ +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Django settings for project_name project. + +Generated by 'django-admin startproject' using Django 1.8.4. + +For more information on this file, see +https://docs.djangoproject.com/en/1.8/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/1.8/ref/settings/ +""" + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +import os + +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'qgw!j*bpxo7g&o1ux-(2ph818ojfj(3c#-#*_8r^8&hq5jg$3@' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = ( + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'helloworld' +) + +MIDDLEWARE = ( + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +) + +ROOT_URLCONF = 'project_name.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'project_name.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/1.8/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + } +} + + +# Internationalization +# https://docs.djangoproject.com/en/1.8/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/1.8/howto/static-files/ + +STATIC_URL = '/static/' diff --git a/managed_vms/hello_world_django/project_name/urls.py b/appengine/flexible/hello_world_django/project_name/urls.py similarity index 100% rename from managed_vms/hello_world_django/project_name/urls.py rename to appengine/flexible/hello_world_django/project_name/urls.py diff --git a/managed_vms/hello_world_django/project_name/wsgi.py b/appengine/flexible/hello_world_django/project_name/wsgi.py similarity index 100% rename from managed_vms/hello_world_django/project_name/wsgi.py rename to appengine/flexible/hello_world_django/project_name/wsgi.py diff --git a/appengine/flexible/hello_world_django/requirements.txt b/appengine/flexible/hello_world_django/requirements.txt new file mode 100644 index 00000000000..110ee9c2e19 --- /dev/null +++ b/appengine/flexible/hello_world_django/requirements.txt @@ -0,0 +1,2 @@ +Django==2.1.5 +gunicorn==19.9.0 diff --git a/appengine/flexible/mailgun/README.md b/appengine/flexible/mailgun/README.md new file mode 100644 index 00000000000..58991f749b4 --- /dev/null +++ b/appengine/flexible/mailgun/README.md @@ -0,0 +1,29 @@ +# Python Mailgun email sample for Google App Engine Flexible Environment + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/flexible/mailgun/README.md + +This sample demonstrates how to use [Mailgun](https://www.mailgun.com) on [Google App Engine Flexible Environment](https://cloud.google.com/appengine). + +For more information about Mail, see their [documentation](https://documentation.mailgun.com/). + +## Setup + +Before you can run or deploy the sample, you will need to do the following: + +1. [Create a Mailgun Account](http://www.mailgun.com/google). As of September 2015, Google users start with 30,000 free emails per month. + +2. Configure your Mailgun settings in the environment variables section in ``app.yaml``. + +## Running locally + +Refer to the [top-level README](../README.md) for instructions on running and deploying. + +You can run the application locally and send emails from your local machine. You +will need to set environment variables before starting your application: + + $ export MAILGUN_API_KEY=[your-mailgun-api-key] + $ export MAILGUN_DOMAIN_NAME=[your-mailgun-domain-name] + $ python main.py diff --git a/appengine/flexible/mailgun/app.yaml b/appengine/flexible/mailgun/app.yaml new file mode 100644 index 00000000000..a3e24700457 --- /dev/null +++ b/appengine/flexible/mailgun/app.yaml @@ -0,0 +1,12 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 + +# [START env_variables] +env_variables: + MAILGUN_DOMAIN_NAME: your-mailgun-domain-name + MAILGUN_API_KEY: your-mailgun-api-key +# [END env_variables] diff --git a/managed_vms/mailgun/example-attachment.txt b/appengine/flexible/mailgun/example-attachment.txt similarity index 100% rename from managed_vms/mailgun/example-attachment.txt rename to appengine/flexible/mailgun/example-attachment.txt diff --git a/appengine/flexible/mailgun/main.py b/appengine/flexible/mailgun/main.py new file mode 100644 index 00000000000..5491221d064 --- /dev/null +++ b/appengine/flexible/mailgun/main.py @@ -0,0 +1,91 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +from flask import Flask, render_template, request +import requests + +MAILGUN_DOMAIN_NAME = os.environ['MAILGUN_DOMAIN_NAME'] +MAILGUN_API_KEY = os.environ['MAILGUN_API_KEY'] + +app = Flask(__name__) + + +# [START gae_flex_mailgun_simple_message] +def send_simple_message(to): + url = 'https://api.mailgun.net/v3/{}/messages'.format(MAILGUN_DOMAIN_NAME) + auth = ('api', MAILGUN_API_KEY) + data = { + 'from': 'Mailgun User '.format(MAILGUN_DOMAIN_NAME), + 'to': to, + 'subject': 'Simple Mailgun Example', + 'text': 'Plaintext content', + } + + response = requests.post(url, auth=auth, data=data) + response.raise_for_status() +# [END gae_flex_mailgun_simple_message] + + +# [START gae_flex_mailgun_complex_message] +def send_complex_message(to): + url = 'https://api.mailgun.net/v3/{}/messages'.format(MAILGUN_DOMAIN_NAME) + auth = ('api', MAILGUN_API_KEY) + data = { + 'from': 'Mailgun User '.format(MAILGUN_DOMAIN_NAME), + 'to': to, + 'subject': 'Complex Mailgun Example', + 'text': 'Plaintext content', + 'html': 'HTML content' + } + files = [("attachment", open("example-attachment.txt"))] + + response = requests.post(url, auth=auth, data=data, files=files) + response.raise_for_status() +# [END gae_flex_mailgun_complex_message] + + +@app.route('/') +def index(): + return render_template('index.html') + + +@app.route('/send/email', methods=['POST']) +def send_email(): + action = request.form.get('submit') + to = request.form.get('to') + + if action == 'Send simple email': + send_simple_message(to) + else: + send_complex_message(to) + + return 'Email sent.' + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
{}
+ See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/appengine/flexible/mailgun/main_test.py b/appengine/flexible/mailgun/main_test.py new file mode 100644 index 00000000000..b92bda25913 --- /dev/null +++ b/appengine/flexible/mailgun/main_test.py @@ -0,0 +1,80 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytest +import requests +import responses + + +@pytest.fixture +def app(monkeypatch): + monkeypatch.setenv('MAILGUN_DOMAIN_NAME', 'example.com') + monkeypatch.setenv('MAILGUN_API_KEY', 'apikey') + + import main + + main.app.testing = True + return main.app.test_client() + + +def test_index(app): + r = app.get('/') + assert r.status_code == 200 + + +@responses.activate +def test_send_error(app): + responses.add( + responses.POST, + 'https://api.mailgun.net/v3/example.com/messages', + body='Test error', + status=500) + + with pytest.raises(requests.exceptions.HTTPError): + app.post('/send/email', data={ + 'recipient': 'user@example.com', + 'submit': 'Send simple email'}) + + +@responses.activate +def test_send_simple(app): + responses.add( + responses.POST, + 'https://api.mailgun.net/v3/example.com/messages', + body='') + + response = app.post('/send/email', data={ + 'recipient': 'user@example.com', + 'submit': 'Send simple email'}) + assert response.status_code == 200 + assert len(responses.calls) == 1 + + +@responses.activate +def test_send_complex(app, monkeypatch): + import main + monkeypatch.chdir(os.path.dirname(main.__file__)) + + responses.add( + responses.POST, + 'https://api.mailgun.net/v3/example.com/messages', + body='') + + response = app.post('/send/email', data={ + 'recipient': 'user@example.com', + 'submit': 'Send complex email'}) + assert response.status_code == 200 + assert len(responses.calls) == 1 diff --git a/appengine/flexible/mailgun/requirements.txt b/appengine/flexible/mailgun/requirements.txt new file mode 100644 index 00000000000..23558464899 --- /dev/null +++ b/appengine/flexible/mailgun/requirements.txt @@ -0,0 +1,3 @@ +Flask==1.0.2 +gunicorn==19.9.0 +requests[security]==2.21.0 diff --git a/appengine/flexible/mailgun/templates/index.html b/appengine/flexible/mailgun/templates/index.html new file mode 100644 index 00000000000..1aa0f319050 --- /dev/null +++ b/appengine/flexible/mailgun/templates/index.html @@ -0,0 +1,30 @@ +{# +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + + + + Mailgun on Google App Engine Flexible Environment + + + +
+ + + +
+ + + diff --git a/appengine/flexible/mailjet/README.md b/appengine/flexible/mailjet/README.md new file mode 100644 index 00000000000..1d9cb379eac --- /dev/null +++ b/appengine/flexible/mailjet/README.md @@ -0,0 +1,28 @@ +# Python Mailjet email sample for Google App Engine Flexible + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/flexible/mailjet/README.md + +This sample demonstrates how to use [Mailjet](https://www.mailjet.com) on [Google App Engine Flexible](https://cloud.google.com/appengine/docs/flexible/). + +## Setup + +Before you can run or deploy the sample, you will need to do the following: + +1. [Create a Mailjet Account](http://www.mailjet.com/google). + +2. Configure your Mailjet settings in the environment variables section in ``app.yaml``. + +## Running locally + +Refer to the [top-level README](../README.md) for instructions on running and deploying. + +You can run the application locally and send emails from your local machine. You +will need to set environment variables before starting your application: + + $ export MAILJET_API_KEY=[your-mailjet-api-key] + $ export MAILJET_API_SECRET=[your-mailjet-secret] + $ export MAILJET_SENDER=[your-sender-address] + $ python main.py diff --git a/appengine/flexible/mailjet/app.yaml b/appengine/flexible/mailjet/app.yaml new file mode 100644 index 00000000000..3d04d758027 --- /dev/null +++ b/appengine/flexible/mailjet/app.yaml @@ -0,0 +1,13 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 + +# [START gae_flex_mailjet_yaml] +env_variables: + MAILJET_API_KEY: your-mailjet-api-key + MAILJET_API_SECRET: your-mailjet-api-secret + MAILJET_SENDER: your-mailjet-sender-address +# [END gae_flex_mailjet_yaml] diff --git a/appengine/flexible/mailjet/main.py b/appengine/flexible/mailjet/main.py new file mode 100644 index 00000000000..8383b1e44e6 --- /dev/null +++ b/appengine/flexible/mailjet/main.py @@ -0,0 +1,82 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +from flask import Flask, render_template, request +# [START gae_flex_mailjet_config] +import mailjet_rest + +MAILJET_API_KEY = os.environ['MAILJET_API_KEY'] +MAILJET_API_SECRET = os.environ['MAILJET_API_SECRET'] +MAILJET_SENDER = os.environ['MAILJET_SENDER'] +# [END gae_flex_mailjet_config] + +app = Flask(__name__) + + +# [START gae_flex_mailjet_send_message] +def send_message(to): + client = mailjet_rest.Client( + auth=(MAILJET_API_KEY, MAILJET_API_SECRET), version='v3.1') + + data = { + 'Messages': [{ + "From": { + "Email": MAILJET_SENDER, + "Name": 'App Engine Flex Mailjet Sample' + }, + "To": [{ + "Email": to + }], + "Subject": 'Example email.', + "TextPart": 'This is an example email.', + "HTMLPart": 'This is an example email.' + }] + } + + result = client.send.create(data=data) + + return result.json() +# [END gae_flex_mailjet_send_message] + + +@app.route('/') +def index(): + return render_template('index.html') + + +@app.route('/send/email', methods=['POST']) +def send_email(): + to = request.form.get('to') + + result = send_message(to) + + return 'Email sent, response: 
{}
'.format(result) + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
{}
+ See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/appengine/flexible/mailjet/main_test.py b/appengine/flexible/mailjet/main_test.py new file mode 100644 index 00000000000..c910f48c544 --- /dev/null +++ b/appengine/flexible/mailjet/main_test.py @@ -0,0 +1,53 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +import pytest +import responses + + +@pytest.fixture +def app(monkeypatch): + monkeypatch.setenv('MAILJET_API_KEY', 'apikey') + monkeypatch.setenv('MAILJET_API_SECRET', 'apisecret') + monkeypatch.setenv('MAILJET_SENDER', 'sender') + + import main + + main.app.testing = True + return main.app.test_client() + + +def test_index(app): + r = app.get('/') + assert r.status_code == 200 + + +@responses.activate +def test_send_email(app): + responses.add( + responses.POST, + re.compile(r'.*'), + body='{"test": "message"}', + content_type='application/json') + + r = app.post('/send/email', data={'to': 'user@example.com'}) + + assert r.status_code == 200 + assert 'test' in r.data.decode('utf-8') + + assert len(responses.calls) == 1 + request_body = responses.calls[0].request.body + assert 'user@example.com' in request_body diff --git a/appengine/flexible/mailjet/requirements.txt b/appengine/flexible/mailjet/requirements.txt new file mode 100644 index 00000000000..024a4aaf232 --- /dev/null +++ b/appengine/flexible/mailjet/requirements.txt @@ -0,0 +1,4 @@ +Flask==1.0.2 +gunicorn==19.9.0 +requests[security]==2.21.0 +mailjet-rest==1.3.0 diff --git a/appengine/flexible/mailjet/templates/index.html b/appengine/flexible/mailjet/templates/index.html new file mode 100644 index 00000000000..19993c3547d --- /dev/null +++ b/appengine/flexible/mailjet/templates/index.html @@ -0,0 +1,29 @@ +{# +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + + + + Mailjet on Google App Engine Flexible + + + +
+ + +
+ + + diff --git a/appengine/flexible/memcache/README.md b/appengine/flexible/memcache/README.md new file mode 100644 index 00000000000..76a70a2657e --- /dev/null +++ b/appengine/flexible/memcache/README.md @@ -0,0 +1,49 @@ +## Note + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/flexible/memcache/README.md + +This sample demonstrates connecting to existing Memcache servers, or the +built-in Memcache server. + +A managed option for Memcache is RedisLabs: + +https://cloud.google.com/appengine/docs/flexible/python/using-redislabs-memcache + +You can install and manage a Memcache server on Google Compute Engine. One way +to do so is to use a Bitnami click-to-deploy: + +https://bitnami.com/stack/memcached/cloud/google + +Built-in Memcache for Flexible environments is currently in a whitelist-only alpha. To have your project whitelisted, +see the signup form here: + +https://cloud.google.com/appengine/docs/flexible/python/upgrading#memcache_service + +## Running locally + +Refer to the [top-level README](../README.md) for instructions on running and deploying. + +When running locally, you can use the [Google Cloud SDK](https://cloud.google.com/sdk) to provide authentication to use Google Cloud APIs: + + $ gcloud init + +Install dependencies, preferably with a virtualenv: + + $ virtualenv env + $ source env/bin/activate + $ pip install -r requirements.txt + +Start your application: + + $ python main.py + +## Deploying on App Engine + +Deploy using `gcloud`: + + gcloud app deploy + +You can now access the application at `https://your-app-id.appspot.com`. diff --git a/appengine/flexible/memcache/app.yaml b/appengine/flexible/memcache/app.yaml new file mode 100644 index 00000000000..e13699063d3 --- /dev/null +++ b/appengine/flexible/memcache/app.yaml @@ -0,0 +1,16 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 + +# [START gae_flex_redislabs_memcache_yaml] +env_variables: + MEMCACHE_SERVER: your-memcache-server + # If you are using a third-party or self-hosted Memcached server with SASL + # authentiation enabled, uncomment and fill in these values with your + # username and password. + # MEMCACHE_USERNAME: your-memcache-username + # MEMCACHE_PASSWORD: your-memcache-password +# [END gae_flex_redislabs_memcache_yaml] diff --git a/appengine/flexible/memcache/main.py b/appengine/flexible/memcache/main.py new file mode 100644 index 00000000000..3fa566a619a --- /dev/null +++ b/appengine/flexible/memcache/main.py @@ -0,0 +1,60 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +from flask import Flask +import pylibmc + +app = Flask(__name__) + + +# [START gae_flex_redislabs_memcache] +# Environment variables are defined in app.yaml. +MEMCACHE_SERVER = os.environ.get('MEMCACHE_SERVER', 'localhost:11211') +MEMCACHE_USERNAME = os.environ.get('MEMCACHE_USERNAME') +MEMCACHE_PASSWORD = os.environ.get('MEMCACHE_PASSWORD') + +memcache_client = pylibmc.Client( + [MEMCACHE_SERVER], binary=True, + username=MEMCACHE_USERNAME, password=MEMCACHE_PASSWORD) +# [END gae_flex_redislabs_memcache] + + +@app.route('/') +def index(): + + # Set initial value if necessary + if not memcache_client.get('counter'): + memcache_client.set('counter', 0) + + value = memcache_client.incr('counter', 1) + + return 'Value is {}'.format(value) + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
{}
+ See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/appengine/flexible/memcache/main_test.py b/appengine/flexible/memcache/main_test.py new file mode 100644 index 00000000000..f78f53ee294 --- /dev/null +++ b/appengine/flexible/memcache/main_test.py @@ -0,0 +1,41 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +try: + import main +except ImportError: + main = None + + +@pytest.mark.skipif( + not main, + reason='pylibmc not installed.') +def test_index(): + try: + main.memcache_client.set('counter', 0) + except Exception: + pytest.skip('Memcache is unavailable.') + + main.app.testing = True + client = main.app.test_client() + + r = client.get('/') + assert r.status_code == 200 + assert '1' in r.data.decode('utf-8') + + r = client.get('/') + assert r.status_code == 200 + assert '2' in r.data.decode('utf-8') diff --git a/appengine/flexible/memcache/requirements.txt b/appengine/flexible/memcache/requirements.txt new file mode 100644 index 00000000000..0e1ba99ac8f --- /dev/null +++ b/appengine/flexible/memcache/requirements.txt @@ -0,0 +1,3 @@ +Flask==1.0.2 +gunicorn==19.9.0 +pylibmc==1.6.0 diff --git a/appengine/flexible/metadata/app.yaml b/appengine/flexible/metadata/app.yaml new file mode 100644 index 00000000000..e5ac514e8b6 --- /dev/null +++ b/appengine/flexible/metadata/app.yaml @@ -0,0 +1,6 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 diff --git a/appengine/flexible/metadata/main.py b/appengine/flexible/metadata/main.py new file mode 100644 index 00000000000..7c1186da2e1 --- /dev/null +++ b/appengine/flexible/metadata/main.py @@ -0,0 +1,68 @@ +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from flask import Flask +import requests + + +logging.basicConfig(level=logging.INFO) +app = Flask(__name__) + + +# [START gae_flex_metadata] +METADATA_NETWORK_INTERFACE_URL = \ + ('http://metadata/computeMetadata/v1/instance/network-interfaces/0/' + 'access-configs/0/external-ip') + + +def get_external_ip(): + """Gets the instance's external IP address from the Compute Engine metadata + server. If the metadata server is unavailable, it assumes that the + application is running locally. + """ + try: + r = requests.get( + METADATA_NETWORK_INTERFACE_URL, + headers={'Metadata-Flavor': 'Google'}, + timeout=2) + return r.text + except requests.RequestException: + logging.info('Metadata server could not be reached, assuming local.') + return 'localhost' +# [END gae_flex_metadata] + + +@app.route('/') +def index(): + # Websocket connections must be made directly to this instance, so the + # external IP address of this instance is needed. + external_ip = get_external_ip() + return 'External IP: {}'.format(external_ip) + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
{}
+ See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/appengine/flexible/metadata/requirements.txt b/appengine/flexible/metadata/requirements.txt new file mode 100644 index 00000000000..23558464899 --- /dev/null +++ b/appengine/flexible/metadata/requirements.txt @@ -0,0 +1,3 @@ +Flask==1.0.2 +gunicorn==19.9.0 +requests[security]==2.21.0 diff --git a/appengine/flexible/multiple_services/README.md b/appengine/flexible/multiple_services/README.md new file mode 100644 index 00000000000..16e794c010a --- /dev/null +++ b/appengine/flexible/multiple_services/README.md @@ -0,0 +1,67 @@ +# Python Google Cloud Microservices Example - API Gateway + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/flexible/multiple_services/README.md + +This example demonstrates how to deploy multiple python services to [App Engine flexible environment](https://cloud.google.com/appengine/docs/flexible/) + +## To Run Locally + +Use [virtualenv](https://virtualenv.pypa.io/en/stable/) to set up each +service's environment and start the each service in a separate terminal. + +Open a terminal and start the first service: + +```Bash +$ cd gateway-service +$ virtualenv -p python3 env +$ source env/bin/activate +$ pip install -r requirements.txt +$ python main.py +``` + +In a separate terminal, start the second service: + +```Bash +$ cd static-service +$ virtualenv -p python3 env +$ source env/bin/activate +$ pip install -r requirements.txt +$ python main.py +``` + +## To Deploy to App Engine + +### YAML Files + +Each directory contains an `app.yaml` file. These files all describe a +separate App Engine service within the same project. + +For the gateway: + +[Gateway service ](gateway/app.yaml) + +This is the `default` service. There must be one (and not more). The deployed +url will be `https://.appspot.com` + +For the static file server: + +[Static file service ](static/app.yaml) + +The deployed url will be `https://-dot-.appspot.com` + +### Deployment + +To deploy a service cd into its directory and run: +```Bash +$ gcloud app deploy app.yaml +``` +and enter `Y` when prompted. Or to skip the check add `-q`. + +To deploy multiple services simultaneously just add the path to each `app.yaml` +file as an argument to `gcloud app deploy `: +```Bash +$ gcloud app deploy gateway-service/app.yaml static-service/app.yaml +``` diff --git a/appengine/flexible/multiple_services/gateway-service/app.yaml b/appengine/flexible/multiple_services/gateway-service/app.yaml new file mode 100644 index 00000000000..d544520a684 --- /dev/null +++ b/appengine/flexible/multiple_services/gateway-service/app.yaml @@ -0,0 +1,10 @@ +service: default +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 + +manual_scaling: + instances: 1 diff --git a/appengine/flexible/multiple_services/gateway-service/main.py b/appengine/flexible/multiple_services/gateway-service/main.py new file mode 100644 index 00000000000..2a233131fe5 --- /dev/null +++ b/appengine/flexible/multiple_services/gateway-service/main.py @@ -0,0 +1,58 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from flask import Flask +import requests + +import services_config + +app = Flask(__name__) +services_config.init_app(app) + + +@app.route('/') +def root(): + """Gets index.html from the static file server""" + res = requests.get(app.config['SERVICE_MAP']['static']) + return res.content + + +@app.route('/hello/') +def say_hello(service): + """Recieves requests from buttons on the front end and resopnds + or sends request to the static file server""" + # If 'gateway' is specified return immediate + if service == 'gateway': + return 'Gateway says hello' + + # Otherwise send request to service indicated by URL param + responses = [] + url = app.config['SERVICE_MAP'][service] + res = requests.get(url + '/hello') + responses.append(res.content) + return '\n'.encode().join(responses) + + +@app.route('/') +def static_file(path): + """Gets static files required by index.html to static file server""" + url = app.config['SERVICE_MAP']['static'] + res = requests.get(url + '/' + path) + return res.content, 200, {'Content-Type': res.headers['Content-Type']} + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8000, debug=True) diff --git a/appengine/flexible/multiple_services/gateway-service/requirements.txt b/appengine/flexible/multiple_services/gateway-service/requirements.txt new file mode 100644 index 00000000000..0ec46311d19 --- /dev/null +++ b/appengine/flexible/multiple_services/gateway-service/requirements.txt @@ -0,0 +1,3 @@ +Flask==1.0.2 +gunicorn==19.9.0 +requests==2.21.0 diff --git a/appengine/flexible/multiple_services/gateway-service/services_config.py b/appengine/flexible/multiple_services/gateway-service/services_config.py new file mode 100644 index 00000000000..1c9871cf333 --- /dev/null +++ b/appengine/flexible/multiple_services/gateway-service/services_config.py @@ -0,0 +1,57 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +# To add services insert key value pair of the name of the service and +# the port you want it to run on when running locally +SERVICES = { + 'default': 8000, + 'static': 8001 +} + + +def init_app(app): + # The GAE_INSTANCE environment variable will be set when deployed to GAE. + gae_instance = os.environ.get( + 'GAE_INSTANCE', os.environ.get('GAE_MODULE_INSTANCE')) + environment = 'production' if gae_instance is not None else 'development' + app.config['SERVICE_MAP'] = map_services(environment) + + +def map_services(environment): + """Generates a map of services to correct urls for running locally + or when deployed.""" + url_map = {} + for service, local_port in SERVICES.items(): + if environment == 'production': + url_map[service] = production_url(service) + if environment == 'development': + url_map[service] = local_url(local_port) + return url_map + + +def production_url(service_name): + """Generates url for a service when deployed to App Engine.""" + project_id = os.environ.get('GAE_LONG_APP_ID') + project_url = '{}.appspot.com'.format(project_id) + if service_name == 'default': + return 'https://{}'.format(project_url) + else: + return 'https://{}-dot-{}'.format(service_name, project_url) + + +def local_url(port): + """Generates url for a service when running locally""" + return 'http://localhost:{}'.format(str(port)) diff --git a/appengine/flexible/multiple_services/static-service/app.yaml b/appengine/flexible/multiple_services/static-service/app.yaml new file mode 100644 index 00000000000..db3f8b4274c --- /dev/null +++ b/appengine/flexible/multiple_services/static-service/app.yaml @@ -0,0 +1,10 @@ +service: static +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 + +manual_scaling: + instances: 1 diff --git a/appengine/flexible/multiple_services/static-service/main.py b/appengine/flexible/multiple_services/static-service/main.py new file mode 100644 index 00000000000..e5ac2be3763 --- /dev/null +++ b/appengine/flexible/multiple_services/static-service/main.py @@ -0,0 +1,46 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from flask import Flask + +app = Flask(__name__) + + +@app.route('/hello') +def say_hello(): + """responds to request from frontend via gateway""" + return 'Static File Server says hello!' + + +@app.route('/') +def root(): + """serves index.html""" + return app.send_static_file('index.html') + + +@app.route('/') +def static_file(path): + """serves static files required by index.html""" + mimetype = '' + if path.split('.')[1] == 'css': + mimetype = 'text/css' + if path.split('.')[1] == 'js': + mimetype = 'application/javascript' + return app.send_static_file(path), 200, {'Content-Type': mimetype} + + +if __name__ == "__main__": + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8001, debug=True) diff --git a/appengine/flexible/multiple_services/static-service/requirements.txt b/appengine/flexible/multiple_services/static-service/requirements.txt new file mode 100644 index 00000000000..0ec46311d19 --- /dev/null +++ b/appengine/flexible/multiple_services/static-service/requirements.txt @@ -0,0 +1,3 @@ +Flask==1.0.2 +gunicorn==19.9.0 +requests==2.21.0 diff --git a/appengine/flexible/multiple_services/static-service/static/index.html b/appengine/flexible/multiple_services/static-service/static/index.html new file mode 100644 index 00000000000..b56df2a8862 --- /dev/null +++ b/appengine/flexible/multiple_services/static-service/static/index.html @@ -0,0 +1,32 @@ + + + + + + + + + API Gateway on App Engine Flexible Environment + + +

API GATEWAY DEMO

+

Say hi to:

+ + +
    + + diff --git a/appengine/flexible/multiple_services/static-service/static/index.js b/appengine/flexible/multiple_services/static-service/static/index.js new file mode 100644 index 00000000000..ef8cdfd5cba --- /dev/null +++ b/appengine/flexible/multiple_services/static-service/static/index.js @@ -0,0 +1,36 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +function handleResponse(resp){ + const li = document.createElement('li'); + li.innerHTML = resp; + document.querySelector('.responses').appendChild(li) +} + +function handleClick(event){ + $.ajax({ + url: `hello/${event.target.id}`, + type: `GET`, + success(resp){ + handleResponse(resp); + } + }); +} + +document.addEventListener('DOMContentLoaded', () => { + const buttons = document.getElementsByTagName('button') + for (var i = 0; i < buttons.length; i++) { + buttons[i].addEventListener('click', handleClick); + } +}); diff --git a/appengine/flexible/multiple_services/static-service/static/style.css b/appengine/flexible/multiple_services/static-service/static/style.css new file mode 100644 index 00000000000..adc68fa6a4d --- /dev/null +++ b/appengine/flexible/multiple_services/static-service/static/style.css @@ -0,0 +1,3 @@ +h1 { + color: red; +} diff --git a/appengine/flexible/numpy/app.yaml b/appengine/flexible/numpy/app.yaml new file mode 100644 index 00000000000..e5ac514e8b6 --- /dev/null +++ b/appengine/flexible/numpy/app.yaml @@ -0,0 +1,6 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 diff --git a/appengine/flexible/numpy/main.py b/appengine/flexible/numpy/main.py new file mode 100644 index 00000000000..0529f932eae --- /dev/null +++ b/appengine/flexible/numpy/main.py @@ -0,0 +1,48 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from flask import Flask +import numpy as np + +app = Flask(__name__) + + +@app.route('/') +def calculate(): + return_str = '' + x = np.array([[1, 2], [3, 4]]) + y = np.array([[5, 6], [7, 8]]) + + return_str += 'x: {} , y: {}
    '.format(str(x), str(y)) + + # Multiply matrices + return_str += 'x dot y : {}'.format(str(np.dot(x, y))) + return return_str + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
    {}
    + See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/appengine/flexible/numpy/main_test.py b/appengine/flexible/numpy/main_test.py new file mode 100644 index 00000000000..c313fd6b582 --- /dev/null +++ b/appengine/flexible/numpy/main_test.py @@ -0,0 +1,24 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import main + + +def test_index(): + main.app.testing = True + client = main.app.test_client() + + r = client.get('/') + assert r.status_code == 200 + assert '[[19 22]\n [43 50]]' in r.data.decode('utf-8') diff --git a/appengine/flexible/numpy/requirements.txt b/appengine/flexible/numpy/requirements.txt new file mode 100644 index 00000000000..ea664fadc21 --- /dev/null +++ b/appengine/flexible/numpy/requirements.txt @@ -0,0 +1,3 @@ +Flask==1.0.2 +gunicorn==19.9.0 +numpy==1.16.1 diff --git a/appengine/flexible/pubsub/README.md b/appengine/flexible/pubsub/README.md new file mode 100644 index 00000000000..c4c65500681 --- /dev/null +++ b/appengine/flexible/pubsub/README.md @@ -0,0 +1,76 @@ +# Python Google Cloud Pub/Sub sample for Google App Engine Flexible Environment + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/flexible/pubsub/README.md + +This demonstrates how to send and receive messages using [Google Cloud Pub/Sub](https://cloud.google.com/pubsub) on [Google App Engine Flexible Environment](https://cloud.google.com/appengine). + +## Setup + +Before you can run or deploy the sample, you will need to do the following: + +1. Enable the Cloud Pub/Sub API in the [Google Developers Console](https://console.developers.google.com/project/_/apiui/apiview/pubsub/overview). + +2. Create a topic and subscription. + + $ gcloud beta pubsub topics create [your-topic-name] + $ gcloud beta pubsub subscriptions create [your-subscription-name] \ + --topic [your-topic-name] \ + --push-endpoint \ + https://[your-app-id].appspot.com/pubsub/push?token=[your-token] \ + --ack-deadline 30 + +3. Update the environment variables in ``app.yaml``. + +## Running locally + +Refer to the [top-level README](../README.md) for instructions on running and deploying. + +When running locally, you can use the [Google Cloud SDK](https://cloud.google.com/sdk) to provide authentication to use Google Cloud APIs: + + $ gcloud init + +Install dependencies, preferably with a virtualenv: + + $ virtualenv env + $ source env/bin/activate + $ pip install -r requirements.txt + +Then set environment variables before starting your application: + + $ export PUBSUB_VERIFICATION_TOKEN=[your-verification-token] + $ export PUBSUB_TOPIC=[your-topic] + $ python main.py + +### Simulating push notifications + +The application can send messages locally, but it is not able to receive push messages locally. You can, however, simulate a push message by making an HTTP request to the local push notification endpoint. There is an included ``sample_message.json``. You can use +``curl`` or [httpie](https://github.com/jkbrzt/httpie) to POST this: + + $ curl -i --data @sample_message.json ":8080/pubsub/push?token=[your-token]" + +Or + + $ http POST ":8080/pubsub/push?token=[your-token]" < sample_message.json + +Response: + + HTTP/1.0 200 OK + Content-Length: 2 + Content-Type: text/html; charset=utf-8 + Date: Mon, 10 Aug 2015 17:52:03 GMT + Server: Werkzeug/0.10.4 Python/2.7.10 + + OK + +After the request completes, you can refresh ``localhost:8080`` and see the message in the list of received messages. + +## Running on App Engine + +Deploy using `gcloud`: + + gcloud app deploy app.yaml + +You can now access the application at `https://your-app-id.appspot.com`. You can use the form to submit messages, but it's non-deterministic which instance of your application will receive the notification. You can send multiple messages and refresh the page to see the received message. diff --git a/appengine/flexible/pubsub/app.yaml b/appengine/flexible/pubsub/app.yaml new file mode 100644 index 00000000000..e9cc58a8252 --- /dev/null +++ b/appengine/flexible/pubsub/app.yaml @@ -0,0 +1,14 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 + +# [START gae_flex_pubsub_env] +env_variables: + PUBSUB_TOPIC: your-topic + # This token is used to verify that requests originate from your + # application. It can be any sufficiently random string. + PUBSUB_VERIFICATION_TOKEN: 1234abc +# [END gae_flex_pubsub_env] diff --git a/appengine/flexible/pubsub/main.py b/appengine/flexible/pubsub/main.py new file mode 100644 index 00000000000..033a345aea8 --- /dev/null +++ b/appengine/flexible/pubsub/main.py @@ -0,0 +1,87 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import json +import logging +import os + +from flask import current_app, Flask, render_template, request +from google.cloud import pubsub_v1 + + +app = Flask(__name__) + +# Configure the following environment variables via app.yaml +# This is used in the push request handler to verify that the request came from +# pubsub and originated from a trusted source. +app.config['PUBSUB_VERIFICATION_TOKEN'] = \ + os.environ['PUBSUB_VERIFICATION_TOKEN'] +app.config['PUBSUB_TOPIC'] = os.environ['PUBSUB_TOPIC'] +app.config['PROJECT'] = os.environ['GOOGLE_CLOUD_PROJECT'] + + +# Global list to storage messages received by this instance. +MESSAGES = [] + + +# [START gae_flex_pubsub_index] +@app.route('/', methods=['GET', 'POST']) +def index(): + if request.method == 'GET': + return render_template('index.html', messages=MESSAGES) + + data = request.form.get('payload', 'Example payload').encode('utf-8') + + publisher = pubsub_v1.PublisherClient() + topic_path = publisher.topic_path( + current_app.config['PROJECT'], + current_app.config['PUBSUB_TOPIC']) + + publisher.publish(topic_path, data=data) + + return 'OK', 200 +# [END gae_flex_pubsub_index] + + +# [START gae_flex_pubsub_push] +@app.route('/pubsub/push', methods=['POST']) +def pubsub_push(): + if (request.args.get('token', '') != + current_app.config['PUBSUB_VERIFICATION_TOKEN']): + return 'Invalid request', 400 + + envelope = json.loads(request.data.decode('utf-8')) + payload = base64.b64decode(envelope['message']['data']) + + MESSAGES.append(payload) + + # Returning any 2xx status indicates successful receipt of the message. + return 'OK', 200 +# [END gae_flex_pubsub_push] + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
    {}
    + See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/appengine/flexible/pubsub/main_test.py b/appengine/flexible/pubsub/main_test.py new file mode 100644 index 00000000000..c2911415275 --- /dev/null +++ b/appengine/flexible/pubsub/main_test.py @@ -0,0 +1,69 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import json +import os + +import pytest + +import main + + +@pytest.fixture +def client(): + main.app.testing = True + return main.app.test_client() + + +def test_index(client): + r = client.get('/') + assert r.status_code == 200 + + +def test_post_index(client): + r = client.post('/', data={'payload': 'Test payload'}) + assert r.status_code == 200 + + +def test_push_endpoint(client): + url = '/pubsub/push?token=' + os.environ['PUBSUB_VERIFICATION_TOKEN'] + + r = client.post( + url, + data=json.dumps({ + "message": { + "data": base64.b64encode( + u'Test message'.encode('utf-8') + ).decode('utf-8') + } + }) + ) + + assert r.status_code == 200 + + # Make sure the message is visible on the home page. + r = client.get('/') + assert r.status_code == 200 + assert 'Test message' in r.data.decode('utf-8') + + +def test_push_endpoint_errors(client): + # no token + r = client.post('/pubsub/push') + assert r.status_code == 400 + + # invalid token + r = client.post('/pubsub/push?token=bad') + assert r.status_code == 400 diff --git a/appengine/flexible/pubsub/requirements.txt b/appengine/flexible/pubsub/requirements.txt new file mode 100644 index 00000000000..7e5bc5578bb --- /dev/null +++ b/appengine/flexible/pubsub/requirements.txt @@ -0,0 +1,3 @@ +Flask==1.0.2 +google-cloud-pubsub==0.39.1 +gunicorn==19.9.0 diff --git a/managed_vms/pubsub/sample_message.json b/appengine/flexible/pubsub/sample_message.json similarity index 100% rename from managed_vms/pubsub/sample_message.json rename to appengine/flexible/pubsub/sample_message.json diff --git a/appengine/flexible/pubsub/templates/index.html b/appengine/flexible/pubsub/templates/index.html new file mode 100644 index 00000000000..18a917593ee --- /dev/null +++ b/appengine/flexible/pubsub/templates/index.html @@ -0,0 +1,36 @@ +{# +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + + + + Pub/Sub Python on Google App Engine Flexible Environment + + +
    +

    Messages received by this instance:

    +
      + {% for message in messages: %} +
    • {{message}}
      • + {% endfor %} +
    +

    Note: because your application is likely running multiple instances, each instance will have a different list of messages.

    +
    +
    + + +
    + + diff --git a/appengine/flexible/redis/app.yaml b/appengine/flexible/redis/app.yaml new file mode 100644 index 00000000000..e2f33bd6638 --- /dev/null +++ b/appengine/flexible/redis/app.yaml @@ -0,0 +1,13 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 + +# [START gae_flex_python_redis_yaml] +env_variables: + REDIS_HOST: your-redis-host + REDIS_PORT: your-redis-port + REDIS_PASSWORD: your-redis-password +# [END gae_flex_python_redis_yaml] diff --git a/appengine/flexible/redis/main.py b/appengine/flexible/redis/main.py new file mode 100644 index 00000000000..494eeaa10a0 --- /dev/null +++ b/appengine/flexible/redis/main.py @@ -0,0 +1,52 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +from flask import Flask +import redis + + +app = Flask(__name__) + + +# [START gae_flex_python_redis] +redis_host = os.environ.get('REDIS_HOST', 'localhost') +redis_port = int(os.environ.get('REDIS_PORT', 6379)) +redis_password = os.environ.get('REDIS_PASSWORD', None) +redis_client = redis.StrictRedis( + host=redis_host, port=redis_port, password=redis_password) +# [END gae_flex_python_redis] + + +@app.route('/') +def index(): + value = redis_client.incr('counter', 1) + return 'Value is {}'.format(value) + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
    {}
    + See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/appengine/flexible/redis/main_test.py b/appengine/flexible/redis/main_test.py new file mode 100644 index 00000000000..f314c04487a --- /dev/null +++ b/appengine/flexible/redis/main_test.py @@ -0,0 +1,35 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +import main + + +def test_index(): + try: + main.redis_client.set('counter', 0) + except Exception: + pytest.skip('Redis is unavailable.') + + main.app.testing = True + client = main.app.test_client() + + r = client.get('/') + assert r.status_code == 200 + assert '1' in r.data.decode('utf-8') + + r = client.get('/') + assert r.status_code == 200 + assert '2' in r.data.decode('utf-8') diff --git a/appengine/flexible/redis/requirements.txt b/appengine/flexible/redis/requirements.txt new file mode 100644 index 00000000000..bea99653b72 --- /dev/null +++ b/appengine/flexible/redis/requirements.txt @@ -0,0 +1,3 @@ +Flask==1.0.2 +gunicorn==19.9.0 +redis==3.1.0 diff --git a/appengine/flexible/scipy/.gitignore b/appengine/flexible/scipy/.gitignore new file mode 100644 index 00000000000..de724cf6213 --- /dev/null +++ b/appengine/flexible/scipy/.gitignore @@ -0,0 +1 @@ +assets/resized_google_logo.jpg diff --git a/appengine/flexible/scipy/README.md b/appengine/flexible/scipy/README.md new file mode 100644 index 00000000000..3217ab68726 --- /dev/null +++ b/appengine/flexible/scipy/README.md @@ -0,0 +1,9 @@ +# SciPy on App Engine Flexible + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/flexible/scipy/README.md + +This sample demonstrates how to use SciPy to resize an image on App Engine Flexible. + diff --git a/appengine/flexible/scipy/app.yaml b/appengine/flexible/scipy/app.yaml new file mode 100644 index 00000000000..e5ac514e8b6 --- /dev/null +++ b/appengine/flexible/scipy/app.yaml @@ -0,0 +1,6 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 diff --git a/appengine/flexible/scipy/assets/google_logo.jpg b/appengine/flexible/scipy/assets/google_logo.jpg new file mode 100644 index 00000000000..5538eaed2bd Binary files /dev/null and b/appengine/flexible/scipy/assets/google_logo.jpg differ diff --git a/appengine/flexible/scipy/assets/resized_google_logo.jpg b/appengine/flexible/scipy/assets/resized_google_logo.jpg new file mode 100644 index 00000000000..971fb566e15 Binary files /dev/null and b/appengine/flexible/scipy/assets/resized_google_logo.jpg differ diff --git a/appengine/flexible/scipy/fixtures/assets/resized_google_logo.jpg b/appengine/flexible/scipy/fixtures/assets/resized_google_logo.jpg new file mode 100644 index 00000000000..971fb566e15 Binary files /dev/null and b/appengine/flexible/scipy/fixtures/assets/resized_google_logo.jpg differ diff --git a/appengine/flexible/scipy/main.py b/appengine/flexible/scipy/main.py new file mode 100644 index 00000000000..2db7eb12656 --- /dev/null +++ b/appengine/flexible/scipy/main.py @@ -0,0 +1,50 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +from flask import Flask +import scipy.misc + +app = Flask(__name__) + + +@app.route('/') +def resize(): + """Demonstrates using scipy to resize an image.""" + app_path = os.path.dirname(os.path.realpath(__file__)) + image_path = os.path.join(app_path, 'assets/google_logo.jpg') + img = scipy.misc.imread(image_path) + img_tinted = scipy.misc.imresize(img, (300, 300)) + output_image_path = os.path.join( + app_path, 'assets/resized_google_logo.jpg') + # Write the tinted image back to disk + scipy.misc.imsave(output_image_path, img_tinted) + return "Image resized." + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
    {}
    + See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/appengine/flexible/scipy/main_test.py b/appengine/flexible/scipy/main_test.py new file mode 100644 index 00000000000..dd67baefeac --- /dev/null +++ b/appengine/flexible/scipy/main_test.py @@ -0,0 +1,35 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import main + + +def test_index(): + main.app.testing = True + client = main.app.test_client() + test_path = os.path.dirname(os.path.realpath(__file__)) + asset_path = os.path.join( + test_path, 'assets/resized_google_logo.jpg') + fixtured_path = os.path.join( + test_path, 'fixtures/assets/resized_google_logo.jpg') + try: + os.remove(asset_path) + except OSError: + pass # if doesn't exist + r = client.get('/') + + assert os.path.isfile(fixtured_path) + assert r.status_code == 200 diff --git a/appengine/flexible/scipy/requirements.txt b/appengine/flexible/scipy/requirements.txt new file mode 100644 index 00000000000..c856f0d8174 --- /dev/null +++ b/appengine/flexible/scipy/requirements.txt @@ -0,0 +1,5 @@ +Flask==1.0.2 +gunicorn==19.9.0 +numpy==1.16.1 +scipy==1.2.0 +Pillow==5.4.1 diff --git a/appengine/flexible/sendgrid/README.md b/appengine/flexible/sendgrid/README.md new file mode 100644 index 00000000000..14d629e58f2 --- /dev/null +++ b/appengine/flexible/sendgrid/README.md @@ -0,0 +1,29 @@ +# Python SendGrid email sample for Google App Engine Flexible + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/flexible/sendgrid/README.md + +This sample demonstrates how to use [SendGrid](https://www.sendgrid.com) on [Google App Engine Flexible](https://cloud.google.com/appengine). + +For more information about SendGrid, see their [documentation](https://sendgrid.com/docs/User_Guide/index.html). + +## Setup + +Before you can run or deploy the sample, you will need to do the following: + +1. [Create a SendGrid Account](http://sendgrid.com/partner/google). As of September 2015, Google users start with 25,000 free emails per month. + +2. Configure your SendGrid settings in the environment variables section in ``app.yaml``. + +## Running locally + +Refer to the [top-level README](../README.md) for instructions on running and deploying. + +You can run the application locally and send emails from your local machine. You +will need to set environment variables before starting your application: + + $ export SENDGRID_API_KEY=[your-sendgrid-api-key] + $ export SENDGRID_SENDER=[your-sendgrid-sender-email-address] + $ python main.py diff --git a/appengine/flexible/sendgrid/app.yaml b/appengine/flexible/sendgrid/app.yaml new file mode 100644 index 00000000000..93fbf68e2fc --- /dev/null +++ b/appengine/flexible/sendgrid/app.yaml @@ -0,0 +1,12 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 + +# [START gae_flex_sendgrid_yaml] +env_variables: + SENDGRID_API_KEY: your-sendgrid-api-key + SENDGRID_SENDER: your-sendgrid-sender +# [END gae_flex_sendgrid_yaml] diff --git a/appengine/flexible/sendgrid/main.py b/appengine/flexible/sendgrid/main.py new file mode 100644 index 00000000000..2b0617f798c --- /dev/null +++ b/appengine/flexible/sendgrid/main.py @@ -0,0 +1,70 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +from flask import Flask, render_template, request +import sendgrid +from sendgrid.helpers import mail + +SENDGRID_API_KEY = os.environ['SENDGRID_API_KEY'] +SENDGRID_SENDER = os.environ['SENDGRID_SENDER'] + +app = Flask(__name__) + + +@app.route('/') +def index(): + return render_template('index.html') + + +# [START gae_flex_sendgrid] +@app.route('/send/email', methods=['POST']) +def send_email(): + to = request.form.get('to') + if not to: + return ('Please provide an email address in the "to" query string ' + 'parameter.'), 400 + + sg = sendgrid.SendGridAPIClient(apikey=SENDGRID_API_KEY) + + to_email = mail.Email(to) + from_email = mail.Email(SENDGRID_SENDER) + subject = 'This is a test email' + content = mail.Content('text/plain', 'Example message.') + message = mail.Mail(from_email, subject, to_email, content) + + response = sg.client.mail.send.post(request_body=message.get()) + + if response.status_code != 202: + return 'An error occurred: {}'.format(response.body), 500 + + return 'Email sent.' +# [END gae_flex_sendgrid] + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
    {}
    + See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/appengine/flexible/sendgrid/main_test.py b/appengine/flexible/sendgrid/main_test.py new file mode 100644 index 00000000000..7577665f83f --- /dev/null +++ b/appengine/flexible/sendgrid/main_test.py @@ -0,0 +1,49 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import pytest + + +@pytest.fixture +def app(monkeypatch): + monkeypatch.setenv('SENDGRID_API_KEY', 'apikey') + monkeypatch.setenv('SENDGRID_SENDER', 'sender@example.com') + + import main + + main.app.testing = True + return main.app.test_client() + + +def test_get(app): + r = app.get('/') + assert r.status_code == 200 + + +@mock.patch('python_http_client.client.Client._make_request') +def test_post(make_request_mock, app): + response = mock.Mock() + response.getcode.return_value = 200 + response.read.return_value = 'OK' + response.info.return_value = {} + make_request_mock.return_value = response + + app.post('/send/email', data={ + 'to': 'user@example.com' + }) + + assert make_request_mock.called + request = make_request_mock.call_args[0][1] + assert 'user@example.com' in request.data.decode('utf-8') diff --git a/appengine/flexible/sendgrid/requirements.txt b/appengine/flexible/sendgrid/requirements.txt new file mode 100644 index 00000000000..56129f05b62 --- /dev/null +++ b/appengine/flexible/sendgrid/requirements.txt @@ -0,0 +1,3 @@ +Flask==1.0.2 +sendgrid==5.6.0 +gunicorn==19.9.0 diff --git a/appengine/flexible/sendgrid/templates/index.html b/appengine/flexible/sendgrid/templates/index.html new file mode 100644 index 00000000000..db095f71789 --- /dev/null +++ b/appengine/flexible/sendgrid/templates/index.html @@ -0,0 +1,29 @@ +{# +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + + + + SendGrid on Google App Engine Flexible Environment + + + +
    + + +
    + + + diff --git a/appengine/flexible/static_files/README.md b/appengine/flexible/static_files/README.md new file mode 100644 index 00000000000..249a478127e --- /dev/null +++ b/appengine/flexible/static_files/README.md @@ -0,0 +1,12 @@ +# Python / Flask static files sample for Google App Engine Flexible Environment + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/flexible/static_files/README.md + +This demonstrates how to use [Flask](http://flask.pocoo.org/) to serve static files in your application. + +Flask automatically makes anything in the ``static`` directory available via the ``/static`` URL. If you plan on using a different framework, it may have different conventions for serving static files. + +Refer to the [top-level README](../README.md) for instructions on running and deploying. diff --git a/appengine/flexible/static_files/app.yaml b/appengine/flexible/static_files/app.yaml new file mode 100644 index 00000000000..e5ac514e8b6 --- /dev/null +++ b/appengine/flexible/static_files/app.yaml @@ -0,0 +1,6 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 diff --git a/appengine/flexible/static_files/main.py b/appengine/flexible/static_files/main.py new file mode 100644 index 00000000000..667d59453d5 --- /dev/null +++ b/appengine/flexible/static_files/main.py @@ -0,0 +1,42 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START gae_flex_python_static_files] +import logging + +from flask import Flask, render_template + + +app = Flask(__name__) + + +@app.route('/') +def hello(): + return render_template('index.html') + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
    {}
    + See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) +# [END gae_flex_python_static_files] diff --git a/managed_vms/static_files/main_test.py b/appengine/flexible/static_files/main_test.py similarity index 100% rename from managed_vms/static_files/main_test.py rename to appengine/flexible/static_files/main_test.py diff --git a/appengine/flexible/static_files/requirements.txt b/appengine/flexible/static_files/requirements.txt new file mode 100644 index 00000000000..a34d076bacf --- /dev/null +++ b/appengine/flexible/static_files/requirements.txt @@ -0,0 +1,2 @@ +Flask==1.0.2 +gunicorn==19.9.0 diff --git a/appengine/flexible/static_files/static/main.css b/appengine/flexible/static_files/static/main.css new file mode 100644 index 00000000000..69c795697f0 --- /dev/null +++ b/appengine/flexible/static_files/static/main.css @@ -0,0 +1,21 @@ +/* +Copyright 2015 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +/* [START gae_flex_python_css] */ +body { + font-family: Verdana, Helvetica, sans-serif; + background-color: #CCCCFF; +} +/* [END gae_flex_python_css] */ diff --git a/managed_vms/static_files/templates/index.html b/appengine/flexible/static_files/templates/index.html similarity index 100% rename from managed_vms/static_files/templates/index.html rename to appengine/flexible/static_files/templates/index.html diff --git a/appengine/flexible/storage/README.md b/appengine/flexible/storage/README.md new file mode 100644 index 00000000000..051e49a2375 --- /dev/null +++ b/appengine/flexible/storage/README.md @@ -0,0 +1,37 @@ +# Python Google Cloud Storage sample for Google App Engine Flexible Environment + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/flexible/storage/README.md + +This sample demonstrates how to use [Google Cloud Storage](https://cloud.google.com/storage/) on [Google App Engine Flexible Environment](https://cloud.google.com/appengine). + +## Setup + +Before you can run or deploy the sample, you will need to do the following: + +1. Enable the Cloud Storage API in the [Google Developers Console](https://console.developers.google.com/project/_/apiui/apiview/storage/overview). + +2. Create a Cloud Storage Bucket. You can do this with the [Google Cloud SDK](https://cloud.google.com/sdk) with the following command: + + $ gsutil mb gs://[your-bucket-name] + +3. Set the default ACL on your bucket to public read in order to serve files directly from Cloud Storage. You can do this with the [Google Cloud SDK](https://cloud.google.com/sdk) with the following command: + + $ gsutil defacl set public-read gs://[your-bucket-name] + +4. Update the environment variables in ``app.yaml``. + +## Running locally + +Refer to the [top-level README](../README.md) for instructions on running and deploying. + +When running locally, you can use the [Google Cloud SDK](https://cloud.google.com/sdk) to provide authentication to use Google Cloud APIs: + + $ gcloud init + +Then set environment variables before starting your application: + + $ export CLOUD_STORAGE_BUCKET=[your-bucket-name] + $ python main.py diff --git a/appengine/flexible/storage/app.yaml b/appengine/flexible/storage/app.yaml new file mode 100644 index 00000000000..d42d550a8ac --- /dev/null +++ b/appengine/flexible/storage/app.yaml @@ -0,0 +1,11 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 + +#[START gae_flex_storage_yaml] +env_variables: + CLOUD_STORAGE_BUCKET: your-bucket-name +#[END gae_flex_storage_yaml] diff --git a/appengine/flexible/storage/main.py b/appengine/flexible/storage/main.py new file mode 100644 index 00000000000..29f62c29642 --- /dev/null +++ b/appengine/flexible/storage/main.py @@ -0,0 +1,77 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START gae_flex_storage_app] +import logging +import os + +from flask import Flask, request +from google.cloud import storage + +app = Flask(__name__) + +# Configure this environment variable via app.yaml +CLOUD_STORAGE_BUCKET = os.environ['CLOUD_STORAGE_BUCKET'] + + +@app.route('/') +def index(): + return """ +
    + + +
    +""" + + +@app.route('/upload', methods=['POST']) +def upload(): + """Process the uploaded file and upload it to Google Cloud Storage.""" + uploaded_file = request.files.get('file') + + if not uploaded_file: + return 'No file uploaded.', 400 + + # Create a Cloud Storage client. + gcs = storage.Client() + + # Get the bucket that the file will be uploaded to. + bucket = gcs.get_bucket(CLOUD_STORAGE_BUCKET) + + # Create a new blob and upload the file's content. + blob = bucket.blob(uploaded_file.filename) + + blob.upload_from_string( + uploaded_file.read(), + content_type=uploaded_file.content_type + ) + + # The public URL can be used to directly access the uploaded file via HTTP. + return blob.public_url + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
    {}
    + See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) +# [END gae_flex_storage_app] diff --git a/appengine/flexible/storage/main_test.py b/appengine/flexible/storage/main_test.py new file mode 100644 index 00000000000..76391d74f64 --- /dev/null +++ b/appengine/flexible/storage/main_test.py @@ -0,0 +1,50 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import requests +from six import BytesIO + +import main + + +@pytest.fixture +def client(): + main.app.testing = True + return main.app.test_client() + + +def test_index(client): + r = client.get('/') + assert r.status_code == 200 + + +def test_upload(client): + # Upload a simple file + file_content = b"This is some test content." + + r = client.post( + '/upload', + data={ + 'file': (BytesIO(file_content), 'example.txt') + } + ) + + assert r.status_code == 200 + + # The app should return the public cloud storage URL for the uploaded + # file. Download and verify it. + cloud_storage_url = r.data.decode('utf-8') + r = requests.get(cloud_storage_url) + assert r.text.encode('utf-8') == file_content diff --git a/appengine/flexible/storage/requirements.txt b/appengine/flexible/storage/requirements.txt new file mode 100644 index 00000000000..b9a55518268 --- /dev/null +++ b/appengine/flexible/storage/requirements.txt @@ -0,0 +1,3 @@ +Flask==1.0.2 +google-cloud-storage==1.13.2 +gunicorn==19.9.0 diff --git a/appengine/flexible/tasks/Dockerfile b/appengine/flexible/tasks/Dockerfile new file mode 100644 index 00000000000..64f160d8253 --- /dev/null +++ b/appengine/flexible/tasks/Dockerfile @@ -0,0 +1,17 @@ +# Use the official Python image. +# https://hub.docker.com/_/python +FROM python:3.7 + +# Copy local code to the container image. +ENV APP_HOME /app +WORKDIR $APP_HOME +COPY . . + +# Install production dependencies. +RUN pip install Flask gunicorn + +# Run the web service on container startup. Here we use the gunicorn +# webserver, with one worker process and 8 threads. +# For environments with multiple CPU cores, increase the number of workers +# to be equal to the cores available. +CMD exec gunicorn --bind :$PORT --workers 1 --threads 8 main:app diff --git a/appengine/flexible/tasks/README.md b/appengine/flexible/tasks/README.md new file mode 100644 index 00000000000..c2888ba647b --- /dev/null +++ b/appengine/flexible/tasks/README.md @@ -0,0 +1,103 @@ +# Google Cloud Tasks Samples + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/flexible/tasks/README.md + +Sample command-line programs for interacting with the Cloud Tasks API +. + +App Engine queues push tasks to an App Engine HTTP target. This directory +contains both the App Engine app to deploy, as well as the snippets to run +locally to push tasks to it, which could also be called on App Engine. + +`create_app_engine_queue_task.py` is a simple command-line program to create +tasks to be pushed to the App Engine app. + +`main.py` is the main App Engine app. This app serves as an endpoint to receive +App Engine task attempts. + +`app.yaml` configures the App Engine app. + + +## Prerequisites to run locally: + +Please refer to [Setting Up a Python Development Environment](https://cloud.google.com/python/setup). + +## Authentication + +To set up authentication, please refer to our +[authentication getting started guide](https://cloud.google.com/docs/authentication/getting-started). + +## Creating a queue + +To create a queue using the Cloud SDK, use the following gcloud command: + +``` +gcloud beta tasks queues create-app-engine-queue my-appengine-queue +``` + +Note: A newly created queue will route to the default App Engine service and +version unless configured to do otherwise. + +## Deploying the App Engine App + +Deploy the App Engine app with gcloud: + +* To deploy to the Standard environment: + ``` + gcloud app deploy app.yaml + ``` +* To deploy to the Flexible environment: + ``` + gcloud app deploy app.flexible.yaml + ``` + +Verify the index page is serving: + +``` +gcloud app browse +``` + +The App Engine app serves as a target for the push requests. It has an +endpoint `/example_task_handler` that reads the payload (i.e., the request body) +of the HTTP POST request and logs it. The log output can be viewed with: + +``` +gcloud app logs read +``` + +## Run the Sample Using the Command Line + +Set environment variables: + +First, your project ID: + +``` +export PROJECT_ID=my-project-id +``` + +Then the queue ID, as specified at queue creation time. Queue IDs already +created can be listed with `gcloud beta tasks queues list`. + +``` +export QUEUE_ID=my-appengine-queue +``` + +And finally the location ID, which can be discovered with +`gcloud beta tasks queues describe $QUEUE_ID`, with the location embedded in +the "name" value (for instance, if the name is +"projects/my-project/locations/us-central1/queues/my-appengine-queue", then the +location is "us-central1"). + +``` +export LOCATION_ID=us-central1 +``` +### Using App Engine Queues +Running the sample will create a task, targeted at the `/example_task_handler` +endpoint, with a payload specified: + +``` +python create_app_engine_queue_task.py --project=$PROJECT_ID --queue=$QUEUE_ID --location=$LOCATION_ID --payload=hello +``` diff --git a/appengine/flexible/tasks/app.flexible.yaml b/appengine/flexible/tasks/app.flexible.yaml new file mode 100644 index 00000000000..fdc2d9a1d4e --- /dev/null +++ b/appengine/flexible/tasks/app.flexible.yaml @@ -0,0 +1,20 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT --threads=4 main:app + +runtime_config: + python_version: 3 diff --git a/appengine/flexible/tasks/app.yaml b/appengine/flexible/tasks/app.yaml new file mode 100644 index 00000000000..620fcb3a3dc --- /dev/null +++ b/appengine/flexible/tasks/app.yaml @@ -0,0 +1,15 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +runtime: python37 diff --git a/appengine/flexible/tasks/create_app_engine_queue_task.py b/appengine/flexible/tasks/create_app_engine_queue_task.py new file mode 100644 index 00000000000..71db1cae288 --- /dev/null +++ b/appengine/flexible/tasks/create_app_engine_queue_task.py @@ -0,0 +1,110 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import argparse +import datetime + + +def create_task(project, queue, location, payload=None, in_seconds=None): + # [START cloud_tasks_appengine_create_task] + """Create a task for a given queue with an arbitrary payload.""" + + from google.cloud import tasks_v2 + from google.protobuf import timestamp_pb2 + + # Create a client. + client = tasks_v2.CloudTasksClient() + + # TODO(developer): Uncomment these lines and replace with your values. + # project = 'my-project-id' + # queue = 'my-appengine-queue' + # location = 'us-central1' + # payload = 'hello' + + # Construct the fully qualified queue name. + parent = client.queue_path(project, location, queue) + + # Construct the request body. + task = { + 'app_engine_http_request': { # Specify the type of request. + 'http_method': 'POST', + 'relative_uri': '/example_task_handler' + } + } + if payload is not None: + # The API expects a payload of type bytes. + converted_payload = payload.encode() + + # Add the payload to the request. + task['app_engine_http_request']['body'] = converted_payload + + if in_seconds is not None: + # Convert "seconds from now" into an rfc3339 datetime string. + d = datetime.datetime.utcnow() + datetime.timedelta(seconds=in_seconds) + + # Create Timestamp protobuf. + timestamp = timestamp_pb2.Timestamp() + timestamp.FromDatetime(d) + + # Add the timestamp to the tasks. + task['schedule_time'] = timestamp + + # Use the client to build and send the task. + response = client.create_task(parent, task) + + print('Created task {}'.format(response.name)) + return response +# [END cloud_tasks_appengine_create_task] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=create_task.__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + + parser.add_argument( + '--project', + help='Project of the queue to add the task to.', + required=True, + ) + + parser.add_argument( + '--queue', + help='ID (short name) of the queue to add the task to.', + required=True, + ) + + parser.add_argument( + '--location', + help='Location of the queue to add the task to.', + required=True, + ) + + parser.add_argument( + '--payload', + help='Optional payload to attach to the push queue.' + ) + + parser.add_argument( + '--in_seconds', type=int, + help='The number of seconds from now to schedule task attempt.' + ) + + args = parser.parse_args() + + create_task( + args.project, args.queue, args.location, + args.payload, args.in_seconds) diff --git a/appengine/flexible/tasks/create_app_engine_queue_task_test.py b/appengine/flexible/tasks/create_app_engine_queue_task_test.py new file mode 100644 index 00000000000..39edff12993 --- /dev/null +++ b/appengine/flexible/tasks/create_app_engine_queue_task_test.py @@ -0,0 +1,27 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import create_app_engine_queue_task + +TEST_PROJECT_ID = os.getenv('GCLOUD_PROJECT') +TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') +TEST_QUEUE_NAME = os.getenv('TEST_QUEUE_NAME', 'my-appengine-queue') + + +def test_create_task(): + result = create_app_engine_queue_task.create_task( + TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION) + assert TEST_QUEUE_NAME in result.name diff --git a/appengine/flexible/tasks/main.py b/appengine/flexible/tasks/main.py new file mode 100644 index 00000000000..5af1777595a --- /dev/null +++ b/appengine/flexible/tasks/main.py @@ -0,0 +1,41 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""App Engine app to serve as an endpoint for App Engine queue samples.""" + +# [START cloud_tasks_appengine_quickstart] +from flask import Flask, request + +app = Flask(__name__) + + +@app.route('/example_task_handler', methods=['POST']) +def example_task_handler(): + """Log the request payload.""" + payload = request.get_data(as_text=True) or '(empty payload)' + print('Received task with payload: {}'.format(payload)) + return 'Printed task payload: {}'.format(payload) +# [END cloud_tasks_appengine_quickstart] + + +@app.route('/') +def hello(): + """Basic index to verify app is serving.""" + return 'Hello World!' + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/appengine/flexible/tasks/main_test.py b/appengine/flexible/tasks/main_test.py new file mode 100644 index 00000000000..916f911241c --- /dev/null +++ b/appengine/flexible/tasks/main_test.py @@ -0,0 +1,45 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + + +@pytest.fixture +def app(): + import main + main.app.testing = True + return main.app.test_client() + + +def test_index(app): + r = app.get('/') + assert r.status_code == 200 + + +def test_log_payload(capsys, app): + payload = 'test_payload' + + r = app.post('/example_task_handler', data=payload) + assert r.status_code == 200 + + out, _ = capsys.readouterr() + assert payload in out + + +def test_empty_payload(capsys, app): + r = app.post('/example_task_handler') + assert r.status_code == 200 + + out, _ = capsys.readouterr() + assert 'empty payload' in out diff --git a/appengine/flexible/tasks/requirements.txt b/appengine/flexible/tasks/requirements.txt new file mode 100644 index 00000000000..fe50a5aa3b2 --- /dev/null +++ b/appengine/flexible/tasks/requirements.txt @@ -0,0 +1,3 @@ +Flask==1.0.2 +gunicorn==19.9.0 +google-cloud-tasks==0.7.0 diff --git a/appengine/flexible/twilio/README.md b/appengine/flexible/twilio/README.md new file mode 100644 index 00000000000..432acef801a --- /dev/null +++ b/appengine/flexible/twilio/README.md @@ -0,0 +1,34 @@ +# Python Twilio voice and SMS sample for Google App Engine Flexible Environment + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/flexible/twilio/README.md + +This sample demonstrates how to use [Twilio](https://www.twilio.com) on [Google App Engine Flexible Environment](https://cloud.google.com/appengine). + +For more information about Twilio, see their [Python quickstart tutorials](https://www.twilio.com/docs/quickstart/python). + +## Setup + +Before you can run or deploy the sample, you will need to do the following: + +1. [Create a Twilio Account](http://ahoy.twilio.com/googlecloudplatform). Google App Engine +customers receive a complimentary credit for SMS messages and inbound messages. + +2. Create a number on twilio, and configure the voice request URL to be ``https://your-app-id.appspot.com/call/receive`` +and the SMS request URL to be ``https://your-app-id.appspot.com/sms/receive``. + +3. Configure your Twilio settings in the environment variables section in ``app.yaml``. + +## Running locally + +Refer to the [top-level README](../README.md) for instructions on running and deploying. + +You can run the application locally to test the callbacks and SMS sending. You +will need to set environment variables before starting your application: + + $ export TWILIO_ACCOUNT_SID=[your-twilio-accoun-sid] + $ export TWILIO_AUTH_TOKEN=[your-twilio-auth-token] + $ export TWILIO_NUMBER=[your-twilio-number] + $ python main.py diff --git a/appengine/flexible/twilio/app.yaml b/appengine/flexible/twilio/app.yaml new file mode 100644 index 00000000000..597c1e6099d --- /dev/null +++ b/appengine/flexible/twilio/app.yaml @@ -0,0 +1,13 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 + +# [START gae_flex_twilio_env] +env_variables: + TWILIO_ACCOUNT_SID: your-account-sid + TWILIO_AUTH_TOKEN: your-auth-token + TWILIO_NUMBER: your-twilio-number +# [END gae_flex_twilio_env] diff --git a/appengine/flexible/twilio/main.py b/appengine/flexible/twilio/main.py new file mode 100644 index 00000000000..f02689245f6 --- /dev/null +++ b/appengine/flexible/twilio/main.py @@ -0,0 +1,87 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +from flask import Flask, request +from twilio import rest +from twilio.twiml import messaging_response, voice_response + + +TWILIO_ACCOUNT_SID = os.environ['TWILIO_ACCOUNT_SID'] +TWILIO_AUTH_TOKEN = os.environ['TWILIO_AUTH_TOKEN'] +TWILIO_NUMBER = os.environ['TWILIO_NUMBER'] + + +app = Flask(__name__) + + +# [START gae_flex_twilio_receive_call] +@app.route('/call/receive', methods=['POST']) +def receive_call(): + """Answers a call and replies with a simple greeting.""" + response = voice_response.VoiceResponse() + response.say('Hello from Twilio!') + return str(response), 200, {'Content-Type': 'application/xml'} +# [END gae_flex_twilio_receive_call] + + +# [START gae_flex_twilio_send_sms] +@app.route('/sms/send') +def send_sms(): + """Sends a simple SMS message.""" + to = request.args.get('to') + if not to: + return ('Please provide the number to message in the "to" query string' + ' parameter.'), 400 + + client = rest.Client(TWILIO_ACCOUNT_SID, TWILIO_AUTH_TOKEN) + rv = client.messages.create( + to=to, + from_=TWILIO_NUMBER, + body='Hello from Twilio!' + ) + return str(rv) +# [END gae_flex_twilio_send_sms] + + +# [START gae_flex_twilio_receive_sms] +@app.route('/sms/receive', methods=['POST']) +def receive_sms(): + """Receives an SMS message and replies with a simple greeting.""" + sender = request.values.get('From') + body = request.values.get('Body') + + message = 'Hello, {}, you said: {}'.format(sender, body) + + response = messaging_response.MessagingResponse() + response.message(message) + return str(response), 200, {'Content-Type': 'application/xml'} +# [END gae_flex_twilio_receive_sms] + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
    {}
    + See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/appengine/flexible/twilio/main_test.py b/appengine/flexible/twilio/main_test.py new file mode 100644 index 00000000000..bfa037e2094 --- /dev/null +++ b/appengine/flexible/twilio/main_test.py @@ -0,0 +1,77 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import re + +import pytest +import responses + + +@pytest.fixture +def app(monkeypatch): + monkeypatch.setenv('TWILIO_ACCOUNT_SID', 'sid123') + monkeypatch.setenv('TWILIO_AUTH_TOKEN', 'auth123') + monkeypatch.setenv('TWILIO_NUMBER', '0123456789') + + import main + + main.app.testing = True + return main.app.test_client() + + +def test_receive_call(app): + r = app.post('/call/receive') + assert 'Hello from Twilio!' in r.data.decode('utf-8') + + +@responses.activate +def test_send_sms(app, monkeypatch): + sample_response = { + "sid": "sid", + "date_created": "Wed, 20 Dec 2017 19:32:14 +0000", + "date_updated": "Wed, 20 Dec 2017 19:32:14 +0000", + "date_sent": None, + "account_sid": "account_sid", + "to": "+1234567890", + "from": "+9876543210", + "messaging_service_sid": None, + "body": "Hello from Twilio!", + "status": "queued", + "num_segments": "1", + "num_media": "0", + "direction": "outbound-api", + "api_version": "2010-04-01", + "price": None, + "price_unit": "USD", + "error_code": None, + "error_message": None, + "uri": "/2010-04-01/Accounts/sample.json", + "subresource_uris": { + "media": "/2010-04-01/Accounts/sample/Media.json" + } + } + responses.add(responses.POST, re.compile('.*'), + json=sample_response, status=200) + + r = app.get('/sms/send') + assert r.status_code == 400 + + r = app.get('/sms/send?to=5558675309') + assert r.status_code == 200 + + +def test_receive_sms(app): + r = app.post('/sms/receive', data={ + 'From': '5558675309', 'Body': 'Jenny, I got your number.'}) + assert r.status_code == 200 + assert 'Jenny, I got your number' in r.data.decode('utf-8') diff --git a/appengine/flexible/twilio/requirements.txt b/appengine/flexible/twilio/requirements.txt new file mode 100644 index 00000000000..c7701ca6849 --- /dev/null +++ b/appengine/flexible/twilio/requirements.txt @@ -0,0 +1,3 @@ +Flask==1.0.2 +gunicorn==19.9.0 +twilio==6.24.0 diff --git a/appengine/flexible/websockets/README.md b/appengine/flexible/websockets/README.md new file mode 100644 index 00000000000..fabd0995a40 --- /dev/null +++ b/appengine/flexible/websockets/README.md @@ -0,0 +1,11 @@ +# Python websockets sample for Google App Engine Flexible Environment + +This sample demonstrates how to use websockets on [Google App Engine Flexible Environment](https://cloud.google.com/appengine). + +## Running locally + +Refer to the [top-level README](../README.md) for instructions on running and deploying. + +To run locally, you need to use gunicorn with the ``flask_socket`` worker: + + $ gunicorn -b 127.0.0.1:8080 -k flask_sockets.worker main:app diff --git a/appengine/flexible/websockets/app.yaml b/appengine/flexible/websockets/app.yaml new file mode 100644 index 00000000000..89d3afa08d9 --- /dev/null +++ b/appengine/flexible/websockets/app.yaml @@ -0,0 +1,22 @@ +runtime: python +env: flex + +# Use a special gunicorn worker class to support websockets. +entrypoint: gunicorn -b :$PORT -k flask_sockets.worker main:app + +runtime_config: + python_version: 3 + +# Use only a single instance, so that this local-memory-only chat app will work +# consistently with multiple users. To work across multiple instances, an +# extra-instance messaging system or data store would be needed. +manual_scaling: + instances: 1 + + +# For applications which can take advantage of session affinity +# (where the load balancer will attempt to route multiple connections from +# the same user to the same App Engine instance), uncomment the folowing: + +# network: +# session_affinity: true diff --git a/appengine/flexible/websockets/main.py b/appengine/flexible/websockets/main.py new file mode 100644 index 00000000000..3fd5da9851d --- /dev/null +++ b/appengine/flexible/websockets/main.py @@ -0,0 +1,53 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +# [START gae_flex_websockets_app] +from flask import Flask, render_template +from flask_sockets import Sockets + + +app = Flask(__name__) +sockets = Sockets(app) + + +@sockets.route('/chat') +def chat_socket(ws): + while not ws.closed: + message = ws.receive() + if message is None: # message is "None" if the client has closed. + continue + # Send the message to all clients connected to this webserver + # process. (To support multiple processes or instances, an + # extra-instance storage or messaging system would be required.) + clients = ws.handler.server.clients.values() + for client in clients: + client.ws.send(message) +# [END gae_flex_websockets_app] + + +@app.route('/') +def index(): + return render_template('index.html') + + +if __name__ == '__main__': + print(""" +This can not be run directly because the Flask development server does not +support web sockets. Instead, use gunicorn: + +gunicorn -b 127.0.0.1:8080 -k flask_sockets.worker main:app + +""") diff --git a/appengine/flexible/websockets/main_test.py b/appengine/flexible/websockets/main_test.py new file mode 100644 index 00000000000..547954bdfb8 --- /dev/null +++ b/appengine/flexible/websockets/main_test.py @@ -0,0 +1,71 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import socket +import subprocess + +import pytest +import requests +from retrying import retry +import websocket + + +@pytest.fixture(scope='module') +def server(): + """Provides the address of a test HTTP/websocket server. + The test server is automatically created before + a test and destroyed at the end. + """ + # Ask the OS to allocate a port. + sock = socket.socket() + sock.bind(('127.0.0.1', 0)) + port = sock.getsockname()[1] + + # Free the port and pass it to a subprocess. + sock.close() + + bind_to = '127.0.0.1:{}'.format(port) + server = subprocess.Popen( + ['gunicorn', '-b', bind_to, '-k' 'flask_sockets.worker', 'main:app']) + + # Wait until the server responds before proceeding. + @retry(wait_fixed=50, stop_max_delay=5000) + def check_server(url): + requests.get(url) + + check_server('http://{}/'.format(bind_to)) + + yield bind_to + + server.kill() + + +def test_http(server): + result = requests.get('http://{}/'.format(server)) + assert 'Python Websockets Chat' in result.text + + +def test_websocket(server): + url = 'ws://{}/chat'.format(server) + ws_one = websocket.WebSocket() + ws_one.connect(url) + + ws_two = websocket.WebSocket() + ws_two.connect(url) + + message = 'Hello, World' + ws_one.send(message) + + assert ws_one.recv() == message + assert ws_two.recv() == message diff --git a/appengine/flexible/websockets/requirements.txt b/appengine/flexible/websockets/requirements.txt new file mode 100644 index 00000000000..43af37c2ae8 --- /dev/null +++ b/appengine/flexible/websockets/requirements.txt @@ -0,0 +1,4 @@ +Flask==1.0.2 +Flask-Sockets==0.2.1 +gunicorn==19.9.0 +requests==2.21.0 diff --git a/appengine/flexible/websockets/templates/index.html b/appengine/flexible/websockets/templates/index.html new file mode 100644 index 00000000000..af6d791f148 --- /dev/null +++ b/appengine/flexible/websockets/templates/index.html @@ -0,0 +1,96 @@ +{# +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + + + + Google App Engine Flexible Environment - Python Websockets Chat + + + + + +

    Chat demo

    +
    + + +
    + +
    +

    Messages:

    +
      +
      + +
      +

      Status:

      +
        +
        + + + + + + diff --git a/appengine/i18n/README.md b/appengine/i18n/README.md deleted file mode 100644 index ab982f2b81e..00000000000 --- a/appengine/i18n/README.md +++ /dev/null @@ -1,128 +0,0 @@ -# App Engine Internationalization Sample - -A simple example app showing how to build an internationalized app -with App Engine. - -## What to internationalize - -There are lots of things to internationalize with your web -applications. - -1. Strings in Python code -2. Strings in HTML template -3. Strings in Javascript -4. Common strings - - Country Names, Language Names, etc. -5. Formatting - - Date/Time formatting - - Number formatting - - Currency -6. Timezone conversion - -This example only covers first 3 basic scenarios above. In order to -cover other aspects, I recommend using -[Babel](http://babel.edgewall.org/) and [pytz] -(http://pypi.python.org/pypi/gaepytz). Also, you may want to use -[webapp2_extras.i18n](http://webapp-improved.appspot.com/tutorials/i18n.html) -module. - -## Wait, so why not webapp2_extras.i18n? - -webapp2_extras.i18n doesn't cover how to internationalize strings in -Javascript code. Additionally it depends on babel and pytz, which -means you need to deploy babel and pytz alongside with your code. I'd -like to show a reasonably minimum example for string -internationalization in Python code, jinja2 templates, as well as -Javascript. - -## How to run this example - -First of all, please install babel in your local Python environment. - -### Wait, you just said I don't need babel, are you crazy? - -As I said before, you don't need to deploy babel with this -application, but you need to locally use pybabel script which is -provided by babel distribution in order to extract the strings, manage -and compile the translations file. - -### Extract strings in Python code and Jinja2 templates to translate - -Move into this project directory and invoke the following command: - - $ env PYTHONPATH=/google_appengine_sdk/lib/jinja2 \ - pybabel extract -o locales/messages.pot -F main.mapping . - -This command creates a `locales/messages.pot` file in the `locales` -directory which contains all the string found in your Python code and -Jija2 tempaltes. - -Since the babel configration file `main.mapping` contains a reference -to `jinja2.ext.babel_extract` helper function which is provided by -jinja2 distribution bundled with the App Engine SDK, you need to add a -PYTHONPATH environment variable pointing to the jinja2 directory in -the SDK. - -### Manage and compile translations. - -Create an initial translation source by the following command: - - $ pybabel init -l ja -d locales -i locales/messages.pot - -Open `locales/ja/LC_MESSAGES/messages.po` with any text editor and -translate the strings, then compile the file by the following command: - - $ pybabel compile -d locales - -If any of the strings changes, you can extract the strings again, and -update the translations by the following command: - - $ pybabel update -l ja -d locales -i locales/messages.pot - -Note: If you run `pybabel init` against an existant translations file, -you will lose your translations. - - -### Extract strings in Javascript code and compile translations - - $ pybabel extract -o locales/jsmessages.pot -F js.mapping . - $ pybabel init -l ja -d locales -i locales/jsmessages.pot -D jsmessages - -Open `locales/ja/LC_MESSAGES/jsmessages.po` and translate it. - - $ pybabel compile -d locales -D jsmessages - -### Running locally & deploying - -Refer to the [App Engine Samples README](../README.md) for information on how to run and deploy this sample. - -## How it works - -As you can see it in the `appengine_config.py` file, our -`main.application` is wrapped by the `i18n_utils.I18nMiddleware` WSGI -middleware. When a request comes in, this middleware parses the -`HTTP_ACCEPT_LANGUAGE` HTTP header, loads available translation -files(`messages.mo`) from the application directory, and install the -`gettext` and `ngettext` functions to the `__builtin__` namespace in -the Python runtime. - -For strings in Jinja2 templates, there is the `i18n_utils.BaseHandler` -class from which you can extend in order to have a handy property -named `jinja2_env` that lazily initializes Jinja2 environment for you -with the `jinja2.ext.i18n` extention, and similar to the -`I18nMiddleware`, installs `gettext` and `ngettext` functions to the -global namespace of the Jinja2 environment. - -## What about Javascript? - -The `BaseHandler` class also installs the `get_i18n_js_tag()` instance -method to the Jinja2 global namespace. When you use this function in -your Jinja2 template (like in the `index.jinja2` file), you will get a -set of Javascript functions; `gettext`, `ngettext`, and `format` on -the string type. The `format` function can be used with `ngettext`ed -strings for number formatting. See this example: - - window.alert(ngettext( - 'You need to provide at least {0} item.', - 'You need to provide at least {0} items.', - n).format(n); diff --git a/appengine/i18n/main.py b/appengine/i18n/main.py deleted file mode 100644 index e50ab1260a0..00000000000 --- a/appengine/i18n/main.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2013 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Sample application that demonstrates how to internationalize and localize -and App Engine application. - -For more information, see README.md -""" - -from i18n_utils import BaseHandler - -import webapp2 - - -class MainHandler(BaseHandler): - """A simple handler with internationalized strings. - - This handler demonstrates how to internationalize strings in - Python, Jinja2 template and Javascript. - """ - - def get(self): - """A get handler for this sample. - - It just shows internationalized strings in Python, Jinja2 - template and Javascript. - """ - - context = dict(message=gettext('Hello World from Python code!')) - template = self.jinja2_env.get_template('index.jinja2') - self.response.out.write(template.render(context)) - - -application = webapp2.WSGIApplication([ - ('/', MainHandler), -], debug=True) diff --git a/appengine/images/README.md b/appengine/images/README.md deleted file mode 100644 index f4164204c54..00000000000 --- a/appengine/images/README.md +++ /dev/null @@ -1,13 +0,0 @@ -## Images Guestbook Sample - -This is a sample app for Google App Engine that demonstrates the [Images Python -API](https://cloud.google.com/appengine/docs/python/images/usingimages). - - -These samples are used on the following documentation page: - -> https://cloud.google.com/appengine/docs/python/images/usingimages - - - -Refer to the [App Engine Samples README](../README.md) for information on how to run and deploy this sample. diff --git a/appengine/images/app.yaml b/appengine/images/app.yaml deleted file mode 100644 index 6033ebaf3c3..00000000000 --- a/appengine/images/app.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# This file specifies your Python application's runtime configuration -# including URL routing, versions, static file uploads, etc. See -# https://developers.google.com/appengine/docs/python/config/appconfig -# for details. - -runtime: python27 -api_version: 1 -threadsafe: yes - -# Handlers define how to route requests to your application. -handlers: - -# This handler tells app engine how to route requests to a WSGI application. -# The script value is in the format . -# where is a WSGI application object. -- url: .* # This regex directs all routes to main.app - script: main.app diff --git a/appengine/images/main_test.py b/appengine/images/main_test.py deleted file mode 100644 index f3315973587..00000000000 --- a/appengine/images/main_test.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import main -import mock -import pytest -import webtest - - -@pytest.fixture -def app(testbed): - return webtest.TestApp(main.app) - - -def test_get(app): - main.Greeting( - parent=main.guestbook_key('default_guestbook'), - author='123', - content='abc' - ).put() - - response = app.get('/') - - # Let's check if the response is correct. - assert response.status_int == 200 - - -def test_post(app): - with mock.patch('main.images') as mock_images: - mock_images.resize.return_value = 'asdf' - - response = app.post('/sign', {'content': 'asdf'}) - mock_images.resize.assert_called_once_with(mock.ANY, 32, 32) - - # Correct response is a redirect - assert response.status_int == 302 - - -def test_img(app): - greeting = main.Greeting( - parent=main.guestbook_key('default_guestbook'), - id=123 - ) - greeting.author = 'asdf' - greeting.content = 'asdf' - greeting.avatar = b'123' - greeting.put() - - response = app.get('/img?img_id=%s' % greeting.key.urlsafe()) - - assert response.status_int == 200 - - -def test_img_missing(app): - # Bogus image id, should get error - app.get('/img?img_id=123', status=500) - - -def test_post_and_get(app): - with mock.patch('main.images') as mock_images: - mock_images.resize.return_value = 'asdf' - - app.post('/sign', {'content': 'asdf'}) - response = app.get('/') - - assert response.status_int == 200 diff --git a/appengine/localtesting/README.md b/appengine/localtesting/README.md deleted file mode 100644 index 0d45442f02b..00000000000 --- a/appengine/localtesting/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# App Engine Local Testing Samples - -These samples show how to do automated testing of App Engine applications. - - -These samples are used on the following documentation page: - -> https://cloud.google.com/appengine/docs/python/tools/localunittesting - - diff --git a/appengine/localtesting/runner.py b/appengine/localtesting/runner.py deleted file mode 100755 index b9cb839e96c..00000000000 --- a/appengine/localtesting/runner.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/python -# Copyright 2015 Google Inc -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START runner] -import optparse -import os -import sys -import unittest - - -USAGE = """%prog SDK_PATH TEST_PATH -Run unit tests for App Engine apps. - -SDK_PATH Path to Google Cloud or Google App Engine SDK installation, usually - ~/google_cloud_sdk -TEST_PATH Path to package containing test modules""" - - -def main(sdk_path, test_path): - # If the sdk path points to a google cloud sdk installation - # then we should alter it to point to the GAE platform location. - if os.path.exists(os.path.join(sdk_path, 'platform/google_appengine')): - sys.path.insert(0, os.path.join(sdk_path, 'platform/google_appengine')) - else: - sys.path.insert(0, sdk_path) - - # Ensure that the google.appengine.* packages are available - # in tests as well as all bundled third-party packages. - import dev_appserver - dev_appserver.fix_sys_path() - - # Loading appengine_config from the current project ensures that any - # changes to configuration there are available to all tests (e.g. - # sys.path modifications, namespaces, etc.) - try: - import appengine_config - (appengine_config) - except ImportError: - print "Note: unable to import appengine_config." - - # Discover and run tests. - suite = unittest.loader.TestLoader().discover(test_path) - unittest.TextTestRunner(verbosity=2).run(suite) - - -if __name__ == '__main__': - parser = optparse.OptionParser(USAGE) - options, args = parser.parse_args() - if len(args) != 2: - print 'Error: Exactly 2 arguments required.' - parser.print_help() - sys.exit(1) - SDK_PATH = args[0] - TEST_PATH = args[1] - main(SDK_PATH, TEST_PATH) - -# [END runner] diff --git a/appengine/logging/reading_logs/main.py b/appengine/logging/reading_logs/main.py deleted file mode 100644 index ef3e9460606..00000000000 --- a/appengine/logging/reading_logs/main.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Sample Google App Engine application that demonstrates how to use the App -Engine Log Service API to read application logs. -""" - -# [START all] -import base64 -import datetime -from itertools import islice -from textwrap import dedent -import time - -from google.appengine.api.logservice import logservice -import webapp2 - - -def get_logs(offset=None): - # Logs are read backwards from the given end time. This specifies to read - # all logs up until now. - end_time = time.time() - - logs = logservice.fetch( - end_time=end_time, - offset=offset, - minimum_log_level=logservice.LOG_LEVEL_INFO, - include_app_logs=True) - - return logs - - -def format_log_entry(entry): - # Format any application logs that happened during this request. - logs = [] - for log in entry.app_logs: - date = datetime.datetime.fromtimestamp( - log.time).strftime('%D %T UTC') - logs.append('Date: {}, Message: {}'.format( - date, log.message)) - - # Format the request log and include the application logs. - date = datetime.datetime.fromtimestamp( - entry.end_time).strftime('%D %T UTC') - - output = dedent(""" - Date: {} - IP: {} - Method: {} - Resource: {} - Logs: - """.format(date, entry.ip, entry.method, entry.resource)) - - output += '\n'.join(logs) - - return output - - -class MainPage(webapp2.RequestHandler): - def get(self): - offset = self.request.get('offset', None) - - if offset: - offset = base64.urlsafe_b64decode(str(offset)) - - # Get the logs given the specified offset. - logs = get_logs(offset=offset) - - # Output the first 10 logs. - log = None - for log in islice(logs, 10): - self.response.write( - '
        {}
        '.format(format_log_entry(log))) - - offset = log.offset - - if not log: - self.response.write('No log entries found.') - - # Add a link to view more log entries. - elif offset: - self.response.write( - 'More'.format(MAILGUN_DOMAIN_NAME), - 'to': recipient, - 'subject': 'This is an example email from Mailgun', - 'text': 'Test message from Mailgun' - } - - resp, content = http.request(url, 'POST', urlencode(data)) - - if resp.status != 200: - raise RuntimeError( - 'Mailgun API error: {} {}'.format(resp.status, content)) -# [END simple_message] - - -# [START complex_message] -def send_complex_message(recipient): - http = httplib2.Http() - http.add_credentials('api', MAILGUN_API_KEY) - - url = 'https://api.mailgun.net/v3/{}/messages'.format(MAILGUN_DOMAIN_NAME) - data = { - 'from': 'Example Sender '.format(MAILGUN_DOMAIN_NAME), - 'to': recipient, - 'subject': 'This is an example email from Mailgun', - 'text': 'Test message from Mailgun', - 'html': 'HTML version of the body' - } - - resp, content = http.request(url, 'POST', urlencode(data)) - - if resp.status != 200: - raise RuntimeError( - 'Mailgun API error: {} {}'.format(resp.status, content)) -# [END complex_message] - - -class MainPage(webapp2.RequestHandler): - def get(self): - self.response.content_type = 'text/html' - self.response.write(""" - - -
        - - - -
        - -""") - - def post(self): - recipient = self.request.get('recipient') - action = self.request.get('submit') - - if action == 'Send simple email': - send_simple_message(recipient) - else: - send_complex_message(recipient) - - self.response.write('Mail sent') - - -app = webapp2.WSGIApplication([ - ('/', MainPage) -], debug=True) diff --git a/appengine/mailgun/main_test.py b/appengine/mailgun/main_test.py deleted file mode 100644 index de92f476f29..00000000000 --- a/appengine/mailgun/main_test.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from googleapiclient.http import HttpMockSequence -import httplib2 -import main -import mock -import pytest -import webtest - - -class HttpMockSequenceWithCredentials(HttpMockSequence): - def add_credentials(self, *args): - pass - - -@pytest.fixture -def app(): - return webtest.TestApp(main.app) - - -def test_get(app): - response = app.get('/') - assert response.status_int == 200 - - -def test_post(app): - http = HttpMockSequenceWithCredentials([ - ({'status': '200'}, '')]) - patch_http = mock.patch.object(httplib2, 'Http', lambda: http) - - with patch_http: - response = app.post('/', { - 'recipient': 'jonwayne@google.com', - 'submit': 'Send simple email'}) - - assert response.status_int == 200 - - http = HttpMockSequenceWithCredentials([ - ({'status': '200'}, '')]) - - with patch_http: - response = app.post('/', { - 'recipient': 'jonwayne@google.com', - 'submit': 'Send complex email'}) - - assert response.status_int == 200 - - http = HttpMockSequenceWithCredentials([ - ({'status': '500'}, 'Test error')]) - - with patch_http, pytest.raises(Exception): - app.post('/', { - 'recipient': 'jonwayne@google.com', - 'submit': 'Send simple email'}) diff --git a/appengine/mailgun/requirements.txt b/appengine/mailgun/requirements.txt deleted file mode 100644 index 9fd60e085e5..00000000000 --- a/appengine/mailgun/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -httplib2==0.9.2 diff --git a/appengine/memcache/guestbook/README.md b/appengine/memcache/guestbook/README.md deleted file mode 100644 index 9adf18d0f74..00000000000 --- a/appengine/memcache/guestbook/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Memcache Guestbook Sample - -This is a sample app for Google App Engine that demonstrates the Memcache Python API. - - -These samples are used on the following documentation page: - -> https://cloud.google.com/appengine/docs/python/memcache/usingmemcache - - - -Refer to the [App Engine Samples README](../../README.md) for information on how to run and deploy this sample. diff --git a/appengine/memcache/guestbook/main.py b/appengine/memcache/guestbook/main.py deleted file mode 100644 index fbe850e4925..00000000000 --- a/appengine/memcache/guestbook/main.py +++ /dev/null @@ -1,147 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Sample application that demonstrates how to use the App Engine Memcache API. - -For more information, see README.md. -""" - -# [START all] - -import cgi -import cStringIO -import logging -import urllib - -from google.appengine.api import memcache -from google.appengine.api import users -from google.appengine.ext import ndb - -import webapp2 - - -class Greeting(ndb.Model): - """Models an individual Guestbook entry with author, content, and date.""" - author = ndb.StringProperty() - content = ndb.StringProperty() - date = ndb.DateTimeProperty(auto_now_add=True) - - -def guestbook_key(guestbook_name=None): - """Constructs a Datastore key for a Guestbook entity with guestbook_name""" - return ndb.Key('Guestbook', guestbook_name or 'default_guestbook') - - -class MainPage(webapp2.RequestHandler): - def get(self): - self.response.out.write('') - guestbook_name = self.request.get('guestbook_name') - - greetings = self.get_greetings(guestbook_name) - stats = memcache.get_stats() - - self.response.write('Cache Hits:{}
        '.format(stats['hits'])) - self.response.write('Cache Misses:{}

        '.format( - stats['misses'])) - self.response.write(greetings) - - self.response.write(""" -
        -
        -
        -
        -
        -
        Guestbook name: -
        - - """.format(urllib.urlencode({'guestbook_name': guestbook_name}), - cgi.escape(guestbook_name))) - - # [START check_memcache] - def get_greetings(self, guestbook_name): - """ - get_greetings() - Checks the cache to see if there are cached greetings. - If not, call render_greetings and set the cache - - Args: - guestbook_name: Guestbook entity group key (string). - - Returns: - A string of HTML containing greetings. - """ - greetings = memcache.get('{}:greetings'.format(guestbook_name)) - if greetings is None: - greetings = self.render_greetings(guestbook_name) - if not memcache.add('{}:greetings'.format(guestbook_name), - greetings, 10): - logging.error('Memcache set failed.') - return greetings - # [END check_memcache] - - # [START query_datastore] - def render_greetings(self, guestbook_name): - """ - render_greetings() - Queries the database for greetings, iterate through the - results and create the HTML. - - Args: - guestbook_name: Guestbook entity group key (string). - - Returns: - A string of HTML containing greetings - """ - greetings = ndb.gql('SELECT * ' - 'FROM Greeting ' - 'WHERE ANCESTOR IS :1 ' - 'ORDER BY date DESC LIMIT 10', - guestbook_key(guestbook_name)) - output = cStringIO.StringIO() - for greeting in greetings: - if greeting.author: - output.write('{} wrote:'.format(greeting.author)) - else: - output.write('An anonymous person wrote:') - output.write('
        {}
        '.format( - cgi.escape(greeting.content))) - return output.getvalue() - # [END query_datastore] - - -class Guestbook(webapp2.RequestHandler): - def post(self): - # We set the same parent key on the 'Greeting' to ensure each greeting - # is in the same entity group. Queries across the single entity group - # are strongly consistent. However, the write rate to a single entity - # group is limited to ~1/second. - guestbook_name = self.request.get('guestbook_name') - greeting = Greeting(parent=guestbook_key(guestbook_name)) - - if users.get_current_user(): - greeting.author = users.get_current_user().nickname() - - greeting.content = self.request.get('content') - greeting.put() - memcache.delete('{}:greetings'.format(guestbook_name)) - self.redirect('/?' + - urllib.urlencode({'guestbook_name': guestbook_name})) - - -app = webapp2.WSGIApplication([('/', MainPage), - ('/sign', Guestbook)], - debug=True) - -# [END all] diff --git a/appengine/memcache/guestbook/main_test.py b/appengine/memcache/guestbook/main_test.py deleted file mode 100644 index 977004c1abe..00000000000 --- a/appengine/memcache/guestbook/main_test.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import main -import webtest - - -def test_app(testbed): - app = webtest.TestApp(main.app) - response = app.get('/') - assert response.status_int == 200 diff --git a/appengine/multitenancy/README.md b/appengine/multitenancy/README.md deleted file mode 100644 index 3097ac92961..00000000000 --- a/appengine/multitenancy/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Google App Engine Namespaces - -This sample demonstrates how to use Google App Engine's [Namespace Manager API](https://cloud.google.com/appengine/docs/python/multitenancy/multitenancy). - - -These samples are used on the following documentation page: - -> https://cloud.google.com/appengine/docs/python/multitenancy/namespaces - - - -Refer to the [App Engine Samples README](../README.md) for information on how to run and deploy this sample. diff --git a/appengine/ndb/modeling/README.md b/appengine/ndb/modeling/README.md deleted file mode 100644 index f4d19f5e586..00000000000 --- a/appengine/ndb/modeling/README.md +++ /dev/null @@ -1,5 +0,0 @@ -## App Engine Datastore NDB Modeling Samples - -These samples demonstrate how to [model entity relationships](https://cloud.google.com/appengine/articles/modeling]) using the Datastore NDB library. - -Refer to the [App Engine Samples README](../../README.md) for information on how to run and deploy this sample. diff --git a/appengine/ndb/overview/README.md b/appengine/ndb/overview/README.md deleted file mode 100644 index 35d6b218e69..00000000000 --- a/appengine/ndb/overview/README.md +++ /dev/null @@ -1,12 +0,0 @@ -## App Engine Datastore NDB Overview Sample - -This is a sample app for Google App Engine that demonstrates the [Datastore NDB Python API](https://cloud.google.com/appengine/docs/python/ndb/). - - -These samples are used on the following documentation page: - -> https://cloud.google.com/appengine/docs/python/ndb/ - - - -Refer to the [App Engine Samples README](../../README.md) for information on how to run and deploy this sample. diff --git a/appengine/ndb/overview/app.yaml b/appengine/ndb/overview/app.yaml deleted file mode 100644 index 5227472ff0c..00000000000 --- a/appengine/ndb/overview/app.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# This file specifies your Python application's runtime configuration -# including URL routing, versions, static file uploads, etc. See -# https://developers.google.com/appengine/docs/python/config/appconfig -# for details. - -version: 1 -runtime: python27 -api_version: 1 -threadsafe: yes - -# Handlers define how to route requests to your application. -handlers: - -# This handler tells app engine how to route requests to a WSGI application. -# The script value is in the format . -# where is a WSGI application object. -- url: .* # This regex directs all routes to main.app - script: main.app - -libraries: -- name: webapp2 - version: "2.5.2" diff --git a/appengine/ndb/overview/main.py b/appengine/ndb/overview/main.py deleted file mode 100644 index f2f6e755c42..00000000000 --- a/appengine/ndb/overview/main.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Cloud Datastore NDB API guestbook sample. - -This sample is used on this page: - https://cloud.google.com/appengine/docs/python/ndb/ - -For more information, see README.md -""" - -# [START all] -import cgi -import urllib - -from google.appengine.ext import ndb - -import webapp2 - - -# [START greeting] -class Greeting(ndb.Model): - """Models an individual Guestbook entry with content and date.""" - content = ndb.StringProperty() - date = ndb.DateTimeProperty(auto_now_add=True) -# [END greeting] - -# [START query] - @classmethod - def query_book(cls, ancestor_key): - return cls.query(ancestor=ancestor_key).order(-cls.date) - - -class MainPage(webapp2.RequestHandler): - def get(self): - self.response.out.write('') - guestbook_name = self.request.get('guestbook_name') - ancestor_key = ndb.Key("Book", guestbook_name or "*notitle*") - greetings = Greeting.query_book(ancestor_key).fetch(20) - - for greeting in greetings: - self.response.out.write('
        %s
        ' % - cgi.escape(greeting.content)) -# [END query] - - self.response.out.write(""" -
        -
        -
        -
        -
        -
        Guestbook name: -
        - - """ % (urllib.urlencode({'guestbook_name': guestbook_name}), - cgi.escape(guestbook_name))) - - -# [START submit] -class SubmitForm(webapp2.RequestHandler): - def post(self): - # We set the parent key on each 'Greeting' to ensure each guestbook's - # greetings are in the same entity group. - guestbook_name = self.request.get('guestbook_name') - greeting = Greeting(parent=ndb.Key("Book", - guestbook_name or "*notitle*"), - content=self.request.get('content')) - greeting.put() -# [END submit] - self.redirect('/?' + urllib.urlencode( - {'guestbook_name': guestbook_name})) - - -app = webapp2.WSGIApplication([ - ('/', MainPage), - ('/sign', SubmitForm) -]) -# [END all] diff --git a/appengine/ndb/overview/main_test.py b/appengine/ndb/overview/main_test.py deleted file mode 100644 index 6abe11e8d42..00000000000 --- a/appengine/ndb/overview/main_test.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import main -import webtest - - -def test_app(testbed): - app = webtest.TestApp(main.app) - response = app.get('/') - assert response.status_int == 200 diff --git a/appengine/ndb/transactions/README.md b/appengine/ndb/transactions/README.md deleted file mode 100644 index 78dc33ab269..00000000000 --- a/appengine/ndb/transactions/README.md +++ /dev/null @@ -1,14 +0,0 @@ -## App Engine Datastore NDB Transactions Sample - -This is a sample app for Google App Engine that demonstrates the [NDB Transactions Python API](https://cloud.google.com/appengine/docs/python/ndb/transactions) - -This app presents a list of notes. After you submit a note with a particular title, you may not change that note or submit a new note with the same title. There are multiple note pages available. - - -These samples are used on the following documentation page: - -> https://cloud.google.com/appengine/docs/python/ndb/transactions - - - -Refer to the [App Engine Samples README](../../README.md) for information on how to run and deploy this sample. diff --git a/appengine/ndb/transactions/main_test.py b/appengine/ndb/transactions/main_test.py deleted file mode 100644 index 43c6349bbbe..00000000000 --- a/appengine/ndb/transactions/main_test.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import main -import pytest - - -@pytest.fixture -def app(testbed): - main.app.config['TESTING'] = True - return main.app.test_client() - - -def test_index(app): - rv = app.get('/') - assert 'Permanent note page' in rv.data - assert rv.status == '200 OK' - - -def test_post(app): - rv = app.post('/add', data=dict( - note_title='Title', - note_text='Text' - ), follow_redirects=True) - assert rv.status == '200 OK' - - -def test_there(app): - rv = app.post('/add', data=dict( - note_title='Title', - note_text='New' - ), follow_redirects=True) - rv = app.post('/add', data=dict( - note_title='Title', - note_text='There' - ), follow_redirects=True) - assert 'Already there' in rv.data - assert rv.status == '200 OK' diff --git a/appengine/ndb/transactions/requirements.txt b/appengine/ndb/transactions/requirements.txt deleted file mode 100644 index 632a1efabce..00000000000 --- a/appengine/ndb/transactions/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -Flask==0.10.1 diff --git a/appengine/standard/README.md b/appengine/standard/README.md new file mode 100644 index 00000000000..3882865cfb3 --- /dev/null +++ b/appengine/standard/README.md @@ -0,0 +1,47 @@ +# Google App Engine Samples + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/README.md + +This section contains samples for [Google App Engine](https://cloud.google.com/appengine). Most of these samples have associated documentation that is linked +within the docstring of the sample itself. + +## Running the samples locally + +1. Download the [Google App Engine Python SDK](https://cloud.google.com/appengine/downloads) for your platform. +2. Many samples require extra libraries to be installed. If there is a `requirements.txt`, you will need to install the dependencies with [`pip`](pip.readthedocs.org). + + pip install -t lib -r requirements.txt + +3. Use `dev_appserver.py` to run the sample: + + dev_appserver.py app.yaml + +4. Visit `http://localhost:8080` to view your application. + +Some samples may require additional setup. Refer to individual sample READMEs. + +## Deploying the samples + +1. Download the [Google App Engine Python SDK](https://cloud.google.com/appengine/downloads) for your platform. +2. Many samples require extra libraries to be installed. If there is a `requirements.txt`, you will need to install the dependencies with [`pip`](pip.readthedocs.org). + + pip install -t lib -r requirements.txt + +3. Use `gcloud` to deploy the sample, you will need to specify your Project ID and a version number: + + gcloud app deploy --project your-app-id -v your-version + +4. Visit `https://your-app-id.appspot.com` to view your application. + +## Additional resources + +For more information on App Engine: + +> https://cloud.google.com/appengine + +For more information on Python on App Engine: + +> https://cloud.google.com/appengine/docs/python diff --git a/appengine/standard/analytics/app.yaml b/appengine/standard/analytics/app.yaml new file mode 100644 index 00000000000..ccce47480a9 --- /dev/null +++ b/appengine/standard/analytics/app.yaml @@ -0,0 +1,10 @@ +runtime: python27 +api_version: 1 +threadsafe: yes + +handlers: +- url: .* + script: main.app + +env_variables: + GA_TRACKING_ID: your-tracking-id diff --git a/appengine/standard/analytics/appengine_config.py b/appengine/standard/analytics/appengine_config.py new file mode 100644 index 00000000000..c903d9a0ac5 --- /dev/null +++ b/appengine/standard/analytics/appengine_config.py @@ -0,0 +1,18 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.ext import vendor + +# Add any libraries installed in the "lib" folder. +vendor.add('lib') diff --git a/appengine/standard/analytics/main.py b/appengine/standard/analytics/main.py new file mode 100644 index 00000000000..d7d829f3063 --- /dev/null +++ b/appengine/standard/analytics/main.py @@ -0,0 +1,77 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +from flask import Flask +import requests +import requests_toolbelt.adapters.appengine + +# Use the App Engine Requests adapter. This makes sure that Requests uses +# URLFetch. +requests_toolbelt.adapters.appengine.monkeypatch() + +app = Flask(__name__) + +# Environment variables are defined in app.yaml. +GA_TRACKING_ID = os.environ['GA_TRACKING_ID'] + + +# [START gae_analytics_track_event] +def track_event(category, action, label=None, value=0): + data = { + 'v': '1', # API Version. + 'tid': GA_TRACKING_ID, # Tracking ID / Property ID. + # Anonymous Client Identifier. Ideally, this should be a UUID that + # is associated with particular user, device, or browser instance. + 'cid': '555', + 't': 'event', # Event hit type. + 'ec': category, # Event category. + 'ea': action, # Event action. + 'el': label, # Event label. + 'ev': value, # Event value, must be an integer + } + + response = requests.post( + 'https://www.google-analytics.com/collect', data=data) + + # If the request fails, this will raise a RequestException. Depending + # on your application's needs, this may be a non-error and can be caught + # by the caller. + response.raise_for_status() + + +@app.route('/') +def track_example(): + track_event( + category='Example', + action='test action') + return 'Event tracked.' +# [END gae_analytics_track_event] + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
        {}
        + See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/appengine/standard/analytics/main_test.py b/appengine/standard/analytics/main_test.py new file mode 100644 index 00000000000..db9d2a93691 --- /dev/null +++ b/appengine/standard/analytics/main_test.py @@ -0,0 +1,47 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +import pytest +import responses + + +@pytest.fixture +def app(monkeypatch, testbed): + monkeypatch.setenv('GA_TRACKING_ID', '1234') + + import main + + main.app.testing = True + return main.app.test_client() + + +@responses.activate +def test_tracking(app): + responses.add( + responses.POST, + re.compile(r'.*'), + body='{}', + content_type='application/json') + + r = app.get('/') + + assert r.status_code == 200 + assert 'Event tracked' in r.data.decode('utf-8') + + assert len(responses.calls) == 1 + request_body = responses.calls[0].request.body + assert 'tid=1234' in request_body + assert 'ea=test+action' in request_body diff --git a/appengine/standard/analytics/requirements.txt b/appengine/standard/analytics/requirements.txt new file mode 100644 index 00000000000..c1089c7d4dc --- /dev/null +++ b/appengine/standard/analytics/requirements.txt @@ -0,0 +1,3 @@ +Flask==1.0.2 +requests==2.21.0 +requests-toolbelt==0.9.1 diff --git a/appengine/standard/angular/README.md b/appengine/standard/angular/README.md new file mode 100644 index 00000000000..cb2132c595e --- /dev/null +++ b/appengine/standard/angular/README.md @@ -0,0 +1,10 @@ +## App Engine & Angular JS + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/angular/README.md + +A simple [AngularJS](http://angularjs.org/) CRUD application for [Google App Engine](https://appengine.google.com/). + +Refer to the [App Engine Samples README](../README.md) for information on how to run and deploy this sample. diff --git a/appengine/standard/angular/app.yaml b/appengine/standard/angular/app.yaml new file mode 100644 index 00000000000..c7f82392acd --- /dev/null +++ b/appengine/standard/angular/app.yaml @@ -0,0 +1,15 @@ +runtime: python27 +threadsafe: true +api_version: 1 + +handlers: +- url: /rest/.* + script: main.APP + +- url: /(.+) + static_files: app/\1 + upload: app/.* + +- url: / + static_files: app/index.html + upload: app/index.html diff --git a/appengine/angular/app/css/app.css b/appengine/standard/angular/app/css/app.css similarity index 100% rename from appengine/angular/app/css/app.css rename to appengine/standard/angular/app/css/app.css diff --git a/appengine/angular/app/index.html b/appengine/standard/angular/app/index.html similarity index 100% rename from appengine/angular/app/index.html rename to appengine/standard/angular/app/index.html diff --git a/appengine/angular/app/js/app.js b/appengine/standard/angular/app/js/app.js similarity index 100% rename from appengine/angular/app/js/app.js rename to appengine/standard/angular/app/js/app.js diff --git a/appengine/angular/app/partials/insert.html b/appengine/standard/angular/app/partials/insert.html similarity index 100% rename from appengine/angular/app/partials/insert.html rename to appengine/standard/angular/app/partials/insert.html diff --git a/appengine/angular/app/partials/main.html b/appengine/standard/angular/app/partials/main.html similarity index 100% rename from appengine/angular/app/partials/main.html rename to appengine/standard/angular/app/partials/main.html diff --git a/appengine/angular/app/partials/update.html b/appengine/standard/angular/app/partials/update.html similarity index 100% rename from appengine/angular/app/partials/update.html rename to appengine/standard/angular/app/partials/update.html diff --git a/appengine/standard/angular/main.py b/appengine/standard/angular/main.py new file mode 100644 index 00000000000..127605e7d21 --- /dev/null +++ b/appengine/standard/angular/main.py @@ -0,0 +1,75 @@ +# Copyright 2013 Google, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import webapp2 + +import model + + +def AsDict(guest): + return {'id': guest.key.id(), 'first': guest.first, 'last': guest.last} + + +class RestHandler(webapp2.RequestHandler): + + def dispatch(self): + # time.sleep(1) + super(RestHandler, self).dispatch() + + def SendJson(self, r): + self.response.headers['content-type'] = 'text/plain' + self.response.write(json.dumps(r)) + + +class QueryHandler(RestHandler): + + def get(self): + guests = model.AllGuests() + r = [AsDict(guest) for guest in guests] + self.SendJson(r) + + +class UpdateHandler(RestHandler): + + def post(self): + r = json.loads(self.request.body) + guest = model.UpdateGuest(r['id'], r['first'], r['last']) + r = AsDict(guest) + self.SendJson(r) + + +class InsertHandler(RestHandler): + + def post(self): + r = json.loads(self.request.body) + guest = model.InsertGuest(r['first'], r['last']) + r = AsDict(guest) + self.SendJson(r) + + +class DeleteHandler(RestHandler): + + def post(self): + r = json.loads(self.request.body) + model.DeleteGuest(r['id']) + + +APP = webapp2.WSGIApplication([ + ('/rest/query', QueryHandler), + ('/rest/insert', InsertHandler), + ('/rest/delete', DeleteHandler), + ('/rest/update', UpdateHandler), +], debug=True) diff --git a/appengine/angular/model.py b/appengine/standard/angular/model.py similarity index 100% rename from appengine/angular/model.py rename to appengine/standard/angular/model.py diff --git a/appengine/angular/scripts/deploy.sh b/appengine/standard/angular/scripts/deploy.sh similarity index 100% rename from appengine/angular/scripts/deploy.sh rename to appengine/standard/angular/scripts/deploy.sh diff --git a/appengine/angular/scripts/run.sh b/appengine/standard/angular/scripts/run.sh similarity index 100% rename from appengine/angular/scripts/run.sh rename to appengine/standard/angular/scripts/run.sh diff --git a/appengine/logging/reading_logs/app.yaml b/appengine/standard/app_identity/asserting/app.yaml similarity index 100% rename from appengine/logging/reading_logs/app.yaml rename to appengine/standard/app_identity/asserting/app.yaml diff --git a/appengine/standard/app_identity/asserting/main.py b/appengine/standard/app_identity/asserting/main.py new file mode 100644 index 00000000000..33bfad0c6e4 --- /dev/null +++ b/appengine/standard/app_identity/asserting/main.py @@ -0,0 +1,62 @@ +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Sample Google App Engine application that demonstrates using the App Engine +identity API to generate an auth token. + +For more information about App Engine, see README.md under /appengine. +""" + +# [START gae_python_app_identity_asserting] +import json +import logging + +from google.appengine.api import app_identity +from google.appengine.api import urlfetch +import webapp2 + + +class MainPage(webapp2.RequestHandler): + def get(self): + auth_token, _ = app_identity.get_access_token( + 'https://www.googleapis.com/auth/cloud-platform') + logging.info( + 'Using token {} to represent identity {}'.format( + auth_token, app_identity.get_service_account_name())) + + response = urlfetch.fetch( + 'https://www.googleapis.com/storage/v1/b?project={}'.format( + app_identity.get_application_id()), + method=urlfetch.GET, + headers={ + 'Authorization': 'Bearer {}'.format(auth_token) + } + ) + + if response.status_code != 200: + raise Exception( + 'Call failed. Status code {}. Body {}'.format( + response.status_code, response.content)) + + result = json.loads(response.content) + self.response.headers['Content-Type'] = 'application/json' + self.response.write(json.dumps(result, indent=2)) + + +app = webapp2.WSGIApplication([ + ('/', MainPage) +], debug=True) + +# [END gae_python_app_identity_asserting] diff --git a/appengine/standard/app_identity/asserting/main_test.py b/appengine/standard/app_identity/asserting/main_test.py new file mode 100644 index 00000000000..66169aa2b82 --- /dev/null +++ b/appengine/standard/app_identity/asserting/main_test.py @@ -0,0 +1,32 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import webtest + +import main + + +def test_app(testbed): + app = webtest.TestApp(main.app) + + with mock.patch('main.urlfetch.fetch') as fetch_mock: + result_mock = mock.Mock() + result_mock.status_code = 200 + result_mock.content = '{}' + fetch_mock.return_value = result_mock + + response = app.get('/') + assert response.status_int == 200 + assert fetch_mock.called diff --git a/appengine/logging/writing_logs/app.yaml b/appengine/standard/app_identity/incoming/app.yaml similarity index 100% rename from appengine/logging/writing_logs/app.yaml rename to appengine/standard/app_identity/incoming/app.yaml diff --git a/appengine/standard/app_identity/incoming/main.py b/appengine/standard/app_identity/incoming/main.py new file mode 100644 index 00000000000..d5daeaa8a2e --- /dev/null +++ b/appengine/standard/app_identity/incoming/main.py @@ -0,0 +1,46 @@ +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Sample Google App Engine application that demonstrates usage of the app +engine inbound app ID header. + +For more information about App Engine, see README.md under /appengine. +""" + +# [START gae_python_app_identity_incoming] +import webapp2 + + +class MainPage(webapp2.RequestHandler): + allowed_app_ids = [ + 'other-app-id', + 'other-app-id-2' + ] + + def get(self): + incoming_app_id = self.request.headers.get( + 'X-Appengine-Inbound-Appid', None) + + if incoming_app_id not in self.allowed_app_ids: + self.abort(403) + + self.response.write('This is a protected page.') + + +app = webapp2.WSGIApplication([ + ('/', MainPage) +], debug=True) + +# [END gae_python_app_identity_incoming] diff --git a/appengine/standard/app_identity/incoming/main_test.py b/appengine/standard/app_identity/incoming/main_test.py new file mode 100644 index 00000000000..38faaa3f015 --- /dev/null +++ b/appengine/standard/app_identity/incoming/main_test.py @@ -0,0 +1,28 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import webtest + +import main + + +def test_app(testbed): + app = webtest.TestApp(main.app) + + response = app.get('/', status=403) + + response = app.get('/', headers={ + 'X-Appengine-Inbound-Appid': 'other-app-id' + }) + assert response.status_int == 200 diff --git a/appengine/app_identity/signing/app.yaml b/appengine/standard/app_identity/signing/app.yaml similarity index 100% rename from appengine/app_identity/signing/app.yaml rename to appengine/standard/app_identity/signing/app.yaml diff --git a/appengine/standard/app_identity/signing/main.py b/appengine/standard/app_identity/signing/main.py new file mode 100644 index 00000000000..863e9e9e482 --- /dev/null +++ b/appengine/standard/app_identity/signing/main.py @@ -0,0 +1,84 @@ +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Sample Google App Engine application that demonstrates usage of the app +identity API to sign bytes and verify signatures. + +For more information about App Engine, see README.md under /appengine. +""" + +# [START gae_python_app_identity_signing] + +import base64 + +from Crypto.Hash import SHA256 +from Crypto.PublicKey import RSA +from Crypto.Signature import PKCS1_v1_5 +from Crypto.Util.asn1 import DerSequence +from google.appengine.api import app_identity +import webapp2 + + +def verify_signature(data, signature, x509_certificate): + """Verifies a signature using the given x.509 public key certificate.""" + + # PyCrypto 2.6 doesn't support x.509 certificates directly, so we'll need + # to extract the public key from it manually. + # This code is based on https://github.com/google/oauth2client/blob/master + # /oauth2client/_pycrypto_crypt.py + pem_lines = x509_certificate.replace(b' ', b'').split() + cert_der = base64.urlsafe_b64decode(b''.join(pem_lines[1:-1])) + cert_seq = DerSequence() + cert_seq.decode(cert_der) + tbs_seq = DerSequence() + tbs_seq.decode(cert_seq[0]) + public_key = RSA.importKey(tbs_seq[6]) + + signer = PKCS1_v1_5.new(public_key) + digest = SHA256.new(data) + + return signer.verify(digest, signature) + + +def verify_signed_by_app(data, signature): + """Checks the signature and data against all currently valid certificates + for the application.""" + public_certificates = app_identity.get_public_certificates() + + for cert in public_certificates: + if verify_signature(data, signature, cert.x509_certificate_pem): + return True + + return False + + +class MainPage(webapp2.RequestHandler): + def get(self): + message = 'Hello, world!' + signing_key_name, signature = app_identity.sign_blob(message) + verified = verify_signed_by_app(message, signature) + + self.response.content_type = 'text/plain' + self.response.write('Message: {}\n'.format(message)) + self.response.write( + 'Signature: {}\n'.format(base64.b64encode(signature))) + self.response.write('Verified: {}\n'.format(verified)) + + +app = webapp2.WSGIApplication([ + ('/', MainPage) +], debug=True) + +# [END gae_python_app_identity_signing] diff --git a/appengine/standard/app_identity/signing/main_test.py b/appengine/standard/app_identity/signing/main_test.py new file mode 100644 index 00000000000..4864f9a18db --- /dev/null +++ b/appengine/standard/app_identity/signing/main_test.py @@ -0,0 +1,24 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import webtest + +import main + + +def test_app(testbed): + app = webtest.TestApp(main.app) + response = app.get('/') + assert response.status_int == 200 + assert 'Verified: True' in response.text diff --git a/appengine/standard/appstats/app.yaml b/appengine/standard/appstats/app.yaml new file mode 100644 index 00000000000..eb89129d68e --- /dev/null +++ b/appengine/standard/appstats/app.yaml @@ -0,0 +1,10 @@ +runtime: python27 +api_version: 1 +threadsafe: yes + +builtins: +- appstats: on + +handlers: +- url: .* + script: main.app diff --git a/appengine/standard/appstats/appengine_config.py b/appengine/standard/appstats/appengine_config.py new file mode 100644 index 00000000000..4c7a268901c --- /dev/null +++ b/appengine/standard/appstats/appengine_config.py @@ -0,0 +1,20 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# The app engine runtime will call this function during instance startup. +def webapp_add_wsgi_middleware(app): + from google.appengine.ext.appstats import recording + app = recording.appstats_wsgi_middleware(app) + return app diff --git a/appengine/standard/appstats/main.py b/appengine/standard/appstats/main.py new file mode 100644 index 00000000000..d4b5cab75f1 --- /dev/null +++ b/appengine/standard/appstats/main.py @@ -0,0 +1,35 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Sample Google App Engine application that demonstrates using appstats. + +For more information about App Engine, see README.md under /appengine. +""" + +from google.appengine.api import memcache +import webapp2 + + +class MainPage(webapp2.RequestHandler): + def get(self): + # Perform some RPCs so that appstats can capture them. + memcache.set('example_key', 50) + value = memcache.get('example_key') + self.response.write('Value is: {}'.format(value)) + + +app = webapp2.WSGIApplication([ + ('/', MainPage) +], debug=True) diff --git a/appengine/standard/appstats/main_test.py b/appengine/standard/appstats/main_test.py new file mode 100644 index 00000000000..9cceda3d317 --- /dev/null +++ b/appengine/standard/appstats/main_test.py @@ -0,0 +1,22 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import webtest + +import main + + +def test_app(testbed): + app = webtest.TestApp(main.app) + app.get('/') diff --git a/appengine/standard/background/README.md b/appengine/standard/background/README.md new file mode 100644 index 00000000000..b945953e913 --- /dev/null +++ b/appengine/standard/background/README.md @@ -0,0 +1,14 @@ +# Using Background Threads from Google App Engine + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/background/README.md + +This example shows how to use manual or basic scaling to start App Engine background threads. + +See the [documentation on modules](https://cloud.google.com/appengine/docs/python/modules/) for +more information. + +Your app.yaml configuration must specify scaling as either manual or basic. The default +automatic scaling does not allow background threads. diff --git a/appengine/standard/background/app.yaml b/appengine/standard/background/app.yaml new file mode 100644 index 00000000000..57cdb0bf57c --- /dev/null +++ b/appengine/standard/background/app.yaml @@ -0,0 +1,10 @@ +runtime: python27 +api_version: 1 +threadsafe: yes + +manual_scaling: + instances: 1 + +handlers: +- url: .* + script: main.app diff --git a/appengine/standard/background/main.py b/appengine/standard/background/main.py new file mode 100644 index 00000000000..a55b0c0554c --- /dev/null +++ b/appengine/standard/background/main.py @@ -0,0 +1,85 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Sample application that demonstrates how to use the App Engine background +threads. + +app.yaml scaling must be set to manual or basic. +""" + +# [START gae_runtime_import] +from google.appengine.api import background_thread +# [END gae_runtime_import] + +import webapp2 + +val = 'Dog' + + +class MainHandler(webapp2.RequestHandler): + def get(self): + self.response.headers['Content-Type'] = 'text/plain' + self.response.write(str(val)) + + +class SetDogHandler(webapp2.RequestHandler): + """ Resets the global val to 'Dog'""" + + def get(self): + global val + val = 'Dog' + self.response.headers['Content-Type'] = 'text/plain' + self.response.write('Done') + + +class SetCatBackgroundHandler(webapp2.RequestHandler): + """ Demonstrates two ways to start new background threads + """ + + def get(self): + """ + Demonstrates using a background thread to change the global + val from 'Dog' to 'Cat' + + The auto GET parameter determines whether to start the thread + automatically or manually + """ + auto = self.request.get('auto') + + # [START gae_runtime] + # sample function to run in a background thread + def change_val(arg): + global val + val = arg + + if auto: + # Start the new thread in one command + background_thread.start_new_background_thread(change_val, ['Cat']) + else: + # create a new thread and start it + t = background_thread.BackgroundThread( + target=change_val, args=['Cat']) + t.start() + # [END gae_runtime] + + self.response.headers['Content-Type'] = 'text/plain' + self.response.write('Done') + + +app = webapp2.WSGIApplication([ + ('/', MainHandler), + ('/dog', SetDogHandler), + ('/cat', SetCatBackgroundHandler), +], debug=True) diff --git a/appengine/standard/background/main_test.py b/appengine/standard/background/main_test.py new file mode 100644 index 00000000000..aa3b78caac7 --- /dev/null +++ b/appengine/standard/background/main_test.py @@ -0,0 +1,57 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from mock import patch +import pytest +import webtest + +import main + +PROJECT = os.environ['GCLOUD_PROJECT'] + + +@pytest.fixture +def app(testbed): + main.PROJECTID = PROJECT + return webtest.TestApp(main.app) + + +@patch("main.background_thread") +def test_background(thread, app): + app.get('/dog') + response = app.get('/') + assert response.status_int == 200 + assert response.body == 'Dog' + app.get('/cat') + # no stub for system so manually set + main.val = 'Cat' + response = app.get('/') + assert response.status_int == 200 + assert response.body == 'Cat' + + +@patch("main.background_thread") +def test_background_auto_start(thread, app): + app.get('/dog') + response = app.get('/') + assert response.status_int == 200 + assert response.body == 'Dog' + app.get('/cat?auto=True') + # no stub for system so manually set + main.val = 'Cat' + response = app.get('/') + assert response.status_int == 200 + assert response.body == 'Cat' diff --git a/appengine/standard/blobstore/api/README.md b/appengine/standard/blobstore/api/README.md new file mode 100644 index 00000000000..d171d4e1d86 --- /dev/null +++ b/appengine/standard/blobstore/api/README.md @@ -0,0 +1,16 @@ +# App Engine Blobstore Sample + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/blobstore/api/README.md + + +These samples are used on the following documentation pages: + +> +* https://cloud.google.com/appengine/docs/python/tools/webapp/blobstorehandlers +* https://cloud.google.com/appengine/docs/python/blobstore/ + + + --> diff --git a/appengine/blobstore/app.yaml b/appengine/standard/blobstore/api/app.yaml similarity index 100% rename from appengine/blobstore/app.yaml rename to appengine/standard/blobstore/api/app.yaml diff --git a/appengine/standard/blobstore/api/main.py b/appengine/standard/blobstore/api/main.py new file mode 100644 index 00000000000..2a0b606c9a3 --- /dev/null +++ b/appengine/standard/blobstore/api/main.py @@ -0,0 +1,81 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Sample application that demonstrates how to use the App Engine Blobstore API. + +For more information, see README.md. +""" + +# [START gae_blobstore_sample] +from google.appengine.api import users +from google.appengine.ext import blobstore +from google.appengine.ext import ndb +from google.appengine.ext.webapp import blobstore_handlers +import webapp2 + + +# This datastore model keeps track of which users uploaded which photos. +class UserPhoto(ndb.Model): + user = ndb.StringProperty() + blob_key = ndb.BlobKeyProperty() + + +class PhotoUploadFormHandler(webapp2.RequestHandler): + def get(self): + # [START gae_blobstore_upload_url] + upload_url = blobstore.create_upload_url('/upload_photo') + # [END gae_blobstore_upload_url] + # [START gae_blobstore_upload_form] + # To upload files to the blobstore, the request method must be "POST" + # and enctype must be set to "multipart/form-data". + self.response.out.write(""" + +
        + Upload File:
        + +
        +""".format(upload_url)) + # [END gae_blobstore_upload_form] + + +# [START gae_blobstore_upload_handler] +class PhotoUploadHandler(blobstore_handlers.BlobstoreUploadHandler): + def post(self): + upload = self.get_uploads()[0] + user_photo = UserPhoto( + user=users.get_current_user().user_id(), + blob_key=upload.key()) + user_photo.put() + + self.redirect('/view_photo/%s' % upload.key()) +# [END gae_blobstore_upload_handler] + + +# [START gae_blobstore_download_handler] +class ViewPhotoHandler(blobstore_handlers.BlobstoreDownloadHandler): + def get(self, photo_key): + if not blobstore.get(photo_key): + self.error(404) + else: + self.send_blob(photo_key) +# [END gae_blobstore_download_handler] + + +app = webapp2.WSGIApplication([ + ('/', PhotoUploadFormHandler), + ('/upload_photo', PhotoUploadHandler), + ('/view_photo/([^/]+)?', ViewPhotoHandler), +], debug=True) +# [END gae_blobstore_sample] diff --git a/appengine/standard/blobstore/api/main_test.py b/appengine/standard/blobstore/api/main_test.py new file mode 100644 index 00000000000..adfc579698e --- /dev/null +++ b/appengine/standard/blobstore/api/main_test.py @@ -0,0 +1,26 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import webtest + +import main + + +def test_app(testbed, login): + app = webtest.TestApp(main.app) + + login() + response = app.get('/') + + assert '/_ah/upload' in response diff --git a/appengine/standard/blobstore/blobreader/app.yaml b/appengine/standard/blobstore/blobreader/app.yaml new file mode 100644 index 00000000000..636a04bae7e --- /dev/null +++ b/appengine/standard/blobstore/blobreader/app.yaml @@ -0,0 +1,8 @@ +runtime: python27 +api_version: 1 +threadsafe: yes + +handlers: +- url: .* + script: main.app + login: required diff --git a/appengine/standard/blobstore/blobreader/appengine_config.py b/appengine/standard/blobstore/blobreader/appengine_config.py new file mode 100644 index 00000000000..c903d9a0ac5 --- /dev/null +++ b/appengine/standard/blobstore/blobreader/appengine_config.py @@ -0,0 +1,18 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.ext import vendor + +# Add any libraries installed in the "lib" folder. +vendor.add('lib') diff --git a/appengine/standard/blobstore/blobreader/main.py b/appengine/standard/blobstore/blobreader/main.py new file mode 100644 index 00000000000..36f10ed1eff --- /dev/null +++ b/appengine/standard/blobstore/blobreader/main.py @@ -0,0 +1,69 @@ +"""A sample app that operates on GCS files with blobstore API's BlobReader.""" + +import cloudstorage +from google.appengine.api import app_identity +from google.appengine.ext import blobstore +import webapp2 + + +class BlobreaderHandler(webapp2.RequestHandler): + def get(self): + # Get the default Cloud Storage Bucket name and create a file name for + # the object in Cloud Storage. + bucket = app_identity.get_default_gcs_bucket_name() + + # Cloud Storage file names are in the format /bucket/object. + filename = '/{}/blobreader_demo'.format(bucket) + + # Create a file in Google Cloud Storage and write something to it. + with cloudstorage.open(filename, 'w') as filehandle: + filehandle.write('abcde\n') + + # In order to read the contents of the file using the Blobstore API, + # you must create a blob_key from the Cloud Storage file name. + # Blobstore expects the filename to be in the format of: + # /gs/bucket/object + blobstore_filename = '/gs{}'.format(filename) + blob_key = blobstore.create_gs_key(blobstore_filename) + + # [START gae_blobstore_reader] + # Instantiate a BlobReader for a given Blobstore blob_key. + blob_reader = blobstore.BlobReader(blob_key) + + # Instantiate a BlobReader for a given Blobstore blob_key, setting the + # buffer size to 1 MB. + blob_reader = blobstore.BlobReader(blob_key, buffer_size=1048576) + + # Instantiate a BlobReader for a given Blobstore blob_key, setting the + # initial read position. + blob_reader = blobstore.BlobReader(blob_key, position=0) + + # Read the entire value into memory. This may take a while depending + # on the size of the value and the size of the read buffer, and is not + # recommended for large values. + blob_reader_data = blob_reader.read() + + # Write the contents to the response. + self.response.headers['Content-Type'] = 'text/plain' + self.response.write(blob_reader_data) + + # Set the read position back to 0, then read and write 3 bytes. + blob_reader.seek(0) + blob_reader_data = blob_reader.read(3) + self.response.write(blob_reader_data) + self.response.write('\n') + + # Set the read position back to 0, then read and write one line (up to + # and including a '\n' character) at a time. + blob_reader.seek(0) + for line in blob_reader: + self.response.write(line) + # [END gae_blobstore_reader] + + # Delete the file from Google Cloud Storage using the blob_key. + blobstore.delete(blob_key) + + +app = webapp2.WSGIApplication([ + ('/', BlobreaderHandler), + ('/blobreader', BlobreaderHandler)], debug=True) diff --git a/appengine/standard/blobstore/blobreader/main_test.py b/appengine/standard/blobstore/blobreader/main_test.py new file mode 100644 index 00000000000..b0cc5898a42 --- /dev/null +++ b/appengine/standard/blobstore/blobreader/main_test.py @@ -0,0 +1,25 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import webtest + +import main + + +def test_blobreader(testbed, login): + app = webtest.TestApp(main.app) + + response = app.get('/blobreader') + + assert 'abcde\nabc\nabcde\n' in response diff --git a/appengine/standard/blobstore/blobreader/requirements.txt b/appengine/standard/blobstore/blobreader/requirements.txt new file mode 100644 index 00000000000..f2ec35f05f9 --- /dev/null +++ b/appengine/standard/blobstore/blobreader/requirements.txt @@ -0,0 +1 @@ +GoogleAppEngineCloudStorageClient==1.9.22.1 diff --git a/appengine/standard/blobstore/gcs/app.yaml b/appengine/standard/blobstore/gcs/app.yaml new file mode 100644 index 00000000000..636a04bae7e --- /dev/null +++ b/appengine/standard/blobstore/gcs/app.yaml @@ -0,0 +1,8 @@ +runtime: python27 +api_version: 1 +threadsafe: yes + +handlers: +- url: .* + script: main.app + login: required diff --git a/appengine/standard/blobstore/gcs/appengine_config.py b/appengine/standard/blobstore/gcs/appengine_config.py new file mode 100644 index 00000000000..c903d9a0ac5 --- /dev/null +++ b/appengine/standard/blobstore/gcs/appengine_config.py @@ -0,0 +1,18 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.ext import vendor + +# Add any libraries installed in the "lib" folder. +vendor.add('lib') diff --git a/appengine/standard/blobstore/gcs/main.py b/appengine/standard/blobstore/gcs/main.py new file mode 100644 index 00000000000..4ba0816ad90 --- /dev/null +++ b/appengine/standard/blobstore/gcs/main.py @@ -0,0 +1,76 @@ +"""A sample app that operates on GCS files with blobstore API.""" + +import cloudstorage +from google.appengine.api import app_identity +from google.appengine.ext import blobstore +from google.appengine.ext.webapp import blobstore_handlers +import webapp2 + + +# This handler creates a file in Cloud Storage using the cloudstorage +# client library and then reads the data back using the Blobstore API. +class CreateAndReadFileHandler(webapp2.RequestHandler): + def get(self): + # Get the default Cloud Storage Bucket name and create a file name for + # the object in Cloud Storage. + bucket = app_identity.get_default_gcs_bucket_name() + + # Cloud Storage file names are in the format /bucket/object. + filename = '/{}/blobstore_demo'.format(bucket) + + # Create a file in Google Cloud Storage and write something to it. + with cloudstorage.open(filename, 'w') as filehandle: + filehandle.write('abcde\n') + + # In order to read the contents of the file using the Blobstore API, + # you must create a blob_key from the Cloud Storage file name. + # Blobstore expects the filename to be in the format of: + # /gs/bucket/object + blobstore_filename = '/gs{}'.format(filename) + blob_key = blobstore.create_gs_key(blobstore_filename) + + # Read the file's contents using the Blobstore API. + # The last two parameters specify the start and end index of bytes we + # want to read. + data = blobstore.fetch_data(blob_key, 0, 6) + + # Write the contents to the response. + self.response.headers['Content-Type'] = 'text/plain' + self.response.write(data) + + # Delete the file from Google Cloud Storage using the blob_key. + blobstore.delete(blob_key) + + +# This handler creates a file in Cloud Storage using the cloudstorage +# client library and then serves the file back using the Blobstore API. +class CreateAndServeFileHandler(blobstore_handlers.BlobstoreDownloadHandler): + + def get(self): + # Get the default Cloud Storage Bucket name and create a file name for + # the object in Cloud Storage. + bucket = app_identity.get_default_gcs_bucket_name() + + # Cloud Storage file names are in the format /bucket/object. + filename = '/{}/blobstore_serving_demo'.format(bucket) + + # Create a file in Google Cloud Storage and write something to it. + with cloudstorage.open(filename, 'w') as filehandle: + filehandle.write('abcde\n') + + # In order to read the contents of the file using the Blobstore API, + # you must create a blob_key from the Cloud Storage file name. + # Blobstore expects the filename to be in the format of: + # /gs/bucket/object + blobstore_filename = '/gs{}'.format(filename) + blob_key = blobstore.create_gs_key(blobstore_filename) + + # BlobstoreDownloadHandler serves the file from Google Cloud Storage to + # your computer using blob_key. + self.send_blob(blob_key) + + +app = webapp2.WSGIApplication([ + ('/', CreateAndReadFileHandler), + ('/blobstore/read', CreateAndReadFileHandler), + ('/blobstore/serve', CreateAndServeFileHandler)], debug=True) diff --git a/appengine/standard/blobstore/gcs/main_test.py b/appengine/standard/blobstore/gcs/main_test.py new file mode 100644 index 00000000000..b5d1c27eb32 --- /dev/null +++ b/appengine/standard/blobstore/gcs/main_test.py @@ -0,0 +1,34 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import webtest + +import main + + +def test_create_and_read(testbed, login): + app = webtest.TestApp(main.app) + + response = app.get('/blobstore/read') + + assert 'abcde' in response + + +def test_create_and_serve(testbed, login): + app = webtest.TestApp(main.app) + + response = app.get('/blobstore/serve') + served_file_header = response.headers['X-AppEngine-BlobKey'] + + assert 'encoded_gs_file' in served_file_header diff --git a/appengine/standard/blobstore/gcs/requirements.txt b/appengine/standard/blobstore/gcs/requirements.txt new file mode 100644 index 00000000000..f2ec35f05f9 --- /dev/null +++ b/appengine/standard/blobstore/gcs/requirements.txt @@ -0,0 +1 @@ +GoogleAppEngineCloudStorageClient==1.9.22.1 diff --git a/appengine/standard/cloudsql/README.md b/appengine/standard/cloudsql/README.md new file mode 100644 index 00000000000..134c69f2844 --- /dev/null +++ b/appengine/standard/cloudsql/README.md @@ -0,0 +1,16 @@ +# Using Cloud SQL from Google App Engine + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/cloudsql/README.md + +This is an example program showing how to use the native MySQL connections from Google App Engine to [Google Cloud SQL](https://cloud.google.com/sql). + +Refer to the [App Engine Samples README](../README.md) for information on how to run and deploy this sample. + +## Setup + +1. You will need to create a [Cloud SQL instance](https://cloud.google.com/sql/docs/create-instance). + +2. Edit the update the `env_variables` section in `app.yaml` with your Cloud SQL configuration. diff --git a/appengine/standard/cloudsql/app.yaml b/appengine/standard/cloudsql/app.yaml new file mode 100644 index 00000000000..ef79e273a5d --- /dev/null +++ b/appengine/standard/cloudsql/app.yaml @@ -0,0 +1,18 @@ +runtime: python27 +api_version: 1 +threadsafe: yes + +handlers: +- url: / + script: main.app + +libraries: +- name: MySQLdb + version: "latest" + +# [START gae_python_mysql_env] +env_variables: + CLOUDSQL_CONNECTION_NAME: your-connection-name + CLOUDSQL_USER: root + CLOUDSQL_PASSWORD: your-cloudsql-user-password +# [END gae_python_mysql_env] diff --git a/appengine/standard/cloudsql/main.py b/appengine/standard/cloudsql/main.py new file mode 100644 index 00000000000..7bc1db3bbed --- /dev/null +++ b/appengine/standard/cloudsql/main.py @@ -0,0 +1,79 @@ +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Sample App Engine application demonstrating how to connect to Google Cloud SQL +using App Engine's native unix socket or using TCP when running locally. + +For more information, see the README.md. +""" + +# [START gae_python_mysql_app] +import os + +import MySQLdb +import webapp2 + + +# These environment variables are configured in app.yaml. +CLOUDSQL_CONNECTION_NAME = os.environ.get('CLOUDSQL_CONNECTION_NAME') +CLOUDSQL_USER = os.environ.get('CLOUDSQL_USER') +CLOUDSQL_PASSWORD = os.environ.get('CLOUDSQL_PASSWORD') + + +def connect_to_cloudsql(): + # When deployed to App Engine, the `SERVER_SOFTWARE` environment variable + # will be set to 'Google App Engine/version'. + if os.getenv('SERVER_SOFTWARE', '').startswith('Google App Engine/'): + # Connect using the unix socket located at + # /cloudsql/cloudsql-connection-name. + cloudsql_unix_socket = os.path.join( + '/cloudsql', CLOUDSQL_CONNECTION_NAME) + + db = MySQLdb.connect( + unix_socket=cloudsql_unix_socket, + user=CLOUDSQL_USER, + passwd=CLOUDSQL_PASSWORD) + + # If the unix socket is unavailable, then try to connect using TCP. This + # will work if you're running a local MySQL server or using the Cloud SQL + # proxy, for example: + # + # $ cloud_sql_proxy -instances=your-connection-name=tcp:3306 + # + else: + db = MySQLdb.connect( + host='127.0.0.1', user=CLOUDSQL_USER, passwd=CLOUDSQL_PASSWORD) + + return db + + +class MainPage(webapp2.RequestHandler): + def get(self): + """Simple request handler that shows all of the MySQL variables.""" + self.response.headers['Content-Type'] = 'text/plain' + + db = connect_to_cloudsql() + cursor = db.cursor() + cursor.execute('SHOW VARIABLES') + + for r in cursor.fetchall(): + self.response.write('{}\n'.format(r)) + + +app = webapp2.WSGIApplication([ + ('/', MainPage), +], debug=True) + +# [END gae_python_mysql_app] diff --git a/appengine/standard/cloudsql/main_test.py b/appengine/standard/cloudsql/main_test.py new file mode 100644 index 00000000000..d20847d02c4 --- /dev/null +++ b/appengine/standard/cloudsql/main_test.py @@ -0,0 +1,40 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +import pytest +import webtest + + +@pytest.fixture +def main(monkeypatch): + monkeypatch.setenv('CLOUDSQL_USER', 'root') + monkeypatch.setenv('CLOUDSQL_PASSWORD', '') + import main + return main + + +@pytest.mark.skipif( + not os.path.exists('/var/run/mysqld/mysqld.sock'), + reason='Local MySQL server not available.') +def test_app(main): + app = webtest.TestApp(main.app) + response = app.get('/') + + assert response.status_int == 200 + assert re.search( + re.compile(r'.*version.*', re.DOTALL), + response.body) diff --git a/appengine/standard/conftest.py b/appengine/standard/conftest.py new file mode 100644 index 00000000000..f600c548ee9 --- /dev/null +++ b/appengine/standard/conftest.py @@ -0,0 +1,40 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +# Import py.test hooks and fixtures for App Engine +from gcp_devrel.testing.appengine import ( + login, + pytest_configure, + pytest_runtest_call, + run_tasks, + testbed) +import six + +(login) +(pytest_configure) +(pytest_runtest_call) +(run_tasks) +(testbed) + + +def pytest_ignore_collect(path, config): + """Skip App Engine tests in python 3 or if no SDK is available.""" + if 'appengine/standard' in str(path): + if six.PY3: + return True + if 'GAE_SDK_PATH' not in os.environ: + return True + return False diff --git a/appengine/standard/django/README.md b/appengine/standard/django/README.md new file mode 100644 index 00000000000..7292e794aaa --- /dev/null +++ b/appengine/standard/django/README.md @@ -0,0 +1,15 @@ +# Getting started with Django on Google Cloud Platform on App Engine Standard + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/django/README.md + +This repository is an example of how to run a [Django](https://www.djangoproject.com/) +app on Google App Engine Standard Environment. It uses the +[Writing your first Django app](https://docs.djangoproject.com/en/1.9/intro/tutorial01/) as the +example app to deploy. + + +# Tutorial +See our [Running Django in the App Engine Standard Environment](https://cloud.google.com/python/django/appengine) tutorial for instructions for setting up and deploying this sample application. diff --git a/appengine/standard/django/app.yaml b/appengine/standard/django/app.yaml new file mode 100644 index 00000000000..15781218755 --- /dev/null +++ b/appengine/standard/django/app.yaml @@ -0,0 +1,31 @@ +# [START django_app] +runtime: python27 +api_version: 1 +threadsafe: yes + +handlers: +- url: /static + static_dir: static/ +- url: .* + script: mysite.wsgi.application + +# Only pure Python libraries can be vendored +# Python libraries that use C extensions can +# only be included if they are part of the App Engine SDK +# Using Third Party Libraries: https://cloud.google.com/appengine/docs/python/tools/using-libraries-python-27 +libraries: +- name: MySQLdb + version: 1.2.5 +# [END django_app] + +# Google App Engine limits application deployments to 10,000 uploaded files per +# version. The skip_files section allows us to skip virtual environment files +# to meet this requirement. The first 5 are the default regular expressions to +# skip, while the last one is for all env/ files. +skip_files: +- ^(.*/)?#.*#$ +- ^(.*/)?.*~$ +- ^(.*/)?.*\.py[co]$ +- ^(.*/)?.*/RCS/.*$ +- ^(.*/)?\..*$ +- ^env/.*$ diff --git a/appengine/standard/django/appengine_config.py b/appengine/standard/django/appengine_config.py new file mode 100644 index 00000000000..f18f4328eb2 --- /dev/null +++ b/appengine/standard/django/appengine_config.py @@ -0,0 +1,19 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START vendor] +from google.appengine.ext import vendor + +vendor.add('lib') +# [END vendor] diff --git a/appengine/standard/django/manage.py b/appengine/standard/django/manage.py new file mode 100755 index 00000000000..834b0091d73 --- /dev/null +++ b/appengine/standard/django/manage.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mysite.settings") + + from django.core.management import execute_from_command_line + + execute_from_command_line(sys.argv) diff --git a/appengine/standard/django/mysite/__init__.py b/appengine/standard/django/mysite/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/appengine/standard/django/mysite/settings.py b/appengine/standard/django/mysite/settings.py new file mode 100644 index 00000000000..58149c6cabf --- /dev/null +++ b/appengine/standard/django/mysite/settings.py @@ -0,0 +1,155 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Django settings for mysite project. + +Generated by 'django-admin startproject' using Django 1.8.5. + +For more information on this file, see +https://docs.djangoproject.com/en/1.8/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/1.8/ref/settings/ +""" + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +import os + +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = '-c&qt=71oi^e5s8(ene*$b89^#%*0xeve$x_trs91veok9#0h0' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +# SECURITY WARNING: App Engine's security features ensure that it is safe to +# have ALLOWED_HOSTS = ['*'] when the app is deployed. If you deploy a Django +# app not on App Engine, make sure to set an appropriate host here. +# See https://docs.djangoproject.com/en/1.10/ref/settings/ +ALLOWED_HOSTS = ['*'] + +# Application definition + +INSTALLED_APPS = ( + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'polls', +) + +MIDDLEWARE_CLASSES = ( + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'django.middleware.security.SecurityMiddleware', +) + +ROOT_URLCONF = 'mysite.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'mysite.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/1.8/ref/settings/#databases + +# Check to see if MySQLdb is available; if not, have pymysql masquerade as +# MySQLdb. This is a convenience feature for developers who cannot install +# MySQLdb locally; when running in production on Google App Engine Standard +# Environment, MySQLdb will be used. +try: + import MySQLdb # noqa: F401 +except ImportError: + import pymysql + pymysql.install_as_MySQLdb() + +# [START db_setup] +if os.getenv('SERVER_SOFTWARE', '').startswith('Google App Engine'): + # Running on production App Engine, so connect to Google Cloud SQL using + # the unix socket at /cloudsql/ + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.mysql', + 'HOST': '/cloudsql/', + 'NAME': 'polls', + 'USER': '', + 'PASSWORD': '', + } + } +else: + # Running locally so connect to either a local MySQL instance or connect to + # Cloud SQL via the proxy. To start the proxy via command line: + # + # $ cloud_sql_proxy -instances=[INSTANCE_CONNECTION_NAME]=tcp:3306 + # + # See https://cloud.google.com/sql/docs/mysql-connect-proxy + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.mysql', + 'HOST': '127.0.0.1', + 'PORT': '3306', + 'NAME': 'polls', + 'USER': '', + 'PASSWORD': '', + } + } +# [END db_setup] + +# Internationalization +# https://docs.djangoproject.com/en/1.8/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/1.8/howto/static-files/ + +STATIC_ROOT = 'static' +STATIC_URL = '/static/' diff --git a/appengine/standard/django/mysite/urls.py b/appengine/standard/django/mysite/urls.py new file mode 100644 index 00000000000..1b387da6467 --- /dev/null +++ b/appengine/standard/django/mysite/urls.py @@ -0,0 +1,23 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from django.conf.urls import include, url +from django.contrib import admin + +from polls.views import index + +urlpatterns = [ + url(r'^$', index), + url(r'^admin/', include(admin.site.urls)), +] diff --git a/appengine/standard/django/mysite/wsgi.py b/appengine/standard/django/mysite/wsgi.py new file mode 100644 index 00000000000..ddc0aaa08b8 --- /dev/null +++ b/appengine/standard/django/mysite/wsgi.py @@ -0,0 +1,29 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +WSGI config for mysite project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/1.8/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mysite.settings") + +application = get_wsgi_application() diff --git a/appengine/standard/django/polls/__init__.py b/appengine/standard/django/polls/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/appengine/standard/django/polls/admin.py b/appengine/standard/django/polls/admin.py new file mode 100644 index 00000000000..0c5b3a09a34 --- /dev/null +++ b/appengine/standard/django/polls/admin.py @@ -0,0 +1,19 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from django.contrib import admin + +from .models import Question + +admin.site.register(Question) diff --git a/appengine/standard/django/polls/models.py b/appengine/standard/django/polls/models.py new file mode 100644 index 00000000000..9e69c2a93ff --- /dev/null +++ b/appengine/standard/django/polls/models.py @@ -0,0 +1,26 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from django.db import models + + +class Question(models.Model): + question_text = models.CharField(max_length=200) + pub_date = models.DateTimeField('date published') + + +class Choice(models.Model): + question = models.ForeignKey(Question) + choice_text = models.CharField(max_length=200) + votes = models.IntegerField(default=0) diff --git a/appengine/standard/django/polls/tests.py b/appengine/standard/django/polls/tests.py new file mode 100644 index 00000000000..c3b029bad97 --- /dev/null +++ b/appengine/standard/django/polls/tests.py @@ -0,0 +1,20 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Uncomment these imports and add tests here + +# from django import http +# from django.test import TestCase + +# from . import views diff --git a/appengine/standard/django/polls/views.py b/appengine/standard/django/polls/views.py new file mode 100644 index 00000000000..595ccf18ed8 --- /dev/null +++ b/appengine/standard/django/polls/views.py @@ -0,0 +1,19 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from django.http import HttpResponse + + +def index(request): + return HttpResponse("Hello, world. You're at the polls index.") diff --git a/appengine/standard/django/requirements-vendor.txt b/appengine/standard/django/requirements-vendor.txt new file mode 100644 index 00000000000..f2292a7fe9a --- /dev/null +++ b/appengine/standard/django/requirements-vendor.txt @@ -0,0 +1 @@ +Django<2.0.0,>=1.11.8 diff --git a/appengine/standard/django/requirements.txt b/appengine/standard/django/requirements.txt new file mode 100644 index 00000000000..6da4b3c39bf --- /dev/null +++ b/appengine/standard/django/requirements.txt @@ -0,0 +1,2 @@ +PyMySQL==0.9.3 +Django<2.0.0,>=1.11.8 diff --git a/appengine/standard/endpoints-frameworks-v2/echo/README.md b/appengine/standard/endpoints-frameworks-v2/echo/README.md new file mode 100644 index 00000000000..54a2274d033 --- /dev/null +++ b/appengine/standard/endpoints-frameworks-v2/echo/README.md @@ -0,0 +1,38 @@ +## Endpoints Frameworks v2 Python Sample + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/endpoints-frameworks-v2/echo/README.md + +This demonstrates how to use Google Cloud Endpoints Frameworks v2 on Google App Engine Standard Environment using Python. + +## Setup + +Create a `lib` directory in which to install the Endpoints Frameworks v2 library. For more info, see [Installing a library](https://cloud.google.com/appengine/docs/python/tools/using-libraries-python-27#installing_a_library). + +Install the Endpoints Frameworks v2 library: + + $ pip install -t lib -r requirements.txt + +## Deploying to Google App Engine + +Generate an OpenAPI file by running: `python lib/endpoints/endpointscfg.py get_openapi_spec main.EchoApi --hostname [YOUR-PROJECT-ID].appspot.com` + +Remember to replace [YOUR-PROJECT-ID] with your project ID. + +To set up OAuth2, replace `your-oauth-client-id.com` under `audiences` in the annotation for `get_user_email` with your OAuth2 client ID. If you want to use Google OAuth2 Playground, use `407408718192.apps.googleusercontent.com` as your audience. To generate a JWT, go to the following address: `https://developers.google.com/oauthplayground`. + +Deploy the generated service spec to Google Cloud Service Management: `gcloud endpoints services deploy echov1openapi.json` + +The command returns several lines of information, including a line similar to the following: + + Service Configuration [2016-08-01r0] uploaded for service "[YOUR-PROJECT-ID].appspot.com" + +Open the `app.yaml` file and in the `env_variables` section, replace [YOUR-PROJECT-ID] in `[YOUR-PROJECT-ID].appspot.com` with your project ID. This is your Endpoints service name. Then replace `2016-08-01r0` with your uploaded service management configuration. + +Then, deploy the sample using `gcloud`: + + $ gcloud app deploy + +Once deployed, you can access the application at https://your-service.appspot.com diff --git a/appengine/standard/endpoints-frameworks-v2/echo/app.yaml b/appengine/standard/endpoints-frameworks-v2/echo/app.yaml new file mode 100644 index 00000000000..8784eba77de --- /dev/null +++ b/appengine/standard/endpoints-frameworks-v2/echo/app.yaml @@ -0,0 +1,34 @@ +runtime: python27 +threadsafe: true +api_version: 1 +basic_scaling: + max_instances: 2 + +#[START_EXCLUDE] +skip_files: +- ^(.*/)?#.*#$ +- ^(.*/)?.*~$ +- ^(.*/)?.*\.py[co]$ +- ^(.*/)?.*/RCS/.*$ +- ^(.*/)?\..*$ +- ^(.*/)?setuptools/script \(dev\).tmpl$ +#[END_EXCLUDE] + +handlers: +# The endpoints handler must be mapped to /_ah/api. +- url: /_ah/api/.* + script: main.api + +libraries: +- name: pycrypto + version: 2.6 +- name: ssl + version: 2.7.11 + +# [START env_vars] +env_variables: + # The following values are to be replaced by information from the output of + # 'gcloud endpoints services deploy swagger.json' command. + ENDPOINTS_SERVICE_NAME: YOUR-PROJECT-ID.appspot.com + ENDPOINTS_SERVICE_VERSION: 2016-08-01r0 + # [END env_vars] diff --git a/appengine/standard/endpoints-frameworks-v2/echo/appengine_config.py b/appengine/standard/endpoints-frameworks-v2/echo/appengine_config.py new file mode 100644 index 00000000000..3bb4ea6e3f1 --- /dev/null +++ b/appengine/standard/endpoints-frameworks-v2/echo/appengine_config.py @@ -0,0 +1,4 @@ +from google.appengine.ext import vendor + +# Add any libraries installed in the `lib` folder. +vendor.add('lib') diff --git a/appengine/standard/endpoints-frameworks-v2/echo/main.py b/appengine/standard/endpoints-frameworks-v2/echo/main.py new file mode 100644 index 00000000000..ce1a144976b --- /dev/null +++ b/appengine/standard/endpoints-frameworks-v2/echo/main.py @@ -0,0 +1,109 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This is a sample Hello World API implemented using Google Cloud +Endpoints.""" + +# [START imports] +import endpoints +from endpoints import message_types +from endpoints import messages +from endpoints import remote +# [END imports] + + +# [START messages] +class EchoRequest(messages.Message): + message = messages.StringField(1) + + +class EchoResponse(messages.Message): + """A proto Message that contains a simple string field.""" + message = messages.StringField(1) + + +ECHO_RESOURCE = endpoints.ResourceContainer( + EchoRequest, + n=messages.IntegerField(2, default=1)) +# [END messages] + + +# [START echo_api_class] +@endpoints.api(name='echo', version='v1') +class EchoApi(remote.Service): + + # [START echo_api_method] + @endpoints.method( + # This method takes a ResourceContainer defined above. + ECHO_RESOURCE, + # This method returns an Echo message. + EchoResponse, + path='echo', + http_method='POST', + name='echo') + def echo(self, request): + output_message = ' '.join([request.message] * request.n) + return EchoResponse(message=output_message) + # [END echo_api_method] + + @endpoints.method( + # This method takes a ResourceContainer defined above. + ECHO_RESOURCE, + # This method returns an Echo message. + EchoResponse, + path='echo/{n}', + http_method='POST', + name='echo_path_parameter') + def echo_path_parameter(self, request): + output_message = ' '.join([request.message] * request.n) + return EchoResponse(message=output_message) + + @endpoints.method( + # This method takes a ResourceContainer defined above. + message_types.VoidMessage, + # This method returns an Echo message. + EchoResponse, + path='echo/getApiKey', + http_method='GET', + name='echo_api_key', + api_key_required=True) + def echo_api_key(self, request): + key, key_type = request.get_unrecognized_field_info('key') + return EchoResponse(message=key) + + @endpoints.method( + # This method takes an empty request body. + message_types.VoidMessage, + # This method returns an Echo message. + EchoResponse, + path='echo/email', + http_method='GET', + # Require auth tokens to have the following scopes to access this API. + scopes=[endpoints.EMAIL_SCOPE], + # OAuth2 audiences allowed in incoming tokens. + audiences=['your-oauth-client-id.com'], + allowed_client_ids=['your-oauth-client-id.com']) + def get_user_email(self, request): + user = endpoints.get_current_user() + # If there's no user defined, the request was unauthenticated, so we + # raise 401 Unauthorized. + if not user: + raise endpoints.UnauthorizedException + return EchoResponse(message=user.email()) +# [END echo_api_class] + + +# [START api_server] +api = endpoints.api_server([EchoApi]) +# [END api_server] diff --git a/appengine/standard/endpoints-frameworks-v2/echo/main_test.py b/appengine/standard/endpoints-frameworks-v2/echo/main_test.py new file mode 100644 index 00000000000..2ed12efdcb5 --- /dev/null +++ b/appengine/standard/endpoints-frameworks-v2/echo/main_test.py @@ -0,0 +1,41 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import endpoints +from endpoints import message_types +import mock +import pytest + +import main + + +def test_echo(): + api = main.EchoApi() + request = main.EchoApi.echo.remote.request_type(message='Hello world!') + response = api.echo(request) + assert 'Hello world!' == response.message + + +def test_get_user_email(): + api = main.EchoApi() + + with mock.patch('main.endpoints.get_current_user') as user_mock: + user_mock.return_value = None + with pytest.raises(endpoints.UnauthorizedException): + api.get_user_email(message_types.VoidMessage()) + + user_mock.return_value = mock.Mock() + user_mock.return_value.email.return_value = 'user@example.com' + response = api.get_user_email(message_types.VoidMessage()) + assert 'user@example.com' == response.message diff --git a/appengine/standard/endpoints-frameworks-v2/echo/requirements.txt b/appengine/standard/endpoints-frameworks-v2/echo/requirements.txt new file mode 100644 index 00000000000..f5818130663 --- /dev/null +++ b/appengine/standard/endpoints-frameworks-v2/echo/requirements.txt @@ -0,0 +1,2 @@ +google-endpoints==4.8.0 +google-endpoints-api-management==1.11.0 diff --git a/appengine/standard/endpoints-frameworks-v2/iata/README.md b/appengine/standard/endpoints-frameworks-v2/iata/README.md new file mode 100644 index 00000000000..f09f51156b4 --- /dev/null +++ b/appengine/standard/endpoints-frameworks-v2/iata/README.md @@ -0,0 +1,36 @@ +## Endpoints Frameworks v2 Python Sample (Airport Information Edition) + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/endpoints-frameworks-v2/echo/README.md + +This demonstrates how to use Google Cloud Endpoints Frameworks v2 on Google App Engine Standard Environment using Python. + +## Setup + +Create a `lib` directory in which to install the Endpoints Frameworks v2 library. For more info, see [Installing a library](https://cloud.google.com/appengine/docs/python/tools/using-libraries-python-27#installing_a_library). + +Install the Endpoints Frameworks v2 library: + + $ pip install -t lib -r requirements.txt + +## Deploying to Google App Engine + +Generate an OpenAPI file by running: `python lib/endpoints/endpointscfg.py get_openapi_spec main.IataApi --hostname [YOUR-PROJECT-ID].appspot.com` + +Remember to replace [YOUR-PROJECT-ID] with your project ID. + +Deploy the generated service spec to Google Cloud Service Management: `gcloud endpoints services deploy iatav1openapi.json` + +The command returns several lines of information, including a line similar to the following: + + Service Configuration [2016-08-01r0] uploaded for service "[YOUR-PROJECT-ID].appspot.com" + +Open the `app.yaml` file and in the `env_variables` section, replace [YOUR-PROJECT-ID] in `[YOUR-PROJECT-ID].appspot.com` with your project ID. This is your Endpoints service name. Then replace `2016-08-01r0` with your uploaded service management configuration. + +Then, deploy the sample using `gcloud`: + + $ gcloud app deploy + +Once deployed, you can access the application at https://your-service.appspot.com diff --git a/appengine/standard/endpoints-frameworks-v2/iata/app.yaml b/appengine/standard/endpoints-frameworks-v2/iata/app.yaml new file mode 100644 index 00000000000..073f7da05b9 --- /dev/null +++ b/appengine/standard/endpoints-frameworks-v2/iata/app.yaml @@ -0,0 +1,30 @@ +runtime: python27 +threadsafe: true +api_version: 1 +basic_scaling: + max_instances: 2 + +skip_files: +- ^(.*/)?#.*#$ +- ^(.*/)?.*~$ +- ^(.*/)?.*\.py[co]$ +- ^(.*/)?.*/RCS/.*$ +- ^(.*/)?\..*$ +- ^(.*/)?setuptools/script \(dev\).tmpl$ + +handlers: +# The endpoints handler must be mapped to /_ah/api. +- url: /_ah/api/.* + script: main.api + +libraries: +- name: pycrypto + version: 2.6 +- name: ssl + version: 2.7.11 + +env_variables: + # The following values are to be replaced by information from the output of + # 'gcloud endpoints services deploy swagger.json' command. + ENDPOINTS_SERVICE_NAME: YOUR-PROJECT-ID.appspot.com + ENDPOINTS_SERVICE_VERSION: 2016-08-01r0 diff --git a/appengine/standard/endpoints-frameworks-v2/iata/appengine_config.py b/appengine/standard/endpoints-frameworks-v2/iata/appengine_config.py new file mode 100644 index 00000000000..6d7813aebeb --- /dev/null +++ b/appengine/standard/endpoints-frameworks-v2/iata/appengine_config.py @@ -0,0 +1,18 @@ +# Copyright 2018 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.ext import vendor + +# Add any libraries installed in the `lib` folder. +vendor.add('lib') diff --git a/appengine/standard/endpoints-frameworks-v2/iata/data.py b/appengine/standard/endpoints-frameworks-v2/iata/data.py new file mode 100644 index 00000000000..8e44728d345 --- /dev/null +++ b/appengine/standard/endpoints-frameworks-v2/iata/data.py @@ -0,0 +1,207 @@ +# Copyright 2018 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +AIRPORTS = { + u'ABQ': u'Albuquerque International Sunport Airport', + u'ACA': u'General Juan N Alvarez International Airport', + u'ADW': u'Andrews Air Force Base', + u'AFW': u'Fort Worth Alliance Airport', + u'AGS': u'Augusta Regional At Bush Field', + u'AMA': u'Rick Husband Amarillo International Airport', + u'ANC': u'Ted Stevens Anchorage International Airport', + u'ATL': u'Hartsfield Jackson Atlanta International Airport', + u'AUS': u'Austin Bergstrom International Airport', + u'AVL': u'Asheville Regional Airport', + u'BAB': u'Beale Air Force Base', + u'BAD': u'Barksdale Air Force Base', + u'BDL': u'Bradley International Airport', + u'BFI': u'Boeing Field King County International Airport', + u'BGR': u'Bangor International Airport', + u'BHM': u'Birmingham-Shuttlesworth International Airport', + u'BIL': u'Billings Logan International Airport', + u'BLV': u'Scott AFB/Midamerica Airport', + u'BMI': u'Central Illinois Regional Airport at Bloomington-Normal', + u'BNA': u'Nashville International Airport', + u'BOI': u'Boise Air Terminal/Gowen field', + u'BOS': u'General Edward Lawrence Logan International Airport', + u'BTR': u'Baton Rouge Metropolitan, Ryan Field', + u'BUF': u'Buffalo Niagara International Airport', + u'BWI': u'Baltimore/Washington International Thurgood Marshall Airport', + u'CAE': u'Columbia Metropolitan Airport', + u'CBM': u'Columbus Air Force Base', + u'CHA': u'Lovell Field', + u'CHS': u'Charleston Air Force Base-International Airport', + u'CID': u'The Eastern Iowa Airport', + u'CLE': u'Cleveland Hopkins International Airport', + u'CLT': u'Charlotte Douglas International Airport', + u'CMH': u'Port Columbus International Airport', + u'COS': u'City of Colorado Springs Municipal Airport', + u'CPR': u'Casper-Natrona County International Airport', + u'CRP': u'Corpus Christi International Airport', + u'CRW': u'Yeager Airport', + u'CUN': u'Canc\xfan International Airport', + u'CVG': u'Cincinnati Northern Kentucky International Airport', + u'CVS': u'Cannon Air Force Base', + u'DAB': u'Daytona Beach International Airport', + u'DAL': u'Dallas Love Field', + u'DAY': u'James M Cox Dayton International Airport', + u'DBQ': u'Dubuque Regional Airport', + u'DCA': u'Ronald Reagan Washington National Airport', + u'DEN': u'Denver International Airport', + u'DFW': u'Dallas Fort Worth International Airport', + u'DLF': u'Laughlin Air Force Base', + u'DLH': u'Duluth International Airport', + u'DOV': u'Dover Air Force Base', + u'DSM': u'Des Moines International Airport', + u'DTW': u'Detroit Metropolitan Wayne County Airport', + u'DYS': u'Dyess Air Force Base', + u'EDW': u'Edwards Air Force Base', + u'END': u'Vance Air Force Base', + u'ERI': u'Erie International Tom Ridge Field', + u'EWR': u'Newark Liberty International Airport', + u'FAI': u'Fairbanks International Airport', + u'FFO': u'Wright-Patterson Air Force Base', + u'FLL': u'Fort Lauderdale Hollywood International Airport', + u'FSM': u'Fort Smith Regional Airport', + u'FTW': u'Fort Worth Meacham International Airport', + u'FWA': u'Fort Wayne International Airport', + u'GDL': u'Don Miguel Hidalgo Y Costilla International Airport', + u'GEG': u'Spokane International Airport', + u'GPT': u'Gulfport Biloxi International Airport', + u'GRB': u'Austin Straubel International Airport', + u'GSB': u'Seymour Johnson Air Force Base', + u'GSO': u'Piedmont Triad International Airport', + u'GSP': u'Greenville Spartanburg International Airport', + u'GUS': u'Grissom Air Reserve Base', + u'HIB': u'Range Regional Airport', + u'HMN': u'Holloman Air Force Base', + u'HMO': u'General Ignacio P. Garcia International Airport', + u'HNL': u'Honolulu International Airport', + u'HOU': u'William P Hobby Airport', + u'HSV': u'Huntsville International Carl T Jones Field', + u'HTS': u'Tri-State/Milton J. Ferguson Field', + u'IAD': u'Washington Dulles International Airport', + u'IAH': u'George Bush Intercontinental Houston Airport', + u'ICT': u'Wichita Mid Continent Airport', + u'IND': u'Indianapolis International Airport', + u'JAN': u'Jackson-Medgar Wiley Evers International Airport', + u'JAX': u'Jacksonville International Airport', + u'JFK': u'John F Kennedy International Airport', + u'JLN': u'Joplin Regional Airport', + u'LAS': u'McCarran International Airport', + u'LAX': u'Los Angeles International Airport', + u'LBB': u'Lubbock Preston Smith International Airport', + u'LCK': u'Rickenbacker International Airport', + u'LEX': u'Blue Grass Airport', + u'LFI': u'Langley Air Force Base', + u'LFT': u'Lafayette Regional Airport', + u'LGA': u'La Guardia Airport', + u'LIT': u'Bill & Hillary Clinton National Airport/Adams Field', + u'LTS': u'Altus Air Force Base', + u'LUF': u'Luke Air Force Base', + u'MBS': u'MBS International Airport', + u'MCF': u'Mac Dill Air Force Base', + u'MCI': u'Kansas City International Airport', + u'MCO': u'Orlando International Airport', + u'MDW': u'Chicago Midway International Airport', + u'MEM': u'Memphis International Airport', + u'MEX': u'Licenciado Benito Juarez International Airport', + u'MGE': u'Dobbins Air Reserve Base', + u'MGM': u'Montgomery Regional (Dannelly Field) Airport', + u'MHT': u'Manchester Airport', + u'MIA': u'Miami International Airport', + u'MKE': u'General Mitchell International Airport', + u'MLI': u'Quad City International Airport', + u'MLU': u'Monroe Regional Airport', + u'MOB': u'Mobile Regional Airport', + u'MSN': u'Dane County Regional Truax Field', + u'MSP': u'Minneapolis-St Paul International/Wold-Chamberlain Airport', + u'MSY': u'Louis Armstrong New Orleans International Airport', + u'MTY': u'General Mariano Escobedo International Airport', + u'MUO': u'Mountain Home Air Force Base', + u'OAK': u'Metropolitan Oakland International Airport', + u'OKC': u'Will Rogers World Airport', + u'ONT': u'Ontario International Airport', + u'ORD': u"Chicago O'Hare International Airport", + u'ORF': u'Norfolk International Airport', + u'PAM': u'Tyndall Air Force Base', + u'PBI': u'Palm Beach International Airport', + u'PDX': u'Portland International Airport', + u'PHF': u'Newport News Williamsburg International Airport', + u'PHL': u'Philadelphia International Airport', + u'PHX': u'Phoenix Sky Harbor International Airport', + u'PIA': u'General Wayne A. Downing Peoria International Airport', + u'PIT': u'Pittsburgh International Airport', + u'PPE': u'Mar de Cort\xe9s International Airport', + u'PVR': u'Licenciado Gustavo D\xedaz Ordaz International Airport', + u'PWM': u'Portland International Jetport Airport', + u'RDU': u'Raleigh Durham International Airport', + u'RFD': u'Chicago Rockford International Airport', + u'RIC': u'Richmond International Airport', + u'RND': u'Randolph Air Force Base', + u'RNO': u'Reno Tahoe International Airport', + u'ROA': u'Roanoke\u2013Blacksburg Regional Airport', + u'ROC': u'Greater Rochester International Airport', + u'RST': u'Rochester International Airport', + u'RSW': u'Southwest Florida International Airport', + u'SAN': u'San Diego International Airport', + u'SAT': u'San Antonio International Airport', + u'SAV': u'Savannah Hilton Head International Airport', + u'SBN': u'South Bend Regional Airport', + u'SDF': u'Louisville International Standiford Field', + u'SEA': u'Seattle Tacoma International Airport', + u'SFB': u'Orlando Sanford International Airport', + u'SFO': u'San Francisco International Airport', + u'SGF': u'Springfield Branson National Airport', + u'SHV': u'Shreveport Regional Airport', + u'SJC': u'Norman Y. Mineta San Jose International Airport', + u'SJD': u'Los Cabos International Airport', + u'SKA': u'Fairchild Air Force Base', + u'SLC': u'Salt Lake City International Airport', + u'SMF': u'Sacramento International Airport', + u'SNA': u'John Wayne Airport-Orange County Airport', + u'SPI': u'Abraham Lincoln Capital Airport', + u'SPS': u'Sheppard Air Force Base-Wichita Falls Municipal Airport', + u'SRQ': u'Sarasota Bradenton International Airport', + u'SSC': u'Shaw Air Force Base', + u'STL': u'Lambert St Louis International Airport', + u'SUS': u'Spirit of St Louis Airport', + u'SUU': u'Travis Air Force Base', + u'SUX': u'Sioux Gateway Col. Bud Day Field', + u'SYR': u'Syracuse Hancock International Airport', + u'SZL': u'Whiteman Air Force Base', + u'TCM': u'McChord Air Force Base', + u'TIJ': u'General Abelardo L. Rodr\xedguez International Airport', + u'TIK': u'Tinker Air Force Base', + u'TLH': u'Tallahassee Regional Airport', + u'TOL': u'Toledo Express Airport', + u'TPA': u'Tampa International Airport', + u'TRI': u'Tri Cities Regional Tn Va Airport', + u'TUL': u'Tulsa International Airport', + u'TUS': u'Tucson International Airport', + u'TYS': u'McGhee Tyson Airport', + u'VBG': u'Vandenberg Air Force Base', + u'VPS': u'Destin-Ft Walton Beach Airport', + u'WRB': u'Robins Air Force Base', + u'YEG': u'Edmonton International Airport', + u'YHZ': u'Halifax / Stanfield International Airport', + u'YOW': u'Ottawa Macdonald-Cartier International Airport', + u'YUL': u'Montreal / Pierre Elliott Trudeau International Airport', + u'YVR': u'Vancouver International Airport', + u'YWG': u'Winnipeg / James Armstrong Richardson International Airport', + u'YYC': u'Calgary International Airport', + u'YYJ': u'Victoria International Airport', + u'YYT': u"St. John's International Airport", + u'YYZ': u'Lester B. Pearson International Airport' +} diff --git a/appengine/standard/endpoints-frameworks-v2/iata/main.py b/appengine/standard/endpoints-frameworks-v2/iata/main.py new file mode 100644 index 00000000000..97ab4a97f98 --- /dev/null +++ b/appengine/standard/endpoints-frameworks-v2/iata/main.py @@ -0,0 +1,119 @@ +# Copyright 2018 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This is a sample Airport Information service implemented using +Google Cloud Endpoints Frameworks for Python.""" + +# [START imports] +import endpoints +from endpoints import message_types +from endpoints import messages +from endpoints import remote + +from data import AIRPORTS +# [END imports] + +# [START messages] +IATA_RESOURCE = endpoints.ResourceContainer( + iata=messages.StringField(1, required=True) +) + + +class Airport(messages.Message): + iata = messages.StringField(1, required=True) + name = messages.StringField(2, required=True) + + +IATA_AIRPORT_RESOURCE = endpoints.ResourceContainer( + Airport, + iata=messages.StringField(1, required=True) +) + + +class AirportList(messages.Message): + airports = messages.MessageField(Airport, 1, repeated=True) +# [END messages] + + +# [START iata_api] +@endpoints.api(name='iata', version='v1') +class IataApi(remote.Service): + @endpoints.method( + IATA_RESOURCE, + Airport, + path='airport/{iata}', + http_method='GET', + name='get_airport') + def get_airport(self, request): + if request.iata not in AIRPORTS: + raise endpoints.NotFoundException() + return Airport(iata=request.iata, name=AIRPORTS[request.iata]) + + @endpoints.method( + message_types.VoidMessage, + AirportList, + path='airports', + http_method='GET', + name='list_airports') + def list_airports(self, request): + codes = AIRPORTS.keys() + codes.sort() + return AirportList(airports=[ + Airport(iata=iata, name=AIRPORTS[iata]) for iata in codes + ]) + + @endpoints.method( + IATA_RESOURCE, + message_types.VoidMessage, + path='airport/{iata}', + http_method='DELETE', + name='delete_airport', + api_key_required=True) + def delete_airport(self, request): + if request.iata not in AIRPORTS: + raise endpoints.NotFoundException() + del AIRPORTS[request.iata] + return message_types.VoidMessage() + + @endpoints.method( + Airport, + Airport, + path='airport', + http_method='POST', + name='create_airport', + api_key_required=True) + def create_airport(self, request): + if request.iata in AIRPORTS: + raise endpoints.BadRequestException() + AIRPORTS[request.iata] = request.name + return Airport(iata=request.iata, name=AIRPORTS[request.iata]) + + @endpoints.method( + IATA_AIRPORT_RESOURCE, + Airport, + path='airport/{iata}', + http_method='POST', + name='update_airport', + api_key_required=True) + def update_airport(self, request): + if request.iata not in AIRPORTS: + raise endpoints.BadRequestException() + AIRPORTS[request.iata] = request.name + return Airport(iata=request.iata, name=AIRPORTS[request.iata]) +# [END iata_api] + + +# [START api_server] +api = endpoints.api_server([IataApi]) +# [END api_server] diff --git a/appengine/standard/endpoints-frameworks-v2/iata/requirements.txt b/appengine/standard/endpoints-frameworks-v2/iata/requirements.txt new file mode 100644 index 00000000000..f5818130663 --- /dev/null +++ b/appengine/standard/endpoints-frameworks-v2/iata/requirements.txt @@ -0,0 +1,2 @@ +google-endpoints==4.8.0 +google-endpoints-api-management==1.11.0 diff --git a/appengine/standard/endpoints-frameworks-v2/quickstart/app.yaml b/appengine/standard/endpoints-frameworks-v2/quickstart/app.yaml new file mode 100644 index 00000000000..e7126cebae9 --- /dev/null +++ b/appengine/standard/endpoints-frameworks-v2/quickstart/app.yaml @@ -0,0 +1,28 @@ +runtime: python27 +threadsafe: true +api_version: 1 + +skip_files: +- ^(.*/)?#.*#$ +- ^(.*/)?.*~$ +- ^(.*/)?.*\.py[co]$ +- ^(.*/)?.*/RCS/.*$ +- ^(.*/)?\..*$ +- ^lib/setuptools/script \(dev\).tmpl$ + +handlers: +# The endpoints handler must be mapped to /_ah/api. +- url: /_ah/api/.* + script: main.api + +libraries: +- name: pycrypto + version: 2.6 +- name: ssl + version: 2.7.11 + +env_variables: + # The following values are to be replaced by information from the output of + # 'gcloud endpoints services deploy swagger.json' command. + ENDPOINTS_SERVICE_NAME: greeting-api.endpoints.[YOUR-PROJECT-ID].cloud.goog + ENDPOINTS_SERVICE_VERSION: 2016-08-01r0 diff --git a/appengine/standard/endpoints-frameworks-v2/quickstart/appengine_config.py b/appengine/standard/endpoints-frameworks-v2/quickstart/appengine_config.py new file mode 100644 index 00000000000..3bb4ea6e3f1 --- /dev/null +++ b/appengine/standard/endpoints-frameworks-v2/quickstart/appengine_config.py @@ -0,0 +1,4 @@ +from google.appengine.ext import vendor + +# Add any libraries installed in the `lib` folder. +vendor.add('lib') diff --git a/appengine/standard/endpoints-frameworks-v2/quickstart/main.py b/appengine/standard/endpoints-frameworks-v2/quickstart/main.py new file mode 100644 index 00000000000..1d98dc4eba0 --- /dev/null +++ b/appengine/standard/endpoints-frameworks-v2/quickstart/main.py @@ -0,0 +1,149 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This is a sample Hello World API implemented using Google Cloud +Endpoints.""" + +# [START imports] +import endpoints +from endpoints import message_types +from endpoints import messages +from endpoints import remote +# [END imports] + + +# [START messages] +class Greeting(messages.Message): + """Greeting that stores a message.""" + message = messages.StringField(1) + + +class GreetingCollection(messages.Message): + """Collection of Greetings.""" + items = messages.MessageField(Greeting, 1, repeated=True) + + +STORED_GREETINGS = GreetingCollection(items=[ + Greeting(message='hello world!'), + Greeting(message='goodbye world!'), +]) +# [END messages] + + +# [START greeting_api] +@endpoints.api(name='greeting', version='v1') +class GreetingApi(remote.Service): + + @endpoints.method( + # This method does not take a request message. + message_types.VoidMessage, + # This method returns a GreetingCollection message. + GreetingCollection, + path='greetings', + http_method='GET', + name='greetings.list') + def list_greetings(self, unused_request): + return STORED_GREETINGS + + # ResourceContainers are used to encapsuate a request body and url + # parameters. This one is used to represent the Greeting ID for the + # greeting_get method. + GET_RESOURCE = endpoints.ResourceContainer( + # The request body should be empty. + message_types.VoidMessage, + # Accept one url parameter: and integer named 'id' + id=messages.IntegerField(1, variant=messages.Variant.INT32)) + + @endpoints.method( + # Use the ResourceContainer defined above to accept an empty body + # but an ID in the query string. + GET_RESOURCE, + # This method returns a Greeting message. + Greeting, + # The path defines the source of the URL parameter 'id'. If not + # specified here, it would need to be in the query string. + path='greetings/{id}', + http_method='GET', + name='greetings.get') + def get_greeting(self, request): + try: + # request.id is used to access the URL parameter. + return STORED_GREETINGS.items[request.id] + except (IndexError, TypeError): + raise endpoints.NotFoundException( + 'Greeting {} not found'.format(request.id)) + # [END greeting_api] + + # [START multiply] + # This ResourceContainer is similar to the one used for get_greeting, but + # this one also contains a request body in the form of a Greeting message. + MULTIPLY_RESOURCE = endpoints.ResourceContainer( + Greeting, + times=messages.IntegerField(2, variant=messages.Variant.INT32, + required=True)) + + @endpoints.method( + # This method accepts a request body containing a Greeting message + # and a URL parameter specifying how many times to multiply the + # message. + MULTIPLY_RESOURCE, + # This method returns a Greeting message. + Greeting, + path='greetings/multiply/{times}', + http_method='POST', + name='greetings.multiply') + def multiply_greeting(self, request): + return Greeting(message=request.message * request.times) + # [END multiply] + + +# [START auth_config] +WEB_CLIENT_ID = 'replace this with your web client application ID' +ANDROID_CLIENT_ID = 'replace this with your Android client ID' +IOS_CLIENT_ID = 'replace this with your iOS client ID' +ANDROID_AUDIENCE = WEB_CLIENT_ID +ALLOWED_CLIENT_IDS = [ + WEB_CLIENT_ID, ANDROID_CLIENT_ID, IOS_CLIENT_ID, + endpoints.API_EXPLORER_CLIENT_ID] +# [END auth_config] + + +# [START authed_greeting_api] +@endpoints.api( + name='authed_greeting', + version='v1', + # Only allowed configured Client IDs to access this API. + allowed_client_ids=ALLOWED_CLIENT_IDS, + # Only allow auth tokens with the given audience to access this API. + audiences=[ANDROID_AUDIENCE], + # Require auth tokens to have the following scopes to access this API. + scopes=[endpoints.EMAIL_SCOPE]) +class AuthedGreetingApi(remote.Service): + + @endpoints.method( + message_types.VoidMessage, + Greeting, + path='greet', + http_method='POST', + name='greet') + def greet(self, request): + user = endpoints.get_current_user() + user_name = user.email() if user else 'Anonymous' + return Greeting(message='Hello, {}'.format(user_name)) +# [END authed_greeting_api] + + +# [START api_server] +api = endpoints.api_server([GreetingApi, AuthedGreetingApi]) +# [END api_server] diff --git a/appengine/standard/endpoints-frameworks-v2/quickstart/main_test.py b/appengine/standard/endpoints-frameworks-v2/quickstart/main_test.py new file mode 100644 index 00000000000..9e9fab2d1cb --- /dev/null +++ b/appengine/standard/endpoints-frameworks-v2/quickstart/main_test.py @@ -0,0 +1,54 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from endpoints import message_types +import mock + +import main + + +def test_list_greetings(testbed): + api = main.GreetingApi() + response = api.list_greetings(message_types.VoidMessage()) + assert len(response.items) == 2 + + +def test_get_greeting(testbed): + api = main.GreetingApi() + request = main.GreetingApi.get_greeting.remote.request_type(id=1) + response = api.get_greeting(request) + assert response.message == 'goodbye world!' + + +def test_multiply_greeting(testbed): + api = main.GreetingApi() + request = main.GreetingApi.multiply_greeting.remote.request_type( + times=4, + message='help I\'m trapped in a test case.') + response = api.multiply_greeting(request) + assert response.message == 'help I\'m trapped in a test case.' * 4 + + +def test_authed_greet(testbed): + api = main.AuthedGreetingApi() + + with mock.patch('main.endpoints.get_current_user') as user_mock: + user_mock.return_value = None + response = api.greet(message_types.VoidMessage()) + assert response.message == 'Hello, Anonymous' + + user_mock.return_value = mock.Mock() + user_mock.return_value.email.return_value = 'user@example.com' + response = api.greet(message_types.VoidMessage()) + assert response.message == 'Hello, user@example.com' diff --git a/appengine/standard/endpoints-frameworks-v2/quickstart/requirements.txt b/appengine/standard/endpoints-frameworks-v2/quickstart/requirements.txt new file mode 100644 index 00000000000..f5818130663 --- /dev/null +++ b/appengine/standard/endpoints-frameworks-v2/quickstart/requirements.txt @@ -0,0 +1,2 @@ +google-endpoints==4.8.0 +google-endpoints-api-management==1.11.0 diff --git a/appengine/standard/endpoints/backend/app.yaml b/appengine/standard/endpoints/backend/app.yaml new file mode 100644 index 00000000000..31030b73367 --- /dev/null +++ b/appengine/standard/endpoints/backend/app.yaml @@ -0,0 +1,16 @@ +runtime: python27 +threadsafe: true +api_version: 1 + +handlers: +# The endpoints handler must be mapped to /_ah/spi. +# Apps send requests to /_ah/api, but the endpoints service handles mapping +# those requests to /_ah/spi. +- url: /_ah/spi/.* + script: main.api + +libraries: +- name: pycrypto + version: 2.6 +- name: endpoints + version: 1.0 diff --git a/appengine/standard/endpoints/backend/main.py b/appengine/standard/endpoints/backend/main.py new file mode 100644 index 00000000000..ced7b778b3a --- /dev/null +++ b/appengine/standard/endpoints/backend/main.py @@ -0,0 +1,149 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This is a sample Hello World API implemented using Google Cloud +Endpoints.""" + +# [START imports] +import endpoints +from protorpc import message_types +from protorpc import messages +from protorpc import remote +# [END imports] + + +# [START messages] +class Greeting(messages.Message): + """Greeting that stores a message.""" + message = messages.StringField(1) + + +class GreetingCollection(messages.Message): + """Collection of Greetings.""" + items = messages.MessageField(Greeting, 1, repeated=True) + + +STORED_GREETINGS = GreetingCollection(items=[ + Greeting(message='hello world!'), + Greeting(message='goodbye world!'), +]) +# [END messages] + + +# [START greeting_api] +@endpoints.api(name='greeting', version='v1') +class GreetingApi(remote.Service): + + @endpoints.method( + # This method does not take a request message. + message_types.VoidMessage, + # This method returns a GreetingCollection message. + GreetingCollection, + path='greetings', + http_method='GET', + name='greetings.list') + def list_greetings(self, unused_request): + return STORED_GREETINGS + + # ResourceContainers are used to encapsuate a request body and url + # parameters. This one is used to represent the Greeting ID for the + # greeting_get method. + GET_RESOURCE = endpoints.ResourceContainer( + # The request body should be empty. + message_types.VoidMessage, + # Accept one url parameter: an integer named 'id' + id=messages.IntegerField(1, variant=messages.Variant.INT32)) + + @endpoints.method( + # Use the ResourceContainer defined above to accept an empty body + # but an ID in the query string. + GET_RESOURCE, + # This method returns a Greeting message. + Greeting, + # The path defines the source of the URL parameter 'id'. If not + # specified here, it would need to be in the query string. + path='greetings/{id}', + http_method='GET', + name='greetings.get') + def get_greeting(self, request): + try: + # request.id is used to access the URL parameter. + return STORED_GREETINGS.items[request.id] + except (IndexError, TypeError): + raise endpoints.NotFoundException( + 'Greeting {} not found'.format(request.id)) + # [END greeting_api] + + # [START multiply] + # This ResourceContainer is similar to the one used for get_greeting, but + # this one also contains a request body in the form of a Greeting message. + MULTIPLY_RESOURCE = endpoints.ResourceContainer( + Greeting, + times=messages.IntegerField(2, variant=messages.Variant.INT32, + required=True)) + + @endpoints.method( + # This method accepts a request body containing a Greeting message + # and a URL parameter specifying how many times to multiply the + # message. + MULTIPLY_RESOURCE, + # This method returns a Greeting message. + Greeting, + path='greetings/multiply/{times}', + http_method='POST', + name='greetings.multiply') + def multiply_greeting(self, request): + return Greeting(message=request.message * request.times) + # [END multiply] + + +# [START auth_config] +WEB_CLIENT_ID = 'replace this with your web client application ID' +ANDROID_CLIENT_ID = 'replace this with your Android client ID' +IOS_CLIENT_ID = 'replace this with your iOS client ID' +ANDROID_AUDIENCE = WEB_CLIENT_ID +ALLOWED_CLIENT_IDS = [ + WEB_CLIENT_ID, ANDROID_CLIENT_ID, IOS_CLIENT_ID, + endpoints.API_EXPLORER_CLIENT_ID] +# [END auth_config] + + +# [START authed_greeting_api] +@endpoints.api( + name='authed_greeting', + version='v1', + # Only allowed configured Client IDs to access this API. + allowed_client_ids=ALLOWED_CLIENT_IDS, + # Only allow auth tokens with the given audience to access this API. + audiences=[ANDROID_AUDIENCE], + # Require auth tokens to have the following scopes to access this API. + scopes=[endpoints.EMAIL_SCOPE]) +class AuthedGreetingApi(remote.Service): + + @endpoints.method( + message_types.VoidMessage, + Greeting, + path='greet', + http_method='POST', + name='greet') + def greet(self, request): + user = endpoints.get_current_user() + user_name = user.email() if user else 'Anonymous' + return Greeting(message='Hello, {}'.format(user_name)) +# [END authed_greeting_api] + + +# [START api_server] +api = endpoints.api_server([GreetingApi, AuthedGreetingApi]) +# [END api_server] diff --git a/appengine/standard/endpoints/backend/main_test.py b/appengine/standard/endpoints/backend/main_test.py new file mode 100644 index 00000000000..c8255a90d30 --- /dev/null +++ b/appengine/standard/endpoints/backend/main_test.py @@ -0,0 +1,54 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +from protorpc import message_types + +import main + + +def test_list_greetings(testbed): + api = main.GreetingApi() + response = api.list_greetings(message_types.VoidMessage()) + assert len(response.items) == 2 + + +def test_get_greeting(testbed): + api = main.GreetingApi() + request = main.GreetingApi.get_greeting.remote.request_type(id=1) + response = api.get_greeting(request) + assert response.message == 'goodbye world!' + + +def test_multiply_greeting(testbed): + api = main.GreetingApi() + request = main.GreetingApi.multiply_greeting.remote.request_type( + times=4, + message='help I\'m trapped in a test case.') + response = api.multiply_greeting(request) + assert response.message == 'help I\'m trapped in a test case.' * 4 + + +def test_authed_greet(testbed): + api = main.AuthedGreetingApi() + + with mock.patch('main.endpoints.get_current_user') as user_mock: + user_mock.return_value = None + response = api.greet(message_types.VoidMessage()) + assert response.message == 'Hello, Anonymous' + + user_mock.return_value = mock.Mock() + user_mock.return_value.email.return_value = 'user@example.com' + response = api.greet(message_types.VoidMessage()) + assert response.message == 'Hello, user@example.com' diff --git a/appengine/standard/endpoints/multiapi/app.yaml b/appengine/standard/endpoints/multiapi/app.yaml new file mode 100644 index 00000000000..31030b73367 --- /dev/null +++ b/appengine/standard/endpoints/multiapi/app.yaml @@ -0,0 +1,16 @@ +runtime: python27 +threadsafe: true +api_version: 1 + +handlers: +# The endpoints handler must be mapped to /_ah/spi. +# Apps send requests to /_ah/api, but the endpoints service handles mapping +# those requests to /_ah/spi. +- url: /_ah/spi/.* + script: main.api + +libraries: +- name: pycrypto + version: 2.6 +- name: endpoints + version: 1.0 diff --git a/appengine/standard/endpoints/multiapi/main.py b/appengine/standard/endpoints/multiapi/main.py new file mode 100644 index 00000000000..f108c4fa68a --- /dev/null +++ b/appengine/standard/endpoints/multiapi/main.py @@ -0,0 +1,59 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This is a sample multi-class API implemented using Cloud Ednpoints""" + +import endpoints +from protorpc import messages +from protorpc import remote + + +class Request(messages.Message): + message = messages.StringField(1) + + +class Response(messages.Message): + message = messages.StringField(1) + + +# [START multiclass] +api_collection = endpoints.api(name='library', version='v1.0') + + +@api_collection.api_class(resource_name='shelves') +class Shelves(remote.Service): + + @endpoints.method(Request, Response) + def list(self, request): + return Response() + + +# [START books] +@api_collection.api_class(resource_name='books', path='books') +class Books(remote.Service): + + @endpoints.method(Request, Response, path='bookmark') + def get_bookmark(self, request): + return Response() + + @endpoints.method(Request, Response) + def best_sellers_list(self, request): + return Response() +# [END books] +# [END multiclass] + + +# [START api_server] +api = endpoints.api_server([api_collection]) +# [END api_server] diff --git a/appengine/standard/endpoints/multiapi/main_test.py b/appengine/standard/endpoints/multiapi/main_test.py new file mode 100644 index 00000000000..18aa8872e72 --- /dev/null +++ b/appengine/standard/endpoints/multiapi/main_test.py @@ -0,0 +1,27 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import main + + +def test_shelves(testbed): + api = main.Shelves() + response = api.list(main.Request()) + assert response + + +def test_books(testbed): + api = main.Books() + response = api.get_bookmark(main.Request()) + assert response diff --git a/appengine/standard/firebase/firenotes/README.md b/appengine/standard/firebase/firenotes/README.md new file mode 100644 index 00000000000..492a27cc5d6 --- /dev/null +++ b/appengine/standard/firebase/firenotes/README.md @@ -0,0 +1,64 @@ +# Firenotes: Firebase Authentication on Google App Engine + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/firebase/firenotes/README.md + +A simple note-taking application that stores users' notes in their own personal +notebooks separated by a unique user ID generated by Firebase. Uses Firebase +Authentication, Google App Engine, and Google Cloud Datastore. + +This sample is used on the following documentation page: + +[https://cloud.google.com/appengine/docs/python/authenticating-users-firebase-appengine/](https://cloud.google.com/appengine/docs/python/authenticating-users-firebase-appengine/) + +You'll need to have [Python 2.7](https://www.python.org/) and the [Google Cloud SDK](https://cloud.google.com/sdk/?hl=en) +installed and initialized to an App Engine project before running the code in +this sample. + +## Setup + +1. Clone this repo: + + git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +1. Navigate to the directory that contains the sample code: + + cd python-docs-samples/appengine/standard/firebase/firenotes + +1. Within a virtualenv, install the dependencies to the backend service: + + pip install -r requirements.txt -t lib + +1. [Add Firebase to your app.](https://firebase.google.com/docs/web/setup#add_firebase_to_your_app) +1. Add your Firebase project ID to the backend’s `app.yaml` file as an +environment variable. +1. Select which providers you want to enable. Delete the providers from +`main.js` that you do no want to offer. Enable the providers you chose to keep +in the Firebase console under **Auth** > **Sign-in Method** > +**Sign-in providers**. +1. In the Firebase console, under **OAuth redirect domains**, click +**Add Domain** and enter the domain of your app on App Engine: +[PROJECT_ID].appspot.com. Do not include "http://" before the domain name. + +## Run Locally +1. Add the backend host URL to `main.js`: http://localhost:8081. +1. Navigate to the root directory of the application and start the development +server with the following command: + + dev_appserver.py frontend/app.yaml backend/app.yaml + +1. Visit [http://localhost:8080/](http://localhost:8080/) in a web browser. + +## Deploy +1. Change the backend host URL in `main.js` to +https://backend-dot-[PROJECT_ID].appspot.com. +1. Deploy the application using the Cloud SDK command-line interface: + + gcloud app deploy backend/index.yaml frontend/app.yaml backend/app.yaml + + The Cloud Datastore indexes can take a while to update, so the application + might not be fully functional immediately after deployment. + +1. View the application live at https://[PROJECT_ID].appspot.com. diff --git a/appengine/bigquery/.gitignore b/appengine/standard/firebase/firenotes/backend/.gitignore similarity index 100% rename from appengine/bigquery/.gitignore rename to appengine/standard/firebase/firenotes/backend/.gitignore diff --git a/appengine/standard/firebase/firenotes/backend/app.yaml b/appengine/standard/firebase/firenotes/backend/app.yaml new file mode 100644 index 00000000000..683445b9f5a --- /dev/null +++ b/appengine/standard/firebase/firenotes/backend/app.yaml @@ -0,0 +1,13 @@ +runtime: python27 +api_version: 1 +threadsafe: true +service: backend + +handlers: +- url: /.* + script: main.app + +env_variables: + # Replace with your Firebase project ID. + FIREBASE_PROJECT_ID: '' + GAE_USE_SOCKETS_HTTPLIB : 'true' diff --git a/appengine/standard/firebase/firenotes/backend/appengine_config.py b/appengine/standard/firebase/firenotes/backend/appengine_config.py new file mode 100644 index 00000000000..c903d9a0ac5 --- /dev/null +++ b/appengine/standard/firebase/firenotes/backend/appengine_config.py @@ -0,0 +1,18 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.ext import vendor + +# Add any libraries installed in the "lib" folder. +vendor.add('lib') diff --git a/appengine/standard/firebase/firenotes/backend/index.yaml b/appengine/standard/firebase/firenotes/backend/index.yaml new file mode 100644 index 00000000000..b0ebb4ffea8 --- /dev/null +++ b/appengine/standard/firebase/firenotes/backend/index.yaml @@ -0,0 +1,22 @@ +indexes: + +# AUTOGENERATED + +# This index.yaml is automatically updated whenever the dev_appserver +# detects that a new type of query is run. If you want to manage the +# index.yaml file manually, remove the above marker line (the line +# saying "# AUTOGENERATED"). If you want to manage some indexes +# manually, move them above the marker line. The index.yaml file is +# automatically uploaded to the admin console when you next deploy +# your application using appcfg.py. + +- kind: Note + ancestor: yes + properties: + - name: created + +- kind: Note + ancestor: yes + properties: + - name: created + direction: desc diff --git a/appengine/standard/firebase/firenotes/backend/main.py b/appengine/standard/firebase/firenotes/backend/main.py new file mode 100644 index 00000000000..35226b6e8bc --- /dev/null +++ b/appengine/standard/firebase/firenotes/backend/main.py @@ -0,0 +1,125 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from flask import Flask, jsonify, request +import flask_cors +from google.appengine.ext import ndb +import google.auth.transport.requests +import google.oauth2.id_token +import requests_toolbelt.adapters.appengine + +# Use the App Engine Requests adapter. This makes sure that Requests uses +# URLFetch. +requests_toolbelt.adapters.appengine.monkeypatch() +HTTP_REQUEST = google.auth.transport.requests.Request() + +app = Flask(__name__) +flask_cors.CORS(app) + + +class Note(ndb.Model): + """NDB model class for a user's note. + + Key is user id from decrypted token. + """ + friendly_id = ndb.StringProperty() + message = ndb.TextProperty() + created = ndb.DateTimeProperty(auto_now_add=True) + + +# [START gae_python_query_database] +def query_database(user_id): + """Fetches all notes associated with user_id. + + Notes are ordered them by date created, with most recent note added + first. + """ + ancestor_key = ndb.Key(Note, user_id) + query = Note.query(ancestor=ancestor_key).order(-Note.created) + notes = query.fetch() + + note_messages = [] + + for note in notes: + note_messages.append({ + 'friendly_id': note.friendly_id, + 'message': note.message, + 'created': note.created + }) + + return note_messages +# [END gae_python_query_database] + + +@app.route('/notes', methods=['GET']) +def list_notes(): + """Returns a list of notes added by the current Firebase user.""" + + # Verify Firebase auth. + # [START gae_python_verify_token] + id_token = request.headers['Authorization'].split(' ').pop() + claims = google.oauth2.id_token.verify_firebase_token( + id_token, HTTP_REQUEST) + if not claims: + return 'Unauthorized', 401 + # [END gae_python_verify_token] + + notes = query_database(claims['sub']) + + return jsonify(notes) + + +@app.route('/notes', methods=['POST', 'PUT']) +def add_note(): + """ + Adds a note to the user's notebook. The request should be in this format: + + { + "message": "note message." + } + """ + + # Verify Firebase auth. + id_token = request.headers['Authorization'].split(' ').pop() + claims = google.oauth2.id_token.verify_firebase_token( + id_token, HTTP_REQUEST) + if not claims: + return 'Unauthorized', 401 + + # [START gae_python_create_entity] + data = request.get_json() + + # Populates note properties according to the model, + # with the user ID as the key name. + note = Note( + parent=ndb.Key(Note, claims['sub']), + message=data['message']) + + # Some providers do not provide one of these so either can be used. + note.friendly_id = claims.get('name', claims.get('email', 'Unknown')) + # [END gae_python_create_entity] + + # Stores note in database. + note.put() + + return 'OK', 200 + + +@app.errorhandler(500) +def server_error(e): + # Log the error and stacktrace. + logging.exception('An error occurred during a request.') + return 'An internal error occurred.', 500 diff --git a/appengine/standard/firebase/firenotes/backend/main_test.py b/appengine/standard/firebase/firenotes/backend/main_test.py new file mode 100644 index 00000000000..3aa7e944734 --- /dev/null +++ b/appengine/standard/firebase/firenotes/backend/main_test.py @@ -0,0 +1,96 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +from google.appengine.ext import ndb +import jwt +import mock +import pytest + + +@pytest.fixture +def app(): + # Remove any existing pyjwt handlers, as firebase_helper will register + # its own. + try: + jwt.unregister_algorithm('RS256') + except KeyError: + pass + + import main + main.app.testing = True + return main.app.test_client() + + +@pytest.fixture +def mock_token(): + patch = mock.patch('google.oauth2.id_token.verify_firebase_token') + with patch as mock_verify: + yield mock_verify + + +@pytest.fixture +def test_data(): + from main import Note + ancestor_key = ndb.Key(Note, '123') + notes = [ + Note(parent=ancestor_key, message='1'), + Note(parent=ancestor_key, message='2') + ] + ndb.put_multi(notes) + yield + + +def test_list_notes_with_mock_token(testbed, app, mock_token, test_data): + mock_token.return_value = {'sub': '123'} + + r = app.get('/notes', headers={'Authorization': 'Bearer 123'}) + assert r.status_code == 200 + + data = json.loads(r.data) + assert len(data) == 2 + assert data[0]['message'] == '2' + + +def test_list_notes_with_bad_mock_token(testbed, app, mock_token): + mock_token.return_value = None + + r = app.get('/notes', headers={'Authorization': 'Bearer 123'}) + assert r.status_code == 401 + + +def test_add_note_with_mock_token(testbed, app, mock_token): + mock_token.return_value = {'sub': '123'} + + r = app.post( + '/notes', + data=json.dumps({'message': 'Hello, world!'}), + content_type='application/json', + headers={'Authorization': 'Bearer 123'}) + + assert r.status_code == 200 + + from main import Note + + results = Note.query().fetch() + assert len(results) == 1 + assert results[0].message == 'Hello, world!' + + +def test_add_note_with_bad_mock_token(testbed, app, mock_token): + mock_token.return_value = None + + r = app.post('/notes', headers={'Authorization': 'Bearer 123'}) + assert r.status_code == 401 diff --git a/appengine/standard/firebase/firenotes/backend/requirements.txt b/appengine/standard/firebase/firenotes/backend/requirements.txt new file mode 100644 index 00000000000..ea3b77a6930 --- /dev/null +++ b/appengine/standard/firebase/firenotes/backend/requirements.txt @@ -0,0 +1,6 @@ +Flask==0.12.4 +pyjwt==1.7.1 +flask-cors==3.0.7 +google-auth==1.6.2 +requests==2.21.0 +requests-toolbelt==0.9.1 diff --git a/appengine/standard/firebase/firenotes/frontend/app.yaml b/appengine/standard/firebase/firenotes/frontend/app.yaml new file mode 100644 index 00000000000..e205ff25c09 --- /dev/null +++ b/appengine/standard/firebase/firenotes/frontend/app.yaml @@ -0,0 +1,15 @@ +runtime: python27 +api_version: 1 +service: default +threadsafe: true + +handlers: + +# root +- url: / + static_files: index.html + upload: index.html + +- url: /(.+) + static_files: \1 + upload: (.+) diff --git a/appengine/standard/firebase/firenotes/frontend/index.html b/appengine/standard/firebase/firenotes/frontend/index.html new file mode 100644 index 00000000000..0f6c00a1897 --- /dev/null +++ b/appengine/standard/firebase/firenotes/frontend/index.html @@ -0,0 +1,43 @@ + + + + + + + + + + + + Firenotes + + +
        +

        Firenotes

        +

        Sign in to access your notebook

        +
        +
        + +
        +

        Welcome, !

        +

        Enter a note and save it to your personal notebook

        +
        +
        +
        + +
        +
        + + +
        +
        +
        + +
        +
        + + diff --git a/appengine/standard/firebase/firenotes/frontend/main.js b/appengine/standard/firebase/firenotes/frontend/main.js new file mode 100644 index 00000000000..1213c66236a --- /dev/null +++ b/appengine/standard/firebase/firenotes/frontend/main.js @@ -0,0 +1,156 @@ +// Copyright 2016, Google, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +$(function(){ + // This is the host for the backend. + // TODO: When running Firenotes locally, set to http://localhost:8081. Before + // deploying the application to a live production environment, change to + // https://backend-dot-.appspot.com as specified in the + // backend's app.yaml file. + var backendHostUrl = ''; + + // [START gae_python_firenotes_config] + // Obtain the following from the "Add Firebase to your web app" dialogue + // Initialize Firebase + var config = { + apiKey: "", + authDomain: ".firebaseapp.com", + databaseURL: "https://.firebaseio.com", + projectId: "", + storageBucket: ".appspot.com", + messagingSenderId: "" + }; + // [END gae_python_firenotes_config] + + // This is passed into the backend to authenticate the user. + var userIdToken = null; + + // Firebase log-in + function configureFirebaseLogin() { + + firebase.initializeApp(config); + + // [START gae_python_state_change] + firebase.auth().onAuthStateChanged(function(user) { + if (user) { + $('#logged-out').hide(); + var name = user.displayName; + + /* If the provider gives a display name, use the name for the + personal welcome message. Otherwise, use the user's email. */ + var welcomeName = name ? name : user.email; + + user.getIdToken().then(function(idToken) { + userIdToken = idToken; + + /* Now that the user is authenicated, fetch the notes. */ + fetchNotes(); + + $('#user').text(welcomeName); + $('#logged-in').show(); + + }); + + } else { + $('#logged-in').hide(); + $('#logged-out').show(); + + } + // [END gae_python_state_change] + + }); + + } + + // [START configureFirebaseLoginWidget] + // Firebase log-in widget + function configureFirebaseLoginWidget() { + var uiConfig = { + 'signInSuccessUrl': '/', + 'signInOptions': [ + // Leave the lines as is for the providers you want to offer your users. + firebase.auth.GoogleAuthProvider.PROVIDER_ID, + firebase.auth.FacebookAuthProvider.PROVIDER_ID, + firebase.auth.TwitterAuthProvider.PROVIDER_ID, + firebase.auth.GithubAuthProvider.PROVIDER_ID, + firebase.auth.EmailAuthProvider.PROVIDER_ID + ], + // Terms of service url + 'tosUrl': '', + }; + + var ui = new firebaseui.auth.AuthUI(firebase.auth()); + ui.start('#firebaseui-auth-container', uiConfig); + } + // [END gae_python_firebase_login] + + // [START gae_python_fetch_notes] + // Fetch notes from the backend. + function fetchNotes() { + $.ajax(backendHostUrl + '/notes', { + /* Set header for the XMLHttpRequest to get data from the web server + associated with userIdToken */ + headers: { + 'Authorization': 'Bearer ' + userIdToken + } + }).then(function(data){ + $('#notes-container').empty(); + // Iterate over user data to display user's notes from database. + data.forEach(function(note){ + $('#notes-container').append($('

        ').text(note.message)); + }); + }); + } + // [END gae_python_fetch_notes] + + // Sign out a user + var signOutBtn =$('#sign-out'); + signOutBtn.click(function(event) { + event.preventDefault(); + + firebase.auth().signOut().then(function() { + console.log("Sign out successful"); + }, function(error) { + console.log(error); + }); + }); + + // Save a note to the backend + var saveNoteBtn = $('#add-note'); + saveNoteBtn.click(function(event) { + event.preventDefault(); + + var noteField = $('#note-content'); + var note = noteField.val(); + noteField.val(""); + + /* Send note data to backend, storing in database with existing data + associated with userIdToken */ + $.ajax(backendHostUrl + '/notes', { + headers: { + 'Authorization': 'Bearer ' + userIdToken + }, + method: 'POST', + data: JSON.stringify({'message': note}), + contentType : 'application/json' + }).then(function(){ + // Refresh notebook display. + fetchNotes(); + }); + + }); + + configureFirebaseLogin(); + configureFirebaseLoginWidget(); + +}); diff --git a/appengine/standard/firebase/firenotes/frontend/style.css b/appengine/standard/firebase/firenotes/frontend/style.css new file mode 100644 index 00000000000..19b4f1d69bd --- /dev/null +++ b/appengine/standard/firebase/firenotes/frontend/style.css @@ -0,0 +1,44 @@ +/* + Copyright 2016, Google, Inc. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +body { + font-family: "helvetica", sans-serif; + text-align: center; +} + +form { + padding: 5px 0 10px; + margin-bottom: 30px; +} +h3,legend { + font-weight: 400; + padding: 18px 0 15px; + margin: 0 0 0; +} + +div.form-group { + margin-bottom: 10px; +} + +input, textarea { + width: 250px; + font-size: 14px; + padding: 6px; +} + +textarea { + vertical-align: top; + height: 75px; +} diff --git a/appengine/standard/firebase/firetactoe/README.md b/appengine/standard/firebase/firetactoe/README.md new file mode 100644 index 00000000000..e52cc1f061f --- /dev/null +++ b/appengine/standard/firebase/firetactoe/README.md @@ -0,0 +1,49 @@ +# Tic Tac Toe, using Firebase, on App Engine Standard + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/firebase/firetactoe/README.md + +This sample shows how to use the [Firebase](https://firebase.google.com/) +realtime database to implement a simple Tic Tac Toe game on [Google App Engine +Standard](https://cloud.google.com/appengine). + +## Setup + +Make sure you have the [Google Cloud SDK](https://cloud.google.com/sdk/) +installed. You'll need this to test and deploy your App Engine app. + +### Authentication + +* Create a project in the [Firebase + console](https://firebase.google.com/console) +* In the Overview section, click 'Add Firebase to your web app' and replace the + contents of the file + [`templates/_firebase_config.html`](templates/_firebase_config.html) with the + given snippet. This provides credentials for the javascript client. +* For running the sample locally, you'll need to download a service account to + provide credentials that would normally be provided automatically in the App + Engine environment. Click the gear icon in the Firebase Console and select + 'Permissions'; then go to the 'Service accounts' tab. Download a new or + existing App Engine service account credentials file. Then set the environment + variable `GOOGLE_APPLICATION_CREDENTIALS` to the path to this file: + + export GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials.json + + This allows the server to create unique secure tokens for each user for + Firebase to validate. + +### Install dependencies + +Before running or deploying this application, install the dependencies using +[pip](http://pip.readthedocs.io/en/stable/): + + pip install -t lib -r requirements.txt + +## Running the sample + + dev_appserver.py . + +For more information on running or deploying the sample, see the [App Engine +Standard README](../../README.md). diff --git a/appengine/standard/firebase/firetactoe/app.yaml b/appengine/standard/firebase/firetactoe/app.yaml new file mode 100644 index 00000000000..4e474a199d9 --- /dev/null +++ b/appengine/standard/firebase/firetactoe/app.yaml @@ -0,0 +1,11 @@ +runtime: python27 +api_version: 1 +threadsafe: true + +handlers: +- url: /static + static_dir: static + +- url: /.* + script: firetactoe.app + login: required diff --git a/appengine/standard/firebase/firetactoe/appengine_config.py b/appengine/standard/firebase/firetactoe/appengine_config.py new file mode 100644 index 00000000000..25d6ed86866 --- /dev/null +++ b/appengine/standard/firebase/firetactoe/appengine_config.py @@ -0,0 +1,10 @@ +import os.path + +from google.appengine.ext import vendor + +# Add any libraries installed in the "lib" folder. +vendor.add('lib') + +# Patch os.path.expanduser. This should be fixed in GAE +# versions released after Nov 2016. +os.path.expanduser = lambda path: path diff --git a/appengine/standard/firebase/firetactoe/firetactoe.py b/appengine/standard/firebase/firetactoe/firetactoe.py new file mode 100644 index 00000000000..ec84abf90c8 --- /dev/null +++ b/appengine/standard/firebase/firetactoe/firetactoe.py @@ -0,0 +1,274 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tic Tac Toe with the Firebase API""" + +import base64 +try: + from functools import lru_cache +except ImportError: + from functools32 import lru_cache +import json +import os +import re +import time +import urllib + + +import flask +from flask import request +from google.appengine.api import app_identity +from google.appengine.api import users +from google.appengine.ext import ndb +import httplib2 +from oauth2client.client import GoogleCredentials + + +_FIREBASE_CONFIG = '_firebase_config.html' + +_IDENTITY_ENDPOINT = ('https://identitytoolkit.googleapis.com/' + 'google.identity.identitytoolkit.v1.IdentityToolkit') +_FIREBASE_SCOPES = [ + 'https://www.googleapis.com/auth/firebase.database', + 'https://www.googleapis.com/auth/userinfo.email'] + +_X_WIN_PATTERNS = [ + 'XXX......', '...XXX...', '......XXX', 'X..X..X..', '.X..X..X.', + '..X..X..X', 'X...X...X', '..X.X.X..'] +_O_WIN_PATTERNS = map(lambda s: s.replace('X', 'O'), _X_WIN_PATTERNS) + +X_WINS = map(lambda s: re.compile(s), _X_WIN_PATTERNS) +O_WINS = map(lambda s: re.compile(s), _O_WIN_PATTERNS) + + +app = flask.Flask(__name__) + + +# Memoize the value, to avoid parsing the code snippet every time +@lru_cache() +def _get_firebase_db_url(): + """Grabs the databaseURL from the Firebase config snippet. Regex looks + scary, but all it is doing is pulling the 'databaseURL' field from the + Firebase javascript snippet""" + regex = re.compile(r'\bdatabaseURL\b.*?["\']([^"\']+)') + cwd = os.path.dirname(__file__) + try: + with open(os.path.join(cwd, 'templates', _FIREBASE_CONFIG)) as f: + url = next(regex.search(line) for line in f if regex.search(line)) + except StopIteration: + raise ValueError( + 'Error parsing databaseURL. Please copy Firebase web snippet ' + 'into templates/{}'.format(_FIREBASE_CONFIG)) + return url.group(1) + + +# Memoize the authorized http, to avoid fetching new access tokens +@lru_cache() +def _get_http(): + """Provides an authed http object.""" + http = httplib2.Http() + # Use application default credentials to make the Firebase calls + # https://firebase.google.com/docs/reference/rest/database/user-auth + creds = GoogleCredentials.get_application_default().create_scoped( + _FIREBASE_SCOPES) + creds.authorize(http) + return http + + +def _send_firebase_message(u_id, message=None): + """Updates data in firebase. If a message is provided, then it updates + the data at /channels/ with the message using the PATCH + http method. If no message is provided, then the data at this location + is deleted using the DELETE http method + """ + url = '{}/channels/{}.json'.format(_get_firebase_db_url(), u_id) + + if message: + return _get_http().request(url, 'PATCH', body=message) + else: + return _get_http().request(url, 'DELETE') + + +def create_custom_token(uid, valid_minutes=60): + """Create a secure token for the given id. + + This method is used to create secure custom JWT tokens to be passed to + clients. It takes a unique id (uid) that will be used by Firebase's + security rules to prevent unauthorized access. In this case, the uid will + be the channel id which is a combination of user_id and game_key + """ + + # use the app_identity service from google.appengine.api to get the + # project's service account email automatically + client_email = app_identity.get_service_account_name() + + now = int(time.time()) + # encode the required claims + # per https://firebase.google.com/docs/auth/server/create-custom-tokens + payload = base64.b64encode(json.dumps({ + 'iss': client_email, + 'sub': client_email, + 'aud': _IDENTITY_ENDPOINT, + 'uid': uid, # the important parameter, as it will be the channel id + 'iat': now, + 'exp': now + (valid_minutes * 60), + })) + # add standard header to identify this as a JWT + header = base64.b64encode(json.dumps({'typ': 'JWT', 'alg': 'RS256'})) + to_sign = '{}.{}'.format(header, payload) + # Sign the jwt using the built in app_identity service + return '{}.{}'.format(to_sign, base64.b64encode( + app_identity.sign_blob(to_sign)[1])) + + +class Game(ndb.Model): + """All the data we store for a game""" + userX = ndb.UserProperty() + userO = ndb.UserProperty() + board = ndb.StringProperty() + moveX = ndb.BooleanProperty() + winner = ndb.StringProperty() + winning_board = ndb.StringProperty() + + def to_json(self): + d = self.to_dict() + d['winningBoard'] = d.pop('winning_board') + return json.dumps(d, default=lambda user: user.user_id()) + + def send_update(self): + """Updates Firebase's copy of the board.""" + message = self.to_json() + # send updated game state to user X + _send_firebase_message( + self.userX.user_id() + self.key.id(), message=message) + # send updated game state to user O + if self.userO: + _send_firebase_message( + self.userO.user_id() + self.key.id(), message=message) + + def _check_win(self): + if self.moveX: + # O just moved, check for O wins + wins = O_WINS + potential_winner = self.userO.user_id() + else: + # X just moved, check for X wins + wins = X_WINS + potential_winner = self.userX.user_id() + + for win in wins: + if win.match(self.board): + self.winner = potential_winner + self.winning_board = win.pattern + return + + # In case of a draw, everyone loses. + if ' ' not in self.board: + self.winner = 'Noone' + + def make_move(self, position, user): + # If the user is a player, and it's their move + if (user in (self.userX, self.userO)) and ( + self.moveX == (user == self.userX)): + boardList = list(self.board) + # If the spot you want to move to is blank + if (boardList[position] == ' '): + boardList[position] = 'X' if self.moveX else 'O' + self.board = ''.join(boardList) + self.moveX = not self.moveX + self._check_win() + self.put() + self.send_update() + return + + +# [START move_route] +@app.route('/move', methods=['POST']) +def move(): + game = Game.get_by_id(request.args.get('g')) + position = int(request.form.get('i')) + if not (game and (0 <= position <= 8)): + return 'Game not found, or invalid position', 400 + game.make_move(position, users.get_current_user()) + return '' +# [END move_route] + + +# [START route_delete] +@app.route('/delete', methods=['POST']) +def delete(): + game = Game.get_by_id(request.args.get('g')) + if not game: + return 'Game not found', 400 + user = users.get_current_user() + _send_firebase_message(user.user_id() + game.key.id(), message=None) + return '' +# [END route_delete] + + +@app.route('/opened', methods=['POST']) +def opened(): + game = Game.get_by_id(request.args.get('g')) + if not game: + return 'Game not found', 400 + game.send_update() + return '' + + +@app.route('/') +def main_page(): + """Renders the main page. When this page is shown, we create a new + channel to push asynchronous updates to the client.""" + user = users.get_current_user() + game_key = request.args.get('g') + + if not game_key: + game_key = user.user_id() + game = Game(id=game_key, userX=user, moveX=True, board=' '*9) + game.put() + else: + game = Game.get_by_id(game_key) + if not game: + return 'No such game', 404 + if not game.userO: + game.userO = user + game.put() + + # [START pass_token] + # choose a unique identifier for channel_id + channel_id = user.user_id() + game_key + # encrypt the channel_id and send it as a custom token to the + # client + # Firebase's data security rules will be able to decrypt the + # token and prevent unauthorized access + client_auth_token = create_custom_token(channel_id) + _send_firebase_message(channel_id, message=game.to_json()) + + # game_link is a url that you can open in another browser to play + # against this player + game_link = '{}?g={}'.format(request.base_url, game_key) + + # push all the data to the html template so the client will + # have access + template_values = { + 'token': client_auth_token, + 'channel_id': channel_id, + 'me': user.user_id(), + 'game_key': game_key, + 'game_link': game_link, + 'initial_message': urllib.unquote(game.to_json()) + } + + return flask.render_template('fire_index.html', **template_values) + # [END pass_token] diff --git a/appengine/standard/firebase/firetactoe/firetactoe_test.py b/appengine/standard/firebase/firetactoe/firetactoe_test.py new file mode 100644 index 00000000000..f5ec1ca9573 --- /dev/null +++ b/appengine/standard/firebase/firetactoe/firetactoe_test.py @@ -0,0 +1,161 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import re + +from google.appengine.api import users +from google.appengine.ext import ndb +import httplib2 +import pytest +import webtest + +import firetactoe + + +class MockHttp(object): + """Mock the Http object, so we can set what the response will be.""" + def __init__(self, status, content=''): + self.content = content + self.status = status + self.request_url = None + + def __call__(self, *args, **kwargs): + return self + + def request(self, url, method, content='', *args, **kwargs): + self.request_url = url + self.request_method = method + self.request_content = content + return self, self.content + + +@pytest.fixture +def app(testbed, monkeypatch, login): + # Don't let the _get_http function memoize its value + firetactoe._get_http.cache_clear() + + # Provide a test firebase config. The following will set the databaseURL + # databaseURL: "http://firebase.com/test-db-url" + monkeypatch.setattr( + firetactoe, '_FIREBASE_CONFIG', '../firetactoe_test.py') + + login(id='38') + + firetactoe.app.debug = True + return webtest.TestApp(firetactoe.app) + + +def test_index_new_game(app, monkeypatch): + mock_http = MockHttp(200, content=json.dumps({'access_token': '123'})) + monkeypatch.setattr(httplib2, 'Http', mock_http) + + response = app.get('/') + + assert 'g=' in response.body + # Look for the unique game token + assert re.search( + r'initGame[^\n]+\'[\w+/=]+\.[\w+/=]+\.[\w+/=]+\'', response.body) + + assert firetactoe.Game.query().count() == 1 + + assert mock_http.request_url.startswith( + 'http://firebase.com/test-db-url/channels/') + assert mock_http.request_method == 'PATCH' + + +def test_index_existing_game(app, monkeypatch): + mock_http = MockHttp(200, content=json.dumps({'access_token': '123'})) + monkeypatch.setattr(httplib2, 'Http', mock_http) + userX = users.User('x@example.com', _user_id='123') + firetactoe.Game(id='razem', userX=userX).put() + + response = app.get('/?g=razem') + + assert 'g=' in response.body + # Look for the unique game token + assert re.search( + r'initGame[^\n]+\'[\w+/=]+\.[\w+/=]+\.[\w+/=]+\'', response.body) + + assert firetactoe.Game.query().count() == 1 + game = ndb.Key('Game', 'razem').get() + assert game is not None + assert game.userO.user_id() == '38' + + assert mock_http.request_url.startswith( + 'http://firebase.com/test-db-url/channels/') + assert mock_http.request_method == 'PATCH' + + +def test_index_nonexisting_game(app, monkeypatch): + mock_http = MockHttp(200, content=json.dumps({'access_token': '123'})) + monkeypatch.setattr(httplib2, 'Http', mock_http) + firetactoe.Game(id='razem', userX=users.get_current_user()).put() + + app.get('/?g=razemfrazem', status=404) + + assert mock_http.request_url is None + + +def test_opened(app, monkeypatch): + mock_http = MockHttp(200, content=json.dumps({'access_token': '123'})) + monkeypatch.setattr(httplib2, 'Http', mock_http) + firetactoe.Game(id='razem', userX=users.get_current_user()).put() + + app.post('/opened?g=razem', status=200) + + assert mock_http.request_url.startswith( + 'http://firebase.com/test-db-url/channels/') + assert mock_http.request_method == 'PATCH' + + +def test_bad_move(app, monkeypatch): + mock_http = MockHttp(200, content=json.dumps({'access_token': '123'})) + monkeypatch.setattr(httplib2, 'Http', mock_http) + firetactoe.Game( + id='razem', userX=users.get_current_user(), board=9*' ', + moveX=True).put() + + app.post('/move?g=razem', {'i': 10}, status=400) + + assert mock_http.request_url is None + + +def test_move(app, monkeypatch): + mock_http = MockHttp(200, content=json.dumps({'access_token': '123'})) + monkeypatch.setattr(httplib2, 'Http', mock_http) + firetactoe.Game( + id='razem', userX=users.get_current_user(), board=9*' ', + moveX=True).put() + + app.post('/move?g=razem', {'i': 0}, status=200) + + game = ndb.Key('Game', 'razem').get() + assert game.board == 'X' + (8 * ' ') + + assert mock_http.request_url.startswith( + 'http://firebase.com/test-db-url/channels/') + assert mock_http.request_method == 'PATCH' + + +def test_delete(app, monkeypatch): + mock_http = MockHttp(200, content=json.dumps({'access_token': '123'})) + monkeypatch.setattr(httplib2, 'Http', mock_http) + firetactoe.Game(id='razem', userX=users.get_current_user()).put() + + app.post('/delete?g=razem', status=200) + + assert mock_http.request_url.startswith( + 'http://firebase.com/test-db-url/channels/') + assert mock_http.request_method == 'DELETE' diff --git a/appengine/standard/firebase/firetactoe/requirements.txt b/appengine/standard/firebase/firetactoe/requirements.txt new file mode 100644 index 00000000000..578fcb891c8 --- /dev/null +++ b/appengine/standard/firebase/firetactoe/requirements.txt @@ -0,0 +1,5 @@ +flask==1.0.2 +requests==2.21.0 +requests_toolbelt==0.9.1 +oauth2client==4.1.3 +functools32==3.2.3.post2; python_version < "3" diff --git a/appengine/standard/firebase/firetactoe/rest_api.py b/appengine/standard/firebase/firetactoe/rest_api.py new file mode 100644 index 00000000000..a067953a8f1 --- /dev/null +++ b/appengine/standard/firebase/firetactoe/rest_api.py @@ -0,0 +1,114 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Demonstration of the Firebase REST API in Python""" + +try: + from functools import lru_cache +except ImportError: + from functools32 import lru_cache +# [START rest_writing_data] +import json + +import httplib2 +from oauth2client.client import GoogleCredentials + +_FIREBASE_SCOPES = [ + 'https://www.googleapis.com/auth/firebase.database', + 'https://www.googleapis.com/auth/userinfo.email'] + + +# Memoize the authorized http, to avoid fetching new access tokens +@lru_cache() +def _get_http(): + """Provides an authed http object.""" + http = httplib2.Http() + # Use application default credentials to make the Firebase calls + # https://firebase.google.com/docs/reference/rest/database/user-auth + creds = GoogleCredentials.get_application_default().create_scoped( + _FIREBASE_SCOPES) + creds.authorize(http) + return http + + +def firebase_put(path, value=None): + """Writes data to Firebase. + + An HTTP PUT writes an entire object at the given database path. Updates to + fields cannot be performed without overwriting the entire object + + Args: + path - the url to the Firebase object to write. + value - a json string. + """ + response, content = _get_http().request(path, method='PUT', body=value) + return json.loads(content) + + +def firebase_patch(path, value=None): + """Update specific children or fields + + An HTTP PATCH allows specific children or fields to be updated without + overwriting the entire object. + + Args: + path - the url to the Firebase object to write. + value - a json string. + """ + response, content = _get_http().request(path, method='PATCH', body=value) + return json.loads(content) + + +def firebase_post(path, value=None): + """Add an object to an existing list of data. + + An HTTP POST allows an object to be added to an existing list of data. + A successful request will be indicated by a 200 OK HTTP status code. The + response content will contain a new attribute "name" which is the key for + the child added. + + Args: + path - the url to the Firebase list to append to. + value - a json string. + """ + response, content = _get_http().request(path, method='POST', body=value) + return json.loads(content) +# [END rest_writing_data] + + +def firebase_get(path): + """Read the data at the given path. + + An HTTP GET request allows reading of data at a particular path. + A successful request will be indicated by a 200 OK HTTP status code. + The response will contain the data being retrieved. + + Args: + path - the url to the Firebase object to read. + """ + response, content = _get_http().request(path, method='GET') + return json.loads(content) + + +def firebase_delete(path): + """Removes the data at a particular path. + + An HTTP DELETE removes the data at a particular path. A successful request + will be indicated by a 200 OK HTTP status code with a response containing + JSON null. + + Args: + path - the url to the Firebase object to delete. + """ + response, content = _get_http().request(path, method='DELETE') diff --git a/appengine/standard/firebase/firetactoe/static/main.css b/appengine/standard/firebase/firetactoe/static/main.css new file mode 100644 index 00000000000..f314eab5b37 --- /dev/null +++ b/appengine/standard/firebase/firetactoe/static/main.css @@ -0,0 +1,82 @@ +body { + font-family: 'Helvetica'; +} + +#board { + width:152px; + height: 152px; + margin: 20px auto; +} + +#display-area { + text-align: center; +} + +#other-player, #your-move, #their-move, #you-won, #you-lost { + display: none; +} + +#display-area.waiting #other-player { + display: block; +} + +#display-area.waiting #board, #display-area.waiting #this-game { + display: none; +} +#display-area.won #you-won { + display: block; +} +#display-area.lost #you-lost { + display: block; +} +#display-area.your-move #your-move { + display: block; +} +#display-area.their-move #their-move { + display: block; +} + + +#this-game { + font-size: 9pt; +} + +div.cell { + float: left; + width: 50px; + height: 50px; + border: none; + margin: 0px; + padding: 0px; + box-sizing: border-box; + + line-height: 50px; + font-family: "Helvetica"; + font-size: 16pt; + text-align: center; +} + +.your-move div.cell:hover { + background: lightgrey; +} + +.your-move div.cell:empty:hover { + background: lightblue; + cursor: pointer; +} + +div.l { + border-right: 1pt solid black; +} + +div.r { + border-left: 1pt solid black; +} + +div.t { + border-bottom: 1pt solid black; +} + +div.b { + border-top: 1pt solid black; +} diff --git a/appengine/standard/firebase/firetactoe/static/main.js b/appengine/standard/firebase/firetactoe/static/main.js new file mode 100644 index 00000000000..980509574f2 --- /dev/null +++ b/appengine/standard/firebase/firetactoe/static/main.js @@ -0,0 +1,178 @@ +/** + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +/** + * @fileoverview Tic-Tac-Toe, using the Firebase API + */ + +/** + * @param gameKey - a unique key for this game. + * @param me - my user id. + * @param token - secure token passed from the server + * @param channelId - id of the 'channel' we'll be listening to + */ +function initGame(gameKey, me, token, channelId, initialMessage) { + var state = { + gameKey: gameKey, + me: me + }; + + // This is our Firebase realtime DB path that we'll listen to for updates + // We'll initialize this later in openChannel() + var channel = null; + + /** + * Updates the displayed game board. + */ + function updateGame(newState) { + $.extend(state, newState); + + $('.cell').each(function(i) { + var square = $(this); + var value = state.board[i]; + square.html(' ' === value ? '' : value); + + if (state.winner && state.winningBoard) { + if (state.winningBoard[i] === value) { + if (state.winner === state.me) { + square.css('background', 'green'); + } else { + square.css('background', 'red'); + } + } else { + square.css('background', ''); + } + } + }); + + var displayArea = $('#display-area'); + + if (!state.userO) { + displayArea[0].className = 'waiting'; + } else if (state.winner === state.me) { + displayArea[0].className = 'won'; + } else if (state.winner) { + displayArea[0].className = 'lost'; + } else if (isMyMove()) { + displayArea[0].className = 'your-move'; + } else { + displayArea[0].className = 'their-move'; + } + } + + function isMyMove() { + return !state.winner && (state.moveX === (state.userX === state.me)); + } + + function myPiece() { + return state.userX === state.me ? 'X' : 'O'; + } + + /** + * Send the user's latest move back to the server + */ + function moveInSquare(e) { + var id = $(e.currentTarget).index(); + if (isMyMove() && state.board[id] === ' ') { + $.post('/move', {i: id}); + } + } + + /** + * This method lets the server know that the user has opened the channel + * After this method is called, the server may begin to send updates + */ + function onOpened() { + $.post('/opened'); + } + + /** + * This deletes the data associated with the Firebase path + * it is critical that this data be deleted since it costs money + */ + function deleteChannel() { + $.post('/delete'); + } + + /** + * This method is called every time an event is fired from Firebase + * it updates the entire game state and checks for a winner + * if a player has won the game, this function calls the server to delete + * the data stored in Firebase + */ + function onMessage(newState) { + updateGame(newState); + + // now check to see if there is a winner + if (channel && state.winner && state.winningBoard) { + channel.off(); //stop listening on this path + deleteChannel(); //delete the data we wrote + } + } + + /** + * This function opens a realtime communication channel with Firebase + * It logs in securely using the client token passed from the server + * then it sets up a listener on the proper database path (also passed by server) + * finally, it calls onOpened() to let the server know it is ready to receive messages + */ + function openChannel() { + // [START auth_login] + // sign into Firebase with the token passed from the server + firebase.auth().signInWithCustomToken(token).catch(function(error) { + console.log('Login Failed!', error.code); + console.log('Error message: ', error.message); + }); + // [END auth_login] + + // [START add_listener] + // setup a database reference at path /channels/channelId + channel = firebase.database().ref('channels/' + channelId); + // add a listener to the path that fires any time the value of the data changes + channel.on('value', function(data) { + onMessage(data.val()); + }); + // [END add_listener] + onOpened(); + // let the server know that the channel is open + } + + /** + * This function opens a communication channel with the server + * then it adds listeners to all the squares on the board + * next it pulls down the initial game state from template values + * finally it updates the game state with those values by calling onMessage() + */ + function initialize() { + // Always include the gamekey in our requests + $.ajaxPrefilter(function(opts) { + if (opts.url.indexOf('?') > 0) + opts.url += '&g=' + state.gameKey; + else + opts.url += '?g=' + state.gameKey; + }); + + $('#board').on('click', '.cell', moveInSquare); + + openChannel(); + + onMessage(initialMessage); + } + + setTimeout(initialize, 100); +} diff --git a/appengine/standard/firebase/firetactoe/templates/_firebase_config.html b/appengine/standard/firebase/firetactoe/templates/_firebase_config.html new file mode 100644 index 00000000000..25898c985af --- /dev/null +++ b/appengine/standard/firebase/firetactoe/templates/_firebase_config.html @@ -0,0 +1,3 @@ +REPLACE ME WITH YOUR FIREBASE WEBAPP CODE SNIPPET: + +https://console.firebase.google.com/project/_/overview diff --git a/appengine/standard/firebase/firetactoe/templates/fire_index.html b/appengine/standard/firebase/firetactoe/templates/fire_index.html new file mode 100644 index 00000000000..3b18380b533 --- /dev/null +++ b/appengine/standard/firebase/firetactoe/templates/fire_index.html @@ -0,0 +1,43 @@ + + + + + {% include "_firebase_config.html" %} + + + + + + +

        +

        Firebase-enabled Tic Tac Toe

        +         +
        Your move! Click a square to place your piece.
        +
        Waiting for other player to move...
        +
        You won this game!
        +
        You lost this game.
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        + Quick link to this game: {{ game_link }} +
        +
        + + diff --git a/appengine/mailgun/.gitignore b/appengine/standard/flask/hello_world/.gitignore similarity index 100% rename from appengine/mailgun/.gitignore rename to appengine/standard/flask/hello_world/.gitignore diff --git a/appengine/standard/flask/hello_world/README.md b/appengine/standard/flask/hello_world/README.md new file mode 100644 index 00000000000..caf101d4db9 --- /dev/null +++ b/appengine/standard/flask/hello_world/README.md @@ -0,0 +1,11 @@ +# App Engine Standard Flask Hello World + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/flask/hello_world/README.md + +This sample shows how to use [Flask](http://flask.pocoo.org/) with Google App +Engine Standard. + +For more information, see the [App Engine Standard README](../../README.md) diff --git a/appengine/standard/flask/hello_world/app.yaml b/appengine/standard/flask/hello_world/app.yaml new file mode 100644 index 00000000000..bab91f281de --- /dev/null +++ b/appengine/standard/flask/hello_world/app.yaml @@ -0,0 +1,11 @@ +runtime: python27 +api_version: 1 +threadsafe: true + +handlers: +- url: /.* + script: main.app + +libraries: +- name: flask + version: 0.12 diff --git a/appengine/standard/flask/hello_world/main.py b/appengine/standard/flask/hello_world/main.py new file mode 100644 index 00000000000..6e5a9d82fd1 --- /dev/null +++ b/appengine/standard/flask/hello_world/main.py @@ -0,0 +1,34 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START app] +import logging + +from flask import Flask + + +app = Flask(__name__) + + +@app.route('/') +def hello(): + return 'Hello World!' + + +@app.errorhandler(500) +def server_error(e): + # Log the error and stacktrace. + logging.exception('An error occurred during a request.') + return 'An internal error occurred.', 500 +# [END app] diff --git a/appengine/standard/flask/hello_world/main_test.py b/appengine/standard/flask/hello_world/main_test.py new file mode 100644 index 00000000000..ab6db4fe5df --- /dev/null +++ b/appengine/standard/flask/hello_world/main_test.py @@ -0,0 +1,27 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + + +@pytest.fixture +def app(): + import main + main.app.testing = True + return main.app.test_client() + + +def test_index(app): + r = app.get('/') + assert r.status_code == 200 diff --git a/appengine/storage/.gitignore b/appengine/standard/flask/tutorial/.gitignore similarity index 100% rename from appengine/storage/.gitignore rename to appengine/standard/flask/tutorial/.gitignore diff --git a/appengine/standard/flask/tutorial/README.md b/appengine/standard/flask/tutorial/README.md new file mode 100644 index 00000000000..f334542e835 --- /dev/null +++ b/appengine/standard/flask/tutorial/README.md @@ -0,0 +1,16 @@ +# App Engine Standard Flask Tutorial App + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/flask/tutorial/README.md + +This sample shows how to use [Flask](http://flask.pocoo.org/) to handle +requests, forms, templates, and static files on Google App Engine Standard. + +Before running or deploying this application, install the dependencies using +[pip](http://pip.readthedocs.io/en/stable/): + + pip install -t lib -r requirements.txt + +For more information, see the [App Engine Standard README](../../README.md) diff --git a/appengine/standard/flask/tutorial/app.yaml b/appengine/standard/flask/tutorial/app.yaml new file mode 100644 index 00000000000..39e2bdf5008 --- /dev/null +++ b/appengine/standard/flask/tutorial/app.yaml @@ -0,0 +1,15 @@ +runtime: python27 +api_version: 1 +threadsafe: true + +libraries: +- name: ssl + version: latest + +# [START handlers] +handlers: +- url: /static + static_dir: static +- url: /.* + script: main.app +# [END handlers] diff --git a/appengine/standard/flask/tutorial/appengine_config.py b/appengine/standard/flask/tutorial/appengine_config.py new file mode 100644 index 00000000000..c03915197f7 --- /dev/null +++ b/appengine/standard/flask/tutorial/appengine_config.py @@ -0,0 +1,20 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START vendor] +from google.appengine.ext import vendor + +# Add any libraries installed in the "lib" folder. +vendor.add('lib') +# [END vendor] diff --git a/appengine/standard/flask/tutorial/main.py b/appengine/standard/flask/tutorial/main.py new file mode 100644 index 00000000000..feed0fbcf62 --- /dev/null +++ b/appengine/standard/flask/tutorial/main.py @@ -0,0 +1,58 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START app] +import logging + +# [START imports] +from flask import Flask, render_template, request +# [END imports] + +# [START create_app] +app = Flask(__name__) +# [END create_app] + + +# [START form] +@app.route('/form') +def form(): + return render_template('form.html') +# [END form] + + +# [START submitted] +@app.route('/submitted', methods=['POST']) +def submitted_form(): + name = request.form['name'] + email = request.form['email'] + site = request.form['site_url'] + comments = request.form['comments'] + + # [END submitted] + # [START render_template] + return render_template( + 'submitted_form.html', + name=name, + email=email, + site=site, + comments=comments) + # [END render_template] + + +@app.errorhandler(500) +def server_error(e): + # Log the error and stacktrace. + logging.exception('An error occurred during a request.') + return 'An internal error occurred.', 500 +# [END app] diff --git a/appengine/standard/flask/tutorial/main_test.py b/appengine/standard/flask/tutorial/main_test.py new file mode 100644 index 00000000000..9dbd16c9458 --- /dev/null +++ b/appengine/standard/flask/tutorial/main_test.py @@ -0,0 +1,38 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + + +@pytest.fixture +def app(): + import main + main.app.testing = True + return main.app.test_client() + + +def test_form(app): + r = app.get('/form') + assert r.status_code == 200 + assert 'Submit a form' in r.data.decode('utf-8') + + +def test_submitted_form(app): + r = app.post('/submitted', data={ + 'name': 'Inigo Montoya', + 'email': 'inigo@example.com', + 'site_url': 'http://example.com', + 'comments': ''}) + assert r.status_code == 200 + assert 'Inigo Montoya' in r.data.decode('utf-8') diff --git a/appengine/standard/flask/tutorial/requirements.txt b/appengine/standard/flask/tutorial/requirements.txt new file mode 100644 index 00000000000..0595fb4ff1c --- /dev/null +++ b/appengine/standard/flask/tutorial/requirements.txt @@ -0,0 +1,2 @@ +Flask==0.12.4 +Werkzeug<0.13.0,>=0.12.0 diff --git a/appengine/standard/flask/tutorial/static/style.css b/appengine/standard/flask/tutorial/static/style.css new file mode 100644 index 00000000000..378932a78f9 --- /dev/null +++ b/appengine/standard/flask/tutorial/static/style.css @@ -0,0 +1,3 @@ +.pagetitle { + color: #800080; +} diff --git a/appengine/standard/flask/tutorial/templates/form.html b/appengine/standard/flask/tutorial/templates/form.html new file mode 100644 index 00000000000..159752799e8 --- /dev/null +++ b/appengine/standard/flask/tutorial/templates/form.html @@ -0,0 +1,26 @@ + + + Submit a form + + + +
        +
        +

        Submit a form

        +
        +
        +
        + +
        + +
        + +
        + +
        + +
        +
        +
        + + diff --git a/appengine/standard/flask/tutorial/templates/submitted_form.html b/appengine/standard/flask/tutorial/templates/submitted_form.html new file mode 100644 index 00000000000..15f530088f6 --- /dev/null +++ b/appengine/standard/flask/tutorial/templates/submitted_form.html @@ -0,0 +1,23 @@ + + + Submitted form + + + +
        +
        +

        Form submitted

        +
        +
        +

        Thanks for your submission, {{name}}!

        +

        Here's a review of the information that you sent:

        +

        + Name: {{name}}
        + Email: {{email}}
        + Website URL: {{site}}
        + Comments: {{comments}} +

        +
        +
        + + diff --git a/appengine/standard/hello_world/app.yaml b/appengine/standard/hello_world/app.yaml new file mode 100644 index 00000000000..f041d384c05 --- /dev/null +++ b/appengine/standard/hello_world/app.yaml @@ -0,0 +1,7 @@ +runtime: python27 +api_version: 1 +threadsafe: true + +handlers: +- url: /.* + script: main.app diff --git a/appengine/standard/hello_world/main.py b/appengine/standard/hello_world/main.py new file mode 100644 index 00000000000..c26ad7a80f6 --- /dev/null +++ b/appengine/standard/hello_world/main.py @@ -0,0 +1,26 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import webapp2 + + +class MainPage(webapp2.RequestHandler): + def get(self): + self.response.headers['Content-Type'] = 'text/plain' + self.response.write('Hello, World!') + + +app = webapp2.WSGIApplication([ + ('/', MainPage), +], debug=True) diff --git a/appengine/standard/hello_world/main_test.py b/appengine/standard/hello_world/main_test.py new file mode 100644 index 00000000000..64670c5981d --- /dev/null +++ b/appengine/standard/hello_world/main_test.py @@ -0,0 +1,26 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import webtest + +import main + + +def test_get(): + app = webtest.TestApp(main.app) + + response = app.get('/') + + assert response.status_int == 200 + assert response.body == 'Hello, World!' diff --git a/appengine/standard/i18n/README.md b/appengine/standard/i18n/README.md new file mode 100644 index 00000000000..04b9063579d --- /dev/null +++ b/appengine/standard/i18n/README.md @@ -0,0 +1,133 @@ +# App Engine Internationalization Sample + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/i18n/README.md + +A simple example app showing how to build an internationalized app +with App Engine. + +## What to internationalize + +There are lots of things to internationalize with your web +applications. + +1. Strings in Python code +2. Strings in HTML template +3. Strings in Javascript +4. Common strings + - Country Names, Language Names, etc. +5. Formatting + - Date/Time formatting + - Number formatting + - Currency +6. Timezone conversion + +This example only covers first 3 basic scenarios above. In order to +cover other aspects, I recommend using +[Babel](http://babel.edgewall.org/) and [pytz] +(http://pypi.python.org/pypi/gaepytz). Also, you may want to use +[webapp2_extras.i18n](http://webapp-improved.appspot.com/tutorials/i18n.html) +module. + +## Wait, so why not webapp2_extras.i18n? + +webapp2_extras.i18n doesn't cover how to internationalize strings in +Javascript code. Additionally it depends on babel and pytz, which +means you need to deploy babel and pytz alongside with your code. I'd +like to show a reasonably minimum example for string +internationalization in Python code, jinja2 templates, as well as +Javascript. + +## How to run this example + +First of all, please install babel in your local Python environment. + +### Wait, you just said I don't need babel, are you crazy? + +As I said before, you don't need to deploy babel with this +application, but you need to locally use pybabel script which is +provided by babel distribution in order to extract the strings, manage +and compile the translations file. + +### Extract strings in Python code and Jinja2 templates to translate + +Move into this project directory and invoke the following command: + + $ env PYTHONPATH=/google_appengine_sdk/lib/jinja2 \ + pybabel extract -o locales/messages.pot -F main.mapping . + +This command creates a `locales/messages.pot` file in the `locales` +directory which contains all the string found in your Python code and +Jija2 tempaltes. + +Since the babel configration file `main.mapping` contains a reference +to `jinja2.ext.babel_extract` helper function which is provided by +jinja2 distribution bundled with the App Engine SDK, you need to add a +PYTHONPATH environment variable pointing to the jinja2 directory in +the SDK. + +### Manage and compile translations. + +Create an initial translation source by the following command: + + $ pybabel init -l ja -d locales -i locales/messages.pot + +Open `locales/ja/LC_MESSAGES/messages.po` with any text editor and +translate the strings, then compile the file by the following command: + + $ pybabel compile -d locales + +If any of the strings changes, you can extract the strings again, and +update the translations by the following command: + + $ pybabel update -l ja -d locales -i locales/messages.pot + +Note: If you run `pybabel init` against an existant translations file, +you will lose your translations. + + +### Extract strings in Javascript code and compile translations + + $ pybabel extract -o locales/jsmessages.pot -F js.mapping . + $ pybabel init -l ja -d locales -i locales/jsmessages.pot -D jsmessages + +Open `locales/ja/LC_MESSAGES/jsmessages.po` and translate it. + + $ pybabel compile -d locales -D jsmessages + +### Running locally & deploying + +Refer to the [App Engine Samples README](../README.md) for information on how to run and deploy this sample. + +## How it works + +As you can see it in the `appengine_config.py` file, our +`main.application` is wrapped by the `i18n_utils.I18nMiddleware` WSGI +middleware. When a request comes in, this middleware parses the +`HTTP_ACCEPT_LANGUAGE` HTTP header, loads available translation +files(`messages.mo`) from the application directory, and install the +`gettext` and `ngettext` functions to the `__builtin__` namespace in +the Python runtime. + +For strings in Jinja2 templates, there is the `i18n_utils.BaseHandler` +class from which you can extend in order to have a handy property +named `jinja2_env` that lazily initializes Jinja2 environment for you +with the `jinja2.ext.i18n` extention, and similar to the +`I18nMiddleware`, installs `gettext` and `ngettext` functions to the +global namespace of the Jinja2 environment. + +## What about Javascript? + +The `BaseHandler` class also installs the `get_i18n_js_tag()` instance +method to the Jinja2 global namespace. When you use this function in +your Jinja2 template (like in the `index.jinja2` file), you will get a +set of Javascript functions; `gettext`, `ngettext`, and `format` on +the string type. The `format` function can be used with `ngettext`ed +strings for number formatting. See this example: + + window.alert(ngettext( + 'You need to provide at least {0} item.', + 'You need to provide at least {0} items.', + n).format(n); diff --git a/appengine/i18n/app.yaml b/appengine/standard/i18n/app.yaml similarity index 100% rename from appengine/i18n/app.yaml rename to appengine/standard/i18n/app.yaml diff --git a/appengine/i18n/appengine_config.py b/appengine/standard/i18n/appengine_config.py similarity index 100% rename from appengine/i18n/appengine_config.py rename to appengine/standard/i18n/appengine_config.py diff --git a/appengine/i18n/i18n_utils.py b/appengine/standard/i18n/i18n_utils.py similarity index 100% rename from appengine/i18n/i18n_utils.py rename to appengine/standard/i18n/i18n_utils.py diff --git a/appengine/i18n/js.mapping b/appengine/standard/i18n/js.mapping similarity index 100% rename from appengine/i18n/js.mapping rename to appengine/standard/i18n/js.mapping diff --git a/appengine/i18n/locales/en/LC_MESSAGES/jsmessages.mo b/appengine/standard/i18n/locales/en/LC_MESSAGES/jsmessages.mo similarity index 100% rename from appengine/i18n/locales/en/LC_MESSAGES/jsmessages.mo rename to appengine/standard/i18n/locales/en/LC_MESSAGES/jsmessages.mo diff --git a/appengine/i18n/locales/en/LC_MESSAGES/jsmessages.po b/appengine/standard/i18n/locales/en/LC_MESSAGES/jsmessages.po similarity index 100% rename from appengine/i18n/locales/en/LC_MESSAGES/jsmessages.po rename to appengine/standard/i18n/locales/en/LC_MESSAGES/jsmessages.po diff --git a/appengine/i18n/locales/en/LC_MESSAGES/messages.mo b/appengine/standard/i18n/locales/en/LC_MESSAGES/messages.mo similarity index 100% rename from appengine/i18n/locales/en/LC_MESSAGES/messages.mo rename to appengine/standard/i18n/locales/en/LC_MESSAGES/messages.mo diff --git a/appengine/i18n/locales/en/LC_MESSAGES/messages.po b/appengine/standard/i18n/locales/en/LC_MESSAGES/messages.po similarity index 100% rename from appengine/i18n/locales/en/LC_MESSAGES/messages.po rename to appengine/standard/i18n/locales/en/LC_MESSAGES/messages.po diff --git a/appengine/i18n/locales/ja/LC_MESSAGES/jsmessages.mo b/appengine/standard/i18n/locales/ja/LC_MESSAGES/jsmessages.mo similarity index 100% rename from appengine/i18n/locales/ja/LC_MESSAGES/jsmessages.mo rename to appengine/standard/i18n/locales/ja/LC_MESSAGES/jsmessages.mo diff --git a/appengine/i18n/locales/ja/LC_MESSAGES/jsmessages.po b/appengine/standard/i18n/locales/ja/LC_MESSAGES/jsmessages.po similarity index 100% rename from appengine/i18n/locales/ja/LC_MESSAGES/jsmessages.po rename to appengine/standard/i18n/locales/ja/LC_MESSAGES/jsmessages.po diff --git a/appengine/i18n/locales/ja/LC_MESSAGES/messages.mo b/appengine/standard/i18n/locales/ja/LC_MESSAGES/messages.mo similarity index 100% rename from appengine/i18n/locales/ja/LC_MESSAGES/messages.mo rename to appengine/standard/i18n/locales/ja/LC_MESSAGES/messages.mo diff --git a/appengine/i18n/locales/ja/LC_MESSAGES/messages.po b/appengine/standard/i18n/locales/ja/LC_MESSAGES/messages.po similarity index 100% rename from appengine/i18n/locales/ja/LC_MESSAGES/messages.po rename to appengine/standard/i18n/locales/ja/LC_MESSAGES/messages.po diff --git a/appengine/i18n/locales/jsmessages.pot b/appengine/standard/i18n/locales/jsmessages.pot similarity index 100% rename from appengine/i18n/locales/jsmessages.pot rename to appengine/standard/i18n/locales/jsmessages.pot diff --git a/appengine/i18n/locales/messages.pot b/appengine/standard/i18n/locales/messages.pot similarity index 100% rename from appengine/i18n/locales/messages.pot rename to appengine/standard/i18n/locales/messages.pot diff --git a/appengine/i18n/locales/pl/LC_MESSAGES/jsmessages.mo b/appengine/standard/i18n/locales/pl/LC_MESSAGES/jsmessages.mo similarity index 100% rename from appengine/i18n/locales/pl/LC_MESSAGES/jsmessages.mo rename to appengine/standard/i18n/locales/pl/LC_MESSAGES/jsmessages.mo diff --git a/appengine/i18n/locales/pl/LC_MESSAGES/jsmessages.po b/appengine/standard/i18n/locales/pl/LC_MESSAGES/jsmessages.po similarity index 100% rename from appengine/i18n/locales/pl/LC_MESSAGES/jsmessages.po rename to appengine/standard/i18n/locales/pl/LC_MESSAGES/jsmessages.po diff --git a/appengine/i18n/main.mapping b/appengine/standard/i18n/main.mapping similarity index 100% rename from appengine/i18n/main.mapping rename to appengine/standard/i18n/main.mapping diff --git a/appengine/standard/i18n/main.py b/appengine/standard/i18n/main.py new file mode 100644 index 00000000000..e2277049510 --- /dev/null +++ b/appengine/standard/i18n/main.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# +# Copyright 2013 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sample application that demonstrates how to internationalize and localize +and App Engine application. + +For more information, see README.md +""" + +import webapp2 + +from i18n_utils import BaseHandler + + +class MainHandler(BaseHandler): + """A simple handler with internationalized strings. + + This handler demonstrates how to internationalize strings in + Python, Jinja2 template and Javascript. + """ + + def get(self): + """A get handler for this sample. + + It just shows internationalized strings in Python, Jinja2 + template and Javascript. + """ + + context = dict(message=gettext('Hello World from Python code!')) + template = self.jinja2_env.get_template('index.jinja2') + self.response.out.write(template.render(context)) + + +application = webapp2.WSGIApplication([ + ('/', MainHandler), +], debug=True) diff --git a/appengine/i18n/static/js/main.js b/appengine/standard/i18n/static/js/main.js similarity index 100% rename from appengine/i18n/static/js/main.js rename to appengine/standard/i18n/static/js/main.js diff --git a/appengine/i18n/templates/i18n_js.jinja2 b/appengine/standard/i18n/templates/i18n_js.jinja2 similarity index 100% rename from appengine/i18n/templates/i18n_js.jinja2 rename to appengine/standard/i18n/templates/i18n_js.jinja2 diff --git a/appengine/i18n/templates/index.jinja2 b/appengine/standard/i18n/templates/index.jinja2 similarity index 100% rename from appengine/i18n/templates/index.jinja2 rename to appengine/standard/i18n/templates/index.jinja2 diff --git a/appengine/i18n/templates/javascript_tag.jinja2 b/appengine/standard/i18n/templates/javascript_tag.jinja2 similarity index 100% rename from appengine/i18n/templates/javascript_tag.jinja2 rename to appengine/standard/i18n/templates/javascript_tag.jinja2 diff --git a/appengine/i18n/templates/null_i18n_js.jinja2 b/appengine/standard/i18n/templates/null_i18n_js.jinja2 similarity index 100% rename from appengine/i18n/templates/null_i18n_js.jinja2 rename to appengine/standard/i18n/templates/null_i18n_js.jinja2 diff --git a/appengine/standard/iap/.gitignore b/appengine/standard/iap/.gitignore new file mode 100644 index 00000000000..a65b41774ad --- /dev/null +++ b/appengine/standard/iap/.gitignore @@ -0,0 +1 @@ +lib diff --git a/appengine/standard/iap/README.md b/appengine/standard/iap/README.md new file mode 100644 index 00000000000..7840922be38 --- /dev/null +++ b/appengine/standard/iap/README.md @@ -0,0 +1,27 @@ +# Identity-Aware Proxy Refresh Session Sample + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/iap/README.md + +This sample is used on the following documentation page: + +* https://cloud.google.com/iap/docs/sessions-howto + + +## Deploy to Google App Engine standard environment + +```shell +$ gcloud app deploy + +``` + +Enable Cloud IAP using the instructions here: +https://cloud.google.com/iap/docs/app-engine-quickstart#enabling_iap + +## Usage + +The app will continually refresh a fake status (always "Success"). After 1 hour, +the AJAX request will fail. The [js/poll.js](js/poll.js) code will detect this +and allow the user to refresh the session. diff --git a/appengine/standard/iap/app.yaml b/appengine/standard/iap/app.yaml new file mode 100644 index 00000000000..a057d5c7690 --- /dev/null +++ b/appengine/standard/iap/app.yaml @@ -0,0 +1,9 @@ +runtime: python27 +threadsafe: yes +api_version: 1 + +handlers: +- url: /js + static_dir: js +- url: .* + script: main.app diff --git a/appengine/standard/iap/appengine_config.py b/appengine/standard/iap/appengine_config.py new file mode 100644 index 00000000000..c903d9a0ac5 --- /dev/null +++ b/appengine/standard/iap/appengine_config.py @@ -0,0 +1,18 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.ext import vendor + +# Add any libraries installed in the "lib" folder. +vendor.add('lib') diff --git a/appengine/standard/iap/js/poll.js b/appengine/standard/iap/js/poll.js new file mode 100644 index 00000000000..706f785b3df --- /dev/null +++ b/appengine/standard/iap/js/poll.js @@ -0,0 +1,69 @@ +// Copyright Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +"use strict"; + +function getStatus() { + var statusElm = document.getElementById('status'); + statusElm.innerHTML = 'Polling'; + fetch('/status').then(function(response) { + if (response.ok) { + return response.text(); + } + // [START handle_error] + if (response.status === 401) { + statusElm.innerHTML = 'Login stale. '; + } + // [END handle_error] + else { + statusElm.innerHTML = response.statusText; + } + throw new Error (response.statusText); + }) + .then(function(text) { + statusElm.innerHTML = text; + }) + .catch(function(statusText) { + }); +} + +getStatus(); +setInterval(getStatus, 10000); // 10 seconds + +// [START refresh_session] +var iapSessionRefreshWindow = null; + +function sessionRefreshClicked() { + if (iapSessionRefreshWindow == null) { + iapSessionRefreshWindow = window.open("/_gcp_iap/do_session_refresh"); + window.setTimeout(checkSessionRefresh, 500); + } + return false; +} + +function checkSessionRefresh() { + if (iapSessionRefreshWindow != null && !iapSessionRefreshWindow.closed) { + fetch('/favicon.ico').then(function(response) { + if (response.status === 401) { + window.setTimeout(checkSessionRefresh, 500); + } else { + iapSessionRefreshWindow.close(); + iapSessionRefreshWindow = null; + } + }); + } else { + iapSessionRefreshWindow = null; + } +} +// [END refresh_session] diff --git a/appengine/standard/iap/main.py b/appengine/standard/iap/main.py new file mode 100644 index 00000000000..19edde411f5 --- /dev/null +++ b/appengine/standard/iap/main.py @@ -0,0 +1,53 @@ +# Copyright Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Sample application that demonstrates refreshing a session when using +Identity Aware Proxy. This application is for App Engine Standard. +""" + +import flask +from google.appengine.api import users + +app = flask.Flask(__name__) + + +@app.route('/') +def index(): + user = users.get_current_user() + if user: + logged_in = True + nickname = user.nickname() + logout_url = users.create_logout_url('/') + login_url = None + else: + logged_in = False + nickname = None + logout_url = None + login_url = users.create_login_url('/') + + template_values = { + 'logged_in': logged_in, + 'nickname': nickname, + 'logout_url': logout_url, + 'login_url': login_url, + } + + return flask.render_template('index.html', **template_values) + + +# Fake status +@app.route('/status') +def status(): + return 'Success' diff --git a/appengine/standard/iap/main_test.py b/appengine/standard/iap/main_test.py new file mode 100644 index 00000000000..a6fb7e924ac --- /dev/null +++ b/appengine/standard/iap/main_test.py @@ -0,0 +1,35 @@ +# Copyright Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import main +import webtest + + +def test_index(testbed, login): + app = webtest.TestApp(main.app) + + response = app.get('/') + assert 'Login' in response.body + + login() + response = app.get('/') + assert 'Logout' in response.body + assert 'user@example.com' in response.body + + +def test_status(testbed): + app = webtest.TestApp(main.app) + + response = app.get('/status') + assert 'Success' in response.body diff --git a/appengine/standard/iap/requirements.txt b/appengine/standard/iap/requirements.txt new file mode 100644 index 00000000000..f2e1e506599 --- /dev/null +++ b/appengine/standard/iap/requirements.txt @@ -0,0 +1 @@ +Flask==1.0.2 diff --git a/appengine/standard/iap/templates/index.html b/appengine/standard/iap/templates/index.html new file mode 100644 index 00000000000..3f6f84080a3 --- /dev/null +++ b/appengine/standard/iap/templates/index.html @@ -0,0 +1,13 @@ + + + Fake Status Page +
        +{% if logged_in %} + Welcome, {{ nickname }} ! (sign out) +{% else %} + Sign in +{% endif %} +
        +
        Live status below:
        +
        No status yet.
        + diff --git a/appengine/standard/images/api/README.md b/appengine/standard/images/api/README.md new file mode 100644 index 00000000000..c4c4bb536c1 --- /dev/null +++ b/appengine/standard/images/api/README.md @@ -0,0 +1,18 @@ +## Images Guestbook Sample + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/images/api/README.md + +This is a sample app for Google App Engine that demonstrates the [Images Python +API](https://cloud.google.com/appengine/docs/python/images/usingimages). + + +These samples are used on the following documentation page: + +> https://cloud.google.com/appengine/docs/python/images/ + + + +Refer to the [App Engine Samples README](../../README.md) for information on how to run and deploy this sample. diff --git a/appengine/standard/images/api/app.yaml b/appengine/standard/images/api/app.yaml new file mode 100644 index 00000000000..697c4e6d018 --- /dev/null +++ b/appengine/standard/images/api/app.yaml @@ -0,0 +1,8 @@ +runtime: python27 +api_version: 1 +threadsafe: yes + +handlers: + +- url: .* + script: main.app diff --git a/appengine/standard/images/api/blobstore.py b/appengine/standard/images/api/blobstore.py new file mode 100644 index 00000000000..1371288337b --- /dev/null +++ b/appengine/standard/images/api/blobstore.py @@ -0,0 +1,73 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Sample application that demonstrates how to use the App Engine Images API. + +For more information, see README.md. +""" + +# [START all] +# [START thumbnailer] +from google.appengine.api import images +from google.appengine.ext import blobstore + +import webapp2 + + +class Thumbnailer(webapp2.RequestHandler): + def get(self): + blob_key = self.request.get("blob_key") + if blob_key: + blob_info = blobstore.get(blob_key) + + if blob_info: + img = images.Image(blob_key=blob_key) + img.resize(width=80, height=100) + img.im_feeling_lucky() + thumbnail = img.execute_transforms(output_encoding=images.JPEG) + + self.response.headers['Content-Type'] = 'image/jpeg' + self.response.out.write(thumbnail) + return + + # Either "blob_key" wasn't provided, or there was no value with that ID + # in the Blobstore. + self.error(404) +# [END thumbnailer] + + +class ServingUrlRedirect(webapp2.RequestHandler): + def get(self): + blob_key = self.request.get("blob_key") + + if blob_key: + blob_info = blobstore.get(blob_key) + + if blob_info: + # [START get_serving_url] + url = images.get_serving_url( + blob_key, size=150, crop=True, secure_url=True) + # [END get_serving_url] + return webapp2.redirect(url) + + # Either "blob_key" wasn't provided, or there was no value with that ID + # in the Blobstore. + self.error(404) + + +app = webapp2.WSGIApplication( + [('/img', Thumbnailer), + ('/redirect', ServingUrlRedirect)], debug=True) +# [END all] diff --git a/appengine/standard/images/api/blobstore_test.py b/appengine/standard/images/api/blobstore_test.py new file mode 100644 index 00000000000..7c62af054a1 --- /dev/null +++ b/appengine/standard/images/api/blobstore_test.py @@ -0,0 +1,67 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import pytest +import webtest + +import blobstore + + +@pytest.fixture +def app(testbed): + return webtest.TestApp(blobstore.app) + + +def test_img(app): + with mock.patch('blobstore.images') as mock_images: + with mock.patch('blobstore.blobstore') as mock_blobstore: + mock_blobstore.get.return_value = b'123' + mock_images.resize.return_value = 'asdf' + mock_images.im_feeling_lucky.return_value = 'gsdf' + + response = app.get('/img?blob_key=123') + + assert response.status_int == 200 + + +def test_img_missing(app): + # Bogus blob_key, should get error + app.get('/img?blob_key=123', status=404) + + +def test_no_img_id(app): + # No blob_key, should get error + app.get('/img', status=404) + + +def test_url_redirect(app): + with mock.patch('blobstore.images') as mock_images: + with mock.patch('blobstore.blobstore') as mock_blobstore: + mock_blobstore.get.return_value = b'123' + mock_images.get_serving_url.return_value = 'http://lh3.ggpht.com/X' + + response = app.get('/redirect?blob_key=123') + + assert response.status_int == 302 + + +def test_url_redirect_missing(app): + # Bogus blob_key, should get error + app.get('/redirect?blob_key=123', status=404) + + +def test_url_redirect_no_key(app): + # No blob_key, should get error + app.get('/redirect', status=404) diff --git a/appengine/images/favicon.ico b/appengine/standard/images/api/favicon.ico similarity index 100% rename from appengine/images/favicon.ico rename to appengine/standard/images/api/favicon.ico diff --git a/appengine/standard/images/api/main.py b/appengine/standard/images/api/main.py new file mode 100644 index 00000000000..fd9432addfa --- /dev/null +++ b/appengine/standard/images/api/main.py @@ -0,0 +1,56 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Sample application that demonstrates how to use the App Engine Images API. + +For more information, see README.md. +""" + +# [START all] +# [START thumbnailer] +from google.appengine.api import images +from google.appengine.ext import ndb + +import webapp2 + + +class Photo(ndb.Model): + title = ndb.StringProperty() + full_size_image = ndb.BlobProperty() + + +class Thumbnailer(webapp2.RequestHandler): + def get(self): + if self.request.get("id"): + photo = Photo.get_by_id(int(self.request.get("id"))) + + if photo: + img = images.Image(photo.full_size_image) + img.resize(width=80, height=100) + img.im_feeling_lucky() + thumbnail = img.execute_transforms(output_encoding=images.JPEG) + + self.response.headers['Content-Type'] = 'image/jpeg' + self.response.out.write(thumbnail) + return + + # Either "id" wasn't provided, or there was no image with that ID + # in the datastore. + self.error(404) +# [END thumbnailer] + + +app = webapp2.WSGIApplication([('/img', Thumbnailer)], debug=True) +# [END all] diff --git a/appengine/standard/images/api/main_test.py b/appengine/standard/images/api/main_test.py new file mode 100644 index 00000000000..33caf1fa2f0 --- /dev/null +++ b/appengine/standard/images/api/main_test.py @@ -0,0 +1,50 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import pytest +import webtest + +import main + + +@pytest.fixture +def app(testbed): + return webtest.TestApp(main.app) + + +def test_img(app): + with mock.patch('main.images') as mock_images: + mock_images.resize.return_value = 'asdf' + mock_images.im_feeling_lucky.return_value = 'gsdf' + photo = main.Photo( + id=234 + ) + photo.title = 'asdf' + photo.full_size_image = b'123' + photo.put() + + response = app.get('/img?id=%s' % photo.key.id()) + + assert response.status_int == 200 + + +def test_img_missing(app): + # Bogus image id, should get error + app.get('/img?id=123', status=404) + + +def test_no_img_id(app): + # No image id, should get error + app.get('/img', status=404) diff --git a/appengine/standard/images/guestbook/README.md b/appengine/standard/images/guestbook/README.md new file mode 100644 index 00000000000..00fbcadfa5f --- /dev/null +++ b/appengine/standard/images/guestbook/README.md @@ -0,0 +1,18 @@ +## Images Guestbook Sample + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/images/guestbook/README.md + +This is a sample app for Google App Engine that demonstrates the [Images Python +API](https://cloud.google.com/appengine/docs/python/images/usingimages). + + +These samples are used on the following documentation page: + +> https://cloud.google.com/appengine/docs/python/images/usingimages + + + +Refer to the [App Engine Samples README](../../README.md) for information on how to run and deploy this sample. diff --git a/appengine/standard/images/guestbook/app.yaml b/appengine/standard/images/guestbook/app.yaml new file mode 100644 index 00000000000..697c4e6d018 --- /dev/null +++ b/appengine/standard/images/guestbook/app.yaml @@ -0,0 +1,8 @@ +runtime: python27 +api_version: 1 +threadsafe: yes + +handlers: + +- url: .* + script: main.app diff --git a/appengine/memcache/guestbook/favicon.ico b/appengine/standard/images/guestbook/favicon.ico similarity index 100% rename from appengine/memcache/guestbook/favicon.ico rename to appengine/standard/images/guestbook/favicon.ico diff --git a/appengine/images/index.yaml b/appengine/standard/images/guestbook/index.yaml similarity index 100% rename from appengine/images/index.yaml rename to appengine/standard/images/guestbook/index.yaml diff --git a/appengine/images/main.py b/appengine/standard/images/guestbook/main.py similarity index 100% rename from appengine/images/main.py rename to appengine/standard/images/guestbook/main.py diff --git a/appengine/standard/images/guestbook/main_test.py b/appengine/standard/images/guestbook/main_test.py new file mode 100644 index 00000000000..8ef5dfc7339 --- /dev/null +++ b/appengine/standard/images/guestbook/main_test.py @@ -0,0 +1,78 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import pytest +import webtest + +import main + + +@pytest.fixture +def app(testbed): + return webtest.TestApp(main.app) + + +def test_get(app): + main.Greeting( + parent=main.guestbook_key('default_guestbook'), + author='123', + content='abc' + ).put() + + response = app.get('/') + + # Let's check if the response is correct. + assert response.status_int == 200 + + +def test_post(app): + with mock.patch('main.images') as mock_images: + mock_images.resize.return_value = 'asdf' + + response = app.post('/sign', {'content': 'asdf'}) + mock_images.resize.assert_called_once_with(mock.ANY, 32, 32) + + # Correct response is a redirect + assert response.status_int == 302 + + +def test_img(app): + greeting = main.Greeting( + parent=main.guestbook_key('default_guestbook'), + id=123 + ) + greeting.author = 'asdf' + greeting.content = 'asdf' + greeting.avatar = b'123' + greeting.put() + + response = app.get('/img?img_id=%s' % greeting.key.urlsafe()) + + assert response.status_int == 200 + + +def test_img_missing(app): + # Bogus image id, should get error + app.get('/img?img_id=123', status=500) + + +def test_post_and_get(app): + with mock.patch('main.images') as mock_images: + mock_images.resize.return_value = 'asdf' + + app.post('/sign', {'content': 'asdf'}) + response = app.get('/') + + assert response.status_int == 200 diff --git a/appengine/standard/localtesting/README.md b/appengine/standard/localtesting/README.md new file mode 100644 index 00000000000..94ffc327f57 --- /dev/null +++ b/appengine/standard/localtesting/README.md @@ -0,0 +1,15 @@ +# App Engine Local Testing Samples + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/localtesting/README.md + +These samples show how to do automated testing of App Engine applications. + + +These samples are used on the following documentation page: + +> https://cloud.google.com/appengine/docs/python/tools/localunittesting + + diff --git a/appengine/localtesting/datastore_test.py b/appengine/standard/localtesting/datastore_test.py similarity index 100% rename from appengine/localtesting/datastore_test.py rename to appengine/standard/localtesting/datastore_test.py diff --git a/appengine/localtesting/env_vars_test.py b/appengine/standard/localtesting/env_vars_test.py similarity index 88% rename from appengine/localtesting/env_vars_test.py rename to appengine/standard/localtesting/env_vars_test.py index d34ed283415..31e374def64 100644 --- a/appengine/localtesting/env_vars_test.py +++ b/appengine/standard/localtesting/env_vars_test.py @@ -32,9 +32,10 @@ def tearDown(self): self.testbed.deactivate() def testEnvVars(self): - assert os.environ['APPLICATION_ID'] == 'your-app-id' - assert os.environ['MY_CONFIG_SETTING'] == 'example' + self.assertEqual(os.environ['APPLICATION_ID'], 'your-app-id') + self.assertEqual(os.environ['MY_CONFIG_SETTING'], 'example') # [END env_example] + if __name__ == '__main__': unittest.main() diff --git a/appengine/localtesting/login_test.py b/appengine/standard/localtesting/login_test.py similarity index 83% rename from appengine/localtesting/login_test.py rename to appengine/standard/localtesting/login_test.py index 545ea2bd837..a5320c0430c 100644 --- a/appengine/localtesting/login_test.py +++ b/appengine/standard/localtesting/login_test.py @@ -20,34 +20,29 @@ class LoginTestCase(unittest.TestCase): - # [START setup] def setUp(self): self.testbed = testbed.Testbed() self.testbed.activate() self.testbed.init_user_stub() - # [END setup] def tearDown(self): self.testbed.deactivate() - # [START login] def loginUser(self, email='user@example.com', id='123', is_admin=False): self.testbed.setup_env( user_email=email, user_id=id, user_is_admin='1' if is_admin else '0', overwrite=True) - # [END login] - # [START test] def testLogin(self): - assert not users.get_current_user() + self.assertFalse(users.get_current_user()) self.loginUser() - assert users.get_current_user().email() == 'user@example.com' + self.assertEquals(users.get_current_user().email(), 'user@example.com') self.loginUser(is_admin=True) - assert users.is_current_user_admin() - # [END test] + self.assertTrue(users.is_current_user_admin()) # [END login_example] + if __name__ == '__main__': unittest.main() diff --git a/appengine/localtesting/mail_test.py b/appengine/standard/localtesting/mail_test.py similarity index 99% rename from appengine/localtesting/mail_test.py rename to appengine/standard/localtesting/mail_test.py index ead48353820..a362aa210eb 100644 --- a/appengine/localtesting/mail_test.py +++ b/appengine/standard/localtesting/mail_test.py @@ -40,5 +40,6 @@ def testMailSent(self): self.assertEqual('alice@example.com', messages[0].to) # [END mail_example] + if __name__ == '__main__': unittest.main() diff --git a/appengine/localtesting/queue.yaml b/appengine/standard/localtesting/queue.yaml similarity index 100% rename from appengine/localtesting/queue.yaml rename to appengine/standard/localtesting/queue.yaml diff --git a/appengine/localtesting/resources/queue.yaml b/appengine/standard/localtesting/resources/queue.yaml similarity index 100% rename from appengine/localtesting/resources/queue.yaml rename to appengine/standard/localtesting/resources/queue.yaml diff --git a/appengine/standard/localtesting/runner.py b/appengine/standard/localtesting/runner.py new file mode 100755 index 00000000000..6cd1aaca7b2 --- /dev/null +++ b/appengine/standard/localtesting/runner.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python2 + +# Copyright 2015 Google Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START runner] +"""App Engine local test runner example. + +This program handles properly importing the App Engine SDK so that test modules +can use google.appengine.* APIs and the Google App Engine testbed. + +Example invocation: + + $ python runner.py ~/google-cloud-sdk +""" + +import argparse +import os +import sys +import unittest + + +def fixup_paths(path): + """Adds GAE SDK path to system path and appends it to the google path + if that already exists.""" + # Not all Google packages are inside namespace packages, which means + # there might be another non-namespace package named `google` already on + # the path and simply appending the App Engine SDK to the path will not + # work since the other package will get discovered and used first. + # This emulates namespace packages by first searching if a `google` package + # exists by importing it, and if so appending to its module search path. + try: + import google + google.__path__.append("{0}/google".format(path)) + except ImportError: + pass + + sys.path.insert(0, path) + + +def main(sdk_path, test_path, test_pattern): + # If the SDK path points to a Google Cloud SDK installation + # then we should alter it to point to the GAE platform location. + if os.path.exists(os.path.join(sdk_path, 'platform/google_appengine')): + sdk_path = os.path.join(sdk_path, 'platform/google_appengine') + + # Make sure google.appengine.* modules are importable. + fixup_paths(sdk_path) + + # Make sure all bundled third-party packages are available. + import dev_appserver + dev_appserver.fix_sys_path() + + # Loading appengine_config from the current project ensures that any + # changes to configuration there are available to all tests (e.g. + # sys.path modifications, namespaces, etc.) + try: + import appengine_config + (appengine_config) + except ImportError: + print('Note: unable to import appengine_config.') + + # Discover and run tests. + suite = unittest.loader.TestLoader().discover(test_path, test_pattern) + return unittest.TextTestRunner(verbosity=2).run(suite) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + 'sdk_path', + help='The path to the Google App Engine SDK or the Google Cloud SDK.') + parser.add_argument( + '--test-path', + help='The path to look for tests, defaults to the current directory.', + default=os.getcwd()) + parser.add_argument( + '--test-pattern', + help='The file pattern for test modules, defaults to *_test.py.', + default='*_test.py') + + args = parser.parse_args() + + result = main(args.sdk_path, args.test_path, args.test_pattern) + + if not result.wasSuccessful(): + sys.exit(1) + +# [END runner] diff --git a/appengine/localtesting/task_queue_test.py b/appengine/standard/localtesting/task_queue_test.py similarity index 82% rename from appengine/localtesting/task_queue_test.py rename to appengine/standard/localtesting/task_queue_test.py index cfa3d85f1aa..2472fe18887 100644 --- a/appengine/localtesting/task_queue_test.py +++ b/appengine/standard/localtesting/task_queue_test.py @@ -40,8 +40,8 @@ def tearDown(self): def testTaskAddedToQueue(self): taskqueue.Task(name='my_task', url='/url/of/my/task/').add() tasks = self.taskqueue_stub.get_filtered_tasks() - assert len(tasks) == 1 - assert tasks[0].name == 'my_task' + self.assertEqual(len(tasks), 1) + self.assertEqual(tasks[0].name, 'my_task') # [END taskqueue] # [START filtering] @@ -51,27 +51,27 @@ def testFiltering(self): # All tasks tasks = self.taskqueue_stub.get_filtered_tasks() - assert len(tasks) == 2 + self.assertEqual(len(tasks), 2) # Filter by name tasks = self.taskqueue_stub.get_filtered_tasks(name='task_one') - assert len(tasks) == 1 - assert tasks[0].name == 'task_one' + self.assertEqual(len(tasks), 1) + self.assertEqual(tasks[0].name, 'task_one') # Filter by URL tasks = self.taskqueue_stub.get_filtered_tasks(url='/url/of/task/1/') - assert len(tasks) == 1 - assert tasks[0].name == 'task_one' + self.assertEqual(len(tasks), 1) + self.assertEqual(tasks[0].name, 'task_one') # Filter by queue tasks = self.taskqueue_stub.get_filtered_tasks(queue_names='queue-1') - assert len(tasks) == 1 - assert tasks[0].name == 'task_one' + self.assertEqual(len(tasks), 1) + self.assertEqual(tasks[0].name, 'task_one') # Multiple queues tasks = self.taskqueue_stub.get_filtered_tasks( queue_names=['queue-1', 'queue-2']) - assert len(tasks) == 2 + self.assertEqual(len(tasks), 2) # [END filtering] # [START deferred] @@ -79,11 +79,12 @@ def testTaskAddedByDeferred(self): deferred.defer(operator.add, 1, 2) tasks = self.taskqueue_stub.get_filtered_tasks() - assert len(tasks) == 1 + self.assertEqual(len(tasks), 1) result = deferred.run(tasks[0].payload) - assert result == 3 + self.assertEqual(result, 3) # [END deferred] + if __name__ == '__main__': unittest.main() diff --git a/appengine/mailgun/app.yaml b/appengine/standard/logging/writing_logs/app.yaml similarity index 100% rename from appengine/mailgun/app.yaml rename to appengine/standard/logging/writing_logs/app.yaml diff --git a/appengine/standard/logging/writing_logs/main.py b/appengine/standard/logging/writing_logs/main.py new file mode 100644 index 00000000000..fd0facd2051 --- /dev/null +++ b/appengine/standard/logging/writing_logs/main.py @@ -0,0 +1,47 @@ +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Sample Google App Engine application that demonstrates using the standard +Python logging package. Logs are automatically collected and available via +the Google Cloud Console. +""" + +# [START gae_writing_log] +import logging + +import webapp2 + + +class MainPage(webapp2.RequestHandler): + def get(self): + logging.debug('This is a debug message') + logging.info('This is an info message') + logging.warning('This is a warning message') + logging.error('This is an error message') + logging.critical('This is a critical message') + + try: + raise ValueError('This is a sample value error.') + except ValueError: + logging.exception('A example exception log.') + + self.response.out.write('Logging example.') + + +app = webapp2.WSGIApplication([ + ('/', MainPage) +], debug=True) + +# [END gae_writing_log] diff --git a/appengine/standard/logging/writing_logs/main_test.py b/appengine/standard/logging/writing_logs/main_test.py new file mode 100644 index 00000000000..181f788d33a --- /dev/null +++ b/appengine/standard/logging/writing_logs/main_test.py @@ -0,0 +1,24 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import webtest + +import main + + +def test_app(testbed): + app = webtest.TestApp(main.app) + response = app.get('/') + assert response.status_int == 200 + assert 'Logging example' in response.text diff --git a/appengine/standard/mail/README.md b/appengine/standard/mail/README.md new file mode 100644 index 00000000000..39deeadfe26 --- /dev/null +++ b/appengine/standard/mail/README.md @@ -0,0 +1,22 @@ +## App Engine Email Docs Snippets + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/mail/README.md + +This sample application demonstrates different ways to send and receive email +on App Engine + + + +These samples are used on the following documentation pages: + +> +* https://cloud.google.com/appengine/docs/python/mail/headers +* https://cloud.google.com/appengine/docs/python/mail/receiving-mail-with-mail-api +* https://cloud.google.com/appengine/docs/python/mail/sending-mail-with-mail-api +* https://cloud.google.com/appengine/docs/python/mail/attachments +* https://cloud.google.com/appengine/docs/python/mail/bounce + + diff --git a/appengine/standard/mail/app.yaml b/appengine/standard/mail/app.yaml new file mode 100644 index 00000000000..71821319635 --- /dev/null +++ b/appengine/standard/mail/app.yaml @@ -0,0 +1,45 @@ +runtime: python27 +api_version: 1 +threadsafe: yes + +# [START mail_service] +inbound_services: +- mail +- mail_bounce # Handle bounced mail notifications +# [END mail_service] + +handlers: +- url: /user/.+ + script: user_signup.app +- url: /send_mail + script: send_mail.app +- url: /send_message + script: send_message.app +# [START handle_incoming_email] +- url: /_ah/mail/.+ + script: handle_incoming_email.app + login: admin +# [END handle_incoming_email] +# [START handle_all_email] +- url: /_ah/mail/owner@.*your_app_id\.appspotmail\.com + script: handle_owner.app + login: admin +- url: /_ah/mail/support@.*your_app_id\.appspotmail\.com + script: handle_support.app + login: admin +- url: /_ah/mail/.+ + script: handle_catchall.app + login: admin +# [END handle_all_email] +# [START handle_bounced_email] +- url: /_ah/bounce + script: handle_bounced_email.app + login: admin +# [END handle_bounced_email] +- url: /attachment + script: attachment.app +- url: /header + script: header.app +- url: / + static_files: index.html + upload: index.html diff --git a/appengine/standard/mail/attachment.py b/appengine/standard/mail/attachment.py new file mode 100644 index 00000000000..8717d0b1cb1 --- /dev/null +++ b/appengine/standard/mail/attachment.py @@ -0,0 +1,50 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.api import app_identity +from google.appengine.api import mail +import webapp2 + + +# [START send_attachment] +class AttachmentHandler(webapp2.RequestHandler): + def post(self): + f = self.request.POST['file'] + mail.send_mail(sender='{}@appspot.gserviceaccount.com'.format( + app_identity.get_application_id()), + to="Albert Johnson ", + subject="The doc you requested", + body=""" +Attached is the document file you requested. + +The example.com Team +""", + attachments=[(f.filename, f.file.read())]) +# [END send_attachment] + self.response.content_type = 'text/plain' + self.response.write('Sent {} to Albert.'.format(f.filename)) + + def get(self): + self.response.content_type = 'text/html' + self.response.write(""" +
        + Send a file to Albert:
        +

        + +
        + Enter an email thread id: + +
        """) + + def post(self): + print repr(self.request.POST) + id = self.request.POST['thread_id'] + send_example_mail('{}@appspot.gserviceaccount.com'.format( + app_identity.get_application_id()), id) + self.response.content_type = 'text/plain' + self.response.write( + 'Sent an email to Albert with Reference header set to {}.' + .format(id)) + + +app = webapp2.WSGIApplication([ + ('/header', SendMailHandler), +], debug=True) diff --git a/appengine/standard/mail/header_test.py b/appengine/standard/mail/header_test.py new file mode 100644 index 00000000000..da514ef0e72 --- /dev/null +++ b/appengine/standard/mail/header_test.py @@ -0,0 +1,27 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import webtest + +import header + + +def test_send_mail(testbed): + testbed.init_mail_stub() + testbed.init_app_identity_stub() + app = webtest.TestApp(header.app) + response = app.post('/header', 'thread_id=42') + assert response.status_int == 200 + assert ('Sent an email to Albert with Reference header set to 42.' + in response.body) diff --git a/appengine/standard/mail/index.html b/appengine/standard/mail/index.html new file mode 100644 index 00000000000..c0b346c2576 --- /dev/null +++ b/appengine/standard/mail/index.html @@ -0,0 +1,29 @@ + + + + + + Google App Engine Mail Samples + + +

        Send email.

        +

        Send email with a message object.

        +

        Confirm a user's email address.

        +

        Send email with attachments.

        +

        Send email with headers.

        + + diff --git a/appengine/standard/mail/send_mail.py b/appengine/standard/mail/send_mail.py new file mode 100644 index 00000000000..81666b68c37 --- /dev/null +++ b/appengine/standard/mail/send_mail.py @@ -0,0 +1,48 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.api import app_identity +from google.appengine.api import mail +import webapp2 + + +def send_approved_mail(sender_address): + # [START send_mail] + mail.send_mail(sender=sender_address, + to="Albert Johnson ", + subject="Your account has been approved", + body="""Dear Albert: + +Your example.com account has been approved. You can now visit +http://www.example.com/ and sign in using your Google Account to +access new features. + +Please let us know if you have any questions. + +The example.com Team +""") + # [END send_mail] + + +class SendMailHandler(webapp2.RequestHandler): + def get(self): + send_approved_mail('{}@appspot.gserviceaccount.com'.format( + app_identity.get_application_id())) + self.response.content_type = 'text/plain' + self.response.write('Sent an email to Albert.') + + +app = webapp2.WSGIApplication([ + ('/send_mail', SendMailHandler), +], debug=True) diff --git a/appengine/standard/mail/send_mail_test.py b/appengine/standard/mail/send_mail_test.py new file mode 100644 index 00000000000..5e5f7e6be02 --- /dev/null +++ b/appengine/standard/mail/send_mail_test.py @@ -0,0 +1,26 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import webtest + +import send_mail + + +def test_send_mail(testbed): + testbed.init_mail_stub() + testbed.init_app_identity_stub() + app = webtest.TestApp(send_mail.app) + response = app.get('/send_mail') + assert response.status_int == 200 + assert 'Sent an email to Albert.' in response.body diff --git a/appengine/standard/mail/send_message.py b/appengine/standard/mail/send_message.py new file mode 100644 index 00000000000..544c057a38d --- /dev/null +++ b/appengine/standard/mail/send_message.py @@ -0,0 +1,51 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.api import app_identity +from google.appengine.api import mail +import webapp2 + + +def send_approved_mail(sender_address): + # [START send_message] + message = mail.EmailMessage( + sender=sender_address, + subject="Your account has been approved") + + message.to = "Albert Johnson " + message.body = """Dear Albert: + +Your example.com account has been approved. You can now visit +http://www.example.com/ and sign in using your Google Account to +access new features. + +Please let us know if you have any questions. + +The example.com Team +""" + message.send() + # [END send_message] + + +class SendMessageHandler(webapp2.RequestHandler): + def get(self): + send_approved_mail('{}@appspot.gserviceaccount.com'.format( + app_identity.get_application_id())) + self.response.content_type = 'text/plain' + self.response.write('Sent an email message to Albert.') + + +app = webapp2.WSGIApplication([ + ('/send_message', SendMessageHandler), +], debug=True) diff --git a/appengine/standard/mail/send_message_test.py b/appengine/standard/mail/send_message_test.py new file mode 100644 index 00000000000..d56f58e0d6f --- /dev/null +++ b/appengine/standard/mail/send_message_test.py @@ -0,0 +1,26 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import webtest + +import send_message + + +def test_send_message(testbed): + testbed.init_mail_stub() + testbed.init_app_identity_stub() + app = webtest.TestApp(send_message.app) + response = app.get('/send_message') + assert response.status_int == 200 + assert 'Sent an email message to Albert.' in response.body diff --git a/appengine/standard/mail/user_signup.py b/appengine/standard/mail/user_signup.py new file mode 100644 index 00000000000..38b05bd2ca4 --- /dev/null +++ b/appengine/standard/mail/user_signup.py @@ -0,0 +1,106 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import random +import socket +import string + +from google.appengine.api import app_identity +from google.appengine.api import mail +from google.appengine.ext import ndb +import webapp2 + + +# [START send-confirm-email] +class UserSignupHandler(webapp2.RequestHandler): + """Serves the email address sign up form.""" + + def post(self): + user_address = self.request.get('email_address') + + if not mail.is_email_valid(user_address): + self.get() # Show the form again. + else: + confirmation_url = create_new_user_confirmation(user_address) + sender_address = ( + 'Example.com Support <{}@appspot.gserviceaccount.com>'.format( + app_identity.get_application_id())) + subject = 'Confirm your registration' + body = """Thank you for creating an account! +Please confirm your email address by clicking on the link below: + +{} +""".format(confirmation_url) + mail.send_mail(sender_address, user_address, subject, body) +# [END send-confirm-email] + self.response.content_type = 'text/plain' + self.response.write('An email has been sent to {}.'.format( + user_address)) + + def get(self): + self.response.content_type = 'text/html' + self.response.write("""
        + Enter your email address: + +
        """) + + +class UserConfirmationRecord(ndb.Model): + """Datastore record with email address and confirmation code.""" + user_address = ndb.StringProperty(indexed=False) + confirmed = ndb.BooleanProperty(indexed=False, default=False) + timestamp = ndb.DateTimeProperty(indexed=False, auto_now_add=True) + + +def create_new_user_confirmation(user_address): + """Create a new user confirmation. + + Args: + user_address: string, an email addres + + Returns: The url to click to confirm the email address.""" + id_chars = string.ascii_letters + string.digits + rand = random.SystemRandom() + random_id = ''.join([rand.choice(id_chars) for i in range(42)]) + record = UserConfirmationRecord(user_address=user_address, + id=random_id) + record.put() + return 'https://{}/user/confirm?code={}'.format( + socket.getfqdn(socket.gethostname()), random_id) + + +class ConfirmUserSignupHandler(webapp2.RequestHandler): + """Invoked when the user clicks on the confirmation link in the email.""" + + def get(self): + code = self.request.get('code') + if code: + record = ndb.Key(UserConfirmationRecord, code).get() + # 2-hour time limit on confirming. + if record and (datetime.datetime.now() - record.timestamp < + datetime.timedelta(hours=2)): + record.confirmed = True + record.put() + self.response.content_type = 'text/plain' + self.response.write('Confirmed {}.' + .format(record.user_address)) + return + self.response.status_int = 404 + + +app = webapp2.WSGIApplication([ + ('/user/signup', UserSignupHandler), + ('/user/confirm', ConfirmUserSignupHandler), +], debug=True) diff --git a/appengine/standard/mail/user_signup_test.py b/appengine/standard/mail/user_signup_test.py new file mode 100644 index 00000000000..c386f2ace15 --- /dev/null +++ b/appengine/standard/mail/user_signup_test.py @@ -0,0 +1,39 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import webtest + +import user_signup + + +def test_user_signup(testbed): + testbed.init_mail_stub() + testbed.init_app_identity_stub() + testbed.init_datastore_v3_stub() + app = webtest.TestApp(user_signup.app) + response = app.post('/user/signup', 'email_address=alice@example.com') + assert response.status_int == 200 + assert 'An email has been sent to alice@example.com.' in response.body + + records = user_signup.UserConfirmationRecord.query().fetch(1) + response = app.get('/user/confirm?code={}'.format(records[0].key.id())) + assert response.status_int == 200 + assert 'Confirmed alice@example.com.' in response.body + + +def test_bad_code(testbed): + testbed.init_datastore_v3_stub() + app = webtest.TestApp(user_signup.app) + response = app.get('/user/confirm?code=garbage', status=404) + assert response.status_int == 404 diff --git a/appengine/standard/mailgun/.gitignore b/appengine/standard/mailgun/.gitignore new file mode 100644 index 00000000000..a65b41774ad --- /dev/null +++ b/appengine/standard/mailgun/.gitignore @@ -0,0 +1 @@ +lib diff --git a/appengine/standard/mailgun/README.md b/appengine/standard/mailgun/README.md new file mode 100644 index 00000000000..b91083e5de0 --- /dev/null +++ b/appengine/standard/mailgun/README.md @@ -0,0 +1,17 @@ +# Mailgun & Google App Engine + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/mailgun/README.md + +This sample application demonstrates how to use [Mailgun with Google App Engine](https://cloud.google.com/appengine/docs/python/mail/mailgun). + +Refer to the [App Engine Samples README](../../README.md) for information on how to run and deploy this sample. + +# Setup + +Before running this sample: + +1. You will need a [Mailgun account](http://www.mailgun.com/google). +2. Update the `MAILGUN_DOMAIN_NAME` and `MAILGUN_API_KEY` constants in `main.py`. You can use your account's sandbox domain. diff --git a/appengine/storage/app.yaml b/appengine/standard/mailgun/app.yaml similarity index 100% rename from appengine/storage/app.yaml rename to appengine/standard/mailgun/app.yaml diff --git a/appengine/mailgun/appengine_config.py b/appengine/standard/mailgun/appengine_config.py similarity index 100% rename from appengine/mailgun/appengine_config.py rename to appengine/standard/mailgun/appengine_config.py diff --git a/appengine/standard/mailgun/main.py b/appengine/standard/mailgun/main.py new file mode 100644 index 00000000000..3ba7e7e206a --- /dev/null +++ b/appengine/standard/mailgun/main.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python + +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Sample Google App Engine application that demonstrates how to send mail using +Mailgun. + +For more information, see README.md. +""" + +from urllib import urlencode + +import httplib2 +import webapp2 + + +# Your Mailgun Domain Name +MAILGUN_DOMAIN_NAME = 'your-mailgun-domain-name' +# Your Mailgun API key +MAILGUN_API_KEY = 'your-mailgun-api-key' + + +# [START simple_message] +def send_simple_message(recipient): + http = httplib2.Http() + http.add_credentials('api', MAILGUN_API_KEY) + + url = 'https://api.mailgun.net/v3/{}/messages'.format(MAILGUN_DOMAIN_NAME) + data = { + 'from': 'Example Sender '.format(MAILGUN_DOMAIN_NAME), + 'to': recipient, + 'subject': 'This is an example email from Mailgun', + 'text': 'Test message from Mailgun' + } + + resp, content = http.request( + url, 'POST', urlencode(data), + headers={"Content-Type": "application/x-www-form-urlencoded"}) + + if resp.status != 200: + raise RuntimeError( + 'Mailgun API error: {} {}'.format(resp.status, content)) +# [END simple_message] + + +# [START complex_message] +def send_complex_message(recipient): + http = httplib2.Http() + http.add_credentials('api', MAILGUN_API_KEY) + + url = 'https://api.mailgun.net/v3/{}/messages'.format(MAILGUN_DOMAIN_NAME) + data = { + 'from': 'Example Sender '.format(MAILGUN_DOMAIN_NAME), + 'to': recipient, + 'subject': 'This is an example email from Mailgun', + 'text': 'Test message from Mailgun', + 'html': 'HTML version of the body' + } + + resp, content = http.request( + url, 'POST', urlencode(data), + headers={"Content-Type": "application/x-www-form-urlencoded"}) + + if resp.status != 200: + raise RuntimeError( + 'Mailgun API error: {} {}'.format(resp.status, content)) +# [END complex_message] + + +class MainPage(webapp2.RequestHandler): + def get(self): + self.response.content_type = 'text/html' + self.response.write(""" + + +
        + + + +
        + +""") + + def post(self): + recipient = self.request.get('recipient') + action = self.request.get('submit') + + if action == 'Send simple email': + send_simple_message(recipient) + else: + send_complex_message(recipient) + + self.response.write('Mail sent') + + +app = webapp2.WSGIApplication([ + ('/', MainPage) +], debug=True) diff --git a/appengine/standard/mailgun/main_test.py b/appengine/standard/mailgun/main_test.py new file mode 100644 index 00000000000..91980afe867 --- /dev/null +++ b/appengine/standard/mailgun/main_test.py @@ -0,0 +1,67 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from googleapiclient.http import HttpMockSequence +import httplib2 +import mock +import pytest +import webtest + +import main + + +class HttpMockSequenceWithCredentials(HttpMockSequence): + def add_credentials(self, *args): + pass + + +@pytest.fixture +def app(): + return webtest.TestApp(main.app) + + +def test_get(app): + response = app.get('/') + assert response.status_int == 200 + + +def test_post(app): + http = HttpMockSequenceWithCredentials([ + ({'status': '200'}, '')]) + patch_http = mock.patch.object(httplib2, 'Http', lambda: http) + + with patch_http: + response = app.post('/', { + 'recipient': 'jonwayne@google.com', + 'submit': 'Send simple email'}) + + assert response.status_int == 200 + + http = HttpMockSequenceWithCredentials([ + ({'status': '200'}, '')]) + + with patch_http: + response = app.post('/', { + 'recipient': 'jonwayne@google.com', + 'submit': 'Send complex email'}) + + assert response.status_int == 200 + + http = HttpMockSequenceWithCredentials([ + ({'status': '500'}, 'Test error')]) + + with patch_http, pytest.raises(Exception): + app.post('/', { + 'recipient': 'jonwayne@google.com', + 'submit': 'Send simple email'}) diff --git a/appengine/standard/mailgun/requirements.txt b/appengine/standard/mailgun/requirements.txt new file mode 100644 index 00000000000..54f62f573fa --- /dev/null +++ b/appengine/standard/mailgun/requirements.txt @@ -0,0 +1 @@ +httplib2==0.12.0 diff --git a/appengine/standard/mailjet/.gitignore b/appengine/standard/mailjet/.gitignore new file mode 100644 index 00000000000..a65b41774ad --- /dev/null +++ b/appengine/standard/mailjet/.gitignore @@ -0,0 +1 @@ +lib diff --git a/appengine/standard/mailjet/README.md b/appengine/standard/mailjet/README.md new file mode 100644 index 00000000000..5e2c9008bbb --- /dev/null +++ b/appengine/standard/mailjet/README.md @@ -0,0 +1,16 @@ +# Python Mailjet email sample for Google App Engine Standard + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/mailjet/README.md + +This sample demonstrates how to use [Mailjet](https://www.mailgun.com) on [Google App Engine Standard](https://cloud.google.com/appengine/docs/). + +## Setup + +Before you can run or deploy the sample, you will need to do the following: + +1. [Create a Mailjet Account](http://www.mailjet.com/google). + +2. Configure your Mailjet settings in the environment variables section in ``app.yaml``. diff --git a/appengine/standard/mailjet/app.yaml b/appengine/standard/mailjet/app.yaml new file mode 100644 index 00000000000..467dd8e55be --- /dev/null +++ b/appengine/standard/mailjet/app.yaml @@ -0,0 +1,14 @@ +runtime: python27 +threadsafe: yes +api_version: 1 + +handlers: +- url: .* + script: main.app + +# [START env_variables] +env_variables: + MAILJET_API_KEY: your-mailjet-api-key + MAILJET_API_SECRET: your-mailjet-api-secret + MAILJET_SENDER: your-mailjet-sender-address +# [END env_variables] diff --git a/appengine/standard/mailjet/appengine_config.py b/appengine/standard/mailjet/appengine_config.py new file mode 100644 index 00000000000..c903d9a0ac5 --- /dev/null +++ b/appengine/standard/mailjet/appengine_config.py @@ -0,0 +1,18 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.ext import vendor + +# Add any libraries installed in the "lib" folder. +vendor.add('lib') diff --git a/appengine/standard/mailjet/main.py b/appengine/standard/mailjet/main.py new file mode 100644 index 00000000000..448444aa039 --- /dev/null +++ b/appengine/standard/mailjet/main.py @@ -0,0 +1,83 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START app] +import logging +import os + +from flask import Flask, render_template, request +# [START config] +import mailjet_rest +import requests_toolbelt.adapters.appengine + +# Use the App Engine requests adapter to allow the requests library to be +# used on App Engine. +requests_toolbelt.adapters.appengine.monkeypatch() + +MAILJET_API_KEY = os.environ['MAILJET_API_KEY'] +MAILJET_API_SECRET = os.environ['MAILJET_API_SECRET'] +MAILJET_SENDER = os.environ['MAILJET_SENDER'] +# [END config] + +app = Flask(__name__) + + +# [START send_message] +def send_message(to): + client = mailjet_rest.Client( + auth=(MAILJET_API_KEY, MAILJET_API_SECRET), version='v3.1') + + data = { + 'Messages': [{ + "From": { + "Email": MAILJET_SENDER, + "Name": 'App Engine Standard Mailjet Sample' + }, + "To": [{ + "Email": to + }], + "Subject": 'Example email.', + "TextPart": 'This is an example email.', + "HTMLPart": 'This is an example email.' + }] + } + + result = client.send.create(data=data) + + return result.json() +# [END send_message] + + +@app.route('/') +def index(): + return render_template('index.html') + + +@app.route('/send/email', methods=['POST']) +def send_email(): + to = request.form.get('to') + + result = send_message(to) + + return 'Email sent, response: 
        {}
        '.format(result) + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
        {}
        + See logs for full stacktrace. + """.format(e), 500 +# [END app] diff --git a/appengine/standard/mailjet/main_test.py b/appengine/standard/mailjet/main_test.py new file mode 100644 index 00000000000..c910f48c544 --- /dev/null +++ b/appengine/standard/mailjet/main_test.py @@ -0,0 +1,53 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +import pytest +import responses + + +@pytest.fixture +def app(monkeypatch): + monkeypatch.setenv('MAILJET_API_KEY', 'apikey') + monkeypatch.setenv('MAILJET_API_SECRET', 'apisecret') + monkeypatch.setenv('MAILJET_SENDER', 'sender') + + import main + + main.app.testing = True + return main.app.test_client() + + +def test_index(app): + r = app.get('/') + assert r.status_code == 200 + + +@responses.activate +def test_send_email(app): + responses.add( + responses.POST, + re.compile(r'.*'), + body='{"test": "message"}', + content_type='application/json') + + r = app.post('/send/email', data={'to': 'user@example.com'}) + + assert r.status_code == 200 + assert 'test' in r.data.decode('utf-8') + + assert len(responses.calls) == 1 + request_body = responses.calls[0].request.body + assert 'user@example.com' in request_body diff --git a/appengine/standard/mailjet/requirements.txt b/appengine/standard/mailjet/requirements.txt new file mode 100644 index 00000000000..ca6ba1e7509 --- /dev/null +++ b/appengine/standard/mailjet/requirements.txt @@ -0,0 +1,4 @@ +Flask==1.0.2 +requests==2.21.0 +requests-toolbelt==0.9.1 +mailjet-rest==1.3.0 diff --git a/appengine/standard/mailjet/templates/index.html b/appengine/standard/mailjet/templates/index.html new file mode 100644 index 00000000000..cd1c93ff5b3 --- /dev/null +++ b/appengine/standard/mailjet/templates/index.html @@ -0,0 +1,29 @@ +{# +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + + + + Mailjet on Google App Engine + + + +
        + + +
        + + + diff --git a/appengine/standard/memcache/best_practices/README.md b/appengine/standard/memcache/best_practices/README.md new file mode 100644 index 00000000000..b9772f74959 --- /dev/null +++ b/appengine/standard/memcache/best_practices/README.md @@ -0,0 +1,10 @@ +# Memcache Best Practices + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/memcache/best_practices/README.md + +Code snippets for [Memcache Cache Best Practices article](https://cloud.google.com/appengine/articles/best-practices-for-app-engine-memcache) + + diff --git a/appengine/standard/memcache/best_practices/batch/app.yaml b/appengine/standard/memcache/best_practices/batch/app.yaml new file mode 100644 index 00000000000..ef0ebcae28b --- /dev/null +++ b/appengine/standard/memcache/best_practices/batch/app.yaml @@ -0,0 +1,7 @@ +runtime: python27 +threadsafe: yes +api_version: 1 + +handlers: +- url: .* + script: batch.app diff --git a/appengine/standard/memcache/best_practices/batch/batch.py b/appengine/standard/memcache/best_practices/batch/batch.py new file mode 100644 index 00000000000..2f0119c579c --- /dev/null +++ b/appengine/standard/memcache/best_practices/batch/batch.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from google.appengine.api import memcache +import webapp2 + + +class MainPage(webapp2.RequestHandler): + def get(self): + # [START batch] + values = {'comment': 'I did not ... ', 'comment_by': 'Bill Holiday'} + if not memcache.set_multi(values): + logging.error('Unable to set Memcache values') + tvalues = memcache.get_multi(('comment', 'comment_by')) + self.response.write(tvalues) + # [END batch] + + +app = webapp2.WSGIApplication([ + ('/', MainPage), +], debug=True) diff --git a/appengine/standard/memcache/best_practices/batch/batch_test.py b/appengine/standard/memcache/best_practices/batch/batch_test.py new file mode 100644 index 00000000000..ca0883f362e --- /dev/null +++ b/appengine/standard/memcache/best_practices/batch/batch_test.py @@ -0,0 +1,28 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import webtest + +import batch + + +@pytest.fixture +def app(testbed): + return webtest.TestApp(batch.app) + + +def test_get(app): + response = app.get('/') + assert 'Bill Holiday' in response.body diff --git a/appengine/standard/memcache/best_practices/failure/app.yaml b/appengine/standard/memcache/best_practices/failure/app.yaml new file mode 100644 index 00000000000..60fabb0490e --- /dev/null +++ b/appengine/standard/memcache/best_practices/failure/app.yaml @@ -0,0 +1,7 @@ +runtime: python27 +threadsafe: yes +api_version: 1 + +handlers: +- url: .* + script: failure.app diff --git a/appengine/standard/memcache/best_practices/failure/failure.py b/appengine/standard/memcache/best_practices/failure/failure.py new file mode 100644 index 00000000000..767d2094a78 --- /dev/null +++ b/appengine/standard/memcache/best_practices/failure/failure.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from google.appengine.api import memcache +import webapp2 + + +def read_from_persistent_store(): + """Fake method for demonstration purposes. Usually would return + a value from a database like Cloud Datastore or MySQL.""" + return "a persistent value" + + +class ReadPage(webapp2.RequestHandler): + def get(self): + key = "some-key" + # [START memcache-read] + v = memcache.get(key) + if v is None: + v = read_from_persistent_store() + memcache.add(key, v) + # [END memcache-read] + + self.response.content_type = 'text/html' + self.response.write(str(v)) + + +class DeletePage(webapp2.RequestHandler): + def get(self): + key = "some key" + seconds = 5 + memcache.set(key, "some value") + # [START memcache-delete] + memcache.delete(key, seconds) # clears cache + # write to persistent datastore + # Do not attempt to put new value in cache, first reader will do that + # [END memcache-delete] + self.response.content_type = 'text/html' + self.response.write('done') + + +class MainPage(webapp2.RequestHandler): + def get(self): + value = 3 + # [START memcache-failure] + if not memcache.set('counter', value): + logging.error("Memcache set failed") + # Other error handling here + # [END memcache-failure] + self.response.content_type = 'text/html' + self.response.write('done') + + +app = webapp2.WSGIApplication([ + ('/', MainPage), + ('/delete', DeletePage), + ('/read', ReadPage), +], debug=True) diff --git a/appengine/standard/memcache/best_practices/failure/failure_test.py b/appengine/standard/memcache/best_practices/failure/failure_test.py new file mode 100644 index 00000000000..125054be64c --- /dev/null +++ b/appengine/standard/memcache/best_practices/failure/failure_test.py @@ -0,0 +1,35 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import webtest + +import failure + + +@pytest.fixture +def app(testbed): + return webtest.TestApp(failure.app) + + +def test_get(app): + app.get('/') + + +def test_read(app): + app.get('/read') + + +def test_delete(app): + app.get('/delete') diff --git a/appengine/standard/memcache/best_practices/migration_step1/app.yaml b/appengine/standard/memcache/best_practices/migration_step1/app.yaml new file mode 100644 index 00000000000..02e9132c4b0 --- /dev/null +++ b/appengine/standard/memcache/best_practices/migration_step1/app.yaml @@ -0,0 +1,7 @@ +runtime: python27 +threadsafe: yes +api_version: 1 + +handlers: +- url: .* + script: migration1.app diff --git a/appengine/standard/memcache/best_practices/migration_step1/migration1.py b/appengine/standard/memcache/best_practices/migration_step1/migration1.py new file mode 100644 index 00000000000..3cd3ad7a725 --- /dev/null +++ b/appengine/standard/memcache/best_practices/migration_step1/migration1.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from google.appengine.api import memcache +from google.appengine.ext import ndb +import webapp2 + + +# [START best-practice-1] +class Person(ndb.Model): + name = ndb.StringProperty(required=True) + + +def get_or_add_person(name): + person = memcache.get(name) + if person is None: + person = Person(name=name) + memcache.add(name, person) + else: + logging.info('Found in cache: ' + name) + return person +# [END best-practice-1] + + +class MainPage(webapp2.RequestHandler): + def get(self): + person = get_or_add_person('Stevie Wonder') + self.response.content_type = 'text/html' + self.response.write(person.name) + + +app = webapp2.WSGIApplication([ + ('/', MainPage), +], debug=True) diff --git a/appengine/standard/memcache/best_practices/migration_step1/migration1_test.py b/appengine/standard/memcache/best_practices/migration_step1/migration1_test.py new file mode 100644 index 00000000000..fe1b4b7acc8 --- /dev/null +++ b/appengine/standard/memcache/best_practices/migration_step1/migration1_test.py @@ -0,0 +1,22 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import webtest + +import migration1 + + +def test_get(testbed): + app = webtest.TestApp(migration1.app) + app.get('/') diff --git a/appengine/standard/memcache/best_practices/migration_step2/app.yaml b/appengine/standard/memcache/best_practices/migration_step2/app.yaml new file mode 100644 index 00000000000..0a2dd7051d8 --- /dev/null +++ b/appengine/standard/memcache/best_practices/migration_step2/app.yaml @@ -0,0 +1,7 @@ +runtime: python27 +threadsafe: yes +api_version: 1 + +handlers: +- url: .* + script: migration2.app diff --git a/appengine/standard/memcache/best_practices/migration_step2/migration2.py b/appengine/standard/memcache/best_practices/migration_step2/migration2.py new file mode 100644 index 00000000000..1f3b0c810ee --- /dev/null +++ b/appengine/standard/memcache/best_practices/migration_step2/migration2.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from google.appengine.api import memcache +from google.appengine.ext import ndb +import webapp2 + + +# [START best-practice-2] +class Person(ndb.Model): + name = ndb.StringProperty(required=True) + userid = ndb.StringProperty(required=True) + + +def get_or_add_person(name, userid): + person = memcache.get(name) + if person is None: + person = Person(name=name, userid=userid) + memcache.add(name, person) + else: + logging.info('Found in cache: ' + name + ', userid: ' + person.userid) + return person +# [END best-practice-2] + + +class MainPage(webapp2.RequestHandler): + def get(self): + person = get_or_add_person('Stevie Wonder', "1") + self.response.content_type = 'text/html' + self.response.write(person.name) + + +app = webapp2.WSGIApplication([ + ('/', MainPage), +], debug=True) diff --git a/appengine/standard/memcache/best_practices/migration_step2/migration2_test.py b/appengine/standard/memcache/best_practices/migration_step2/migration2_test.py new file mode 100644 index 00000000000..b2441c4331f --- /dev/null +++ b/appengine/standard/memcache/best_practices/migration_step2/migration2_test.py @@ -0,0 +1,22 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import webtest + +import migration2 + + +def test_get(testbed): + app = webtest.TestApp(migration2.app) + app.get('/') diff --git a/appengine/standard/memcache/best_practices/sharing/app.yaml b/appengine/standard/memcache/best_practices/sharing/app.yaml new file mode 100644 index 00000000000..17dc52934b5 --- /dev/null +++ b/appengine/standard/memcache/best_practices/sharing/app.yaml @@ -0,0 +1,7 @@ +runtime: python27 +threadsafe: yes +api_version: 1 + +handlers: +- url: .* + script: sharing.app diff --git a/appengine/standard/memcache/best_practices/sharing/sharing.py b/appengine/standard/memcache/best_practices/sharing/sharing.py new file mode 100644 index 00000000000..3371c0ef1eb --- /dev/null +++ b/appengine/standard/memcache/best_practices/sharing/sharing.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from google.appengine.api import memcache +import webapp2 + + +class MainPage(webapp2.RequestHandler): + def get(self): + # [START sharing] + self.response.headers['Content-Type'] = 'text/plain' + + who = memcache.get('who') + self.response.write('Previously incremented by %s\n' % who) + memcache.set('who', 'Python') + + count = memcache.incr('count', 1, initial_value=0) + self.response.write('Count incremented by Python = %s\n' % count) + # [END sharing] + + +app = webapp2.WSGIApplication([ + ('/', MainPage), +], debug=True) diff --git a/appengine/standard/memcache/best_practices/sharing/sharing_test.py b/appengine/standard/memcache/best_practices/sharing/sharing_test.py new file mode 100644 index 00000000000..f1cb8d0a1d6 --- /dev/null +++ b/appengine/standard/memcache/best_practices/sharing/sharing_test.py @@ -0,0 +1,23 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import webtest + +import sharing + + +def test_get(testbed): + app = webtest.TestApp(sharing.app) + response = app.get('/') + assert 'Previously incremented by ' in response.body diff --git a/appengine/standard/memcache/guestbook/README.md b/appengine/standard/memcache/guestbook/README.md new file mode 100644 index 00000000000..206a9eb920d --- /dev/null +++ b/appengine/standard/memcache/guestbook/README.md @@ -0,0 +1,18 @@ +# Memcache Guestbook Sample + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/memcache/guestbook/README.md + +This is a sample app for Google App Engine that demonstrates the Memcache Python API. + + +These samples are used on the following documentation page: + +> https://cloud.google.com/appengine/docs/python/memcache/examples + + + +Refer to the [App Engine Samples README](../../README.md) for information on how to run and deploy this sample. +ple. diff --git a/appengine/standard/memcache/guestbook/app.yaml b/appengine/standard/memcache/guestbook/app.yaml new file mode 100644 index 00000000000..1676c4ac14c --- /dev/null +++ b/appengine/standard/memcache/guestbook/app.yaml @@ -0,0 +1,21 @@ +# This file specifies your Python application's runtime configuration +# including URL routing, versions, static file uploads, etc. See +# https://developers.google.com/appengine/docs/python/config/appconfig +# for details. + +runtime: python27 +api_version: 1 +threadsafe: yes + +# Handlers define how to route requests to your application. +handlers: + +# This handler tells app engine how to route requests to a WSGI application. +# The script value is in the format . +# where is a WSGI application object. +- url: .* # This regex directs all routes to main.app + script: main.app + +libraries: +- name: webapp2 + version: "2.5.2" diff --git a/appengine/ndb/overview/favicon.ico b/appengine/standard/memcache/guestbook/favicon.ico similarity index 100% rename from appengine/ndb/overview/favicon.ico rename to appengine/standard/memcache/guestbook/favicon.ico diff --git a/appengine/memcache/guestbook/index.yaml b/appengine/standard/memcache/guestbook/index.yaml similarity index 100% rename from appengine/memcache/guestbook/index.yaml rename to appengine/standard/memcache/guestbook/index.yaml diff --git a/appengine/standard/memcache/guestbook/main.py b/appengine/standard/memcache/guestbook/main.py new file mode 100644 index 00000000000..ad087b2ac72 --- /dev/null +++ b/appengine/standard/memcache/guestbook/main.py @@ -0,0 +1,151 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Sample application that demonstrates how to use the App Engine Memcache API. + +For more information, see README.md. +""" + +# [START all] + +import cgi +import cStringIO +import logging +import urllib + +from google.appengine.api import memcache +from google.appengine.api import users +from google.appengine.ext import ndb + +import webapp2 + + +class Greeting(ndb.Model): + """Models an individual Guestbook entry with author, content, and date.""" + author = ndb.StringProperty() + content = ndb.StringProperty() + date = ndb.DateTimeProperty(auto_now_add=True) + + +def guestbook_key(guestbook_name=None): + """Constructs a Datastore key for a Guestbook entity with guestbook_name""" + return ndb.Key('Guestbook', guestbook_name or 'default_guestbook') + + +class MainPage(webapp2.RequestHandler): + def get(self): + self.response.out.write('') + guestbook_name = self.request.get('guestbook_name') + + greetings = self.get_greetings(guestbook_name) + stats = memcache.get_stats() + + self.response.write('Cache Hits:{}
        '.format(stats['hits'])) + self.response.write('Cache Misses:{}

        '.format( + stats['misses'])) + self.response.write(greetings) + + self.response.write(""" +
        +
        +
        +
        +
        +
        Guestbook name: +
        + + """.format(urllib.urlencode({'guestbook_name': guestbook_name}), + cgi.escape(guestbook_name))) + + # [START check_memcache] + def get_greetings(self, guestbook_name): + """ + get_greetings() + Checks the cache to see if there are cached greetings. + If not, call render_greetings and set the cache + + Args: + guestbook_name: Guestbook entity group key (string). + + Returns: + A string of HTML containing greetings. + """ + greetings = memcache.get('{}:greetings'.format(guestbook_name)) + if greetings is None: + greetings = self.render_greetings(guestbook_name) + try: + added = memcache.add( + '{}:greetings'.format(guestbook_name), greetings, 10) + if not added: + logging.error('Memcache set failed.') + except ValueError: + logging.error('Memcache set failed - data larger than 1MB') + return greetings + # [END check_memcache] + + # [START query_datastore] + def render_greetings(self, guestbook_name): + """ + render_greetings() + Queries the database for greetings, iterate through the + results and create the HTML. + + Args: + guestbook_name: Guestbook entity group key (string). + + Returns: + A string of HTML containing greetings + """ + greetings = ndb.gql('SELECT * ' + 'FROM Greeting ' + 'WHERE ANCESTOR IS :1 ' + 'ORDER BY date DESC LIMIT 10', + guestbook_key(guestbook_name)) + output = cStringIO.StringIO() + for greeting in greetings: + if greeting.author: + output.write('{} wrote:'.format(greeting.author)) + else: + output.write('An anonymous person wrote:') + output.write('
        {}
        '.format( + cgi.escape(greeting.content))) + return output.getvalue() + # [END query_datastore] + + +class Guestbook(webapp2.RequestHandler): + def post(self): + # We set the same parent key on the 'Greeting' to ensure each greeting + # is in the same entity group. Queries across the single entity group + # are strongly consistent. However, the write rate to a single entity + # group is limited to ~1/second. + guestbook_name = self.request.get('guestbook_name') + greeting = Greeting(parent=guestbook_key(guestbook_name)) + + if users.get_current_user(): + greeting.author = users.get_current_user().nickname() + + greeting.content = self.request.get('content') + greeting.put() + memcache.delete('{}:greetings'.format(guestbook_name)) + self.redirect('/?' + + urllib.urlencode({'guestbook_name': guestbook_name})) + + +app = webapp2.WSGIApplication([('/', MainPage), + ('/sign', Guestbook)], + debug=True) + +# [END all] diff --git a/appengine/standard/memcache/guestbook/main_test.py b/appengine/standard/memcache/guestbook/main_test.py new file mode 100644 index 00000000000..1851b78db88 --- /dev/null +++ b/appengine/standard/memcache/guestbook/main_test.py @@ -0,0 +1,23 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import webtest + +import main + + +def test_app(testbed): + app = webtest.TestApp(main.app) + response = app.get('/') + assert response.status_int == 200 diff --git a/appengine/standard/memcache/snippets/snippets.py b/appengine/standard/memcache/snippets/snippets.py new file mode 100644 index 00000000000..502d4fbc441 --- /dev/null +++ b/appengine/standard/memcache/snippets/snippets.py @@ -0,0 +1,57 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START get_data] +# [START add_values] +from google.appengine.api import memcache + +# [END get_data] +# [END add_values] + + +def query_for_data(): + return 'data' + + +# [START get_data] +def get_data(): + data = memcache.get('key') + if data is not None: + return data + else: + data = query_for_data() + memcache.add('key', data, 60) + return data +# [END get_data] + + +def add_values(): + # [START add_values] + # Add a value if it doesn't exist in the cache + # with a cache expiration of 1 hour. + memcache.add(key="weather_USA_98105", value="raining", time=3600) + + # Set several values, overwriting any existing values for these keys. + memcache.set_multi( + {"USA_98115": "cloudy", "USA_94105": "foggy", "USA_94043": "sunny"}, + key_prefix="weather_", + time=3600 + ) + + # Atomically increment an integer value. + memcache.set(key="counter", value=0) + memcache.incr("counter") + memcache.incr("counter") + memcache.incr("counter") + # [END add_values] diff --git a/appengine/standard/memcache/snippets/snippets_test.py b/appengine/standard/memcache/snippets/snippets_test.py new file mode 100644 index 00000000000..8d34d8f408b --- /dev/null +++ b/appengine/standard/memcache/snippets/snippets_test.py @@ -0,0 +1,50 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.api import memcache +from mock import patch + +import snippets + + +SNIPPET_VALUES = { + "weather_USA_98105": "raining", + "weather_USA_98115": "cloudy", + "weather_USA_94105": "foggy", + "weather_USA_94043": "sunny", + "counter": 3, +} + + +@patch('snippets.query_for_data', return_value='data') +def test_get_data_not_present(query_fn, testbed): + data = snippets.get_data() + query_fn.assert_called_once_with() + assert data == 'data' + memcache.delete('key') + + +@patch('snippets.query_for_data', return_value='data') +def test_get_data_present(query_fn, testbed): + memcache.add('key', 'data', 9000) + data = snippets.get_data() + query_fn.assert_not_called() + assert data == 'data' + memcache.delete('key') + + +def test_add_values(testbed): + snippets.add_values() + for key, value in SNIPPET_VALUES.iteritems(): + assert memcache.get(key) == value diff --git a/appengine/standard/modules/README.md b/appengine/standard/modules/README.md new file mode 100644 index 00000000000..663a1ea6fbe --- /dev/null +++ b/appengine/standard/modules/README.md @@ -0,0 +1,10 @@ +## App Engine Modules Docs Snippets + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/modules/README.md + +This sample application demonstrates how to use Google App Engine's modules API. + + diff --git a/appengine/standard/modules/app.yaml b/appengine/standard/modules/app.yaml new file mode 100644 index 00000000000..102ed60d1b5 --- /dev/null +++ b/appengine/standard/modules/app.yaml @@ -0,0 +1,7 @@ +runtime: python27 +api_version: 1 +threadsafe: yes + +handlers: +- url: .* + script: main.app diff --git a/appengine/standard/modules/backend.py b/appengine/standard/modules/backend.py new file mode 100644 index 00000000000..efe8318deff --- /dev/null +++ b/appengine/standard/modules/backend.py @@ -0,0 +1,29 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Sample backend module deployed by backend.yaml and accessed in main.py +""" + +import webapp2 + + +class BackendHandler(webapp2.RequestHandler): + def get(self): + self.response.write("hello world") + + +app = webapp2.WSGIApplication([ + ('/', BackendHandler), +], debug=True) diff --git a/appengine/standard/modules/backend.yaml b/appengine/standard/modules/backend.yaml new file mode 100644 index 00000000000..adccf139879 --- /dev/null +++ b/appengine/standard/modules/backend.yaml @@ -0,0 +1,8 @@ +runtime: python27 +api_version: 1 +threadsafe: yes +service: my-backend + +handlers: +- url: .* + script: backend.app diff --git a/appengine/standard/modules/backend_test.py b/appengine/standard/modules/backend_test.py new file mode 100644 index 00000000000..e4022c95586 --- /dev/null +++ b/appengine/standard/modules/backend_test.py @@ -0,0 +1,29 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import webtest + +import backend + + +@pytest.fixture +def app(): + return webtest.TestApp(backend.app) + + +def test_get_module_info(app): + result = app.get('/') + assert result.status_code == 200 + assert 'hello world' in result.body diff --git a/appengine/standard/modules/main.py b/appengine/standard/modules/main.py new file mode 100644 index 00000000000..202daa39346 --- /dev/null +++ b/appengine/standard/modules/main.py @@ -0,0 +1,53 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Sample application that demonstrates getting information about this +AppEngine modules and accessing other modules in the same project. +""" + +import urllib2 +# [START modules_import] +from google.appengine.api import modules +# [END modules_import] +import webapp2 + + +class GetModuleInfoHandler(webapp2.RequestHandler): + def get(self): + # [START module_info] + module = modules.get_current_module_name() + instance_id = modules.get_current_instance_id() + self.response.write( + 'module_id={}&instance_id={}'.format(module, instance_id)) + # [END module_info] + + +class GetBackendHandler(webapp2.RequestHandler): + def get(self): + # [START access_another_module] + backend_hostname = modules.get_hostname(module='my-backend') + url = "http://{}/".format(backend_hostname) + try: + result = urllib2.urlopen(url).read() + self.response.write('Got response {}'.format(result)) + except urllib2.URLError: + pass + # [END access_another_module] + + +app = webapp2.WSGIApplication([ + ('/', GetModuleInfoHandler), + ('/access_backend', GetBackendHandler), +], debug=True) diff --git a/appengine/standard/modules/main_test.py b/appengine/standard/modules/main_test.py new file mode 100644 index 00000000000..6ea602d425b --- /dev/null +++ b/appengine/standard/modules/main_test.py @@ -0,0 +1,46 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import pytest +import webtest + +import main + + +@pytest.fixture +def app(): + return webtest.TestApp(main.app) + + +@mock.patch("main.modules") +def test_get_module_info(modules_mock, app): + modules_mock.get_current_module_name.return_value = "default" + modules_mock.get_current_instance_id.return_value = 1 + response = app.get('/') + assert response.status_int == 200 + results = response.body.split('&') + assert results[0].split('=')[1] == 'default' + assert results[1].split('=')[1] == '1' + + +@mock.patch("main.modules") +@mock.patch("urllib2.urlopen") +def test_get_backend(url_open_mock, modules_mock, app): + url_read_mock = mock.Mock(read=mock.Mock(return_value='hello world')) + url_open_mock.return_value = url_read_mock + response = app.get('/access_backend') + + assert response.status_int == 200 + assert response.body == 'Got response hello world' diff --git a/appengine/standard/multitenancy/README.md b/appengine/standard/multitenancy/README.md new file mode 100644 index 00000000000..214bf6df894 --- /dev/null +++ b/appengine/standard/multitenancy/README.md @@ -0,0 +1,17 @@ +# Google App Engine Namespaces + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/multitenancy/README.md + +This sample demonstrates how to use Google App Engine's [Namespace Manager API](https://cloud.google.com/appengine/docs/python/multitenancy/multitenancy). + + +These samples are used on the following documentation page: + +> https://cloud.google.com/appengine/docs/python/multitenancy/multitenancy + + + +Refer to the [App Engine Samples README](../README.md) for information on how to run and deploy this sample. diff --git a/appengine/multitenancy/app.yaml b/appengine/standard/multitenancy/app.yaml similarity index 100% rename from appengine/multitenancy/app.yaml rename to appengine/standard/multitenancy/app.yaml diff --git a/appengine/multitenancy/datastore.py b/appengine/standard/multitenancy/datastore.py similarity index 100% rename from appengine/multitenancy/datastore.py rename to appengine/standard/multitenancy/datastore.py diff --git a/appengine/multitenancy/datastore_test.py b/appengine/standard/multitenancy/datastore_test.py similarity index 99% rename from appengine/multitenancy/datastore_test.py rename to appengine/standard/multitenancy/datastore_test.py index 3c3dcc2f284..ff23e9b301f 100644 --- a/appengine/multitenancy/datastore_test.py +++ b/appengine/standard/multitenancy/datastore_test.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import datastore import webtest +import datastore + def test_datastore(testbed): app = webtest.TestApp(datastore.app) diff --git a/appengine/multitenancy/memcache.py b/appengine/standard/multitenancy/memcache.py similarity index 100% rename from appengine/multitenancy/memcache.py rename to appengine/standard/multitenancy/memcache.py diff --git a/appengine/multitenancy/memcache_test.py b/appengine/standard/multitenancy/memcache_test.py similarity index 99% rename from appengine/multitenancy/memcache_test.py rename to appengine/standard/multitenancy/memcache_test.py index 97bc7504e17..27410ad2b4d 100644 --- a/appengine/multitenancy/memcache_test.py +++ b/appengine/standard/multitenancy/memcache_test.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import memcache import webtest +import memcache + def test_memcache(testbed): app = webtest.TestApp(memcache.app) diff --git a/appengine/multitenancy/taskqueue.py b/appengine/standard/multitenancy/taskqueue.py similarity index 100% rename from appengine/multitenancy/taskqueue.py rename to appengine/standard/multitenancy/taskqueue.py diff --git a/appengine/multitenancy/taskqueue_test.py b/appengine/standard/multitenancy/taskqueue_test.py similarity index 99% rename from appengine/multitenancy/taskqueue_test.py rename to appengine/standard/multitenancy/taskqueue_test.py index c0f658bc2ac..636478a41b0 100644 --- a/appengine/multitenancy/taskqueue_test.py +++ b/appengine/standard/multitenancy/taskqueue_test.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import taskqueue import webtest +import taskqueue + def test_taskqueue(testbed, run_tasks): app = webtest.TestApp(taskqueue.app) diff --git a/appengine/standard/ndb/async/README.md b/appengine/standard/ndb/async/README.md new file mode 100644 index 00000000000..5f84454b150 --- /dev/null +++ b/appengine/standard/ndb/async/README.md @@ -0,0 +1,16 @@ +## App Engine Datastore NDB Asynchronous Operations Samples + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/ndb/async/README.md + +This contains snippets used in the NDB asynchronous operations documentation, +demonstrating various ways to make asynchronous ndb operations. + + +These samples are used on the following documentation page: + +> https://cloud.google.com/appengine/docs/python/ndb/async + + diff --git a/appengine/standard/ndb/async/app_async.py b/appengine/standard/ndb/async/app_async.py new file mode 100644 index 00000000000..c8062cdc4da --- /dev/null +++ b/appengine/standard/ndb/async/app_async.py @@ -0,0 +1,36 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.api import users +from google.appengine.ext import ndb +import webapp2 + + +class Account(ndb.Model): + view_counter = ndb.IntegerProperty() + + +class MyRequestHandler(webapp2.RequestHandler): + def get(self): + acct = Account.get_by_id(users.get_current_user().user_id()) + acct.view_counter += 1 + future = acct.put_async() + + # ...read something else from Datastore... + + self.response.out.write('Content of the page') + future.get_result() + + +app = webapp2.WSGIApplication([('/', MyRequestHandler)]) diff --git a/appengine/standard/ndb/async/app_async_test.py b/appengine/standard/ndb/async/app_async_test.py new file mode 100644 index 00000000000..57c84c18571 --- /dev/null +++ b/appengine/standard/ndb/async/app_async_test.py @@ -0,0 +1,36 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import webtest + +import app_async + + +@pytest.fixture +def app(testbed): + return webtest.TestApp(app_async.app) + + +def test_main(app, testbed, login): + app_async.Account(id='123', view_counter=4).put() + + # Log the user in + login(id='123') + + response = app.get('/') + + assert response.status_int == 200 + account = app_async.Account.get_by_id('123') + assert account.view_counter == 5 diff --git a/appengine/standard/ndb/async/app_sync.py b/appengine/standard/ndb/async/app_sync.py new file mode 100644 index 00000000000..8cf53f0c15c --- /dev/null +++ b/appengine/standard/ndb/async/app_sync.py @@ -0,0 +1,35 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.api import users +from google.appengine.ext import ndb +import webapp2 + + +class Account(ndb.Model): + view_counter = ndb.IntegerProperty() + + +class MyRequestHandler(webapp2.RequestHandler): + def get(self): + acct = Account.get_by_id(users.get_current_user().user_id()) + acct.view_counter += 1 + acct.put() + + # ...read something else from Datastore... + + self.response.out.write('Content of the page') + + +app = webapp2.WSGIApplication([('/', MyRequestHandler)]) diff --git a/appengine/standard/ndb/async/app_sync_test.py b/appengine/standard/ndb/async/app_sync_test.py new file mode 100644 index 00000000000..27ed85b68d9 --- /dev/null +++ b/appengine/standard/ndb/async/app_sync_test.py @@ -0,0 +1,36 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import webtest + +import app_sync + + +@pytest.fixture +def app(testbed): + return webtest.TestApp(app_sync.app) + + +def test_main(app, testbed, login): + app_sync.Account(id='123', view_counter=4).put() + + # Log the user in + login(id='123') + + response = app.get('/') + + assert response.status_int == 200 + account = app_sync.Account.get_by_id('123') + assert account.view_counter == 5 diff --git a/appengine/standard/ndb/async/app_toplevel/README.md b/appengine/standard/ndb/async/app_toplevel/README.md new file mode 100644 index 00000000000..cc5257c48f7 --- /dev/null +++ b/appengine/standard/ndb/async/app_toplevel/README.md @@ -0,0 +1,7 @@ +This is in a separate folder to isolate it from the other apps. + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/ndb/async/app_toplevel/README.md +This is necessary because the test won't pass when run with the other tests. diff --git a/appengine/standard/ndb/async/app_toplevel/app_toplevel.py b/appengine/standard/ndb/async/app_toplevel/app_toplevel.py new file mode 100644 index 00000000000..a1a4a2ac774 --- /dev/null +++ b/appengine/standard/ndb/async/app_toplevel/app_toplevel.py @@ -0,0 +1,39 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.api import users +from google.appengine.ext import ndb +import webapp2 + + +class Account(ndb.Model): + view_counter = ndb.IntegerProperty() + + +class MyRequestHandler(webapp2.RequestHandler): + @ndb.toplevel + def get(self): + acct = Account.get_by_id(users.get_current_user().user_id()) + acct.view_counter += 1 + acct.put_async() # Ignoring the Future this returns + + # ...read something else from Datastore... + + self.response.out.write('Content of the page') + + +# This is actually redundant, since the `get` decorator already handles it, but +# for demonstration purposes, you can also make the entire app toplevel with +# the following. +app = ndb.toplevel(webapp2.WSGIApplication([('/', MyRequestHandler)])) diff --git a/appengine/standard/ndb/async/app_toplevel/app_toplevel_test.py b/appengine/standard/ndb/async/app_toplevel/app_toplevel_test.py new file mode 100644 index 00000000000..967a1474451 --- /dev/null +++ b/appengine/standard/ndb/async/app_toplevel/app_toplevel_test.py @@ -0,0 +1,36 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import webtest + +import app_toplevel + + +@pytest.fixture +def app(testbed): + return webtest.TestApp(app_toplevel.app) + + +def test_main(app, testbed, login): + app_toplevel.Account(id='123', view_counter=4).put() + + # Log the user in + login(id='123') + + response = app.get('/') + + assert response.status_int == 200 + account = app_toplevel.Account.get_by_id('123') + assert account.view_counter == 5 diff --git a/appengine/standard/ndb/async/app_toplevel/index.html b/appengine/standard/ndb/async/app_toplevel/index.html new file mode 100644 index 00000000000..e69de29bb2d diff --git a/appengine/standard/ndb/async/guestbook.py b/appengine/standard/ndb/async/guestbook.py new file mode 100644 index 00000000000..4c2d452c97f --- /dev/null +++ b/appengine/standard/ndb/async/guestbook.py @@ -0,0 +1,101 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.api import users +from google.appengine.ext import ndb +import webapp2 + + +class Guestbook(ndb.Model): + content = ndb.StringProperty() + post_date = ndb.DateTimeProperty(auto_now_add=True) + + +class Account(ndb.Model): + email = ndb.StringProperty() + nickname = ndb.StringProperty() + + def nick(self): + return self.nickname or self.email # Whichever is non-empty + + +class Message(ndb.Model): + text = ndb.StringProperty() + when = ndb.DateTimeProperty(auto_now_add=True) + author = ndb.KeyProperty(kind=Account) # references Account + + +class MainPage(webapp2.RequestHandler): + def get(self): + if self.request.path == '/guestbook': + if self.request.get('async'): + self.get_guestbook_async() + else: + self.get_guestbook_sync() + elif self.request.path == '/messages': + if self.request.get('async'): + self.get_messages_async() + else: + self.get_messages_sync() + + def get_guestbook_sync(self): + uid = users.get_current_user().user_id() + acct = Account.get_by_id(uid) # I/O action 1 + qry = Guestbook.query().order(-Guestbook.post_date) + recent_entries = qry.fetch(10) # I/O action 2 + + # ...render HTML based on this data... + self.response.out.write('{}'.format(''.join( + '

        {}

        '.format(entry.content) for entry in recent_entries))) + + return acct, qry + + def get_guestbook_async(self): + uid = users.get_current_user().user_id() + acct_future = Account.get_by_id_async(uid) # Start I/O action #1 + qry = Guestbook.query().order(-Guestbook.post_date) + recent_entries_future = qry.fetch_async(10) # Start I/O action #2 + acct = acct_future.get_result() # Complete #1 + recent_entries = recent_entries_future.get_result() # Complete #2 + + # ...render HTML based on this data... + self.response.out.write('{}'.format(''.join( + '

        {}

        '.format(entry.content) for entry in recent_entries))) + + return acct, recent_entries + + def get_messages_sync(self): + qry = Message.query().order(-Message.when) + for msg in qry.fetch(20): + acct = msg.author.get() + self.response.out.write( + '

        On {}, {} wrote:'.format(msg.when, acct.nick())) + self.response.out.write('

        {}'.format(msg.text)) + + def get_messages_async(self): + @ndb.tasklet + def callback(msg): + acct = yield msg.author.get_async() + raise ndb.Return('On {}, {} wrote:\n{}'.format( + msg.when, acct.nick(), msg.text)) + + qry = Message.query().order(-Message.when) + outputs = qry.map(callback, limit=20) + for output in outputs: + self.response.out.write('

        {}

        '.format(output)) + + +app = webapp2.WSGIApplication([ + ('/.*', MainPage), +], debug=True) diff --git a/appengine/standard/ndb/async/guestbook_test.py b/appengine/standard/ndb/async/guestbook_test.py new file mode 100644 index 00000000000..e1c51a71476 --- /dev/null +++ b/appengine/standard/ndb/async/guestbook_test.py @@ -0,0 +1,74 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import webtest + +import guestbook + + +@pytest.fixture +def app(testbed): + return webtest.TestApp(guestbook.app) + + +def test_get_guestbook_sync(app, testbed, login): + guestbook.Account(id='123').put() + # Log the user in + login(id='123') + + for i in range(11): + guestbook.Guestbook(content='Content {}'.format(i)).put() + + response = app.get('/guestbook') + + assert response.status_int == 200 + assert 'Content 1' in response.body + + +def test_get_guestbook_async(app, testbed, login): + guestbook.Account(id='123').put() + # Log the user in + login(id='123') + for i in range(11): + guestbook.Guestbook(content='Content {}'.format(i)).put() + + response = app.get('/guestbook?async=1') + + assert response.status_int == 200 + assert 'Content 1' in response.body + + +def test_get_messages_sync(app, testbed): + for i in range(21): + account_key = guestbook.Account(nickname='Nick {}'.format(i)).put() + guestbook.Message(author=account_key, text='Text {}'.format(i)).put() + + response = app.get('/messages') + + assert response.status_int == 200 + assert 'Nick 1 wrote:' in response.body + assert '

        Text 1' in response.body + + +def test_get_messages_async(app, testbed): + for i in range(21): + account_key = guestbook.Account(nickname='Nick {}'.format(i)).put() + guestbook.Message(author=account_key, text='Text {}'.format(i)).put() + + response = app.get('/messages?async=1') + + assert response.status_int == 200 + assert 'Nick 1 wrote:' in response.body + assert '\nText 1' in response.body diff --git a/appengine/standard/ndb/async/shopping_cart.py b/appengine/standard/ndb/async/shopping_cart.py new file mode 100644 index 00000000000..1ddf1306d42 --- /dev/null +++ b/appengine/standard/ndb/async/shopping_cart.py @@ -0,0 +1,137 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.ext import ndb + + +# [START models] +class Account(ndb.Model): + pass + + +class InventoryItem(ndb.Model): + name = ndb.StringProperty() + + +class CartItem(ndb.Model): + account = ndb.KeyProperty(kind=Account) + inventory = ndb.KeyProperty(kind=InventoryItem) + quantity = ndb.IntegerProperty() + + +class SpecialOffer(ndb.Model): + inventory = ndb.KeyProperty(kind=InventoryItem) +# [END models] + + +def get_cart_plus_offers(acct): + cart = CartItem.query(CartItem.account == acct.key).fetch() + offers = SpecialOffer.query().fetch(10) + ndb.get_multi([item.inventory for item in cart] + + [offer.inventory for offer in offers]) + return cart, offers + + +def get_cart_plus_offers_async(acct): + cart_future = CartItem.query(CartItem.account == acct.key).fetch_async() + offers_future = SpecialOffer.query().fetch_async(10) + cart = cart_future.get_result() + offers = offers_future.get_result() + ndb.get_multi([item.inventory for item in cart] + + [offer.inventory for offer in offers]) + return cart, offers + + +# [START cart_offers_tasklets] +@ndb.tasklet +def get_cart_tasklet(acct): + cart = yield CartItem.query(CartItem.account == acct.key).fetch_async() + yield ndb.get_multi_async([item.inventory for item in cart]) + raise ndb.Return(cart) + + +@ndb.tasklet +def get_offers_tasklet(acct): + offers = yield SpecialOffer.query().fetch_async(10) + yield ndb.get_multi_async([offer.inventory for offer in offers]) + raise ndb.Return(offers) + + +@ndb.tasklet +def get_cart_plus_offers_tasklet(acct): + cart, offers = yield get_cart_tasklet(acct), get_offers_tasklet(acct) + raise ndb.Return((cart, offers)) +# [END cart_offers_tasklets] + + +@ndb.tasklet +def iterate_over_query_results_in_tasklet(Model, is_the_entity_i_want): + qry = Model.query() + qit = qry.iter() + while (yield qit.has_next_async()): + entity = qit.next() + # Do something with entity + if is_the_entity_i_want(entity): + raise ndb.Return(entity) + + +@ndb.tasklet +def blocking_iteration_over_query_results(Model, is_the_entity_i_want): + # DO NOT DO THIS IN A TASKLET + qry = Model.query() + for entity in qry: + # Do something with entity + if is_the_entity_i_want(entity): + raise ndb.Return(entity) + + +def define_get_google(): + @ndb.tasklet + def get_google(): + context = ndb.get_context() + result = yield context.urlfetch("http://www.google.com/") + if result.status_code == 200: + raise ndb.Return(result.content) + # else return None + + return get_google + + +def define_update_counter_async(): + @ndb.transactional_async + def update_counter(counter_key): + counter = counter_key.get() + counter.value += 1 + counter.put() + return counter.value + + return update_counter + + +def define_update_counter_tasklet(): + @ndb.transactional_tasklet + def update_counter(counter_key): + counter = yield counter_key.get_async() + counter.value += 1 + yield counter.put_async() + + return update_counter + + +def get_first_ready(): + urls = ["http://www.google.com/", "http://www.blogspot.com/"] + context = ndb.get_context() + futures = [context.urlfetch(url) for url in urls] + first_future = ndb.Future.wait_any(futures) + return first_future.get_result().content diff --git a/appengine/standard/ndb/async/shopping_cart_test.py b/appengine/standard/ndb/async/shopping_cart_test.py new file mode 100644 index 00000000000..78f519baba6 --- /dev/null +++ b/appengine/standard/ndb/async/shopping_cart_test.py @@ -0,0 +1,144 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.ext import ndb +import pytest + +import shopping_cart + + +@pytest.fixture +def items(testbed): + account = shopping_cart.Account(id='123') + account.put() + + items = [shopping_cart.InventoryItem(name='Item {}'.format(i)) + for i in range(6)] + special_items = [shopping_cart.InventoryItem(name='Special {}'.format(i)) + for i in range(6)] + for i in items + special_items: + i.put() + + special_offers = [shopping_cart.SpecialOffer(inventory=item.key) + for item in special_items] + cart_items = [ + shopping_cart.CartItem( + account=account.key, inventory=item.key, quantity=i) + for i, item in enumerate(items[:6] + special_items[:6])] + for i in special_offers + cart_items: + i.put() + + return account, items, special_items, cart_items, special_offers + + +def test_get_cart_plus_offers(items): + account, items, special_items, cart_items, special_offers = items + + cart, offers = shopping_cart.get_cart_plus_offers(account) + + assert len(cart) == 12 + assert len(offers) == 6 + + +def test_get_cart_plus_offers_async(items): + account, items, special_items, cart_items, special_offers = items + + cart, offers = shopping_cart.get_cart_plus_offers_async(account) + + assert len(cart) == 12 + assert len(offers) == 6 + + +def test_get_cart_tasklet(items): + account, items, special_items, cart_items, special_offers = items + + future = shopping_cart.get_cart_tasklet(account) + cart = future.get_result() + + assert len(cart) == 12 + + +def test_get_offers_tasklet(items): + account, items, special_items, cart_items, special_offers = items + + future = shopping_cart.get_offers_tasklet(account) + offers = future.get_result() + + assert len(offers) == 6 + + +def test_get_cart_plus_offers_tasklet(items): + account, items, special_items, cart_items, special_offers = items + + future = shopping_cart.get_cart_plus_offers_tasklet( + account) + cart, offers = future.get_result() + + assert len(cart) == 12 + assert len(offers) == 6 + + +def test_iterate_over_query_results_in_tasklet(items): + account, items, special_items, cart_items, special_offers = items + + future = shopping_cart.iterate_over_query_results_in_tasklet( + shopping_cart.InventoryItem, lambda item: '3' in item.name) + + assert '3' in future.get_result().name + + +def test_do_not_iterate_over_tasklet_like_this(items): + account, items, special_items, cart_items, special_offers = items + + future = shopping_cart.blocking_iteration_over_query_results( + shopping_cart.InventoryItem, lambda item: '3' in item.name) + + assert '3' in future.get_result().name + + +def test_get_google(testbed): + testbed.init_urlfetch_stub() + + get_google = shopping_cart.define_get_google() + future = get_google() + assert 'Google' in future.get_result() + + +class Counter(ndb.Model): + value = ndb.IntegerProperty() + + +def test_update_counter_async(testbed): + counter_key = Counter(value=1).put() + update_counter = shopping_cart.define_update_counter_async() + future = update_counter(counter_key) + assert counter_key.get().value == 1 + assert future.get_result() == 2 + assert counter_key.get().value == 2 + + +def test_update_counter_tasklet(testbed): + counter_key = Counter(value=1).put() + update_counter = shopping_cart.define_update_counter_tasklet() + future = update_counter(counter_key) + assert counter_key.get().value == 1 + future.get_result() + assert counter_key.get().value == 2 + + +def test_get_first_ready(testbed): + testbed.init_urlfetch_stub() + + content = shopping_cart.get_first_ready() + assert 'html' in content.lower() diff --git a/appengine/standard/ndb/cache/README.md b/appengine/standard/ndb/cache/README.md new file mode 100644 index 00000000000..a702c647819 --- /dev/null +++ b/appengine/standard/ndb/cache/README.md @@ -0,0 +1,16 @@ +## App Engine Datastore NDB Cache Samples + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/ndb/cache/README.md + +This contains snippets used in the NDB cache documentation, demonstrating +various operations on ndb caches. + + +These samples are used on the following documentation page: + +> https://cloud.google.com/appengine/docs/python/ndb/cache + + diff --git a/appengine/standard/ndb/cache/snippets.py b/appengine/standard/ndb/cache/snippets.py new file mode 100644 index 00000000000..391ad3f89a0 --- /dev/null +++ b/appengine/standard/ndb/cache/snippets.py @@ -0,0 +1,45 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.ext import ndb + + +def set_in_process_cache_policy(func): + context = ndb.get_context() + context.set_cache_policy(func) + + +def set_memcache_policy(func): + context = ndb.get_context() + context.set_memcache_policy(func) + + +def bypass_in_process_cache_for_account_entities(): + context = ndb.get_context() + context.set_cache_policy(lambda key: key.kind() != 'Account') + + +def set_datastore_policy(func): + context = ndb.get_context() + context.set_datastore_policy(func) + + +def set_memcache_timeout_policy(func): + context = ndb.get_context() + context.set_memcache_timeout_policy(func) + + +def clear_cache(): + context = ndb.get_context() + context.clear_cache() diff --git a/appengine/standard/ndb/cache/snippets_test.py b/appengine/standard/ndb/cache/snippets_test.py new file mode 100644 index 00000000000..257525895ee --- /dev/null +++ b/appengine/standard/ndb/cache/snippets_test.py @@ -0,0 +1,60 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.ext import ndb + +import snippets + + +def test_set_in_process_cache_policy(testbed): + def policy(key): + return 1 == 1 + + snippets.set_in_process_cache_policy(policy) + assert policy == ndb.get_context().get_cache_policy() + + +def test_set_memcache_policy(testbed): + def policy(key): + return 1 == 2 + + snippets.set_memcache_policy(policy) + assert policy == ndb.get_context().get_memcache_policy() + + +def test_bypass_in_process_cache_for_account_entities(testbed): + context = ndb.get_context() + assert context.get_cache_policy() == context.default_cache_policy + snippets.bypass_in_process_cache_for_account_entities() + assert context.get_cache_policy() != context.default_cache_policy + + +def test_set_datastore_policy(testbed): + def policy(key): + return key is None + + snippets.set_datastore_policy(policy) + assert ndb.get_context().get_datastore_policy() == policy + + +def test_set_memcache_timeout_policy(testbed): + def policy(key): + return 1 + + snippets.set_memcache_timeout_policy(policy) + assert ndb.get_context().get_memcache_timeout_policy() == policy + + +def test_clear_cache(testbed): + snippets.clear_cache() diff --git a/appengine/standard/ndb/entities/README.md b/appengine/standard/ndb/entities/README.md new file mode 100644 index 00000000000..31bee0e7844 --- /dev/null +++ b/appengine/standard/ndb/entities/README.md @@ -0,0 +1,20 @@ +## App Engine Datastore NDB Entities Samples + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/ndb/entities/README.md + +This contains snippets used in the NDB entity documentation, demonstrating +various operations on ndb entities. + + +These samples are used on the following documentation pages: + +> +* https://cloud.google.com/appengine/docs/python/datastore/creating-entities +* https://cloud.google.com/appengine/docs/python/datastore/creating-entity-models +* https://cloud.google.com/appengine/docs/python/users/userobjects +* https://cloud.google.com/appengine/docs/python/datastore/creating-entity-keys + + diff --git a/appengine/standard/ndb/entities/snippets.py b/appengine/standard/ndb/entities/snippets.py new file mode 100644 index 00000000000..a8ecaf0d352 --- /dev/null +++ b/appengine/standard/ndb/entities/snippets.py @@ -0,0 +1,294 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.ext import ndb + + +class Account(ndb.Model): + username = ndb.StringProperty() + userid = ndb.IntegerProperty() + email = ndb.StringProperty() + + +def create_entity_using_keyword_arguments(): + sandy = Account( + username='Sandy', userid=123, email='sandy@example.com') + return sandy + + +def create_entity_using_attributes(): + sandy = Account() + sandy.username = 'Sandy' + sandy.userid = 123 + sandy.email = 'sandy@example.com' + return sandy + + +def create_entity_using_populate(): + sandy = Account() + sandy.populate( + username='Sandy', + userid=123, + email='sandy@gmail.com') + return sandy + + +def demonstrate_model_constructor_type_checking(): + bad = Account( + username='Sandy', userid='not integer') # raises an exception + return bad + + +def demonstrate_entity_attribute_type_checking(sandy): + sandy.username = 42 # raises an exception + + +def save_entity(sandy): + sandy_key = sandy.put() + return sandy_key + + +def get_entity(sandy_key): + sandy = sandy_key.get() + return sandy + + +def get_key_kind_and_id(sandy_key): + kind_string = sandy_key.kind() # returns 'Account' + ident = sandy_key.id() # returns '2' + return kind_string, ident + + +def get_url_safe_key(sandy_key): + url_string = sandy_key.urlsafe() + return url_string + + +def get_entity_from_url_safe_key(url_string): + sandy_key = ndb.Key(urlsafe=url_string) + sandy = sandy_key.get() + return sandy + + +def get_key_and_numeric_id_from_url_safe_key(url_string): + key = ndb.Key(urlsafe=url_string) + kind_string = key.kind() + ident = key.id() + return key, ident, kind_string + + +def update_entity_from_key(key): + sandy = key.get() + sandy.email = 'sandy@example.co.uk' + sandy.put() + + +def delete_entity(sandy): + sandy.key.delete() + + +def create_entity_with_named_key(): + account = Account( + username='Sandy', userid=1234, email='sandy@example.com', + id='sandy@example.com') + + return account.key.id() # returns 'sandy@example.com' + + +def set_key_directly(account): + account.key = ndb.Key('Account', 'sandy@example.com') + + # You can also use the model class object itself, rather than its name, + # to specify the entity's kind: + account.key = ndb.Key(Account, 'sandy@example.com') + + +def create_entity_with_generated_id(): + # note: no id kwarg + account = Account(username='Sandy', userid=1234, email='sandy@example.com') + account.put() + # account.key will now have a key of the form: ndb.Key(Account, 71321839) + # where the value 71321839 was generated by Datastore for us. + return account + + +class Revision(ndb.Model): + message_text = ndb.StringProperty() + + +def demonstrate_entities_with_parent_hierarchy(): + ndb.Key('Account', 'sandy@example.com', 'Message', 123, 'Revision', '1') + ndb.Key('Account', 'sandy@example.com', 'Message', 123, 'Revision', '2') + ndb.Key('Account', 'larry@example.com', 'Message', 456, 'Revision', '1') + ndb.Key('Account', 'larry@example.com', 'Message', 789, 'Revision', '2') + + +def equivalent_ways_to_define_key_with_parent(): + ndb.Key('Account', 'sandy@example.com', 'Message', 123, 'Revision', '1') + + ndb.Key('Revision', '1', parent=ndb.Key( + 'Account', 'sandy@example.com', 'Message', 123)) + + ndb.Key('Revision', '1', parent=ndb.Key( + 'Message', 123, parent=ndb.Key('Account', 'sandy@example.com'))) + + +def create_root_key(): + sandy_key = ndb.Key(Account, 'sandy@example.com') + return sandy_key + + +def create_entity_with_parent_keys(): + account_key = ndb.Key(Account, 'sandy@example.com') + + # Ask Datastore to allocate an ID. + new_id = ndb.Model.allocate_ids(size=1, parent=account_key)[0] + + # Datastore returns us an integer ID that we can use to create the message + # key + message_key = ndb.Key('Message', new_id, parent=account_key) + + # Now we can put the message into Datastore + initial_revision = Revision( + message_text='Hello', id='1', parent=message_key) + initial_revision.put() + + return initial_revision + + +def get_parent_key_of_entity(initial_revision): + message_key = initial_revision.key.parent() + return message_key + + +def operate_on_multiple_keys_at_once(list_of_entities): + list_of_keys = ndb.put_multi(list_of_entities) + list_of_entities = ndb.get_multi(list_of_keys) + ndb.delete_multi(list_of_keys) + + +class Mine(ndb.Expando): + pass + + +def create_entity_using_expando_model(): + e = Mine() + e.foo = 1 + e.bar = 'blah' + e.tags = ['exp', 'and', 'oh'] + e.put() + + return e + + +def get_properties_defined_on_expando(e): + return e._properties + # { + # 'foo': GenericProperty('foo'), + # 'bar': GenericProperty('bar'), + # 'tags': GenericProperty('tags', repeated=True) + # } + + +class FlexEmployee(ndb.Expando): + name = ndb.StringProperty() + age = ndb.IntegerProperty() + + +def create_expando_model_entity_with_defined_properties(): + employee = FlexEmployee(name='Sandy', location='SF') + return employee + + +class Specialized(ndb.Expando): + _default_indexed = False + + +def create_expando_model_entity_that_isnt_indexed_by_default(): + e = Specialized(foo='a', bar=['b']) + return e._properties + # { + # 'foo': GenericProperty('foo', indexed=False), + # 'bar': GenericProperty('bar', indexed=False, repeated=True) + # } + + +def demonstrate_wrong_way_to_query_expando(): + FlexEmployee.query(FlexEmployee.location == 'SF') + + +def demonstrate_right_way_to_query_expando(): + FlexEmployee.query(ndb.GenericProperty('location') == 'SF') + + +notification = None + + +def _notify(message): + global notification + notification = message + + +class Friend(ndb.Model): + name = ndb.StringProperty() + + def _pre_put_hook(self): + _notify('Gee wiz I have a new friend!') + + @classmethod + def _post_delete_hook(cls, key, future): + _notify('I have found occasion to rethink our friendship.') + + +def demonstrate_model_put_and_delete_hooks(): + f = Friend() + f.name = 'Carole King' + f.put() # _pre_put_hook is called + yield f + fut = f.key.delete_async() # _post_delete_hook not yet called + fut.get_result() # _post_delete_hook is called + yield f + + +class MyModel(ndb.Model): + pass + + +def reserve_model_ids(): + first, last = MyModel.allocate_ids(100) + return first, last + + +def reserve_model_ids_with_a_parent(p): + first, last = MyModel.allocate_ids(100, parent=p) + return first, last + + +def construct_keys_from_range_of_reserved_ids(first, last): + keys = [ndb.Key(MyModel, id) for id in range(first, last+1)] + return keys + + +def reserve_model_ids_up_to(N): + first, last = MyModel.allocate_ids(max=N) + return first, last + + +class ModelWithUser(ndb.Model): + user_id = ndb.StringProperty() + color = ndb.StringProperty() + + @classmethod + def get_by_user(cls, user): + return cls.query().filter(cls.user_id == user.user_id()).get() diff --git a/appengine/standard/ndb/entities/snippets_test.py b/appengine/standard/ndb/entities/snippets_test.py new file mode 100644 index 00000000000..e93ffc6b81a --- /dev/null +++ b/appengine/standard/ndb/entities/snippets_test.py @@ -0,0 +1,228 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.api import users +from google.appengine.ext import ndb +from google.appengine.ext.ndb.google_imports import datastore_errors +import pytest + +import snippets + + +def test_create_entity_using_keyword_arguments(testbed): + result = snippets.create_entity_using_keyword_arguments() + assert isinstance(result, snippets.Account) + + +def test_create_entity_using_attributes(testbed): + result = snippets.create_entity_using_attributes() + assert isinstance(result, snippets.Account) + + +def test_create_entity_using_populate(testbed): + result = snippets.create_entity_using_populate() + assert isinstance(result, snippets.Account) + + +def test_demonstrate_model_constructor_type_checking(testbed): + with pytest.raises(datastore_errors.BadValueError): + snippets.demonstrate_model_constructor_type_checking() + + +def test_demonstrate_entity_attribute_type_checking(testbed): + with pytest.raises(datastore_errors.BadValueError): + snippets.demonstrate_entity_attribute_type_checking( + snippets.create_entity_using_keyword_arguments()) + + +def test_save_entity(testbed): + result = snippets.save_entity( + snippets.create_entity_using_keyword_arguments()) + assert isinstance(result, snippets.ndb.Key) + + +def test_get_entity(testbed): + sandy_key = snippets.save_entity( + snippets.create_entity_using_keyword_arguments()) + result = snippets.get_entity(sandy_key) + assert isinstance(result, snippets.Account) + + +def test_get_key_kind_and_id(testbed): + sandy_key = snippets.save_entity( + snippets.create_entity_using_keyword_arguments()) + kind_string, ident = snippets.get_key_kind_and_id(sandy_key) + assert kind_string == 'Account' + assert isinstance(ident, long) + + +def test_get_url_safe_key(testbed): + sandy_key = snippets.save_entity( + snippets.create_entity_using_keyword_arguments()) + result = snippets.get_url_safe_key(sandy_key) + assert isinstance(result, str) + + +def test_get_entity_from_url_safe_key(testbed): + sandy_key = snippets.save_entity( + snippets.create_entity_using_keyword_arguments()) + result = snippets.get_entity_from_url_safe_key( + snippets.get_url_safe_key(sandy_key)) + assert isinstance(result, snippets.Account) + assert result.username == 'Sandy' + + +def test_get_key_and_numeric_id_from_url_safe_key(testbed): + sandy_key = snippets.save_entity( + snippets.create_entity_using_keyword_arguments()) + urlsafe = snippets.get_url_safe_key(sandy_key) + key, ident, kind_string = ( + snippets.get_key_and_numeric_id_from_url_safe_key(urlsafe)) + assert isinstance(key, ndb.Key) + assert isinstance(ident, long) + assert isinstance(kind_string, str) + + +def test_update_entity_from_key(testbed): + sandy = snippets.create_entity_using_keyword_arguments() + sandy_key = snippets.save_entity(sandy) + urlsafe = snippets.get_url_safe_key(sandy_key) + key, ident, kind_string = ( + snippets.get_key_and_numeric_id_from_url_safe_key(urlsafe)) + snippets.update_entity_from_key(key) + assert key.get().email == 'sandy@example.co.uk' + + +def test_delete_entity(testbed): + sandy = snippets.create_entity_using_keyword_arguments() + snippets.save_entity(sandy) + snippets.delete_entity(sandy) + assert sandy.key.get() is None + + +def test_create_entity_with_named_key(testbed): + result = snippets.create_entity_with_named_key() + assert 'sandy@example.com' == result + + +def test_set_key_directly(testbed): + account = snippets.Account() + snippets.set_key_directly(account) + assert account.key.id() == 'sandy@example.com' + + +def test_create_entity_with_generated_id(testbed): + result = snippets.create_entity_with_generated_id() + assert isinstance(result.key.id(), long) + + +def test_demonstrate_entities_with_parent_hierarchy(testbed): + snippets.demonstrate_entities_with_parent_hierarchy() + + +def test_equivalent_ways_to_define_key_with_parent(testbed): + snippets.equivalent_ways_to_define_key_with_parent() + + +def test_create_root_key(testbed): + result = snippets.create_root_key() + assert result.id() == 'sandy@example.com' + + +def test_create_entity_with_parent_keys(testbed): + result = snippets.create_entity_with_parent_keys() + assert result.message_text == 'Hello' + + +def test_get_parent_key_of_entity(testbed): + initial_revision = snippets.create_entity_with_parent_keys() + result = snippets.get_parent_key_of_entity(initial_revision) + assert result.kind() == 'Message' + + +def test_operate_on_multiple_keys_at_once(testbed): + snippets.operate_on_multiple_keys_at_once([ + snippets.Account(email='a@a.com'), snippets.Account(email='b@b.com')]) + + +def test_create_entity_using_expando_model(testbed): + result = snippets.create_entity_using_expando_model() + assert result.foo == 1 + + +def test_get_properties_defined_on_expando(testbed): + result = snippets.get_properties_defined_on_expando( + snippets.create_entity_using_expando_model()) + assert result['foo'] is not None + assert result['bar'] is not None + assert result['tags'] is not None + + +def test_create_entity_using_expando_model_with_defined_properties(testbed): + result = snippets.create_expando_model_entity_with_defined_properties() + assert result.name == 'Sandy' + + +def test_create_expando_model_entity_that_isnt_indexed_by_default(testbed): + result = ( + snippets.create_expando_model_entity_that_isnt_indexed_by_default()) + assert result['foo'] + assert result['bar'] + + +def test_demonstrate_wrong_way_to_query_expando(testbed): + with pytest.raises(AttributeError): + snippets.demonstrate_wrong_way_to_query_expando() + + +def test_demonstrate_right_way_to_query_expando(testbed): + snippets.demonstrate_right_way_to_query_expando() + + +def test_demonstrate_model_put_and_delete_hooks(testbed): + iterator = snippets.demonstrate_model_put_and_delete_hooks() + iterator.next() + assert snippets.notification == 'Gee wiz I have a new friend!' + iterator.next() + assert snippets.notification == ( + 'I have found occasion to rethink our friendship.') + + +def test_reserve_model_ids(testbed): + first, last = snippets.reserve_model_ids() + assert last - first >= 99 + + +def test_reserve_model_ids_with_a_parent(testbed): + first, last = snippets.reserve_model_ids_with_a_parent( + snippets.Friend().key) + assert last - first >= 99 + + +def test_construct_keys_from_range_of_reserved_ids(testbed): + result = snippets.construct_keys_from_range_of_reserved_ids( + *snippets.reserve_model_ids()) + assert len(result) == 100 + + +def test_reserve_model_ids_up_to(testbed): + first, last = snippets.reserve_model_ids_up_to(5) + assert last - first >= 4 + + +def test_model_with_user(testbed): + user = users.User(email='user@example.com', _user_id='123') + item = snippets.ModelWithUser(user_id=user.user_id()) + item.put() + assert snippets.ModelWithUser.get_by_user(user) == item diff --git a/appengine/standard/ndb/modeling/README.md b/appengine/standard/ndb/modeling/README.md new file mode 100644 index 00000000000..c13c59bbb04 --- /dev/null +++ b/appengine/standard/ndb/modeling/README.md @@ -0,0 +1,10 @@ +## App Engine Datastore NDB Modeling Samples + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/ndb/modeling/README.md + +These samples demonstrate how to [model entity relationships](https://cloud.google.com/appengine/articles/modeling) using the Datastore NDB library. + +Refer to the [App Engine Samples README](../../README.md) for information on how to run and deploy this sample. diff --git a/appengine/ndb/modeling/contact_with_group_models.py b/appengine/standard/ndb/modeling/contact_with_group_models.py similarity index 100% rename from appengine/ndb/modeling/contact_with_group_models.py rename to appengine/standard/ndb/modeling/contact_with_group_models.py diff --git a/appengine/ndb/modeling/contact_with_group_models_test.py b/appengine/standard/ndb/modeling/contact_with_group_models_test.py similarity index 99% rename from appengine/ndb/modeling/contact_with_group_models_test.py rename to appengine/standard/ndb/modeling/contact_with_group_models_test.py index 0ec40949d1c..1afe71682df 100644 --- a/appengine/ndb/modeling/contact_with_group_models_test.py +++ b/appengine/standard/ndb/modeling/contact_with_group_models_test.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import contact_with_group_models as models from google.appengine.ext import ndb +import contact_with_group_models as models + def test_models(testbed): # Creates 3 contacts and 1 group. diff --git a/appengine/ndb/modeling/keyproperty_models.py b/appengine/standard/ndb/modeling/keyproperty_models.py similarity index 100% rename from appengine/ndb/modeling/keyproperty_models.py rename to appengine/standard/ndb/modeling/keyproperty_models.py diff --git a/appengine/ndb/modeling/keyproperty_models_test.py b/appengine/standard/ndb/modeling/keyproperty_models_test.py similarity index 99% rename from appengine/ndb/modeling/keyproperty_models_test.py rename to appengine/standard/ndb/modeling/keyproperty_models_test.py index 66c983ac4d5..a2a4d5a3bd9 100644 --- a/appengine/ndb/modeling/keyproperty_models_test.py +++ b/appengine/standard/ndb/modeling/keyproperty_models_test.py @@ -14,9 +14,10 @@ """Test classes for code snippet for modeling article.""" -import keyproperty_models as models import pytest +import keyproperty_models as models + def test_models(testbed): name = 'Takashi Matsuo' diff --git a/appengine/ndb/modeling/naive_models.py b/appengine/standard/ndb/modeling/naive_models.py similarity index 100% rename from appengine/ndb/modeling/naive_models.py rename to appengine/standard/ndb/modeling/naive_models.py diff --git a/appengine/ndb/modeling/naive_models_test.py b/appengine/standard/ndb/modeling/naive_models_test.py similarity index 100% rename from appengine/ndb/modeling/naive_models_test.py rename to appengine/standard/ndb/modeling/naive_models_test.py diff --git a/appengine/ndb/modeling/parent_child_models.py b/appengine/standard/ndb/modeling/parent_child_models.py similarity index 100% rename from appengine/ndb/modeling/parent_child_models.py rename to appengine/standard/ndb/modeling/parent_child_models.py diff --git a/appengine/ndb/modeling/parent_child_models_test.py b/appengine/standard/ndb/modeling/parent_child_models_test.py similarity index 99% rename from appengine/ndb/modeling/parent_child_models_test.py rename to appengine/standard/ndb/modeling/parent_child_models_test.py index cae2c6bedc7..49afc670c57 100644 --- a/appengine/ndb/modeling/parent_child_models_test.py +++ b/appengine/standard/ndb/modeling/parent_child_models_test.py @@ -15,9 +15,10 @@ """Test classes for code snippet for modeling article.""" from google.appengine.ext import ndb -import parent_child_models as models import pytest +import parent_child_models as models + NAME = 'Takashi Matsuo' diff --git a/appengine/ndb/modeling/relation_model_models.py b/appengine/standard/ndb/modeling/relation_model_models.py similarity index 100% rename from appengine/ndb/modeling/relation_model_models.py rename to appengine/standard/ndb/modeling/relation_model_models.py diff --git a/appengine/ndb/modeling/relation_model_models_test.py b/appengine/standard/ndb/modeling/relation_model_models_test.py similarity index 99% rename from appengine/ndb/modeling/relation_model_models_test.py rename to appengine/standard/ndb/modeling/relation_model_models_test.py index 14d4359a298..8310132d133 100644 --- a/appengine/ndb/modeling/relation_model_models_test.py +++ b/appengine/standard/ndb/modeling/relation_model_models_test.py @@ -15,6 +15,7 @@ """Test classes for code snippet for modeling article.""" from google.appengine.ext import ndb + import relation_model_models as models diff --git a/appengine/ndb/modeling/structured_property_models.py b/appengine/standard/ndb/modeling/structured_property_models.py similarity index 100% rename from appengine/ndb/modeling/structured_property_models.py rename to appengine/standard/ndb/modeling/structured_property_models.py diff --git a/appengine/ndb/modeling/structured_property_models_test.py b/appengine/standard/ndb/modeling/structured_property_models_test.py similarity index 100% rename from appengine/ndb/modeling/structured_property_models_test.py rename to appengine/standard/ndb/modeling/structured_property_models_test.py diff --git a/appengine/standard/ndb/overview/README.md b/appengine/standard/ndb/overview/README.md new file mode 100644 index 00000000000..2dd7a8aaff5 --- /dev/null +++ b/appengine/standard/ndb/overview/README.md @@ -0,0 +1,17 @@ +## App Engine Datastore NDB Overview Sample + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/ndb/overview/README.md + +This is a sample app for Google App Engine that demonstrates the [Datastore NDB Python API](https://cloud.google.com/appengine/docs/python/ndb/). + + +These samples are used on the following documentation page: + +> https://cloud.google.com/appengine/docs/python/ndb/ + + + +Refer to the [App Engine Samples README](../../README.md) for information on how to run and deploy this sample. diff --git a/appengine/memcache/guestbook/app.yaml b/appengine/standard/ndb/overview/app.yaml similarity index 100% rename from appengine/memcache/guestbook/app.yaml rename to appengine/standard/ndb/overview/app.yaml diff --git a/appengine/ndb/transactions/favicon.ico b/appengine/standard/ndb/overview/favicon.ico similarity index 100% rename from appengine/ndb/transactions/favicon.ico rename to appengine/standard/ndb/overview/favicon.ico diff --git a/appengine/ndb/overview/index.yaml b/appengine/standard/ndb/overview/index.yaml similarity index 100% rename from appengine/ndb/overview/index.yaml rename to appengine/standard/ndb/overview/index.yaml diff --git a/appengine/standard/ndb/overview/main.py b/appengine/standard/ndb/overview/main.py new file mode 100644 index 00000000000..629d5fae169 --- /dev/null +++ b/appengine/standard/ndb/overview/main.py @@ -0,0 +1,104 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Cloud Datastore NDB API guestbook sample. + +This sample is used on this page: + https://cloud.google.com/appengine/docs/python/ndb/ + +For more information, see README.md +""" + +# [START all] +import cgi +import textwrap +import urllib + +from google.appengine.ext import ndb + +import webapp2 + + +# [START greeting] +class Greeting(ndb.Model): + """Models an individual Guestbook entry with content and date.""" + content = ndb.StringProperty() + date = ndb.DateTimeProperty(auto_now_add=True) +# [END greeting] + +# [START query] + @classmethod + def query_book(cls, ancestor_key): + return cls.query(ancestor=ancestor_key).order(-cls.date) + + +class MainPage(webapp2.RequestHandler): + def get(self): + self.response.out.write('') + guestbook_name = self.request.get('guestbook_name') + ancestor_key = ndb.Key("Book", guestbook_name or "*notitle*") + greetings = Greeting.query_book(ancestor_key).fetch(20) +# [END query] + + greeting_blockquotes = [] + for greeting in greetings: + greeting_blockquotes.append( + '

        %s
        ' % cgi.escape(greeting.content)) + + self.response.out.write(textwrap.dedent("""\ + + + {blockquotes} +
        +
        + +
        +
        + +
        +
        +
        +
        + Guestbook name: + + +
        + + """).format( + blockquotes='\n'.join(greeting_blockquotes), + sign=urllib.urlencode({'guestbook_name': guestbook_name}), + guestbook_name=cgi.escape(guestbook_name))) + + +# [START submit] +class SubmitForm(webapp2.RequestHandler): + def post(self): + # We set the parent key on each 'Greeting' to ensure each guestbook's + # greetings are in the same entity group. + guestbook_name = self.request.get('guestbook_name') + greeting = Greeting(parent=ndb.Key("Book", + guestbook_name or "*notitle*"), + content=self.request.get('content')) + greeting.put() +# [END submit] + self.redirect('/?' + urllib.urlencode( + {'guestbook_name': guestbook_name})) + + +app = webapp2.WSGIApplication([ + ('/', MainPage), + ('/sign', SubmitForm) +]) +# [END all] diff --git a/appengine/standard/ndb/overview/main_test.py b/appengine/standard/ndb/overview/main_test.py new file mode 100644 index 00000000000..1851b78db88 --- /dev/null +++ b/appengine/standard/ndb/overview/main_test.py @@ -0,0 +1,23 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import webtest + +import main + + +def test_app(testbed): + app = webtest.TestApp(main.app) + response = app.get('/') + assert response.status_int == 200 diff --git a/appengine/standard/ndb/projection_queries/README.md b/appengine/standard/ndb/projection_queries/README.md new file mode 100644 index 00000000000..ca2320b8212 --- /dev/null +++ b/appengine/standard/ndb/projection_queries/README.md @@ -0,0 +1,16 @@ +## App Engine Datastore NDB Projection Queries Samples + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/ndb/projection_queries/README.md + +This contains snippets used in the NDB projection queries documentation, +demonstrating various ways to make ndb projection queries. + + +These samples are used on the following documentation page: + +> https://cloud.google.com/appengine/docs/python/datastore/projectionqueries + + diff --git a/appengine/standard/ndb/projection_queries/snippets.py b/appengine/standard/ndb/projection_queries/snippets.py new file mode 100644 index 00000000000..fb0c3254351 --- /dev/null +++ b/appengine/standard/ndb/projection_queries/snippets.py @@ -0,0 +1,61 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.ext import ndb + + +class Article(ndb.Model): + title = ndb.StringProperty() + author = ndb.StringProperty() + tags = ndb.StringProperty(repeated=True) + + +def print_author_tags(): + query = Article.query() + articles = query.fetch(20, projection=[Article.author, Article.tags]) + for article in articles: + print(article.author) + print(article.tags) + # article.title will raise a ndb.UnprojectedPropertyError + + +class Address(ndb.Model): + type = ndb.StringProperty() # E.g., 'home', 'work' + street = ndb.StringProperty() + city = ndb.StringProperty() + + +class Contact(ndb.Model): + name = ndb.StringProperty() + addresses = ndb.StructuredProperty(Address, repeated=True) + + +def fetch_sub_properties(): + Contact.query().fetch(projection=["name", "addresses.city"]) + Contact.query().fetch(projection=[Contact.name, Contact.addresses.city]) + + +def demonstrate_ndb_grouping(): + Article.query(projection=[Article.author], group_by=[Article.author]) + Article.query(projection=[Article.author], distinct=True) + + +class Foo(ndb.Model): + A = ndb.IntegerProperty(repeated=True) + B = ndb.StringProperty(repeated=True) + + +def declare_multiple_valued_property(): + entity = Foo(A=[1, 1, 2, 3], B=['x', 'y', 'x']) + return entity diff --git a/appengine/standard/ndb/projection_queries/snippets_test.py b/appengine/standard/ndb/projection_queries/snippets_test.py new file mode 100644 index 00000000000..e21a5c414e4 --- /dev/null +++ b/appengine/standard/ndb/projection_queries/snippets_test.py @@ -0,0 +1,46 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import snippets + + +def test_print_author_tags(testbed, capsys): + snippets.Article(title="one", author="Two", tags=["three"]).put() + + snippets.print_author_tags() + + stdout, _ = capsys.readouterr() + assert 'Two' in stdout + assert 'three' in stdout + assert 'one' not in stdout + + +def test_fetch_sub_properties(testbed): + address = snippets.Address(type="home", street="college", city="staten") + address.put() + address2 = snippets.Address(type="home", street="brighton", city="staten") + address2.put() + snippets.Contact(name="one", addresses=[address, address2]).put() + + snippets.fetch_sub_properties() + + +def test_demonstrate_ndb_grouping(testbed): + snippets.Article(title="one", author="Two", tags=["three"]).put() + + snippets.demonstrate_ndb_grouping() + + +def test_declare_multiple_valued_property(testbed): + snippets.declare_multiple_valued_property() diff --git a/appengine/standard/ndb/properties/README.md b/appengine/standard/ndb/properties/README.md new file mode 100644 index 00000000000..a0f6ec2cee9 --- /dev/null +++ b/appengine/standard/ndb/properties/README.md @@ -0,0 +1,16 @@ +## App Engine Datastore NDB Properties Samples + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/ndb/properties/README.md + +This contains snippets used in the NDB properties documentation, demonstrating +various operations on ndb properties. + + +These samples are used on the following documentation page: + +> https://cloud.google.com/appengine/docs/python/datastore/entity-property-reference + + diff --git a/appengine/standard/ndb/properties/snippets.py b/appengine/standard/ndb/properties/snippets.py new file mode 100644 index 00000000000..9c98ac6701d --- /dev/null +++ b/appengine/standard/ndb/properties/snippets.py @@ -0,0 +1,150 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START notestore_imports] +from google.appengine.ext import ndb +from google.appengine.ext.ndb import msgprop +# [END notestore_imports] +from protorpc import messages + + +class Account(ndb.Model): + username = ndb.StringProperty() + userid = ndb.IntegerProperty() + email = ndb.StringProperty() + + +class Employee(ndb.Model): + full_name = ndb.StringProperty('n') + retirement_age = ndb.IntegerProperty('r') + + +class Article(ndb.Model): + title = ndb.StringProperty() + stars = ndb.IntegerProperty() + tags = ndb.StringProperty(repeated=True) + + +def create_article(): + article = Article( + title='Python versus Ruby', + stars=3, + tags=['python', 'ruby']) + article.put() + return article + + +class Address(ndb.Model): + type = ndb.StringProperty() # E.g., 'home', 'work' + street = ndb.StringProperty() + city = ndb.StringProperty() + + +class Contact(ndb.Model): + name = ndb.StringProperty() + addresses = ndb.StructuredProperty(Address, repeated=True) + + +class ContactWithLocalStructuredProperty(ndb.Model): + name = ndb.StringProperty() + addresses = ndb.LocalStructuredProperty(Address, repeated=True) + + +def create_contact(): + guido = Contact( + name='Guido', + addresses=[ + Address( + type='home', + city='Amsterdam'), + Address( + type='work', + street='Spear St', + city='SF')]) + + guido.put() + return guido + + +def create_contact_with_local_structured_property(): + guido = ContactWithLocalStructuredProperty( + name='Guido', + addresses=[ + Address( + type='home', + city='Amsterdam'), + Address( + type='work', + street='Spear St', + city='SF')]) + + guido.put() + return guido + + +class SomeEntity(ndb.Model): + name = ndb.StringProperty() + name_lower = ndb.ComputedProperty(lambda self: self.name.lower()) + + +def create_some_entity(): + entity = SomeEntity(name='Nick') + entity.put() + return entity + + +class Note(messages.Message): + text = messages.StringField(1, required=True) + when = messages.IntegerField(2) + + +class NoteStore(ndb.Model): + note = msgprop.MessageProperty(Note, indexed_fields=['when']) + name = ndb.StringProperty() + + +def create_note_store(): + my_note = Note(text='Excellent note', when=50) + + ns = NoteStore(note=my_note, name='excellent') + key = ns.put() + + new_notes = NoteStore.query(NoteStore.note.when >= 10).fetch() + return new_notes, key + + +class Notebook(messages.Message): + notes = messages.MessageField(Note, 1, repeated=True) + + +class SignedStorableNotebook(ndb.Model): + author = ndb.StringProperty() + nb = msgprop.MessageProperty( + Notebook, indexed_fields=['notes.text', 'notes.when']) + + +class Color(messages.Enum): + RED = 620 + GREEN = 495 + BLUE = 450 + + +class Part(ndb.Model): + name = ndb.StringProperty() + color = msgprop.EnumProperty(Color, required=True) + + +def print_part(): + p1 = Part(name='foo', color=Color.RED) + print p1.color # prints "RED" diff --git a/appengine/standard/ndb/properties/snippets_test.py b/appengine/standard/ndb/properties/snippets_test.py new file mode 100644 index 00000000000..0ac3e5bb8ba --- /dev/null +++ b/appengine/standard/ndb/properties/snippets_test.py @@ -0,0 +1,106 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import snippets + + +def test_account(testbed): + account = snippets.Account( + username='flan', + userid=123, + email='flan@example.com') + account.put() + + +def test_employee(testbed): + employee = snippets.Employee( + full_name='Hob Gadling', + retirement_age=600) + employee.put() + + +def test_article(testbed): + article = snippets.create_article() + assert article.title == 'Python versus Ruby' + assert article.stars == 3 + assert sorted(article.tags) == sorted(['python', 'ruby']) + + +def test_create_contact(testbed): + guido = snippets.create_contact() + assert guido.name == 'Guido' + addresses = guido.addresses + assert addresses[0].type == 'home' + assert addresses[1].type == 'work' + assert addresses[0].street is None + assert addresses[1].street == 'Spear St' + assert addresses[0].city == 'Amsterdam' + assert addresses[1].city == 'SF' + + +def test_contact_with_local_structured_property(testbed): + guido = snippets.create_contact_with_local_structured_property() + assert guido.name == 'Guido' + addresses = guido.addresses + assert addresses[0].type == 'home' + assert addresses[1].type == 'work' + + +def test_create_some_entity(testbed): + entity = snippets.create_some_entity() + assert entity.name == 'Nick' + assert entity.name_lower == 'nick' + + +def test_computed_property(testbed): + entity = snippets.create_some_entity() + entity.name = 'Nick' + assert entity.name_lower == 'nick' + entity.name = 'Nickie' + assert entity.name_lower == 'nickie' + + +def test_create_note_store(testbed): + note_stores, _ = snippets.create_note_store() + assert len(note_stores) == 1 + assert note_stores[0].name == 'excellent' + assert note_stores[0].name == 'excellent' + assert note_stores[0].note.text == 'Excellent note' + assert note_stores[0].note.when == 50 + + +def test_notebook(testbed): + note1 = snippets.Note( + text='Refused to die.', + when=1389) + note2 = snippets.Note( + text='Printed some things', + when=1489) + note3 = snippets.Note( + text='Learned to use a sword', + when=1589) + + notebook = snippets.Notebook( + notes=[note1, note2, note3]) + stored_notebook = snippets.SignedStorableNotebook( + author='Hob Gadling', + nb=notebook) + + stored_notebook.put() + + +def test_part(testbed, capsys): + snippets.print_part() + stdout, _ = capsys.readouterr() + assert stdout.strip() == 'RED' diff --git a/appengine/standard/ndb/property_subclasses/README.md b/appengine/standard/ndb/property_subclasses/README.md new file mode 100644 index 00000000000..4bb3e2fc3df --- /dev/null +++ b/appengine/standard/ndb/property_subclasses/README.md @@ -0,0 +1,16 @@ +## App Engine Datastore NDB Property Subclasses Samples + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/ndb/property_subclasses/README.md + +This contains snippets used in the NDB property subclasses documentation, +demonstrating various operations on ndb property subclasses. + + +These samples are used on the following documentation page: + +> https://cloud.google.com/appengine/docs/python/ndb/subclassprop + + diff --git a/appengine/standard/ndb/property_subclasses/my_models.py b/appengine/standard/ndb/property_subclasses/my_models.py new file mode 100644 index 00000000000..ef1a6e8bdd7 --- /dev/null +++ b/appengine/standard/ndb/property_subclasses/my_models.py @@ -0,0 +1,106 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import date + +from google.appengine.ext import ndb + + +class LongIntegerProperty(ndb.StringProperty): + def _validate(self, value): + if not isinstance(value, (int, long)): + raise TypeError('expected an integer, got %s' % repr(value)) + + def _to_base_type(self, value): + return str(value) # Doesn't matter if it's an int or a long + + def _from_base_type(self, value): + return long(value) # Always return a long + + +class BoundedLongIntegerProperty(ndb.StringProperty): + def __init__(self, bits, **kwds): + assert isinstance(bits, int) + assert bits > 0 and bits % 4 == 0 # Make it simple to use hex + super(BoundedLongIntegerProperty, self).__init__(**kwds) + self._bits = bits + + def _validate(self, value): + assert -(2 ** (self._bits - 1)) <= value < 2 ** (self._bits - 1) + + def _to_base_type(self, value): + # convert from signed -> unsigned + if value < 0: + value += 2 ** self._bits + assert 0 <= value < 2 ** self._bits + # Return number as a zero-padded hex string with correct number of + # digits: + return '%0*x' % (self._bits // 4, value) + + def _from_base_type(self, value): + value = int(value, 16) + if value >= 2 ** (self._bits - 1): + value -= 2 ** self._bits + return value + + +# Define an entity class holding some long integers. +class MyModel(ndb.Model): + name = ndb.StringProperty() + abc = LongIntegerProperty(default=0) + xyz = LongIntegerProperty(repeated=True) + + +class FuzzyDate(object): + def __init__(self, first, last=None): + assert isinstance(first, date) + assert last is None or isinstance(last, date) + self.first = first + self.last = last or first + + +class FuzzyDateModel(ndb.Model): + first = ndb.DateProperty() + last = ndb.DateProperty() + + +class FuzzyDateProperty(ndb.StructuredProperty): + def __init__(self, **kwds): + super(FuzzyDateProperty, self).__init__(FuzzyDateModel, **kwds) + + def _validate(self, value): + assert isinstance(value, FuzzyDate) + + def _to_base_type(self, value): + return FuzzyDateModel(first=value.first, last=value.last) + + def _from_base_type(self, value): + return FuzzyDate(value.first, value.last) + + +class MaybeFuzzyDateProperty(FuzzyDateProperty): + def _validate(self, value): + if isinstance(value, date): + return FuzzyDate(value) # Must return the converted value! + # Otherwise, return None and leave validation to the base class + + +# Class to record historic people and events in their life. +class HistoricPerson(ndb.Model): + name = ndb.StringProperty() + birth = FuzzyDateProperty() + death = FuzzyDateProperty() + # Parallel lists: + event_dates = FuzzyDateProperty(repeated=True) + event_names = ndb.StringProperty(repeated=True) diff --git a/appengine/standard/ndb/property_subclasses/snippets.py b/appengine/standard/ndb/property_subclasses/snippets.py new file mode 100644 index 00000000000..50c04a192d5 --- /dev/null +++ b/appengine/standard/ndb/property_subclasses/snippets.py @@ -0,0 +1,57 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import date + +import my_models + + +def create_entity(): + # Create an entity and write it to the Datastore. + entity = my_models.MyModel(name='booh', xyz=[10**100, 6**666]) + assert entity.abc == 0 + key = entity.put() + return key + + +def read_and_update_entity(key): + # Read an entity back from the Datastore and update it. + entity = key.get() + entity.abc += 1 + entity.xyz.append(entity.abc//3) + entity.put() + + +def query_entity(): + # Query for a MyModel entity whose xyz contains 6**666. + # (NOTE: using ordering operations don't work, but == does.) + results = my_models.MyModel.query( + my_models.MyModel.xyz == 6**666).fetch(10) + return results + + +def create_and_query_columbus(): + columbus = my_models.HistoricPerson( + name='Christopher Columbus', + birth=my_models.FuzzyDate(date(1451, 8, 22), date(1451, 10, 31)), + death=my_models.FuzzyDate(date(1506, 5, 20)), + event_dates=[my_models.FuzzyDate( + date(1492, 1, 1), date(1492, 12, 31))], + event_names=['Discovery of America']) + columbus.put() + + # Query for historic people born no later than 1451. + results = my_models.HistoricPerson.query( + my_models.HistoricPerson.birth.last <= date(1451, 12, 31)).fetch() + return results diff --git a/appengine/standard/ndb/property_subclasses/snippets_test.py b/appengine/standard/ndb/property_subclasses/snippets_test.py new file mode 100644 index 00000000000..8aa621620fa --- /dev/null +++ b/appengine/standard/ndb/property_subclasses/snippets_test.py @@ -0,0 +1,98 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime + +from google.appengine.ext import ndb +import pytest + +import my_models +import snippets + + +def test_create_entity(testbed): + assert my_models.MyModel.query().count() == 0 + snippets.create_entity() + entities = my_models.MyModel.query().fetch() + assert len(entities) == 1 + assert entities[0].name == 'booh' + + +def test_read_and_update_entity(testbed): + key = snippets.create_entity() + entities = my_models.MyModel.query().fetch() + assert len(entities) == 1 + assert entities[0].abc == 0 + len_xyz = len(entities[0].xyz) + + snippets.read_and_update_entity(key) + entities = my_models.MyModel.query().fetch() + assert len(entities) == 1 + assert entities[0].abc == 1 + assert len(entities[0].xyz) == len_xyz + 1 + + +def test_query_entity(testbed): + results = snippets.query_entity() + assert len(results) == 0 + + snippets.create_entity() + results = snippets.query_entity() + assert len(results) == 1 + + +def test_create_columbus(testbed): + entities = snippets.create_and_query_columbus() + assert len(entities) == 1 + assert entities[0].name == 'Christopher Columbus' + assert (entities[0].birth.first < entities[0].birth.last < + entities[0].death.first) + + +def test_long_integer_property(testbed): + with pytest.raises(TypeError): + my_models.MyModel( + name='not integer test', + xyz=['not integer']) + + +def test_bounded_long_integer_property(testbed): + class TestBoundedLongIntegerProperty(ndb.Model): + num = my_models.BoundedLongIntegerProperty(4) + + # Test out of the bounds + with pytest.raises(AssertionError): + TestBoundedLongIntegerProperty(num=0xF) + with pytest.raises(AssertionError): + TestBoundedLongIntegerProperty(num=-0xF) + + # This should work + working_instance = TestBoundedLongIntegerProperty(num=0b111) + assert working_instance.num == 0b111 + working_instance.num = 0b10 + assert working_instance.num == 2 + + +def test_maybe_fuzzy_date_property(testbed): + class TestMaybeFuzzyDateProperty(ndb.Model): + first_date = my_models.MaybeFuzzyDateProperty() + second_date = my_models.MaybeFuzzyDateProperty() + + two_types_of_dates = TestMaybeFuzzyDateProperty( + first_date=my_models.FuzzyDate( + datetime.date(1984, 2, 27), datetime.date(1984, 2, 29)), + second_date=datetime.date(2015, 6, 27)) + + assert isinstance(two_types_of_dates.first_date, my_models.FuzzyDate) + assert isinstance(two_types_of_dates.second_date, my_models.FuzzyDate) diff --git a/appengine/standard/ndb/queries/README.md b/appengine/standard/ndb/queries/README.md new file mode 100644 index 00000000000..1e13adceaa0 --- /dev/null +++ b/appengine/standard/ndb/queries/README.md @@ -0,0 +1,16 @@ +## App Engine Datastore NDB Queries Samples + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/ndb/queries/README.md + +This contains snippets used in the NDB queries documentation, demonstrating +various ways to make ndb queries. + + +These samples are used on the following documentation page: + +> https://cloud.google.com/appengine/docs/python/ndb/queries + + diff --git a/appengine/standard/ndb/queries/guestbook.py b/appengine/standard/ndb/queries/guestbook.py new file mode 100644 index 00000000000..b99948f8b54 --- /dev/null +++ b/appengine/standard/ndb/queries/guestbook.py @@ -0,0 +1,75 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import cgi + +from google.appengine.datastore.datastore_query import Cursor +from google.appengine.ext import ndb +import webapp2 + + +class Greeting(ndb.Model): + """Models an individual Guestbook entry with content and date.""" + content = ndb.StringProperty() + date = ndb.DateTimeProperty(auto_now_add=True) + + @classmethod + def query_book(cls, ancestor_key): + return cls.query(ancestor=ancestor_key).order(-cls.date) + + +class MainPage(webapp2.RequestHandler): + GREETINGS_PER_PAGE = 20 + + def get(self): + guestbook_name = self.request.get('guestbook_name') + ancestor_key = ndb.Key('Book', guestbook_name or '*notitle*') + greetings = Greeting.query_book(ancestor_key).fetch( + self.GREETINGS_PER_PAGE) + + self.response.out.write('') + + for greeting in greetings: + self.response.out.write( + '
        %s
        ' % cgi.escape(greeting.content)) + + self.response.out.write('') + + +class List(webapp2.RequestHandler): + GREETINGS_PER_PAGE = 10 + + def get(self): + """Handles requests like /list?cursor=1234567.""" + cursor = Cursor(urlsafe=self.request.get('cursor')) + greets, next_cursor, more = Greeting.query().fetch_page( + self.GREETINGS_PER_PAGE, start_cursor=cursor) + + self.response.out.write('') + + for greeting in greets: + self.response.out.write( + '
        %s
        ' % cgi.escape(greeting.content)) + + if more and next_cursor: + self.response.out.write('More...' % + next_cursor.urlsafe()) + + self.response.out.write('') + + +app = webapp2.WSGIApplication([ + ('/', MainPage), + ('/list', List), +], debug=True) diff --git a/appengine/standard/ndb/queries/guestbook_test.py b/appengine/standard/ndb/queries/guestbook_test.py new file mode 100644 index 00000000000..9ddfcd26902 --- /dev/null +++ b/appengine/standard/ndb/queries/guestbook_test.py @@ -0,0 +1,67 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +from google.appengine.ext import ndb +import pytest +import webtest + +import guestbook + + +@pytest.fixture +def app(testbed): + return webtest.TestApp(guestbook.app) + + +def test_main(app): + # Add a greeting to find + guestbook.Greeting( + content='Hello world', + parent=ndb.Key('Book', 'brane3')).put() + + # Add a greeting to not find. + guestbook.Greeting( + content='Flat sheet', + parent=ndb.Key('Book', 'brane2')).put() + + response = app.get('/?guestbook_name=brane3') + + assert response.status_int == 200 + assert 'Hello world' in response.body + assert 'Flat sheet' not in response.body + + +def test_list(app): + # Add greetings to find + for i in range(11): + guestbook.Greeting(content='Greeting {}'.format(i)).put() + + response = app.get('/list') + assert response.status_int == 200 + + assert 'Greeting 0' in response.body + assert 'Greeting 9' in response.body + assert 'Greeting 10' not in response.body + + next_page = re.search(r'href="([^"]+)"', response.body).group(1) + assert next_page is not None + + response = app.get(next_page) + assert response.status_int == 200 + + assert 'Greeting 0' not in response.body + assert 'Greeting 10' in response.body + assert 'More' not in response.body diff --git a/appengine/standard/ndb/queries/snippets.py b/appengine/standard/ndb/queries/snippets.py new file mode 100644 index 00000000000..987bd8c1c62 --- /dev/null +++ b/appengine/standard/ndb/queries/snippets.py @@ -0,0 +1,236 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.ext import ndb + +from guestbook import Greeting +from snippets_models import (Account, Address, Article, + Bar, Contact, Employee, FlexEmployee, Manager) + + +def query_account_equality(): + query = Account.query(Account.userid == 42) + return query + + +def query_account_inequality(): + query = Account.query(Account.userid >= 40) + return query + + +def query_account_multiple_filters(): + query = Account.query(Account.userid >= 40, Account.userid < 50) + return query + + +def query_account_in_steps(): + query1 = Account.query() # Retrieve all Account entitites + query2 = query1.filter(Account.userid >= 40) # Filter on userid >= 40 + query3 = query2.filter(Account.userid < 50) # Filter on userid < 50 too + return query1, query2, query3 + + +def query_article_inequality(): + query = Article.query(Article.tags != 'perl') + return query + + +def query_article_inequality_explicit(): + query = Article.query(ndb.OR(Article.tags < 'perl', + Article.tags > 'perl')) + return query + + +def articles_with_tags_example(): + # [START included_in_inequality] + Article(title='Perl + Python = Parrot', + stars=5, + tags=['python', 'perl']) + # [END included_in_inequality] + # [START excluded_from_inequality] + Article(title='Introduction to Perl', + stars=3, + tags=['perl']) + # [END excluded_from_inequality] + + +def query_article_in(): + query = Article.query(Article.tags.IN(['python', 'ruby', 'php'])) + return query + + +def query_article_in_equivalent(): + query = Article.query(ndb.OR(Article.tags == 'python', + Article.tags == 'ruby', + Article.tags == 'php')) + return query + + +def query_article_nested(): + query = Article.query(ndb.AND(Article.tags == 'python', + ndb.OR(Article.tags.IN(['ruby', 'jruby']), + ndb.AND(Article.tags == 'php', + Article.tags != 'perl')))) + return query + + +def query_greeting_order(): + query = Greeting.query().order(Greeting.content, -Greeting.date) + return query + + +def query_greeting_multiple_orders(): + query = Greeting.query().order(Greeting.content).order(-Greeting.date) + return query + + +def query_purchase_with_customer_key(): + # [START purchase_with_customer_key_models] + class Customer(ndb.Model): + name = ndb.StringProperty() + + class Purchase(ndb.Model): + customer = ndb.KeyProperty(kind=Customer) + price = ndb.IntegerProperty() + # [END purchase_with_customer_key_models] + + def query_purchases_for_customer_via_key(customer_entity): + purchases = Purchase.query( + Purchase.customer == customer_entity.key).fetch() + return purchases + + return Customer, Purchase, query_purchases_for_customer_via_key + + +def query_purchase_with_ancestor_key(): + # [START purchase_with_ancestor_key_models] + class Customer(ndb.Model): + name = ndb.StringProperty() + + class Purchase(ndb.Model): + price = ndb.IntegerProperty() + # [END purchase_with_ancestor_key_models] + + def create_purchase_for_customer_with_ancestor(customer_entity): + purchase = Purchase(parent=customer_entity.key) + return purchase + + def query_for_purchases_of_customer_with_ancestor(customer_entity): + purchases = Purchase.query(ancestor=customer_entity.key).fetch() + return purchases + + return (Customer, Purchase, + create_purchase_for_customer_with_ancestor, + query_for_purchases_of_customer_with_ancestor) + + +def print_query(): + print(Employee.query()) + # -> Query(kind='Employee') + print(Employee.query(ancestor=ndb.Key(Manager, 1))) + # -> Query(kind='Employee', ancestor=Key('Manager', 1)) + + +def query_contact_with_city(): + query = Contact.query(Contact.addresses.city == 'Amsterdam') + return query + + +def query_contact_sub_entities_beware(): + query = Contact.query(Contact.addresses.city == 'Amsterdam', # Beware! + Contact.addresses.street == 'Spear St') + return query + + +def query_contact_multiple_values_in_single_sub_entity(): + query = Contact.query(Contact.addresses == Address(city='San Francisco', + street='Spear St')) + return query + + +def query_properties_named_by_string_on_expando(): + property_to_query = 'location' + query = FlexEmployee.query(ndb.GenericProperty(property_to_query) == 'SF') + return query + + +def query_properties_named_by_string_for_defined_properties(keyword, value): + query = Article.query(Article._properties[keyword] == value) + return query + + +def query_properties_named_by_string_using_getattr(keyword, value): + query = Article.query(getattr(Article, keyword) == value) + return query + + +def order_query_results_by_property(keyword): + expando_query = FlexEmployee.query().order(ndb.GenericProperty('location')) + + property_query = Article.query().order(Article._properties[keyword]) + + return expando_query, property_query + + +def print_query_keys(query): + for key in query.iter(keys_only=True): + print(key) + + +def reverse_queries(): + # Set up. + q = Bar.query() + q_forward = q.order(Bar.key) + q_reverse = q.order(-Bar.key) + + # Fetch a page going forward. + bars, cursor, more = q_forward.fetch_page(10) + + # Fetch the same page going backward. + r_bars, r_cursor, r_more = q_reverse.fetch_page(10, start_cursor=cursor) + + return (bars, cursor, more), (r_bars, r_cursor, r_more) + + +def fetch_message_accounts_inefficient(message_query): + message_account_pairs = [] + for message in message_query: + key = ndb.Key('Account', message.userid) + account = key.get() + message_account_pairs.append((message, account)) + + return message_account_pairs + + +def fetch_message_accounts_efficient(message_query): + def callback(message): + key = ndb.Key('Account', message.userid) + account = key.get() + return message, account + + message_account_pairs = message_query.map(callback) + # Now message_account_pairs is a list of (message, account) tuples. + return message_account_pairs + + +def fetch_good_articles_using_gql_with_explicit_bind(): + query = ndb.gql("SELECT * FROM Article WHERE stars > :1") + query2 = query.bind(3) + + return query, query2 + + +def fetch_good_articles_using_gql_with_inlined_bind(): + query = ndb.gql("SELECT * FROM Article WHERE stars > :1", 3) + return query diff --git a/appengine/standard/ndb/queries/snippets_models.py b/appengine/standard/ndb/queries/snippets_models.py new file mode 100644 index 00000000000..be497850e34 --- /dev/null +++ b/appengine/standard/ndb/queries/snippets_models.py @@ -0,0 +1,65 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.ext import ndb + + +class Account(ndb.Model): + username = ndb.StringProperty() + userid = ndb.IntegerProperty() + email = ndb.StringProperty() + + +class Address(ndb.Model): + type = ndb.StringProperty() # E.g., 'home', 'work' + street = ndb.StringProperty() + city = ndb.StringProperty() + + +class Contact(ndb.Model): + name = ndb.StringProperty() + addresses = ndb.StructuredProperty(Address, repeated=True) + + +class Article(ndb.Model): + title = ndb.StringProperty() + stars = ndb.IntegerProperty() + tags = ndb.StringProperty(repeated=True) + + +class ArticleWithDifferentDatastoreName(ndb.Model): + title = ndb.StringProperty('t') + + +class Employee(ndb.Model): + full_name = ndb.StringProperty('n') + retirement_age = ndb.IntegerProperty('r') + + +class Manager(ndb.Model): + pass + + +class FlexEmployee(ndb.Expando): + name = ndb.StringProperty() + age = ndb.IntegerProperty() + + +class Bar(ndb.Model): + pass + + +class Message(ndb.Model): + content = ndb.StringProperty() + userid = ndb.IntegerProperty() diff --git a/appengine/standard/ndb/queries/snippets_test.py b/appengine/standard/ndb/queries/snippets_test.py new file mode 100644 index 00000000000..d8e6ddef79c --- /dev/null +++ b/appengine/standard/ndb/queries/snippets_test.py @@ -0,0 +1,388 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from guestbook import Greeting + +import snippets +from snippets_models import (Account, Address, Article, + Bar, Contact, FlexEmployee, Message) + + +def test_query_account_equality(testbed): + Account(userid=42).put() + Account(userid=43).put() + + query = snippets.query_account_equality() + accounts = query.fetch() + + assert len(accounts) == 1 + assert accounts[0].userid == 42 + + +def test_query_account_inequality(testbed): + Account(userid=32).put() + Account(userid=42).put() + Account(userid=43).put() + + query = snippets.query_account_inequality() + accounts = query.fetch() + + assert len(accounts) == 2 + assert all(a.userid > 40 for a in accounts) + + +def test_query_account_multiple_filters(testbed): + Account(userid=40).put() + Account(userid=49).put() + Account(userid=50).put() + Account(userid=6).put() + Account(userid=62).put() + + query = snippets.query_account_multiple_filters() + accounts = query.fetch() + + assert len(accounts) == 2 + assert all(40 <= a.userid < 50 for a in accounts) + + +def test_query_account_in_steps(testbed): + Account(userid=40).put() + Account(userid=49).put() + Account(userid=50).put() + Account(userid=6).put() + Account(userid=62).put() + + _, _, query = snippets.query_account_in_steps() + accounts = query.fetch() + + assert len(accounts) == 2 + assert all(40 <= a.userid < 50 for a in accounts) + + +def test_query_article_inequality(testbed): + Article(tags=['perl']).put() + Article(tags=['perl', 'python']).put() + + query = snippets.query_article_inequality() + articles = query.fetch() + + assert len(articles) == 1 + + +def test_query_article_inequality_explicit(testbed): + Article(tags=['perl']).put() + Article(tags=['perl', 'python']).put() + + query = snippets.query_article_inequality_explicit() + articles = query.fetch() + + assert len(articles) == 1 + + +def test_articles_with_tags_example(testbed): + snippets.articles_with_tags_example() + + +def test_query_article_in(testbed): + Article(tags=['perl']).put() + Article(tags=['perl', 'python']).put() + Article(tags=['ruby']).put() + Article(tags=['php']).put() + + query = snippets.query_article_in() + articles = query.fetch() + + assert len(articles) == 3 + + +def test_query_article_in_equivalent(testbed): + Article(tags=['perl']).put() + Article(tags=['perl', 'python']).put() + Article(tags=['ruby']).put() + Article(tags=['php']).put() + + query = snippets.query_article_in_equivalent() + articles = query.fetch() + + assert len(articles) == 3 + + +def test_query_article_nested(testbed): + Article(tags=['python']).put() # excluded - no non-python + Article(tags=['ruby']).put() # excluded - no python + Article(tags=['python', 'ruby']).put() # included + Article(tags=['python', 'jruby']).put() # included + Article(tags=['python', 'ruby', 'jruby']).put() # included + Article(tags=['python', 'php']).put() # included + Article(tags=['python', 'perl']).put() # excluded + + query = snippets.query_article_nested() + articles = query.fetch() + assert len(articles) == 4 + + +def test_query_greeting_order(testbed): + Greeting(content='3').put() + Greeting(content='2').put() + Greeting(content='1').put() + Greeting(content='2').put() + + query = snippets.query_greeting_order() + greetings = query.fetch() + + assert (greetings[0].content < greetings[1].content < greetings[3].content) + assert greetings[1].content == greetings[2].content + assert greetings[1].date > greetings[2].date + + +def test_query_greeting_multiple_orders(testbed): + Greeting(content='3').put() + Greeting(content='2').put() + Greeting(content='1').put() + Greeting(content='2').put() + + query = snippets.query_greeting_multiple_orders() + greetings = query.fetch() + + assert (greetings[0].content < greetings[1].content < greetings[3].content) + assert greetings[1].content == greetings[2].content + assert greetings[1].date > greetings[2].date + + +def test_query_purchase_with_customer_key(testbed): + Customer, Purchase, do_query = snippets.query_purchase_with_customer_key() + charles = Customer(name='Charles') + charles.put() + snoop_key = Customer(name='Snoop').put() + Purchase(price=123, customer=charles.key).put() + Purchase(price=234, customer=snoop_key).put() + + purchases = do_query(charles) + assert len(purchases) == 1 + assert purchases[0].price == 123 + + +def test_query_purchase_with_ancestor_key(testbed): + Customer, Purchase, create_purchase, do_query = ( + snippets.query_purchase_with_ancestor_key()) + charles = Customer(name='Charles') + charles.put() + snoop = Customer(name='Snoop') + snoop.put() + + charles_purchase = create_purchase(charles) + charles_purchase.price = 123 + charles_purchase.put() + + snoop_purchase = create_purchase(snoop) + snoop_purchase.price = 234 + snoop_purchase.put() + + purchases = do_query(snoop) + assert len(purchases) == 1 + assert purchases[0].price == 234 + + +def test_print_query(testbed, capsys): + snippets.print_query() + stdout, _ = capsys.readouterr() + + assert '' in stdout + + +def test_query_contact_with_city(testbed): + address = Address(type='home', street='Spear St', city='Amsterdam') + address.put() + Contact(name='Bertus Aafjes', addresses=[address]).put() + address1 = Address(type='home', street='Damrak St', city='Amsterdam') + address1.put() + address2 = Address(type='work', street='Spear St', city='San Francisco') + address2.put() + Contact(name='Willem Jan Aalders', addresses=[address1, address2]).put() + address = Address(type='home', street='29th St', city='San Francisco') + address.put() + Contact(name='Hans Aarsman', addresses=[address]).put() + + query = snippets.query_contact_with_city() + contacts = query.fetch() + + assert len(contacts) == 2 + + +def test_query_contact_sub_entities_beware(testbed): + address = Address(type='home', street='Spear St', city='Amsterdam') + address.put() + Contact(name='Bertus Aafjes', addresses=[address]).put() + address1 = Address(type='home', street='Damrak St', city='Amsterdam') + address1.put() + address2 = Address(type='work', street='Spear St', city='San Francisco') + address2.put() + Contact(name='Willem Jan Aalders', addresses=[address1, address2]).put() + address = Address(type='home', street='29th St', city='San Francisco') + address.put() + Contact(name='Hans Aarsman', addresses=[address]).put() + + query = snippets.query_contact_sub_entities_beware() + contacts = query.fetch() + + assert len(contacts) == 2 + for contact in contacts: + assert ('Spear St' in [a.street for a in contact.addresses] or + 'Amsterdam' in [a.city for a in contact.addresses]) + + +def test_query_contact_multiple_values_in_single_sub_entity(testbed): + address = Address(type='home', street='Spear St', city='Amsterdam') + address.put() + Contact(name='Bertus Aafjes', addresses=[address]).put() + address1 = Address(type='home', street='Damrak St', city='Amsterdam') + address1.put() + address2 = Address(type='work', street='Spear St', city='San Francisco') + address2.put() + Contact(name='Willem Jan Aalders', addresses=[address1, address2]).put() + address = Address(type='home', street='29th St', city='San Francisco') + address.put() + Contact(name='Hans Aarsman', addresses=[address]).put() + + query = snippets.query_contact_multiple_values_in_single_sub_entity() + contacts = query.fetch() + + assert len(contacts) == 1 + assert any(a.city == 'San Francisco' and a.street == 'Spear St' + for a in contacts[0].addresses) + + +def test_query_properties_named_by_string_on_expando(testbed): + FlexEmployee(location='SF').put() + FlexEmployee(location='Amsterdam').put() + + query = snippets.query_properties_named_by_string_on_expando() + employees = query.fetch() + assert len(employees) == 1 + + +def test_query_properties_named_by_string_for_defined_properties(testbed): + Article(title='from').put() + Article(title='to').put() + + query = snippets.query_properties_named_by_string_for_defined_properties( + 'title', 'from') + articles = query.fetch() + + assert len(articles) == 1 + + +def test_query_properties_named_by_string_using_getattr(testbed): + Article(title='from').put() + Article(title='to').put() + + query = snippets.query_properties_named_by_string_using_getattr( + 'title', 'from') + articles = query.fetch() + + assert len(articles) == 1 + + +def test_order_query_results_by_property(testbed): + Article(title='2').put() + Article(title='1').put() + FlexEmployee(location=2).put() + FlexEmployee(location=1).put() + expando_query, property_query = snippets.order_query_results_by_property( + 'title') + + assert expando_query.fetch()[0].location == 1 + assert property_query.fetch()[0].title == '1' + + +def test_print_query_keys(testbed, capsys): + for i in range(3): + Article(title='title {}'.format(i)).put() + + snippets.print_query_keys(Article.query()) + + stdout, _ = capsys.readouterr() + assert "Key('Article'" in stdout + + +def test_reverse_queries(testbed): + for i in range(11): + Bar().put() + + (bars, cursor, more), (r_bars, r_cursor, r_more) = ( + snippets.reverse_queries()) + + assert len(bars) == 10 + assert len(r_bars) == 10 + + for prev_bar, bar in zip(bars, bars[1:]): + assert prev_bar.key < bar.key + + for prev_bar, bar in zip(r_bars, r_bars[1:]): + assert prev_bar.key > bar.key + + +def test_fetch_message_accounts_inefficient(testbed): + for i in range(1, 6): + Account(username='Account %s' % i, id=i).put() + Message(content='Message %s' % i, userid=i).put() + + message_account_pairs = snippets.fetch_message_accounts_inefficient( + Message.query().order(Message.userid)) + + assert len(message_account_pairs) == 5 + + print repr(message_account_pairs) + for i in range(1, 6): + message, account = message_account_pairs[i - 1] + assert message.content == 'Message %s' % i + assert account.username == 'Account %s' % i + + +def test_fetch_message_accounts_efficient(testbed): + for i in range(1, 6): + Account(username='Account %s' % i, id=i).put() + Message(content='Message %s' % i, userid=i).put() + + message_account_pairs = snippets.fetch_message_accounts_efficient( + Message.query().order(Message.userid)) + + assert len(message_account_pairs) == 5 + + for i in range(1, 6): + message, account = message_account_pairs[i - 1] + assert message.content == 'Message %s' % i + assert account.username == 'Account %s' % i + + +def test_fetch_good_articles_using_gql_with_explicit_bind(testbed): + for i in range(1, 6): + Article(stars=i).put() + + query, query2 = snippets.fetch_good_articles_using_gql_with_explicit_bind() + articles = query2.fetch() + + assert len(articles) == 2 + assert all(a.stars > 3 for a in articles) + + +def test_fetch_good_articles_using_gql_with_inlined_bind(testbed): + for i in range(1, 6): + Article(stars=i).put() + + query = snippets.fetch_good_articles_using_gql_with_inlined_bind() + articles = query.fetch() + + assert len(articles) == 2 + assert all(a.stars > 3 for a in articles) diff --git a/appengine/standard/ndb/schema_update/app.yaml b/appengine/standard/ndb/schema_update/app.yaml new file mode 100644 index 00000000000..f20b30eb39e --- /dev/null +++ b/appengine/standard/ndb/schema_update/app.yaml @@ -0,0 +1,17 @@ +runtime: python27 +api_version: 1 +threadsafe: true + +builtins: +# Deferred is required to use google.appengine.ext.deferred. +- deferred: on + +handlers: +- url: /.* + script: main.app + +libraries: +- name: webapp2 + version: "2.5.2" +- name: jinja2 + version: "2.6" diff --git a/appengine/standard/ndb/schema_update/main.py b/appengine/standard/ndb/schema_update/main.py new file mode 100644 index 00000000000..60071850539 --- /dev/null +++ b/appengine/standard/ndb/schema_update/main.py @@ -0,0 +1,139 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sample application that shows how to perform a "schema migration" using +Google Cloud Datastore. + +This application uses one model named "Pictures" but two different versions +of it. v2 contains two extra fields. The application shows how to +populate these new fields onto entities that existed prior to adding the +new fields to the model class. +""" +# [START imports] +import logging +import os + +from google.appengine.ext import deferred +from google.appengine.ext import ndb +import jinja2 +import webapp2 + +import models_v1 +import models_v2 + + +JINJA_ENVIRONMENT = jinja2.Environment( + loader=jinja2.FileSystemLoader( + os.path.join(os.path.dirname(__file__), 'templates')), + extensions=['jinja2.ext.autoescape'], + autoescape=True) +# [END imports] + + +# [START display_entities] +class DisplayEntitiesHandler(webapp2.RequestHandler): + """Displays the current set of entities and options to add entities + or update the schema.""" + def get(self): + # Force ndb to use v2 of the model by re-loading it. + reload(models_v2) + + entities = models_v2.Picture.query().fetch() + template_values = { + 'entities': entities, + } + + template = JINJA_ENVIRONMENT.get_template('index.html') + self.response.write(template.render(template_values)) +# [END display_entities] + + +# [START add_entities] +class AddEntitiesHandler(webapp2.RequestHandler): + """Adds new entities using the v1 schema.""" + def post(self): + # Force ndb to use v1 of the model by re-loading it. + reload(models_v1) + + # Save some example data. + ndb.put_multi([ + models_v1.Picture(author='Alice', name='Sunset'), + models_v1.Picture(author='Bob', name='Sunrise') + ]) + + self.response.write(""" + Entities created. View entities. + """) +# [END add_entities] + + +# [START update_schema] +class UpdateSchemaHandler(webapp2.RequestHandler): + """Queues a task to start updating the model schema.""" + def post(self): + deferred.defer(update_schema_task) + self.response.write(""" + Schema update started. Check the console for task progress. + View entities. + """) + + +def update_schema_task(cursor=None, num_updated=0, batch_size=100): + """Task that handles updating the models' schema. + + This is started by + UpdateSchemaHandler. It scans every entity in the datastore for the + Picture model and re-saves it so that it has the new schema fields. + """ + + # Force ndb to use v2 of the model by re-loading it. + reload(models_v2) + + # Get all of the entities for this Model. + query = models_v2.Picture.query() + pictures, next_cursor, more = query.fetch_page( + batch_size, start_cursor=cursor) + + to_put = [] + for picture in pictures: + # Give the new fields default values. + # If you added new fields and were okay with the default values, you + # would not need to do this. + picture.num_votes = 1 + picture.avg_rating = 5 + to_put.append(picture) + + # Save the updated entities. + if to_put: + ndb.put_multi(to_put) + num_updated += len(to_put) + logging.info( + 'Put {} entities to Datastore for a total of {}'.format( + len(to_put), num_updated)) + + # If there are more entities, re-queue this task for the next page. + if more: + deferred.defer( + update_schema_task, cursor=next_cursor, num_updated=num_updated) + else: + logging.debug( + 'update_schema_task complete with {0} updates!'.format( + num_updated)) +# [END update_schema] + + +app = webapp2.WSGIApplication([ + ('/', DisplayEntitiesHandler), + ('/add_entities', AddEntitiesHandler), + ('/update_schema', UpdateSchemaHandler)]) diff --git a/appengine/standard/ndb/schema_update/main_test.py b/appengine/standard/ndb/schema_update/main_test.py new file mode 100644 index 00000000000..eb4a0aa15a3 --- /dev/null +++ b/appengine/standard/ndb/schema_update/main_test.py @@ -0,0 +1,62 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.ext import deferred +import pytest +import webtest + +import main +import models_v1 +import models_v2 + + +@pytest.fixture +def app(testbed): + yield webtest.TestApp(main.app) + + +def test_app(app): + response = app.get('/') + assert response.status_int == 200 + + +def test_add_entities(app): + response = app.post('/add_entities') + assert response.status_int == 200 + response = app.get('/') + assert response.status_int == 200 + assert 'Author: Bob' in response.body + assert 'Name: Sunrise' in response.body + assert 'Author: Alice' in response.body + assert 'Name: Sunset' in response.body + + +def test_update_schema(app, testbed): + reload(models_v1) + test_model = models_v1.Picture(author='Test', name='Test') + test_model.put() + + response = app.post('/update_schema') + assert response.status_int == 200 + + # Run the queued task. + tasks = testbed.taskqueue_stub.get_filtered_tasks() + assert len(tasks) == 1 + deferred.run(tasks[0].payload) + + # Check the updated items + reload(models_v2) + updated_model = test_model.key.get() + assert updated_model.num_votes == 1 + assert updated_model.avg_rating == 5.0 diff --git a/appengine/standard/ndb/schema_update/models_v1.py b/appengine/standard/ndb/schema_update/models_v1.py new file mode 100644 index 00000000000..ea84bc9b869 --- /dev/null +++ b/appengine/standard/ndb/schema_update/models_v1.py @@ -0,0 +1,20 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.ext import ndb + + +class Picture(ndb.Model): + author = ndb.StringProperty() + name = ndb.StringProperty(default='') diff --git a/appengine/standard/ndb/schema_update/models_v2.py b/appengine/standard/ndb/schema_update/models_v2.py new file mode 100644 index 00000000000..8f2c79ff3c6 --- /dev/null +++ b/appengine/standard/ndb/schema_update/models_v2.py @@ -0,0 +1,23 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.ext import ndb + + +class Picture(ndb.Model): + author = ndb.StringProperty() + name = ndb.StringProperty(default='') + # Two new fields + num_votes = ndb.IntegerProperty(default=0) + avg_rating = ndb.FloatProperty(default=0) diff --git a/appengine/standard/ndb/schema_update/templates/index.html b/appengine/standard/ndb/schema_update/templates/index.html new file mode 100644 index 00000000000..cc3b7ca8782 --- /dev/null +++ b/appengine/standard/ndb/schema_update/templates/index.html @@ -0,0 +1,47 @@ +{# +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + + + + + +

        If you've just added or updated entities, you may need to refresh + the page to see the changes due to + eventual consitency.

        + {% for entity in entities %} +

        + Author: {{entity.author}}, + Name: {{entity.name}}, + {% if 'num_votes' in entity._values %} + Votes: {{entity.num_votes}}, + {% endif %} + {% if 'avg_rating' in entity._values %} + Average Rating: {{entity.avg_rating}} + {% endif %} +

        + {% endfor %} + {% if entities|length == 0 %} +
        + +
        + {% endif %} + {% if entities|length > 0 %} +
        + +
        + {% endif %} + + diff --git a/appengine/standard/ndb/transactions/README.md b/appengine/standard/ndb/transactions/README.md new file mode 100644 index 00000000000..30620273f2f --- /dev/null +++ b/appengine/standard/ndb/transactions/README.md @@ -0,0 +1,19 @@ +## App Engine Datastore NDB Transactions Sample + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/ndb/transactions/README.md + +This is a sample app for Google App Engine that demonstrates the [NDB Transactions Python API](https://cloud.google.com/appengine/docs/python/ndb/transactions) + +This app presents a list of notes. After you submit a note with a particular title, you may not change that note or submit a new note with the same title. There are multiple note pages available. + + +These samples are used on the following documentation page: + +> https://cloud.google.com/appengine/docs/python/ndb/transactions + + + +Refer to the [App Engine Samples README](../../README.md) for information on how to run and deploy this sample. diff --git a/appengine/ndb/transactions/app.yaml b/appengine/standard/ndb/transactions/app.yaml similarity index 100% rename from appengine/ndb/transactions/app.yaml rename to appengine/standard/ndb/transactions/app.yaml diff --git a/appengine/ndb/transactions/appengine_config.py b/appengine/standard/ndb/transactions/appengine_config.py similarity index 100% rename from appengine/ndb/transactions/appengine_config.py rename to appengine/standard/ndb/transactions/appengine_config.py diff --git a/appengine/standard/ndb/transactions/favicon.ico b/appengine/standard/ndb/transactions/favicon.ico new file mode 100644 index 00000000000..23c553a2966 Binary files /dev/null and b/appengine/standard/ndb/transactions/favicon.ico differ diff --git a/appengine/ndb/transactions/main.py b/appengine/standard/ndb/transactions/main.py similarity index 100% rename from appengine/ndb/transactions/main.py rename to appengine/standard/ndb/transactions/main.py diff --git a/appengine/standard/ndb/transactions/main_test.py b/appengine/standard/ndb/transactions/main_test.py new file mode 100644 index 00000000000..cd19af5bffc --- /dev/null +++ b/appengine/standard/ndb/transactions/main_test.py @@ -0,0 +1,50 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +import main + + +@pytest.fixture +def app(testbed): + main.app.config['TESTING'] = True + return main.app.test_client() + + +def test_index(app): + rv = app.get('/') + assert 'Permanent note page' in rv.data + assert rv.status == '200 OK' + + +def test_post(app): + rv = app.post('/add', data=dict( + note_title='Title', + note_text='Text' + ), follow_redirects=True) + assert rv.status == '200 OK' + + +def test_there(app): + rv = app.post('/add', data=dict( + note_title='Title', + note_text='New' + ), follow_redirects=True) + rv = app.post('/add', data=dict( + note_title='Title', + note_text='There' + ), follow_redirects=True) + assert 'Already there' in rv.data + assert rv.status == '200 OK' diff --git a/appengine/standard/ndb/transactions/requirements.txt b/appengine/standard/ndb/transactions/requirements.txt new file mode 100644 index 00000000000..f2e1e506599 --- /dev/null +++ b/appengine/standard/ndb/transactions/requirements.txt @@ -0,0 +1 @@ +Flask==1.0.2 diff --git a/appengine/standard/pubsub/README.md b/appengine/standard/pubsub/README.md new file mode 100755 index 00000000000..1b5e1f0c886 --- /dev/null +++ b/appengine/standard/pubsub/README.md @@ -0,0 +1,77 @@ +# Python Google Cloud Pub/Sub sample for Google App Engine Standard Environment + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/pubsub/README.md + +This demonstrates how to send and receive messages using [Google Cloud Pub/Sub](https://cloud.google.com/pubsub) on [Google App Engine Standard Environment](https://cloud.google.com/appengine/docs/standard/). + +## Setup + +Before you can run or deploy the sample, you will need to do the following: + +1. Enable the Cloud Pub/Sub API in the [Google Developers Console](https://console.developers.google.com/project/_/apiui/apiview/pubsub/overview). + +2. Create a topic and subscription. + + $ gcloud pubsub topics create [your-topic-name] + $ gcloud pubsub subscriptions create [your-subscription-name] \ + --topic [your-topic-name] \ + --push-endpoint \ + https://[your-app-id].appspot.com/_ah/push-handlers/receive_messages/token=[your-token] \ + --ack-deadline 30 + +3. Update the environment variables in ``app.yaml``. + +## Running locally + +Refer to the [top-level README](../README.md) for instructions on running and deploying. + +When running locally, you can use the [Google Cloud SDK](https://cloud.google.com/sdk) to provide authentication to use Google Cloud APIs: + + $ gcloud init + +Install dependencies, preferably with a virtualenv: + + $ virtualenv env + $ source env/bin/activate + $ pip install -r requirements.txt + +Then set environment variables before starting your application: + + $ export GOOGLE_CLOUD_PROJECT=[your-project-name] + $ export PUBSUB_VERIFICATION_TOKEN=[your-verification-token] + $ export PUBSUB_TOPIC=[your-topic] + $ python main.py + +### Simulating push notifications + +The application can send messages locally, but it is not able to receive push messages locally. You can, however, simulate a push message by making an HTTP request to the local push notification endpoint. There is an included ``sample_message.json``. You can use +``curl`` or [httpie](https://github.com/jkbrzt/httpie) to POST this: + + $ curl -i --data @sample_message.json ":8080/_ah/push-handlers/receive_messages?token=[your-token]" + +Or + + $ http POST ":8080/_ah/push-handlers/receive_messages?token=[your-token]" < sample_message.json + +Response: + + HTTP/1.0 200 OK + Content-Length: 2 + Content-Type: text/html; charset=utf-8 + Date: Mon, 10 Aug 2015 17:52:03 GMT + Server: Werkzeug/0.10.4 Python/2.7.10 + + OK + +After the request completes, you can refresh ``localhost:8080`` and see the message in the list of received messages. + +## Running on App Engine + +Deploy using `gcloud`: + + gcloud app deploy app.yaml + +You can now access the application at `https://your-app-id.appspot.com`. You can use the form to submit messages, but it's non-deterministic which instance of your application will receive the notification. You can send multiple messages and refresh the page to see the received message. diff --git a/appengine/standard/pubsub/app.yaml b/appengine/standard/pubsub/app.yaml new file mode 100755 index 00000000000..f750c378b42 --- /dev/null +++ b/appengine/standard/pubsub/app.yaml @@ -0,0 +1,19 @@ +runtime: python27 +api_version: 1 +threadsafe: yes + +handlers: +- url: / + script: main.app + +- url: /_ah/push-handlers/.* + script: main.app + login: admin + +#[START env] +env_variables: + PUBSUB_TOPIC: your-topic + # This token is used to verify that requests originate from your + # application. It can be any sufficiently random string. + PUBSUB_VERIFICATION_TOKEN: 1234abc +#[END env] diff --git a/appengine/standard/pubsub/main.py b/appengine/standard/pubsub/main.py new file mode 100755 index 00000000000..54c08810d29 --- /dev/null +++ b/appengine/standard/pubsub/main.py @@ -0,0 +1,94 @@ +# Copyright 2018 Google, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START app] +import base64 +import json +import logging +import os + +from flask import current_app, Flask, render_template, request +from googleapiclient.discovery import build + + +app = Flask(__name__) + +# Configure the following environment variables via app.yaml +# This is used in the push request handler to verify that the request came from +# pubsub and originated from a trusted source. +app.config['PUBSUB_VERIFICATION_TOKEN'] = \ + os.environ['PUBSUB_VERIFICATION_TOKEN'] +app.config['PUBSUB_TOPIC'] = os.environ['PUBSUB_TOPIC'] +app.config['GCLOUD_PROJECT'] = os.environ['GOOGLE_CLOUD_PROJECT'] + + +# Global list to storage messages received by this instance. +MESSAGES = [] + + +# [START index] +@app.route('/', methods=['GET', 'POST']) +def index(): + if request.method == 'GET': + return render_template('index.html', messages=MESSAGES) + + data = request.form.get('payload', 'Example payload').encode('utf-8') + + service = build('pubsub', 'v1') + topic_path = 'projects/{project_id}/topics/{topic}'.format( + project_id=app.config['GCLOUD_PROJECT'], + topic=app.config['PUBSUB_TOPIC'] + ) + service.projects().topics().publish( + topic=topic_path, body={ + "messages": [{ + "data": base64.b64encode(data) + }] + }).execute() + + return 'OK', 200 +# [END index] + + +# [START push] +@app.route('/_ah/push-handlers/receive_messages', methods=['POST']) +def receive_messages_handler(): + if (request.args.get('token', '') != + current_app.config['PUBSUB_VERIFICATION_TOKEN']): + return 'Invalid request', 400 + + envelope = json.loads(request.data.decode('utf-8')) + payload = base64.b64decode(envelope['message']['data']) + + MESSAGES.append(payload) + + # Returning any 2xx status indicates successful receipt of the message. + return 'OK', 200 +# [END push] + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
        {}
        + See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) +# [END app] diff --git a/appengine/standard/pubsub/main_test.py b/appengine/standard/pubsub/main_test.py new file mode 100755 index 00000000000..f2afb5e4a69 --- /dev/null +++ b/appengine/standard/pubsub/main_test.py @@ -0,0 +1,70 @@ +# Copyright 2018 Google, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import json +import os + +import pytest + +import main + + +@pytest.fixture +def client(): + main.app.testing = True + return main.app.test_client() + + +def test_index(client): + r = client.get('/') + assert r.status_code == 200 + + +def test_post_index(client): + r = client.post('/', data={'payload': 'Test payload'}) + assert r.status_code == 200 + + +def test_push_endpoint(client): + url = '/_ah/push-handlers/receive_messages?token=' + \ + os.environ['PUBSUB_VERIFICATION_TOKEN'] + + r = client.post( + url, + data=json.dumps({ + "message": { + "data": base64.b64encode( + u'Test message'.encode('utf-8') + ).decode('utf-8') + } + }) + ) + + assert r.status_code == 200 + + # Make sure the message is visible on the home page. + r = client.get('/') + assert r.status_code == 200 + assert 'Test message' in r.data.decode('utf-8') + + +def test_push_endpoint_errors(client): + # no token + r = client.post('/_ah/push-handlers/receive_messages') + assert r.status_code == 400 + + # invalid token + r = client.post('/_ah/push-handlers/receive_messages?token=bad') + assert r.status_code == 400 diff --git a/appengine/standard/pubsub/requirements.txt b/appengine/standard/pubsub/requirements.txt new file mode 100755 index 00000000000..1e5f24522d4 --- /dev/null +++ b/appengine/standard/pubsub/requirements.txt @@ -0,0 +1,2 @@ +Flask==1.0.2 +google-api-python-client==1.7.8 diff --git a/appengine/standard/pubsub/sample_message.json b/appengine/standard/pubsub/sample_message.json new file mode 100755 index 00000000000..8fe62d23fb9 --- /dev/null +++ b/appengine/standard/pubsub/sample_message.json @@ -0,0 +1,5 @@ +{ + "message": { + "data": "SGVsbG8sIFdvcmxkIQ==" + } +} diff --git a/appengine/standard/pubsub/templates/index.html b/appengine/standard/pubsub/templates/index.html new file mode 100755 index 00000000000..70ebcdf43a6 --- /dev/null +++ b/appengine/standard/pubsub/templates/index.html @@ -0,0 +1,38 @@ +{# +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + + + + Pub/Sub Python on Google App Engine Standard Environment + + +
        +

        Messages received by this instance:

        +
          + {% for message in messages: %} +
        • {{message}}
          • + {% endfor %} +
        +

        Note: because your application is likely running multiple instances, each instance will have a different list of messages.

        +
        + +
        + + +
        + + + diff --git a/appengine/standard/remote_api/app.yaml b/appengine/standard/remote_api/app.yaml new file mode 100644 index 00000000000..fd511db7564 --- /dev/null +++ b/appengine/standard/remote_api/app.yaml @@ -0,0 +1,6 @@ +runtime: python27 +api_version: 1 +threadsafe: true + +builtins: +- remote_api: on diff --git a/appengine/standard/remote_api/client.py b/appengine/standard/remote_api/client.py new file mode 100644 index 00000000000..e8463e15cb9 --- /dev/null +++ b/appengine/standard/remote_api/client.py @@ -0,0 +1,54 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sample app that uses the Google App Engine Remote API to make calls to the +live App Engine APIs.""" + +# [START all] + +import argparse + +try: + import dev_appserver + dev_appserver.fix_sys_path() +except ImportError: + print('Please make sure the App Engine SDK is in your PYTHONPATH.') + raise + +from google.appengine.ext import ndb +from google.appengine.ext.remote_api import remote_api_stub + + +def main(project_id): + remote_api_stub.ConfigureRemoteApiForOAuth( + '{}.appspot.com'.format(project_id), + '/_ah/remote_api') + + # List the first 10 keys in the datastore. + keys = ndb.Query().fetch(10, keys_only=True) + + for key in keys: + print(key) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('project_id', help='Your Project ID.') + + args = parser.parse_args() + + main(args.project_id) +# [END all] diff --git a/appengine/standard/requests/README.md b/appengine/standard/requests/README.md new file mode 100644 index 00000000000..5c3ad04397c --- /dev/null +++ b/appengine/standard/requests/README.md @@ -0,0 +1,15 @@ +## App Engine Requests Docs Snippets + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/requests/README.md + +These snippets demonstrate various aspects of App Engine Python request handling. + + +These samples are used on the following documentation page: + +> https://cloud.google.com/appengine/docs/python/how-requests-are-handled + + diff --git a/appengine/standard/requests/app.yaml b/appengine/standard/requests/app.yaml new file mode 100644 index 00000000000..bdd5bf04dc2 --- /dev/null +++ b/appengine/standard/requests/app.yaml @@ -0,0 +1,7 @@ +runtime: python27 +api_version: 1 +threadsafe: yes + +handlers: +- url: .* # This regex directs all routes to main.app + script: main.app diff --git a/appengine/standard/requests/main.py b/appengine/standard/requests/main.py new file mode 100644 index 00000000000..fe3c1667248 --- /dev/null +++ b/appengine/standard/requests/main.py @@ -0,0 +1,66 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Sample application that demonstrates various aspects of App Engine's request +handling. +""" + +import os +import time + +import webapp2 + + +# [START gae_python_request_timer] +class TimerHandler(webapp2.RequestHandler): + def get(self): + from google.appengine.runtime import DeadlineExceededError + + try: + time.sleep(70) + self.response.write('Completed.') + except DeadlineExceededError: + self.response.clear() + self.response.set_status(500) + self.response.out.write( + 'The request did not complete in time.') +# [END gae_python_request_timer] + + +# [START gae_python_environment] +class PrintEnvironmentHandler(webapp2.RequestHandler): + def get(self): + self.response.headers['Content-Type'] = 'text/plain' + for key, value in os.environ.iteritems(): + self.response.out.write( + "{} = {}\n".format(key, value)) +# [END gae_python_environment] + + +# [START gae_python_request_ids] +class RequestIdHandler(webapp2.RequestHandler): + def get(self): + self.response.headers['Content-Type'] = 'text/plain' + request_id = os.environ.get('REQUEST_LOG_ID') + self.response.write( + 'REQUEST_LOG_ID={}'.format(request_id)) +# [END gae_python_request_ids] + + +app = webapp2.WSGIApplication([ + ('/timer', TimerHandler), + ('/environment', PrintEnvironmentHandler), + ('/requestid', RequestIdHandler) +], debug=True) diff --git a/appengine/standard/requests/main_test.py b/appengine/standard/requests/main_test.py new file mode 100644 index 00000000000..c978856824f --- /dev/null +++ b/appengine/standard/requests/main_test.py @@ -0,0 +1,45 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from google.appengine.runtime import DeadlineExceededError +import mock +import webtest + +import main + + +def test_timer(testbed): + app = webtest.TestApp(main.app) + + with mock.patch('main.time.sleep') as sleep_mock: + sleep_mock.side_effect = DeadlineExceededError() + app.get('/timer', status=500) + assert sleep_mock.called + + +def test_environment(testbed): + app = webtest.TestApp(main.app) + response = app.get('/environment') + assert response.headers['Content-Type'] == 'text/plain' + assert response.body + + +def test_request_id(testbed): + app = webtest.TestApp(main.app) + os.environ['REQUEST_LOG_ID'] = '1234' + response = app.get('/requestid') + assert response.headers['Content-Type'] == 'text/plain' + assert '1234' in response.body diff --git a/appengine/resources/app.yaml b/appengine/standard/resources/app.yaml similarity index 100% rename from appengine/resources/app.yaml rename to appengine/standard/resources/app.yaml diff --git a/appengine/standard/search/snippets/snippets.py b/appengine/standard/search/snippets/snippets.py new file mode 100644 index 00000000000..de4d373ad26 --- /dev/null +++ b/appengine/standard/search/snippets/snippets.py @@ -0,0 +1,287 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime + +from google.appengine.api import search + + +def simple_search(index): + index.search('rose water') + + +def search_date(index): + index.search('1776-07-04') + + +def search_terms(index): + # search for documents with pianos that cost less than $5000 + index.search("product = piano AND price < 5000") + + +def create_document(): + document = search.Document( + # Setting the doc_id is optional. If omitted, the search service will + # create an identifier. + doc_id='PA6-5000', + fields=[ + search.TextField(name='customer', value='Joe Jackson'), + search.HtmlField( + name='comment', value='this is marked up text'), + search.NumberField(name='number_of_visits', value=7), + search.DateField(name='last_visit', value=datetime.now()), + search.DateField( + name='birthday', value=datetime(year=1960, month=6, day=19)), + search.GeoField( + name='home_location', value=search.GeoPoint(37.619, -122.37)) + ]) + return document + + +def add_document_to_index(document): + index = search.Index('products') + index.put(document) + + +def add_document_and_get_doc_id(documents): + index = search.Index('products') + results = index.put(documents) + document_ids = [document.id for document in results] + return document_ids + + +def get_document_by_id(): + index = search.Index('products') + + # Get a single document by ID. + document = index.get("AZ125") + + # Get a range of documents starting with a given ID. + documents = index.get_range(start_id="AZ125", limit=100) + + return document, documents + + +def query_index(): + index = search.Index('products') + query_string = 'product: piano AND price < 5000' + + results = index.search(query_string) + + for scored_document in results: + print(scored_document) + + +def delete_all_in_index(index): + # index.get_range by returns up to 100 documents at a time, so we must + # loop until we've deleted all items. + while True: + # Use ids_only to get the list of document IDs in the index without + # the overhead of getting the entire document. + document_ids = [ + document.doc_id + for document + in index.get_range(ids_only=True)] + + # If no IDs were returned, we've deleted everything. + if not document_ids: + break + + # Delete the documents for the given IDs + index.delete(document_ids) + + +def async_query(index): + futures = [index.search_async('foo'), index.search_async('bar')] + results = [future.get_result() for future in futures] + return results + + +def query_options(): + index = search.Index('products') + query_string = "product: piano AND price < 5000" + + # Create sort options to sort on price and brand. + sort_price = search.SortExpression( + expression='price', + direction=search.SortExpression.DESCENDING, + default_value=0) + sort_brand = search.SortExpression( + expression='brand', + direction=search.SortExpression.DESCENDING, + default_value="") + sort_options = search.SortOptions(expressions=[sort_price, sort_brand]) + + # Create field expressions to add new fields to the scored documents. + price_per_note_expression = search.FieldExpression( + name='price_per_note', expression='price/88') + ivory_expression = search.FieldExpression( + name='ivory', expression='snippet("ivory", summary, 120)') + + # Create query options using the sort options and expressions created + # above. + query_options = search.QueryOptions( + limit=25, + returned_fields=['model', 'price', 'description'], + returned_expressions=[price_per_note_expression, ivory_expression], + sort_options=sort_options) + + # Build the Query and run the search + query = search.Query(query_string=query_string, options=query_options) + results = index.search(query) + for scored_document in results: + print(scored_document) + + +def query_results(index, query_string): + result = index.search(query_string) + total_matches = result.number_found + list_of_docs = result.results + number_of_docs_returned = len(list_of_docs) + return total_matches, list_of_docs, number_of_docs_returned + + +def query_offset(index, query_string): + offset = 0 + + while True: + # Build the query using the current offset. + options = search.QueryOptions(offset=offset) + query = search.Query(query_string=query_string, options=options) + + # Get the results + results = index.search(query) + + number_retrieved = len(results.results) + if number_retrieved == 0: + break + + # Add the number of documents found to the offset, so that the next + # iteration will grab the next page of documents. + offset += number_retrieved + + # Process the matched documents + for document in results: + print(document) + + +def query_cursor(index, query_string): + cursor = search.Cursor() + + while cursor: + # Build the query using the cursor. + options = search.QueryOptions(cursor=cursor) + query = search.Query(query_string=query_string, options=options) + + # Get the results and the next cursor + results = index.search(query) + cursor = results.cursor + + for document in results: + print(document) + + +def query_per_document_cursor(index, query_string): + cursor = search.Cursor(per_result=True) + + # Build the query using the cursor. + options = search.QueryOptions(cursor=cursor) + query = search.Query(query_string=query_string, options=options) + + # Get the results. + results = index.search(query) + + document_cursor = None + for document in results: + # discover some document of interest and grab its cursor, for this + # sample we'll just use the first document. + document_cursor = document.cursor + break + + # Start the next search from the document of interest. + if document_cursor is None: + return + + options = search.QueryOptions(cursor=document_cursor) + query = search.Query(query_string=query_string, options=options) + results = index.search(query) + + for document in results: + print(document) + + +def saving_and_restoring_cursor(cursor): + # Convert the cursor to a web-safe string. + cursor_string = cursor.web_safe_string + # Restore the cursor from a web-safe string. + cursor = search.Cursor(web_safe_string=cursor_string) + + +def add_faceted_document(index): + document = search.Document( + doc_id='doc1', + fields=[ + search.AtomField(name='name', value='x86')], + facets=[ + search.AtomFacet(name='type', value='computer'), + search.NumberFacet(name='ram_size_gb', value=8)]) + + index.put(document) + + +def facet_discovery(index): + # Create the query and enable facet discovery. + query = search.Query('name:x86', enable_facet_discovery=True) + results = index.search(query) + + for facet in results.facets: + print('facet {}.'.format(facet.name)) + for value in facet.values: + print('{}: count={}, refinement_token={}'.format( + value.label, value.count, value.refinement_token)) + + +def facet_by_name(index): + # Create the query and specify to only return the "type" and "ram_size_gb" + # facets. + query = search.Query('name:x86', return_facets=['type', 'ram_size_gb']) + results = index.search(query) + + for facet in results.facets: + print('facet {}'.format(facet.name)) + for value in facet.values: + print('{}: count={}, refinement_token={}'.format( + value.label, value.count, value.refinement_token)) + + +def facet_by_name_and_value(index): + # Create the query and specify to return the "type" facet with values + # "computer" and "printer" and the "ram_size_gb" facet with value in the + # ranges [0,4), [4, 8), and [8, max]. + query = search.Query( + 'name:x86', + return_facets=[ + search.FacetRequest('type', values=['computer', 'printer']), + search.FacetRequest('ram_size_gb', ranges=[ + search.FacetRange(end=4), + search.FacetRange(start=4, end=8), + search.FacetRange(start=8)]) + ]) + + results = index.search(query) + for facet in results.facets: + print('facet {}'.format(facet.name)) + for value in facet.values: + print('{}: count={}, refinement_token={}'.format( + value.label, value.count, value.refinement_token)) diff --git a/appengine/standard/search/snippets/snippets_test.py b/appengine/standard/search/snippets/snippets_test.py new file mode 100644 index 00000000000..3d22bbb5561 --- /dev/null +++ b/appengine/standard/search/snippets/snippets_test.py @@ -0,0 +1,140 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from google.appengine.api import search +import pytest + +import snippets + + +@pytest.fixture +def search_stub(testbed): + testbed.init_search_stub() + + +@pytest.fixture +def index(search_stub): + return search.Index('products') + + +@pytest.fixture +def document(): + return search.Document( + doc_id='doc1', + fields=[ + search.TextField(name='title', value='Meep: A biography')]) + + +def test_simple_search(index): + snippets.simple_search(index) + + +def test_search_date(index): + snippets.search_date(index) + + +def test_search_terms(index): + snippets.search_terms(index) + + +def test_create_document(): + assert snippets.create_document() + + +def test_add_document_to_index(index, document): + snippets.add_document_to_index(document) + assert index.get(document.doc_id) + + +def test_add_document_and_get_doc_id(index, document): + ids = snippets.add_document_and_get_doc_id([document]) + assert ids == [document.doc_id] + + +def test_get_document_by_id(index): + index.put(search.Document(doc_id='AZ124')) + index.put(search.Document(doc_id='AZ125')) + index.put(search.Document(doc_id='AZ126')) + + doc, docs = snippets.get_document_by_id() + + assert doc.doc_id == 'AZ125' + assert [x.doc_id for x in docs] == ['AZ125', 'AZ126'] + + +def test_query_index(index): + snippets.query_index() + + +def test_delete_all_in_index(index, document): + index.put(document) + snippets.delete_all_in_index(index) + assert not index.get(document.doc_id) + + +def test_async_query(index): + snippets.async_query(index) + + +def test_query_options(index): + snippets.query_options() + + +def test_query_results(index, document): + index.put(document) + total_matches, list_of_docs, number_of_docs_returned = ( + snippets.query_results(index, 'meep')) + + assert total_matches == 1 + assert list_of_docs + assert number_of_docs_returned == 1 + + +def test_query_offset(index, document): + index.put(document) + snippets.query_offset(index, 'meep') + + +def test_query_cursor(index, document): + index.put(document) + snippets.query_cursor(index, 'meep') + + +def test_query_per_document_cursor(index, document): + index.put(document) + snippets.query_per_document_cursor(index, 'meep') + + +def test_saving_and_restoring_cursor(index): + snippets.saving_and_restoring_cursor(search.Cursor()) + + +def test_add_faceted_document(index): + snippets.add_faceted_document(index) + + +def test_facet_discovery(index): + snippets.add_faceted_document(index) + snippets.facet_discovery(index) + + +def test_facet_by_name(index): + snippets.add_faceted_document(index) + snippets.facet_by_name(index) + + +def test_facet_by_name_and_value(index): + snippets.add_faceted_document(index) + snippets.facet_by_name_and_value(index) diff --git a/appengine/standard/sendgrid/README.md b/appengine/standard/sendgrid/README.md new file mode 100644 index 00000000000..07eb5a32a88 --- /dev/null +++ b/appengine/standard/sendgrid/README.md @@ -0,0 +1,18 @@ +# Sendgrid & Google App Engine + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/sendgrid/README.md + +This sample application demonstrates how to use [Sendgrid with Google App Engine](https://cloud.google.com/appengine/docs/python/mail/sendgrid) + +Refer to the [App Engine Samples README](../../README.md) for information on how to run and deploy this sample. + +# Setup + +Before running this sample: + +1. You will need a [Sendgrid account](http://sendgrid.com/partner/google). +2. Update the `SENGRID_DOMAIN_NAME` and `SENGRID_API_KEY` constants in `main.py`. You can use +the [Sendgrid sandbox domain](https://support.sendgrid.com/hc/en-us/articles/201995663-Safely-Test-Your-Sending-Speed). diff --git a/appengine/standard/sendgrid/app.yaml b/appengine/standard/sendgrid/app.yaml new file mode 100644 index 00000000000..42ad35ed2a8 --- /dev/null +++ b/appengine/standard/sendgrid/app.yaml @@ -0,0 +1,7 @@ +runtime: python27 +threadsafe: yes +api_version: 1 + +handlers: +- url: .* + script: main.app diff --git a/appengine/standard/sendgrid/appengine_config.py b/appengine/standard/sendgrid/appengine_config.py new file mode 100644 index 00000000000..c903d9a0ac5 --- /dev/null +++ b/appengine/standard/sendgrid/appengine_config.py @@ -0,0 +1,18 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.ext import vendor + +# Add any libraries installed in the "lib" folder. +vendor.add('lib') diff --git a/appengine/standard/sendgrid/main.py b/appengine/standard/sendgrid/main.py new file mode 100644 index 00000000000..1191b3e75cc --- /dev/null +++ b/appengine/standard/sendgrid/main.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START sendgrid-imp] +import sendgrid +from sendgrid.helpers import mail +# [END sendgrid-imp] +import webapp2 + +# make a secure connection to SendGrid +# [START sendgrid-config] +SENDGRID_API_KEY = 'your-sendgrid-api-key' +SENDGRID_SENDER = 'your-sendgrid-sender' +# [END sendgrid-config] + + +def send_simple_message(recipient): + # [START sendgrid-send] + + sg = sendgrid.SendGridAPIClient(apikey=SENDGRID_API_KEY) + + to_email = mail.Email(recipient) + from_email = mail.Email(SENDGRID_SENDER) + subject = 'This is a test email' + content = mail.Content('text/plain', 'Example message.') + message = mail.Mail(from_email, subject, to_email, content) + + response = sg.client.mail.send.post(request_body=message.get()) + + return response + # [END sendgrid-send] + + +class MainPage(webapp2.RequestHandler): + def get(self): + self.response.content_type = 'text/html' + self.response.write(""" + + +
        + + +
        + +""") + + +class SendEmailHandler(webapp2.RequestHandler): + def post(self): + recipient = self.request.get('recipient') + sg_response = send_simple_message(recipient) + self.response.set_status(sg_response.status_code) + self.response.write(sg_response.body) + + +app = webapp2.WSGIApplication([ + ('/', MainPage), + ('/send', SendEmailHandler) +], debug=True) diff --git a/appengine/standard/sendgrid/main_test.py b/appengine/standard/sendgrid/main_test.py new file mode 100644 index 00000000000..72c128fd71a --- /dev/null +++ b/appengine/standard/sendgrid/main_test.py @@ -0,0 +1,46 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import pytest +import webtest + +import main + + +@pytest.fixture +def app(): + return webtest.TestApp(main.app) + + +def test_get(app): + response = app.get('/') + assert response.status_int == 200 + + +@mock.patch('python_http_client.client.Client._make_request') +def test_post(make_request_mock, app): + response = mock.Mock() + response.getcode.return_value = 200 + response.read.return_value = 'OK' + response.info.return_value = {} + make_request_mock.return_value = response + + app.post('/send', { + 'recipient': 'user@example.com' + }) + + assert make_request_mock.called + request = make_request_mock.call_args[0][1] + assert 'user@example.com' in request.data diff --git a/appengine/standard/sendgrid/requirements.txt b/appengine/standard/sendgrid/requirements.txt new file mode 100644 index 00000000000..7fb6ea201dd --- /dev/null +++ b/appengine/standard/sendgrid/requirements.txt @@ -0,0 +1 @@ +sendgrid==5.6.0 diff --git a/appengine/standard/storage/.gitignore b/appengine/standard/storage/.gitignore new file mode 100644 index 00000000000..a65b41774ad --- /dev/null +++ b/appengine/standard/storage/.gitignore @@ -0,0 +1 @@ +lib diff --git a/appengine/standard/storage/api-client/README.md b/appengine/standard/storage/api-client/README.md new file mode 100644 index 00000000000..ea5e9ed6ea3 --- /dev/null +++ b/appengine/standard/storage/api-client/README.md @@ -0,0 +1,20 @@ +# Cloud Storage & Google App Engine + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/storage/api-client/README.md + +This sample demonstrates how to use the [Google Cloud Storage API](https://cloud.google.com/storage/docs/json_api/) from Google App Engine. + +Refer to the [App Engine Samples README](../README.md) for information on how to run and deploy this sample. + +## Setup + +Before running the sample: + +1. You need a Cloud Storage Bucket. You create one with [`gsutil`](https://cloud.google.com/storage/docs/gsutil): + + gsutil mb gs://your-bucket-name + +2. Update `main.py` and replace `` with your Cloud Storage bucket. diff --git a/appengine/standard/storage/api-client/app.yaml b/appengine/standard/storage/api-client/app.yaml new file mode 100644 index 00000000000..42ad35ed2a8 --- /dev/null +++ b/appengine/standard/storage/api-client/app.yaml @@ -0,0 +1,7 @@ +runtime: python27 +threadsafe: yes +api_version: 1 + +handlers: +- url: .* + script: main.app diff --git a/appengine/bigquery/appengine_config.py b/appengine/standard/storage/api-client/appengine_config.py similarity index 100% rename from appengine/bigquery/appengine_config.py rename to appengine/standard/storage/api-client/appengine_config.py diff --git a/appengine/standard/storage/api-client/main.py b/appengine/standard/storage/api-client/main.py new file mode 100644 index 00000000000..bb053ed1198 --- /dev/null +++ b/appengine/standard/storage/api-client/main.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python + +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Sample Google App Engine application that lists the objects in a Google Cloud +Storage bucket. + +For more information about Cloud Storage, see README.md in /storage. +For more information about Google App Engine, see README.md in /appengine. +""" + +import json +import StringIO + +import googleapiclient.discovery +import googleapiclient.http +import webapp2 + + +# The bucket that will be used to list objects. +BUCKET_NAME = '' + +storage = googleapiclient.discovery.build('storage', 'v1') + + +class MainPage(webapp2.RequestHandler): + def upload_object(self, bucket, file_object): + body = { + 'name': 'storage-api-client-sample-file.txt', + } + req = storage.objects().insert( + bucket=bucket, body=body, + media_body=googleapiclient.http.MediaIoBaseUpload( + file_object, 'application/octet-stream')) + resp = req.execute() + return resp + + def delete_object(self, bucket, filename): + req = storage.objects().delete(bucket=bucket, object=filename) + resp = req.execute() + return resp + + def get(self): + string_io_file = StringIO.StringIO('Hello World!') + self.upload_object(BUCKET_NAME, string_io_file) + + response = storage.objects().list(bucket=BUCKET_NAME).execute() + self.response.write( + '

        Objects.list raw response:

        ' + '
        {}
        '.format( + json.dumps(response, sort_keys=True, indent=2))) + + self.delete_object(BUCKET_NAME, 'storage-api-client-sample-file.txt') + + +app = webapp2.WSGIApplication([ + ('/', MainPage) +], debug=True) diff --git a/appengine/standard/storage/api-client/main_test.py b/appengine/standard/storage/api-client/main_test.py new file mode 100644 index 00000000000..1fc960987f6 --- /dev/null +++ b/appengine/standard/storage/api-client/main_test.py @@ -0,0 +1,34 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +import webtest + +import main + +PROJECT = os.environ['GCLOUD_PROJECT'] + + +def test_get(): + main.BUCKET_NAME = PROJECT + app = webtest.TestApp(main.app) + + response = app.get('/') + + assert response.status_int == 200 + assert re.search( + re.compile(r'.*.*items.*etag.*', re.DOTALL), + response.body) diff --git a/appengine/standard/storage/api-client/requirements.txt b/appengine/standard/storage/api-client/requirements.txt new file mode 100644 index 00000000000..7e4359ce08d --- /dev/null +++ b/appengine/standard/storage/api-client/requirements.txt @@ -0,0 +1,3 @@ +google-api-python-client==1.7.8 +google-auth==1.6.2 +google-auth-httplib2==0.0.3 diff --git a/appengine/standard/storage/appengine-client/__init__.py b/appengine/standard/storage/appengine-client/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/appengine/standard/storage/appengine-client/app.yaml b/appengine/standard/storage/appengine-client/app.yaml new file mode 100644 index 00000000000..3ec099ad09c --- /dev/null +++ b/appengine/standard/storage/appengine-client/app.yaml @@ -0,0 +1,12 @@ +runtime: python27 +api_version: 1 +threadsafe: yes + +env_variables: + +handlers: +- url: /blobstore.* + script: blobstore.app + +- url: /.* + script: main.app diff --git a/appengine/storage/appengine_config.py b/appengine/standard/storage/appengine-client/appengine_config.py similarity index 100% rename from appengine/storage/appengine_config.py rename to appengine/standard/storage/appengine-client/appengine_config.py diff --git a/appengine/standard/storage/appengine-client/main.py b/appengine/standard/storage/appengine-client/main.py new file mode 100644 index 00000000000..e5eb54aceaf --- /dev/null +++ b/appengine/standard/storage/appengine-client/main.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python + +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START sample] +"""A sample app that uses GCS client to operate on bucket and file.""" + +# [START imports] +import os + +import cloudstorage +from google.appengine.api import app_identity + +import webapp2 + +# [END imports] + +# [START retries] +cloudstorage.set_default_retry_params( + cloudstorage.RetryParams( + initial_delay=0.2, max_delay=5.0, backoff_factor=2, max_retry_period=15 + )) +# [END retries] + + +class MainPage(webapp2.RequestHandler): + """Main page for GCS demo application.""" + +# [START get_default_bucket] + def get(self): + bucket_name = os.environ.get( + 'BUCKET_NAME', app_identity.get_default_gcs_bucket_name()) + + self.response.headers['Content-Type'] = 'text/plain' + self.response.write( + 'Demo GCS Application running from Version: {}\n'.format( + os.environ['CURRENT_VERSION_ID'])) + self.response.write('Using bucket name: {}\n\n'.format(bucket_name)) +# [END get_default_bucket] + + bucket = '/' + bucket_name + filename = bucket + '/demo-testfile' + self.tmp_filenames_to_clean_up = [] + + self.create_file(filename) + self.response.write('\n\n') + + self.read_file(filename) + self.response.write('\n\n') + + self.stat_file(filename) + self.response.write('\n\n') + + self.create_files_for_list_bucket(bucket) + self.response.write('\n\n') + + self.list_bucket(bucket) + self.response.write('\n\n') + + self.list_bucket_directory_mode(bucket) + self.response.write('\n\n') + + self.delete_files() + self.response.write('\n\nThe demo ran successfully!\n') + +# [START write] + def create_file(self, filename): + """Create a file.""" + + self.response.write('Creating file {}\n'.format(filename)) + + # The retry_params specified in the open call will override the default + # retry params for this particular file handle. + write_retry_params = cloudstorage.RetryParams(backoff_factor=1.1) + with cloudstorage.open( + filename, 'w', content_type='text/plain', options={ + 'x-goog-meta-foo': 'foo', 'x-goog-meta-bar': 'bar'}, + retry_params=write_retry_params) as cloudstorage_file: + cloudstorage_file.write('abcde\n') + cloudstorage_file.write('f'*1024*4 + '\n') + self.tmp_filenames_to_clean_up.append(filename) +# [END write] + +# [START read] + def read_file(self, filename): + self.response.write( + 'Abbreviated file content (first line and last 1K):\n') + + with cloudstorage.open(filename) as cloudstorage_file: + self.response.write(cloudstorage_file.readline()) + cloudstorage_file.seek(-1024, os.SEEK_END) + self.response.write(cloudstorage_file.read()) +# [END read] + + def stat_file(self, filename): + self.response.write('File stat:\n') + + stat = cloudstorage.stat(filename) + self.response.write(repr(stat)) + + def create_files_for_list_bucket(self, bucket): + self.response.write('Creating more files for listbucket...\n') + filenames = [bucket + n for n in [ + '/foo1', '/foo2', '/bar', '/bar/1', '/bar/2', '/boo/']] + for f in filenames: + self.create_file(f) + +# [START list_bucket] + def list_bucket(self, bucket): + """Create several files and paginate through them.""" + + self.response.write('Listbucket result:\n') + + # Production apps should set page_size to a practical value. + page_size = 1 + stats = cloudstorage.listbucket(bucket + '/foo', max_keys=page_size) + while True: + count = 0 + for stat in stats: + count += 1 + self.response.write(repr(stat)) + self.response.write('\n') + + if count != page_size or count == 0: + break + stats = cloudstorage.listbucket( + bucket + '/foo', max_keys=page_size, marker=stat.filename) +# [END list_bucket] + + def list_bucket_directory_mode(self, bucket): + self.response.write('Listbucket directory mode result:\n') + for stat in cloudstorage.listbucket(bucket + '/b', delimiter='/'): + self.response.write(stat) + self.response.write('\n') + if stat.is_dir: + for subdir_file in cloudstorage.listbucket( + stat.filename, delimiter='/'): + self.response.write(' {}'.format(subdir_file)) + self.response.write('\n') + +# [START delete_files] + def delete_files(self): + self.response.write('Deleting files...\n') + for filename in self.tmp_filenames_to_clean_up: + self.response.write('Deleting file {}\n'.format(filename)) + try: + cloudstorage.delete(filename) + except cloudstorage.NotFoundError: + pass +# [END delete_files] + + +app = webapp2.WSGIApplication( + [('/', MainPage)], debug=True) +# [END sample] diff --git a/appengine/standard/storage/appengine-client/main_test.py b/appengine/standard/storage/appengine-client/main_test.py new file mode 100644 index 00000000000..c3a05cecc57 --- /dev/null +++ b/appengine/standard/storage/appengine-client/main_test.py @@ -0,0 +1,31 @@ +# Copyright 2017 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import webtest + +import main + +PROJECT = os.environ['GCLOUD_PROJECT'] + + +def test_get(testbed): + main.BUCKET_NAME = PROJECT + app = webtest.TestApp(main.app) + + response = app.get('/') + + assert response.status_int == 200 + assert 'The demo ran successfully!' in response.body diff --git a/appengine/standard/storage/appengine-client/requirements.txt b/appengine/standard/storage/appengine-client/requirements.txt new file mode 100644 index 00000000000..f2ec35f05f9 --- /dev/null +++ b/appengine/standard/storage/appengine-client/requirements.txt @@ -0,0 +1 @@ +GoogleAppEngineCloudStorageClient==1.9.22.1 diff --git a/appengine/standard/taskqueue/counter/README.md b/appengine/standard/taskqueue/counter/README.md new file mode 100644 index 00000000000..1fab717e93c --- /dev/null +++ b/appengine/standard/taskqueue/counter/README.md @@ -0,0 +1,23 @@ +# App Engine Task Queue Counter + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/taskqueue/counter/README.md + +To run this app locally, specify both `.yaml` files to `dev_appserver.py`: + + dev_appserver.py -A your-app-id app.yaml worker.yaml + +To deploy this application, specify both `.yaml` files to `appcfg.py`: + + appcfg.py update -A your-app-id -V 1 app.yaml worker.yaml + + +These samples are used on the following documentation pages: + +> +* https://cloud.google.com/appengine/docs/python/taskqueue/push/creating-handlers +* https://cloud.google.com/appengine/docs/python/taskqueue/push/creating-tasks + + diff --git a/appengine/standard/taskqueue/counter/app.yaml b/appengine/standard/taskqueue/counter/app.yaml new file mode 100644 index 00000000000..386f52e9620 --- /dev/null +++ b/appengine/standard/taskqueue/counter/app.yaml @@ -0,0 +1,8 @@ +runtime: python27 +api_version: 1 +threadsafe: true +service: default + +handlers: +- url: /.* + script: application.app diff --git a/appengine/standard/taskqueue/counter/application.py b/appengine/standard/taskqueue/counter/application.py new file mode 100644 index 00000000000..d4e82891be1 --- /dev/null +++ b/appengine/standard/taskqueue/counter/application.py @@ -0,0 +1,82 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.api import taskqueue +from google.appengine.ext import ndb +import webapp2 + + +COUNTER_KEY = 'default counter' + + +class Counter(ndb.Model): + count = ndb.IntegerProperty(indexed=False) + + +class MainPageHandler(webapp2.RequestHandler): + def get(self): + counter = Counter.get_by_id(COUNTER_KEY) + count = counter.count if counter else 0 + + self.response.write(""" + Count: {count}
        +
        + + + +
        + """.format(count=count)) + + +class EnqueueTaskHandler(webapp2.RequestHandler): + def post(self): + amount = int(self.request.get('amount')) + + task = taskqueue.add( + url='/update_counter', + target='worker', + params={'amount': amount}) + + self.response.write( + 'Task {} enqueued, ETA {}.'.format(task.name, task.eta)) + + +# AsyncEnqueueTaskHandler behaves the same as EnqueueTaskHandler, but shows +# how to queue the task using the asyncronous API. This is not wired up by +# default. To use this, change the MainPageHandler's form action to +# /enqueue_async +class AsyncEnqueueTaskHandler(webapp2.RequestHandler): + def post(self): + amount = int(self.request.get('amount')) + + queue = taskqueue.Queue(name='default') + task = taskqueue.Task( + url='/update_counter', + target='worker', + params={'amount': amount}) + + rpc = queue.add_async(task) + + # Wait for the rpc to complete and return the queued task. + task = rpc.get_result() + + self.response.write( + 'Task {} enqueued, ETA {}.'.format(task.name, task.eta)) + + +app = webapp2.WSGIApplication([ + ('/', MainPageHandler), + ('/enqueue', EnqueueTaskHandler), + ('/enqueue_async', AsyncEnqueueTaskHandler) +], debug=True) diff --git a/appengine/standard/taskqueue/counter/application_test.py b/appengine/standard/taskqueue/counter/application_test.py new file mode 100644 index 00000000000..09a19abbd72 --- /dev/null +++ b/appengine/standard/taskqueue/counter/application_test.py @@ -0,0 +1,32 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import webtest + +import application +import worker + + +def test_all(testbed, run_tasks): + test_app = webtest.TestApp(application.app) + test_worker = webtest.TestApp(worker.app) + + response = test_app.get('/') + assert '0' in response.body + + test_app.post('/enqueue', {'amount': 5}) + run_tasks(test_worker) + + response = test_app.get('/') + assert '5' in response.body diff --git a/appengine/standard/taskqueue/counter/queue.yaml b/appengine/standard/taskqueue/counter/queue.yaml new file mode 100644 index 00000000000..6b677c17a2f --- /dev/null +++ b/appengine/standard/taskqueue/counter/queue.yaml @@ -0,0 +1,4 @@ +queue: +# Change the refresh rate of the default queue from 5/s to 1/s. +- name: default + rate: 1/s diff --git a/appengine/standard/taskqueue/counter/worker.py b/appengine/standard/taskqueue/counter/worker.py new file mode 100644 index 00000000000..21e560892c4 --- /dev/null +++ b/appengine/standard/taskqueue/counter/worker.py @@ -0,0 +1,46 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START all] + +from google.appengine.ext import ndb +import webapp2 + + +COUNTER_KEY = 'default counter' + + +class Counter(ndb.Model): + count = ndb.IntegerProperty(indexed=False) + + +class UpdateCounterHandler(webapp2.RequestHandler): + def post(self): + amount = int(self.request.get('amount')) + + # This task should run at most once per second because of the datastore + # transaction write throughput. + @ndb.transactional + def update_counter(): + counter = Counter.get_or_insert(COUNTER_KEY, count=0) + counter.count += amount + counter.put() + + update_counter() + + +app = webapp2.WSGIApplication([ + ('/update_counter', UpdateCounterHandler) +], debug=True) +# [END all] diff --git a/appengine/standard/taskqueue/counter/worker.yaml b/appengine/standard/taskqueue/counter/worker.yaml new file mode 100644 index 00000000000..1994abd980f --- /dev/null +++ b/appengine/standard/taskqueue/counter/worker.yaml @@ -0,0 +1,9 @@ +runtime: python27 +api_version: 1 +threadsafe: true +service: worker + +handlers: +- url: /.* + script: worker.app + login: admin diff --git a/appengine/standard/taskqueue/pull-counter/README.md b/appengine/standard/taskqueue/pull-counter/README.md new file mode 100644 index 00000000000..f32cb1b597b --- /dev/null +++ b/appengine/standard/taskqueue/pull-counter/README.md @@ -0,0 +1,13 @@ +# App Engine Task Queue Pull Counter + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/taskqueue/pull-counter/README.md + + +These samples are used on the following documentation page: + +> https://cloud.google.com/appengine/docs/python/taskqueue/overview-pull + + diff --git a/appengine/standard/taskqueue/pull-counter/app.yaml b/appengine/standard/taskqueue/pull-counter/app.yaml new file mode 100644 index 00000000000..e2b5e55709c --- /dev/null +++ b/appengine/standard/taskqueue/pull-counter/app.yaml @@ -0,0 +1,11 @@ +runtime: python27 +api_version: 1 +threadsafe: true + +handlers: +- url: /.* + script: main.app + +libraries: +- name: jinja2 + version: 2.6 diff --git a/appengine/standard/taskqueue/pull-counter/counter.html b/appengine/standard/taskqueue/pull-counter/counter.html new file mode 100644 index 00000000000..a9f6a39d360 --- /dev/null +++ b/appengine/standard/taskqueue/pull-counter/counter.html @@ -0,0 +1,15 @@ + + + +
        + + +
        +
          +{% for counter in counters %} +
        • + {{counter.key.id()|e}}: {{counter.count|e}} +
          • +{% endfor %} + + diff --git a/appengine/standard/taskqueue/pull-counter/main.py b/appengine/standard/taskqueue/pull-counter/main.py new file mode 100644 index 00000000000..a4dfcb38ce9 --- /dev/null +++ b/appengine/standard/taskqueue/pull-counter/main.py @@ -0,0 +1,91 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START all] +"""A simple counter with App Engine pull queue.""" + +import logging +import os +import time + +from google.appengine.api import taskqueue +from google.appengine.ext import ndb +from google.appengine.runtime import apiproxy_errors +import jinja2 +import webapp2 + + +JINJA_ENV = jinja2.Environment( + loader=jinja2.FileSystemLoader(os.path.dirname(__file__))) + + +class Counter(ndb.Model): + count = ndb.IntegerProperty(indexed=False) + + +class CounterHandler(webapp2.RequestHandler): + def get(self): + template_values = {'counters': Counter.query()} + counter_template = JINJA_ENV.get_template('counter.html') + self.response.out.write(counter_template.render(template_values)) + + # [START adding_task] + def post(self): + key = self.request.get('key') + if key: + queue = taskqueue.Queue('pullq') + queue.add(taskqueue.Task(payload='', method='PULL', tag=key)) + self.redirect('/') + # [END adding_task] + + +@ndb.transactional +def update_counter(key, tasks): + counter = Counter.get_or_insert(key, count=0) + counter.count += len(tasks) + counter.put() + + +class CounterWorker(webapp2.RequestHandler): + def get(self): + """Indefinitely fetch tasks and update the datastore.""" + queue = taskqueue.Queue('pullq') + while True: + try: + tasks = queue.lease_tasks_by_tag(3600, 1000, deadline=60) + except (taskqueue.TransientError, + apiproxy_errors.DeadlineExceededError) as e: + logging.exception(e) + time.sleep(1) + continue + + if tasks: + key = tasks[0].tag + + try: + update_counter(key, tasks) + except Exception as e: + logging.exception(e) + raise + finally: + queue.delete_tasks(tasks) + + time.sleep(1) + + +app = webapp2.WSGIApplication([ + ('/', CounterHandler), + ('/_ah/start', CounterWorker) +], debug=True) +# [END all] diff --git a/appengine/standard/taskqueue/pull-counter/pullcounter_test.py b/appengine/standard/taskqueue/pull-counter/pullcounter_test.py new file mode 100644 index 00000000000..6bda3e2f40e --- /dev/null +++ b/appengine/standard/taskqueue/pull-counter/pullcounter_test.py @@ -0,0 +1,43 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from google.appengine.ext import testbed as gaetestbed +import mock +import webtest + +import main + + +def test_app(testbed): + key_name = 'foo' + + testbed.init_taskqueue_stub(root_path=os.path.dirname(__file__)) + + app = webtest.TestApp(main.app) + app.post('/', {'key': key_name}) + + tq_stub = testbed.get_stub(gaetestbed.TASKQUEUE_SERVICE_NAME) + tasks = tq_stub.get_filtered_tasks() + assert len(tasks) == 1 + assert tasks[0].name == 'task1' + + with mock.patch('main.update_counter') as mock_update: + # Force update to fail, otherwise the loop will go forever. + mock_update.side_effect = RuntimeError() + + app.get('/_ah/start', status=500) + + assert mock_update.called diff --git a/appengine/standard/taskqueue/pull-counter/queue.yaml b/appengine/standard/taskqueue/pull-counter/queue.yaml new file mode 100644 index 00000000000..f78fe75fe9e --- /dev/null +++ b/appengine/standard/taskqueue/pull-counter/queue.yaml @@ -0,0 +1,3 @@ +queue: +- name: pullq + mode: pull diff --git a/appengine/standard/taskqueue/pull-counter/worker.yaml b/appengine/standard/taskqueue/pull-counter/worker.yaml new file mode 100644 index 00000000000..05fc21d0d00 --- /dev/null +++ b/appengine/standard/taskqueue/pull-counter/worker.yaml @@ -0,0 +1,15 @@ +service: worker +api_version: 1 +runtime: python27 +instance_class: B1 +threadsafe: yes +manual_scaling: + instances: 1 + +handlers: +- url: /.* + script: main.app + +libraries: +- name: jinja2 + version: 2.6 diff --git a/appengine/standard/urlfetch/README.md b/appengine/standard/urlfetch/README.md new file mode 100644 index 00000000000..aca473b0f99 --- /dev/null +++ b/appengine/standard/urlfetch/README.md @@ -0,0 +1,12 @@ +## App Engine UrlFetch Docs Snippets + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/urlfetch/README.md + +This sample application demonstrates different ways to request a URL +on App Engine + + + diff --git a/appengine/standard/urlfetch/async/app.yaml b/appengine/standard/urlfetch/async/app.yaml new file mode 100644 index 00000000000..e93e25d8389 --- /dev/null +++ b/appengine/standard/urlfetch/async/app.yaml @@ -0,0 +1,7 @@ +runtime: python27 +api_version: 1 +threadsafe: yes + +handlers: +- url: .* + script: rpc.app diff --git a/appengine/standard/urlfetch/async/rpc.py b/appengine/standard/urlfetch/async/rpc.py new file mode 100644 index 00000000000..297e1466b65 --- /dev/null +++ b/appengine/standard/urlfetch/async/rpc.py @@ -0,0 +1,83 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +import logging + +# [START urlfetch-import] +from google.appengine.api import urlfetch +# [END urlfetch-import] +import webapp2 + + +class UrlFetchRpcHandler(webapp2.RequestHandler): + """ Demonstrates an asynchronous HTTP query using urlfetch""" + + def get(self): + # [START urlfetch-rpc] + rpc = urlfetch.create_rpc() + urlfetch.make_fetch_call(rpc, 'http://www.google.com/') + + # ... do other things ... + try: + result = rpc.get_result() + if result.status_code == 200: + text = result.content + self.response.write(text) + else: + self.response.status_int = result.status_code + self.response.write('URL returned status code {}'.format( + result.status_code)) + except urlfetch.DownloadError: + self.response.status_int = 500 + self.response.write('Error fetching URL') + # [END urlfetch-rpc] + + +class UrlFetchRpcCallbackHandler(webapp2.RequestHandler): + """ Demonstrates an asynchronous HTTP query with a callback using + urlfetch""" + + def get(self): + # [START urlfetch-rpc-callback] + def handle_result(rpc): + result = rpc.get_result() + self.response.write(result.content) + logging.info('Handling RPC in callback: result {}'.format(result)) + + urls = ['http://www.google.com', + 'http://www.github.com', + 'http://www.travis-ci.org'] + rpcs = [] + for url in urls: + rpc = urlfetch.create_rpc() + rpc.callback = functools.partial(handle_result, rpc) + urlfetch.make_fetch_call(rpc, url) + rpcs.append(rpc) + + # ... do other things ... + + # Finish all RPCs, and let callbacks process the results. + + for rpc in rpcs: + rpc.wait() + + logging.info('Done waiting for RPCs') + # [END urlfetch-rpc-callback] + + +app = webapp2.WSGIApplication([ + ('/', UrlFetchRpcHandler), + ('/callback', UrlFetchRpcCallbackHandler), +], debug=True) diff --git a/appengine/standard/urlfetch/async/rpc_test.py b/appengine/standard/urlfetch/async/rpc_test.py new file mode 100644 index 00000000000..ed729a969b4 --- /dev/null +++ b/appengine/standard/urlfetch/async/rpc_test.py @@ -0,0 +1,78 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from google.appengine.api import urlfetch +import mock +import pytest +import webtest + +import rpc + + +@pytest.fixture +def app(): + return webtest.TestApp(rpc.app) + + +@mock.patch('rpc.urlfetch') +def test_url_fetch(urlfetch_mock, app): + get_result_mock = mock.Mock( + return_value=mock.Mock( + status_code=200, + content='I\'m Feeling Lucky')) + urlfetch_mock.create_rpc = mock.Mock( + return_value=mock.Mock(get_result=get_result_mock)) + response = app.get('/') + assert response.status_int == 200 + assert 'I\'m Feeling Lucky' in response.body + + +@mock.patch('rpc.urlfetch') +def test_url_fetch_rpc_error(urlfetch_mock, app): + urlfetch_mock.DownloadError = urlfetch.DownloadError + get_result_mock = mock.Mock( + side_effect=urlfetch.DownloadError()) + urlfetch_mock.create_rpc = mock.Mock( + return_value=mock.Mock(get_result=get_result_mock)) + response = app.get('/', status=500) + assert 'Error fetching URL' in response.body + + +@mock.patch('rpc.urlfetch') +def test_url_fetch_http_error(urlfetch_mock, app): + get_result_mock = mock.Mock( + return_value=mock.Mock( + status_code=404, + content='Not Found')) + urlfetch_mock.create_rpc = mock.Mock( + return_value=mock.Mock(get_result=get_result_mock)) + response = app.get('/', status=404) + assert '404' in response.body + + +@mock.patch('rpc.urlfetch') +def test_url_post(urlfetch_mock, app): + get_result_mock = mock.Mock( + return_value=mock.Mock( + status_code=200, + content='I\'m Feeling Lucky')) + urlfetch_mock.create_rpc = mock.Mock( + return_value=mock.Mock(get_result=get_result_mock)) + + rpc_mock = mock.Mock() + urlfetch_mock.create_rpc.return_value = rpc_mock + + app.get('/callback') + rpc_mock.wait.assert_called_with() diff --git a/appengine/standard/urlfetch/requests/.gitignore b/appengine/standard/urlfetch/requests/.gitignore new file mode 100644 index 00000000000..a65b41774ad --- /dev/null +++ b/appengine/standard/urlfetch/requests/.gitignore @@ -0,0 +1 @@ +lib diff --git a/appengine/standard/urlfetch/requests/app.yaml b/appengine/standard/urlfetch/requests/app.yaml new file mode 100644 index 00000000000..102ed60d1b5 --- /dev/null +++ b/appengine/standard/urlfetch/requests/app.yaml @@ -0,0 +1,7 @@ +runtime: python27 +api_version: 1 +threadsafe: yes + +handlers: +- url: .* + script: main.app diff --git a/appengine/standard/urlfetch/requests/appengine_config.py b/appengine/standard/urlfetch/requests/appengine_config.py new file mode 100644 index 00000000000..c903d9a0ac5 --- /dev/null +++ b/appengine/standard/urlfetch/requests/appengine_config.py @@ -0,0 +1,18 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.appengine.ext import vendor + +# Add any libraries installed in the "lib" folder. +vendor.add('lib') diff --git a/appengine/standard/urlfetch/requests/main.py b/appengine/standard/urlfetch/requests/main.py new file mode 100644 index 00000000000..05b395050ea --- /dev/null +++ b/appengine/standard/urlfetch/requests/main.py @@ -0,0 +1,49 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START app] +import logging + +from flask import Flask + +# [START imports] +import requests +import requests_toolbelt.adapters.appengine + +# Use the App Engine Requests adapter. This makes sure that Requests uses +# URLFetch. +requests_toolbelt.adapters.appengine.monkeypatch() +# [END imports] + +app = Flask(__name__) + + +@app.route('/') +def index(): + # [START requests_get] + url = 'http://www.google.com/humans.txt' + response = requests.get(url) + response.raise_for_status() + return response.text + # [END requests_get] + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
          {}
          + See logs for full stacktrace. + """.format(e), 500 +# [END app] diff --git a/appengine/standard/urlfetch/requests/main_test.py b/appengine/standard/urlfetch/requests/main_test.py new file mode 100644 index 00000000000..23666009aa4 --- /dev/null +++ b/appengine/standard/urlfetch/requests/main_test.py @@ -0,0 +1,24 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def test_index(testbed): + # Import main here so that the testbed is active, otherwise none of the + # stubs will be available and it will error. + import main + + app = main.app.test_client() + response = app.get('/') + assert response.status_code == 200 + assert 'Google' in response.data.decode('utf-8') diff --git a/appengine/standard/urlfetch/requests/requirements.txt b/appengine/standard/urlfetch/requests/requirements.txt new file mode 100644 index 00000000000..c1089c7d4dc --- /dev/null +++ b/appengine/standard/urlfetch/requests/requirements.txt @@ -0,0 +1,3 @@ +Flask==1.0.2 +requests==2.21.0 +requests-toolbelt==0.9.1 diff --git a/appengine/standard/urlfetch/snippets/app.yaml b/appengine/standard/urlfetch/snippets/app.yaml new file mode 100644 index 00000000000..102ed60d1b5 --- /dev/null +++ b/appengine/standard/urlfetch/snippets/app.yaml @@ -0,0 +1,7 @@ +runtime: python27 +api_version: 1 +threadsafe: yes + +handlers: +- url: .* + script: main.app diff --git a/appengine/standard/urlfetch/snippets/main.py b/appengine/standard/urlfetch/snippets/main.py new file mode 100644 index 00000000000..00950102429 --- /dev/null +++ b/appengine/standard/urlfetch/snippets/main.py @@ -0,0 +1,100 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Sample application that demonstrates different ways of fetching +URLS on App Engine +""" + +import logging +import urllib + +# [START urllib2-imports] +import urllib2 +# [END urllib2-imports] + +# [START urlfetch-imports] +from google.appengine.api import urlfetch +# [END urlfetch-imports] +import webapp2 + + +class UrlLibFetchHandler(webapp2.RequestHandler): + """ Demonstrates an HTTP query using urllib2""" + + def get(self): + # [START urllib-get] + url = 'http://www.google.com/humans.txt' + try: + result = urllib2.urlopen(url) + self.response.write(result.read()) + except urllib2.URLError: + logging.exception('Caught exception fetching url') + # [END urllib-get] + + +class UrlFetchHandler(webapp2.RequestHandler): + """ Demonstrates an HTTP query using urlfetch""" + + def get(self): + # [START urlfetch-get] + url = 'http://www.google.com/humans.txt' + try: + result = urlfetch.fetch(url) + if result.status_code == 200: + self.response.write(result.content) + else: + self.response.status_code = result.status_code + except urlfetch.Error: + logging.exception('Caught exception fetching url') + # [END urlfetch-get] + + +class UrlPostHandler(webapp2.RequestHandler): + """ Demonstrates an HTTP POST form query using urlfetch""" + + form_fields = { + 'first_name': 'Albert', + 'last_name': 'Johnson', + } + + def get(self): + # [START urlfetch-post] + try: + form_data = urllib.urlencode(UrlPostHandler.form_fields) + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + result = urlfetch.fetch( + url='http://localhost:8080/submit_form', + payload=form_data, + method=urlfetch.POST, + headers=headers) + self.response.write(result.content) + except urlfetch.Error: + logging.exception('Caught exception fetching url') + # [END urlfetch-post] + + +class SubmitHandler(webapp2.RequestHandler): + """ Handler that receives UrlPostHandler POST request""" + + def post(self): + self.response.out.write((self.request.get('first_name'))) + + +app = webapp2.WSGIApplication([ + ('/', UrlLibFetchHandler), + ('/url_fetch', UrlFetchHandler), + ('/url_post', UrlPostHandler), + ('/submit_form', SubmitHandler) +], debug=True) diff --git a/appengine/standard/urlfetch/snippets/main_test.py b/appengine/standard/urlfetch/snippets/main_test.py new file mode 100644 index 00000000000..4ffdf21dfac --- /dev/null +++ b/appengine/standard/urlfetch/snippets/main_test.py @@ -0,0 +1,43 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import pytest +import webtest + +import main + + +@pytest.fixture +def app(testbed): + return webtest.TestApp(main.app) + + +def test_url_lib(app): + response = app.get('/') + assert 'Google' in response.body + + +def test_url_fetch(app): + response = app.get('/url_fetch') + assert 'Google' in response.body + + +@mock.patch("main.urlfetch") +def test_url_post(urlfetch_mock, app): + urlfetch_mock.fetch = mock.Mock( + return_value=mock.Mock(content='Albert', + status_code=200)) + response = app.get('/url_post') + assert 'Albert' in response.body diff --git a/appengine/standard/users/app.yaml b/appengine/standard/users/app.yaml new file mode 100644 index 00000000000..42ad35ed2a8 --- /dev/null +++ b/appengine/standard/users/app.yaml @@ -0,0 +1,7 @@ +runtime: python27 +threadsafe: yes +api_version: 1 + +handlers: +- url: .* + script: main.app diff --git a/appengine/standard/users/main.py b/appengine/standard/users/main.py new file mode 100644 index 00000000000..0146055e6a2 --- /dev/null +++ b/appengine/standard/users/main.py @@ -0,0 +1,61 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Sample Google App Engine application that demonstrates using the Users API + +For more information about App Engine, see README.md under /appengine. +""" + +# [START all] + +from google.appengine.api import users +import webapp2 + + +class MainPage(webapp2.RequestHandler): + def get(self): + # [START user_details] + user = users.get_current_user() + if user: + nickname = user.nickname() + logout_url = users.create_logout_url('/') + greeting = 'Welcome, {}! (sign out)'.format( + nickname, logout_url) + else: + login_url = users.create_login_url('/') + greeting = 'Sign in'.format(login_url) + # [END user_details] + self.response.write( + '{}'.format(greeting)) + + +class AdminPage(webapp2.RequestHandler): + def get(self): + user = users.get_current_user() + if user: + if users.is_current_user_admin(): + self.response.write('You are an administrator.') + else: + self.response.write('You are not an administrator.') + else: + self.response.write('You are not logged in.') + + +app = webapp2.WSGIApplication([ + ('/', MainPage), + ('/admin', AdminPage) +], debug=True) + +# [END all] diff --git a/appengine/standard/users/main_test.py b/appengine/standard/users/main_test.py new file mode 100644 index 00000000000..e6cb557f35f --- /dev/null +++ b/appengine/standard/users/main_test.py @@ -0,0 +1,44 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import webtest + +import main + + +def test_index(testbed, login): + app = webtest.TestApp(main.app) + + response = app.get('/') + assert 'Login' in response.body + + login() + response = app.get('/') + assert 'Logout' in response.body + assert 'user@example.com' in response.body + + +def test_admin(testbed, login): + app = webtest.TestApp(main.app) + + response = app.get('/admin') + assert 'You are not logged in' in response.body + + login() + response = app.get('/admin') + assert 'You are not an administrator' in response.body + + login(is_admin=True) + response = app.get('/admin') + assert 'You are an administrator' in response.body diff --git a/appengine/standard/xmpp/README.md b/appengine/standard/xmpp/README.md new file mode 100644 index 00000000000..5aae873bda3 --- /dev/null +++ b/appengine/standard/xmpp/README.md @@ -0,0 +1,15 @@ +# Google App Engine XMPP + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/xmpp/README.md + +This sample includes snippets used in the [App Engine XMPP Docs](https://cloud.google.com/appengine/docs/python/xmpp/). + + +These samples are used on the following documentation page: + +> https://cloud.google.com/appengine/docs/python/xmpp/ + + diff --git a/appengine/standard/xmpp/app.yaml b/appengine/standard/xmpp/app.yaml new file mode 100644 index 00000000000..5fb971c52b5 --- /dev/null +++ b/appengine/standard/xmpp/app.yaml @@ -0,0 +1,15 @@ +runtime: python27 +threadsafe: yes +api_version: 1 + +handlers: +- url: .* + script: xmpp.app + +# [START inbound-services] +inbound_services: +- xmpp_message +# [END inbound-services] +- xmpp_presence +- xmpp_subscribe +- xmpp_error diff --git a/appengine/standard/xmpp/xmpp.py b/appengine/standard/xmpp/xmpp.py new file mode 100644 index 00000000000..fbc7b95f375 --- /dev/null +++ b/appengine/standard/xmpp/xmpp.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +# [START xmpp-imports] +from google.appengine.api import xmpp +# [END xmpp-imports] +import mock +import webapp2 + +# Mock roster of users +roster = mock.Mock() + + +class SubscribeHandler(webapp2.RequestHandler): + def post(self): + # [START track] + # Split the bare XMPP address (e.g., user@gmail.com) + # from the resource (e.g., gmail), and then add the + # address to the roster. + sender = self.request.get('from').split('/')[0] + roster.add_contact(sender) + # [END track] + + +class PresenceHandler(webapp2.RequestHandler): + def post(self): + # [START presence] + # Split the bare XMPP address (e.g., user@gmail.com) + # from the resource (e.g., gmail), and then add the + # address to the roster. + sender = self.request.get('from').split('/')[0] + xmpp.send_presence(sender, status=self.request.get('status'), + presence_show=self.request.get('show')) + # [END presence] + + +class SendPresenceHandler(webapp2.RequestHandler): + def post(self): + # [START send-presence] + jid = self.request.get('jid') + xmpp.send_presence(jid, status="My app's status") + # [END send-presence] + + +class ErrorHandler(webapp2.RequestHandler): + def post(self): + # [START error] + # In the handler for _ah/xmpp/error + # Log an error + error_sender = self.request.get('from') + error_stanza = self.request.get('stanza') + logging.error('XMPP error received from {} ({})' + .format(error_sender, error_stanza)) + # [END error] + + +class SendChatHandler(webapp2.RequestHandler): + def post(self): + # [START send-chat-to-user] + user_address = 'example@gmail.com' + msg = ('Someone has sent you a gift on Example.com. ' + 'To view: http://example.com/gifts/') + status_code = xmpp.send_message(user_address, msg) + chat_message_sent = (status_code == xmpp.NO_ERROR) + + if not chat_message_sent: + # Send an email message instead... + # [END send-chat-to-user] + pass +# [END send-chat-to-user] + + +# [START chat] +class XMPPHandler(webapp2.RequestHandler): + def post(self): + message = xmpp.Message(self.request.POST) + if message.body[0:5].lower() == 'hello': + message.reply("Greetings!") +# [END chat] + + +app = webapp2.WSGIApplication([ + ('/_ah/xmpp/message/chat/', XMPPHandler), + ('/_ah/xmpp/subscribe', SubscribeHandler), + ('/_ah/xmpp/presence/available', PresenceHandler), + ('/_ah/xmpp/error/', ErrorHandler), + ('/send_presence', SendPresenceHandler), + ('/send_chat', SendChatHandler), +]) diff --git a/appengine/standard/xmpp/xmpp_test.py b/appengine/standard/xmpp/xmpp_test.py new file mode 100644 index 00000000000..ddbd8bacd00 --- /dev/null +++ b/appengine/standard/xmpp/xmpp_test.py @@ -0,0 +1,66 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import pytest +import webtest + +import xmpp + + +@pytest.fixture +def app(testbed): + return webtest.TestApp(xmpp.app) + + +@mock.patch('xmpp.xmpp') +def test_chat(xmpp_mock, app): + app.post('/_ah/xmpp/message/chat/', { + 'from': 'sender@example.com', + 'to': 'recipient@example.com', + 'body': 'hello', + }) + + +@mock.patch('xmpp.xmpp') +def test_subscribe(xmpp_mock, app): + app.post('/_ah/xmpp/subscribe') + + +@mock.patch('xmpp.xmpp') +def test_check_presence(xmpp_mock, app): + + app.post('/_ah/xmpp/presence/available', { + 'from': 'sender@example.com' + }) + + +@mock.patch('xmpp.xmpp') +def test_send_presence(xmpp_mock, app): + app.post('/send_presence', { + 'jid': 'node@domain/resource' + }) + + +@mock.patch('xmpp.xmpp') +def test_error(xmpp_mock, app): + app.post('/_ah/xmpp/error/', { + 'from': 'sender@example.com', + 'stanza': 'hello world' + }) + + +@mock.patch('xmpp.xmpp') +def test_send_chat(xmpp_mock, app): + app.post('/send_chat') diff --git a/appengine/standard_python37/bigquery/.gcloudignore b/appengine/standard_python37/bigquery/.gcloudignore new file mode 100644 index 00000000000..a987f1123d8 --- /dev/null +++ b/appengine/standard_python37/bigquery/.gcloudignore @@ -0,0 +1,19 @@ +# This file specifies files that are *not* uploaded to Google Cloud Platform +# using gcloud. It follows the same syntax as .gitignore, with the addition of +# "#!include" directives (which insert the entries of the given .gitignore-style +# file at that point). +# +# For more information, run: +# $ gcloud topic gcloudignore +# +.gcloudignore +# If you would like to upload your .git directory, .gitignore file or files +# from your .gitignore file, remove the corresponding line +# below: +.git +.gitignore + +# Python pycache: +__pycache__/ +# Ignored by the build system +/setup.cfg \ No newline at end of file diff --git a/appengine/standard_python37/bigquery/app.yaml b/appengine/standard_python37/bigquery/app.yaml new file mode 100644 index 00000000000..a0b719d6dd4 --- /dev/null +++ b/appengine/standard_python37/bigquery/app.yaml @@ -0,0 +1 @@ +runtime: python37 diff --git a/appengine/standard_python37/bigquery/main.py b/appengine/standard_python37/bigquery/main.py new file mode 100644 index 00000000000..fc2ec8d4ad0 --- /dev/null +++ b/appengine/standard_python37/bigquery/main.py @@ -0,0 +1,78 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START gae_python37_bigquery] +import concurrent.futures + +import flask +from google.cloud import bigquery + + +app = flask.Flask(__name__) +bigquery_client = bigquery.Client() + + +@app.route("/") +def main(): + query_job = bigquery_client.query( + """ + SELECT + CONCAT( + 'https://stackoverflow.com/questions/', + CAST(id as STRING)) as url, + view_count + FROM `bigquery-public-data.stackoverflow.posts_questions` + WHERE tags like '%google-bigquery%' + ORDER BY view_count DESC + LIMIT 10 + """ + ) + + return flask.redirect( + flask.url_for( + "results", + project_id=query_job.project, + job_id=query_job.job_id, + location=query_job.location, + ) + ) + + +@app.route("/results") +def results(): + project_id = flask.request.args.get("project_id") + job_id = flask.request.args.get("job_id") + location = flask.request.args.get("location") + + query_job = bigquery_client.get_job( + job_id, + project=project_id, + location=location, + ) + + try: + # Set a timeout because queries could take longer than one minute. + results = query_job.result(timeout=30) + except concurrent.futures.TimeoutError: + return flask.render_template("timeout.html", job_id=query_job.job_id) + + return flask.render_template("query_result.html", results=results) + + +if __name__ == "__main__": + # This is used when running locally only. When deploying to Google App + # Engine, a webserver process such as Gunicorn will serve the app. This + # can be configured by adding an `entrypoint` to app.yaml. + app.run(host="127.0.0.1", port=8080, debug=True) +# [END gae_python37_bigquery] diff --git a/appengine/standard_python37/bigquery/main_test.py b/appengine/standard_python37/bigquery/main_test.py new file mode 100644 index 00000000000..b00f7cd8aac --- /dev/null +++ b/appengine/standard_python37/bigquery/main_test.py @@ -0,0 +1,73 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import concurrent.futures +from unittest import mock + +from google.cloud import bigquery +import pytest + + +@pytest.fixture +def flask_client(): + import main + + main.app.testing = True + return main.app.test_client() + + +def test_main(flask_client): + r = flask_client.get("/") + assert r.status_code == 302 + assert "/results" in r.headers.get("location", "") + + +def test_results(flask_client, monkeypatch): + import main + + fake_job = mock.create_autospec(bigquery.QueryJob) + fake_rows = [("example1.com", "42"), ("example2.com", "38")] + fake_job.result.return_value = fake_rows + + def fake_get_job(self, job_id, **kwargs): + return fake_job + + monkeypatch.setattr(main.bigquery.Client, "get_job", fake_get_job) + + r = flask_client.get( + "/results?project_id=123&job_id=456&location=my_location" + ) + response_body = r.data.decode("utf-8") + + assert r.status_code == 200 + assert "Query Result" in response_body # verifies header + assert "example2.com" in response_body + assert "42" in response_body + + +def test_results_timeout(flask_client, monkeypatch): + import main + + fake_job = mock.create_autospec(bigquery.QueryJob) + fake_job.result.side_effect = concurrent.futures.TimeoutError() + + def fake_get_job(self, job_id, **kwargs): + return fake_job + + monkeypatch.setattr(main.bigquery.Client, "get_job", fake_get_job) + + r = flask_client.get("/results", follow_redirects=True) + + assert r.status_code == 200 + assert "Query Timeout" in r.data.decode("utf-8") diff --git a/appengine/standard_python37/bigquery/requirements.txt b/appengine/standard_python37/bigquery/requirements.txt new file mode 100644 index 00000000000..690ee60f6be --- /dev/null +++ b/appengine/standard_python37/bigquery/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-bigquery==1.9.0 +Flask==1.0.2 diff --git a/appengine/standard_python37/bigquery/templates/query_result.html b/appengine/standard_python37/bigquery/templates/query_result.html new file mode 100644 index 00000000000..aa727e5baea --- /dev/null +++ b/appengine/standard_python37/bigquery/templates/query_result.html @@ -0,0 +1,31 @@ + +{# +Copyright 2019 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} + +Query Result + + + + + + + {% for result in results %} + + + + + {% endfor %} +
          URLView Count
          {{ result[0] }}{{ result[1] }}
          diff --git a/appengine/standard_python37/bigquery/templates/timeout.html b/appengine/standard_python37/bigquery/templates/timeout.html new file mode 100644 index 00000000000..b59a22a97bc --- /dev/null +++ b/appengine/standard_python37/bigquery/templates/timeout.html @@ -0,0 +1,20 @@ + +{# +Copyright 2019 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} + +Query Timeout + +

          Query job {{ job_id }} timed out. diff --git a/appengine/standard_python37/building-an-app/building-an-app-1/.gcloudignore b/appengine/standard_python37/building-an-app/building-an-app-1/.gcloudignore new file mode 100644 index 00000000000..60a5f008535 --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-1/.gcloudignore @@ -0,0 +1,14 @@ +# This file specifies files that are *not* uploaded to Google Cloud Platform +# using gcloud. It follows the same syntax as .gitignore, with the addition of +# "#!include" directives (which insert the entries of the given .gitignore-style +# file at that point). +# +# For more information, run: +# $ gcloud topic gcloudignore +# +.gcloudignore +# If you would like to upload your .git directory, .gitignore file or files +# from your .gitignore file, remove the corresponding line +# below: +.git +.gitignore diff --git a/appengine/standard_python37/building-an-app/building-an-app-1/app.yaml b/appengine/standard_python37/building-an-app/building-an-app-1/app.yaml new file mode 100644 index 00000000000..f8b10563663 --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-1/app.yaml @@ -0,0 +1,13 @@ +runtime: python37 + +handlers: + # This configures Google App Engine to serve the files in the app's static + # directory. +- url: /static + static_dir: static + + # This handler routes all requests not caught above to your main app. It is + # required when static routes are defined, but can be omitted (along with + # the entire handlers section) when there are no static files defined. +- url: /.* + script: auto diff --git a/appengine/standard_python37/building-an-app/building-an-app-1/main.py b/appengine/standard_python37/building-an-app/building-an-app-1/main.py new file mode 100644 index 00000000000..e161f10e18b --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-1/main.py @@ -0,0 +1,44 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START gae_python37_render_template] +import datetime + +from flask import Flask, render_template + +app = Flask(__name__) + + +@app.route('/') +def root(): + # For the sake of example, use static information to inflate the template. + # This will be replaced with real information in later steps. + dummy_times = [datetime.datetime(2018, 1, 1, 10, 0, 0), + datetime.datetime(2018, 1, 2, 10, 30, 0), + datetime.datetime(2018, 1, 3, 11, 0, 0), + ] + + return render_template('index.html', times=dummy_times) + + +if __name__ == '__main__': + # This is used when running locally only. When deploying to Google App + # Engine, a webserver process such as Gunicorn will serve the app. This + # can be configured by adding an `entrypoint` to app.yaml. + # Flask's development server will automatically serve static files in + # the "static" directory. See: + # http://flask.pocoo.org/docs/1.0/quickstart/#static-files. Once deployed, + # App Engine itself will serve those files as configured in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) +# [START gae_python37_render_template] diff --git a/appengine/standard_python37/building-an-app/building-an-app-1/main_test.py b/appengine/standard_python37/building-an-app/building-an-app-1/main_test.py new file mode 100644 index 00000000000..380ee880562 --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-1/main_test.py @@ -0,0 +1,23 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import main + + +def test_index(): + main.app.testing = True + client = main.app.test_client() + + r = client.get('/') + assert r.status_code == 200 diff --git a/appengine/standard_python37/building-an-app/building-an-app-1/requirements.txt b/appengine/standard_python37/building-an-app/building-an-app-1/requirements.txt new file mode 100644 index 00000000000..f2e1e506599 --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-1/requirements.txt @@ -0,0 +1 @@ +Flask==1.0.2 diff --git a/appengine/standard_python37/building-an-app/building-an-app-1/static/script.js b/appengine/standard_python37/building-an-app/building-an-app-1/static/script.js new file mode 100644 index 00000000000..4ec5e25bd52 --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-1/static/script.js @@ -0,0 +1,24 @@ +/** + * Copyright 2018, Google LLC + * Licensed under the Apache License, Version 2.0 (the `License`); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an `AS IS` BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// [START gae_python37_log] +'use strict'; + +window.addEventListener('load', function () { + + console.log("Hello World!"); + +}); +// [END gae_python37_log] diff --git a/appengine/standard_python37/building-an-app/building-an-app-1/static/style.css b/appengine/standard_python37/building-an-app/building-an-app-1/static/style.css new file mode 100644 index 00000000000..84017459eba --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-1/static/style.css @@ -0,0 +1,4 @@ +body { + font-family: "helvetica", sans-serif; + text-align: center; +} diff --git a/appengine/standard_python37/building-an-app/building-an-app-1/templates/index.html b/appengine/standard_python37/building-an-app/building-an-app-1/templates/index.html new file mode 100644 index 00000000000..f159e282cfb --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-1/templates/index.html @@ -0,0 +1,18 @@ + + + + Datastore and Firebase Auth Example + + + + + +

          Datastore and Firebase Auth Example

          + +

          Last 10 visits

          + {% for time in times %} +

          {{ time }}

          + {% endfor %} + + + diff --git a/appengine/standard_python37/building-an-app/building-an-app-2/.gcloudignore b/appengine/standard_python37/building-an-app/building-an-app-2/.gcloudignore new file mode 100644 index 00000000000..60a5f008535 --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-2/.gcloudignore @@ -0,0 +1,14 @@ +# This file specifies files that are *not* uploaded to Google Cloud Platform +# using gcloud. It follows the same syntax as .gitignore, with the addition of +# "#!include" directives (which insert the entries of the given .gitignore-style +# file at that point). +# +# For more information, run: +# $ gcloud topic gcloudignore +# +.gcloudignore +# If you would like to upload your .git directory, .gitignore file or files +# from your .gitignore file, remove the corresponding line +# below: +.git +.gitignore diff --git a/appengine/standard_python37/building-an-app/building-an-app-2/app.yaml b/appengine/standard_python37/building-an-app/building-an-app-2/app.yaml new file mode 100644 index 00000000000..f8b10563663 --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-2/app.yaml @@ -0,0 +1,13 @@ +runtime: python37 + +handlers: + # This configures Google App Engine to serve the files in the app's static + # directory. +- url: /static + static_dir: static + + # This handler routes all requests not caught above to your main app. It is + # required when static routes are defined, but can be omitted (along with + # the entire handlers section) when there are no static files defined. +- url: /.* + script: auto diff --git a/appengine/standard_python37/building-an-app/building-an-app-2/main.py b/appengine/standard_python37/building-an-app/building-an-app-2/main.py new file mode 100644 index 00000000000..ed3fccdf395 --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-2/main.py @@ -0,0 +1,71 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime + +from flask import Flask, render_template + +# [START gae_python37_datastore_store_and_fetch_times] +from google.cloud import datastore + +datastore_client = datastore.Client() + +# [END gae_python37_datastore_store_and_fetch_times] +app = Flask(__name__) + + +# [START gae_python37_datastore_store_and_fetch_times] +def store_time(dt): + entity = datastore.Entity(key=datastore_client.key('visit')) + entity.update({ + 'timestamp': dt + }) + + datastore_client.put(entity) + + +def fetch_times(limit): + query = datastore_client.query(kind='visit') + query.order = ['-timestamp'] + + times = query.fetch(limit=limit) + + return times +# [END gae_python37_datastore_store_and_fetch_times] + + +# [START gae_python37_datastore_render_times] +@app.route('/') +def root(): + # Store the current access time in Datastore. + store_time(datetime.datetime.now()) + + # Fetch the most recent 10 access times from Datastore. + times = fetch_times(10) + + return render_template( + 'index.html', times=times) +# [END gae_python37_datastore_render_times] + + +if __name__ == '__main__': + # This is used when running locally only. When deploying to Google App + # Engine, a webserver process such as Gunicorn will serve the app. This + # can be configured by adding an `entrypoint` to app.yaml. + + # Flask's development server will automatically serve static files in + # the "static" directory. See: + # http://flask.pocoo.org/docs/1.0/quickstart/#static-files. Once deployed, + # App Engine itself will serve those files as configured in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/appengine/standard_python37/building-an-app/building-an-app-2/main_test.py b/appengine/standard_python37/building-an-app/building-an-app-2/main_test.py new file mode 100644 index 00000000000..380ee880562 --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-2/main_test.py @@ -0,0 +1,23 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import main + + +def test_index(): + main.app.testing = True + client = main.app.test_client() + + r = client.get('/') + assert r.status_code == 200 diff --git a/appengine/standard_python37/building-an-app/building-an-app-2/requirements.txt b/appengine/standard_python37/building-an-app/building-an-app-2/requirements.txt new file mode 100644 index 00000000000..31fb29dc887 --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-2/requirements.txt @@ -0,0 +1,2 @@ +Flask==1.0.2 +google-cloud-datastore==1.7.3 diff --git a/appengine/standard_python37/building-an-app/building-an-app-2/static/script.js b/appengine/standard_python37/building-an-app/building-an-app-2/static/script.js new file mode 100644 index 00000000000..4ec5e25bd52 --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-2/static/script.js @@ -0,0 +1,24 @@ +/** + * Copyright 2018, Google LLC + * Licensed under the Apache License, Version 2.0 (the `License`); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an `AS IS` BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// [START gae_python37_log] +'use strict'; + +window.addEventListener('load', function () { + + console.log("Hello World!"); + +}); +// [END gae_python37_log] diff --git a/appengine/standard_python37/building-an-app/building-an-app-2/static/style.css b/appengine/standard_python37/building-an-app/building-an-app-2/static/style.css new file mode 100644 index 00000000000..84017459eba --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-2/static/style.css @@ -0,0 +1,4 @@ +body { + font-family: "helvetica", sans-serif; + text-align: center; +} diff --git a/appengine/standard_python37/building-an-app/building-an-app-2/templates/index.html b/appengine/standard_python37/building-an-app/building-an-app-2/templates/index.html new file mode 100644 index 00000000000..90df3819bb3 --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-2/templates/index.html @@ -0,0 +1,18 @@ + + + + Datastore and Firebase Auth Example + + + + + +

          Datastore and Firebase Auth Example

          + +

          Last 10 visits

          + {% for time in times %} +

          {{ time['timestamp'] }}

          + {% endfor %} + + + diff --git a/appengine/standard_python37/building-an-app/building-an-app-3/.gcloudignore b/appengine/standard_python37/building-an-app/building-an-app-3/.gcloudignore new file mode 100644 index 00000000000..60a5f008535 --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-3/.gcloudignore @@ -0,0 +1,14 @@ +# This file specifies files that are *not* uploaded to Google Cloud Platform +# using gcloud. It follows the same syntax as .gitignore, with the addition of +# "#!include" directives (which insert the entries of the given .gitignore-style +# file at that point). +# +# For more information, run: +# $ gcloud topic gcloudignore +# +.gcloudignore +# If you would like to upload your .git directory, .gitignore file or files +# from your .gitignore file, remove the corresponding line +# below: +.git +.gitignore diff --git a/appengine/standard_python37/building-an-app/building-an-app-3/app.yaml b/appengine/standard_python37/building-an-app/building-an-app-3/app.yaml new file mode 100644 index 00000000000..f8b10563663 --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-3/app.yaml @@ -0,0 +1,13 @@ +runtime: python37 + +handlers: + # This configures Google App Engine to serve the files in the app's static + # directory. +- url: /static + static_dir: static + + # This handler routes all requests not caught above to your main app. It is + # required when static routes are defined, but can be omitted (along with + # the entire handlers section) when there are no static files defined. +- url: /.* + script: auto diff --git a/appengine/standard_python37/building-an-app/building-an-app-3/main.py b/appengine/standard_python37/building-an-app/building-an-app-3/main.py new file mode 100644 index 00000000000..95ae05bccd2 --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-3/main.py @@ -0,0 +1,93 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime + +# [START gae_python37_auth_verify_token] +from flask import Flask, render_template, request +from google.auth.transport import requests +from google.cloud import datastore +import google.oauth2.id_token + +firebase_request_adapter = requests.Request() +# [END gae_python37_auth_verify_token] + +datastore_client = datastore.Client() + +app = Flask(__name__) + + +def store_time(dt): + entity = datastore.Entity(key=datastore_client.key('visit')) + entity.update({ + 'timestamp': dt + }) + + datastore_client.put(entity) + + +def fetch_times(limit): + query = datastore_client.query(kind='visit') + query.order = ['-timestamp'] + + times = query.fetch(limit=limit) + + return times + + +# [START gae_python37_auth_verify_token] +@app.route('/') +def root(): + # Verify Firebase auth. + id_token = request.cookies.get("token") + error_message = None + claims = None + times = None + + if id_token: + try: + # Verify the token against the Firebase Auth API. This example + # verifies the token on each page load. For improved performance, + # some applications may wish to cache results in an encrypted + # session store (see for instance + # http://flask.pocoo.org/docs/1.0/quickstart/#sessions). + claims = google.oauth2.id_token.verify_firebase_token( + id_token, firebase_request_adapter) + except ValueError as exc: + # This will be raised if the token is expired or any other + # verification checks fail. + error_message = str(exc) + + # Record and fetch the recent times a logged-in user has accessed + # the site. This is currently shared amongst all users, but will be + # individualized in a following step. + store_time(datetime.datetime.now()) + times = fetch_times(10) + + return render_template( + 'index.html', + user_data=claims, error_message=error_message, times=times) +# [END gae_python37_auth_verify_token] + + +if __name__ == '__main__': + # This is used when running locally only. When deploying to Google App + # Engine, a webserver process such as Gunicorn will serve the app. This + # can be configured by adding an `entrypoint` to app.yaml. + + # Flask's development server will automatically serve static files in + # the "static" directory. See: + # http://flask.pocoo.org/docs/1.0/quickstart/#static-files. Once deployed, + # App Engine itself will serve those files as configured in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/appengine/standard_python37/building-an-app/building-an-app-3/main_test.py b/appengine/standard_python37/building-an-app/building-an-app-3/main_test.py new file mode 100644 index 00000000000..380ee880562 --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-3/main_test.py @@ -0,0 +1,23 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import main + + +def test_index(): + main.app.testing = True + client = main.app.test_client() + + r = client.get('/') + assert r.status_code == 200 diff --git a/appengine/standard_python37/building-an-app/building-an-app-3/requirements.txt b/appengine/standard_python37/building-an-app/building-an-app-3/requirements.txt new file mode 100644 index 00000000000..029eca50f33 --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-3/requirements.txt @@ -0,0 +1,4 @@ +Flask==1.0.2 +google-cloud-datastore==1.7.3 +google-auth==1.6.2 +requests==2.21.0 diff --git a/appengine/standard_python37/building-an-app/building-an-app-3/static/script.js b/appengine/standard_python37/building-an-app/building-an-app-3/static/script.js new file mode 100644 index 00000000000..739289f3460 --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-3/static/script.js @@ -0,0 +1,73 @@ +/** + * Copyright 2018, Google LLC + * Licensed under the Apache License, Version 2.0 (the `License`); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an `AS IS` BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +// [START gae_python37_auth_javascript] +window.addEventListener('load', function () { + document.getElementById('sign-out').onclick = function () { + firebase.auth().signOut(); + }; + + // FirebaseUI config. + var uiConfig = { + signInSuccessUrl: '/', + signInOptions: [ + // Comment out any lines corresponding to providers you did not check in + // the Firebase console. + firebase.auth.GoogleAuthProvider.PROVIDER_ID, + firebase.auth.EmailAuthProvider.PROVIDER_ID, + //firebase.auth.FacebookAuthProvider.PROVIDER_ID, + //firebase.auth.TwitterAuthProvider.PROVIDER_ID, + //firebase.auth.GithubAuthProvider.PROVIDER_ID, + //firebase.auth.PhoneAuthProvider.PROVIDER_ID + + ], + // Terms of service url. + tosUrl: '' + }; + + firebase.auth().onAuthStateChanged(function (user) { + if (user) { + // User is signed in, so display the "sign out" button and login info. + document.getElementById('sign-out').hidden = false; + document.getElementById('login-info').hidden = false; + console.log(`Signed in as ${user.displayName} (${user.email})`); + user.getIdToken().then(function (token) { + // Add the token to the browser's cookies. The server will then be + // able to verify the token against the API. + // SECURITY NOTE: As cookies can easily be modified, only put the + // token (which is verified server-side) in a cookie; do not add other + // user information. + document.cookie = "token=" + token; + }); + } else { + // User is signed out. + // Initialize the FirebaseUI Widget using Firebase. + var ui = new firebaseui.auth.AuthUI(firebase.auth()); + // Show the Firebase login button. + ui.start('#firebaseui-auth-container', uiConfig); + // Update the login state indicators. + document.getElementById('sign-out').hidden = true; + document.getElementById('login-info').hidden = true; + // Clear the token cookie. + document.cookie = "token="; + } + }, function (error) { + console.log(error); + alert('Unable to log in: ' + error) + }); +}); +// [END gae_python37_auth_javascript] diff --git a/appengine/standard_python37/building-an-app/building-an-app-3/static/style.css b/appengine/standard_python37/building-an-app/building-an-app-3/static/style.css new file mode 100644 index 00000000000..84017459eba --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-3/static/style.css @@ -0,0 +1,4 @@ +body { + font-family: "helvetica", sans-serif; + text-align: center; +} diff --git a/appengine/standard_python37/building-an-app/building-an-app-3/templates/index.html b/appengine/standard_python37/building-an-app/building-an-app-3/templates/index.html new file mode 100644 index 00000000000..f0426537203 --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-3/templates/index.html @@ -0,0 +1,54 @@ + + + + Datastore and Firebase Auth Example + + + + + + + + + + + + + + + + + +

          Datastore and Firebase Auth Example

          + +
          + + + + + + + diff --git a/appengine/standard_python37/building-an-app/building-an-app-4/app.yaml b/appengine/standard_python37/building-an-app/building-an-app-4/app.yaml new file mode 100644 index 00000000000..f8b10563663 --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-4/app.yaml @@ -0,0 +1,13 @@ +runtime: python37 + +handlers: + # This configures Google App Engine to serve the files in the app's static + # directory. +- url: /static + static_dir: static + + # This handler routes all requests not caught above to your main app. It is + # required when static routes are defined, but can be omitted (along with + # the entire handlers section) when there are no static files defined. +- url: /.* + script: auto diff --git a/appengine/standard_python37/building-an-app/building-an-app-4/index.yaml b/appengine/standard_python37/building-an-app/building-an-app-4/index.yaml new file mode 100644 index 00000000000..fc8d0c9810e --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-4/index.yaml @@ -0,0 +1,7 @@ +indexes: + +- kind: visit + ancestor: yes + properties: + - name: timestamp + direction: desc diff --git a/appengine/standard_python37/building-an-app/building-an-app-4/main.py b/appengine/standard_python37/building-an-app/building-an-app-4/main.py new file mode 100644 index 00000000000..7b05e70695f --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-4/main.py @@ -0,0 +1,94 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime + +from flask import Flask, render_template, request +from google.auth.transport import requests +from google.cloud import datastore +import google.oauth2.id_token + +firebase_request_adapter = requests.Request() + +# [START gae_python37_datastore_store_and_fetch_user_times] +datastore_client = datastore.Client() + +# [END gae_python37_datastore_store_and_fetch_user_times] +app = Flask(__name__) + + +# [START gae_python37_datastore_store_and_fetch_user_times] +def store_time(email, dt): + entity = datastore.Entity(key=datastore_client.key('User', email, 'visit')) + entity.update({ + 'timestamp': dt + }) + + datastore_client.put(entity) + + +def fetch_times(email, limit): + ancestor = datastore_client.key('User', email) + query = datastore_client.query(kind='visit', ancestor=ancestor) + query.order = ['-timestamp'] + + times = query.fetch(limit=limit) + + return times +# [END gae_python37_datastore_store_and_fetch_user_times] + + +# [START gae_python37_datastore_render_user_times] +@app.route('/') +def root(): + # Verify Firebase auth. + id_token = request.cookies.get("token") + error_message = None + claims = None + times = None + + if id_token: + try: + # Verify the token against the Firebase Auth API. This example + # verifies the token on each page load. For improved performance, + # some applications may wish to cache results in an encrypted + # session store (see for instance + # http://flask.pocoo.org/docs/1.0/quickstart/#sessions). + claims = google.oauth2.id_token.verify_firebase_token( + id_token, firebase_request_adapter) + + store_time(claims['email'], datetime.datetime.now()) + times = fetch_times(claims['email'], 10) + + except ValueError as exc: + # This will be raised if the token is expired or any other + # verification checks fail. + error_message = str(exc) + + return render_template( + 'index.html', + user_data=claims, error_message=error_message, times=times) +# [END gae_python37_datastore_render_user_times] + + +if __name__ == '__main__': + # This is used when running locally only. When deploying to Google App + # Engine, a webserver process such as Gunicorn will serve the app. This + # can be configured by adding an `entrypoint` to app.yaml. + + # Flask's development server will automatically serve static files in + # the "static" directory. See: + # http://flask.pocoo.org/docs/1.0/quickstart/#static-files. Once deployed, + # App Engine itself will serve those files as configured in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/appengine/standard_python37/building-an-app/building-an-app-4/main_test.py b/appengine/standard_python37/building-an-app/building-an-app-4/main_test.py new file mode 100644 index 00000000000..380ee880562 --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-4/main_test.py @@ -0,0 +1,23 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import main + + +def test_index(): + main.app.testing = True + client = main.app.test_client() + + r = client.get('/') + assert r.status_code == 200 diff --git a/appengine/standard_python37/building-an-app/building-an-app-4/requirements.txt b/appengine/standard_python37/building-an-app/building-an-app-4/requirements.txt new file mode 100644 index 00000000000..029eca50f33 --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-4/requirements.txt @@ -0,0 +1,4 @@ +Flask==1.0.2 +google-cloud-datastore==1.7.3 +google-auth==1.6.2 +requests==2.21.0 diff --git a/appengine/standard_python37/building-an-app/building-an-app-4/static/script.js b/appengine/standard_python37/building-an-app/building-an-app-4/static/script.js new file mode 100644 index 00000000000..3ffcdc3fc86 --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-4/static/script.js @@ -0,0 +1,73 @@ +/** + * Copyright 2018, Google LLC + * Licensed under the Apache License, Version 2.0 (the `License`); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an `AS IS` BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +window.addEventListener('load', function () { + + // [START gae_python37_auth_signout] + document.getElementById('sign-out').onclick = function () { + firebase.auth().signOut(); + }; + // [END gae_python37_auth_signout] + + // [START gae_python37_auth_UIconfig_variable] + // FirebaseUI config. + var uiConfig = { + signInSuccessUrl: '/', + signInOptions: [ + // Remove any lines corresponding to providers you did not check in + // the Firebase console. + firebase.auth.GoogleAuthProvider.PROVIDER_ID, + firebase.auth.EmailAuthProvider.PROVIDER_ID, + ], + // Terms of service url. + tosUrl: '' + }; + // [END gae_python37_auth_UIconfig_variable] + + // [START gae_python37_auth_request] + firebase.auth().onAuthStateChanged(function (user) { + if (user) { + // User is signed in, so display the "sign out" button and login info. + document.getElementById('sign-out').hidden = false; + document.getElementById('login-info').hidden = false; + console.log(`Signed in as ${user.displayName} (${user.email})`); + user.getIdToken().then(function (token) { + // Add the token to the browser's cookies. The server will then be + // able to verify the token against the API. + // SECURITY NOTE: As cookies can easily be modified, only put the + // token (which is verified server-side) in a cookie; do not add other + // user information. + document.cookie = "token=" + token; + }); + } else { + // User is signed out. + // Initialize the FirebaseUI Widget using Firebase. + var ui = new firebaseui.auth.AuthUI(firebase.auth()); + // Show the Firebase login button. + ui.start('#firebaseui-auth-container', uiConfig); + // Update the login state indicators. + document.getElementById('sign-out').hidden = true; + document.getElementById('login-info').hidden = true; + // Clear the token cookie. + document.cookie = "token="; + } + }, function (error) { + console.log(error); + alert('Unable to log in: ' + error) + }); + // [END gae_python37_auth_request] +}); diff --git a/appengine/standard_python37/building-an-app/building-an-app-4/static/style.css b/appengine/standard_python37/building-an-app/building-an-app-4/static/style.css new file mode 100644 index 00000000000..84017459eba --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-4/static/style.css @@ -0,0 +1,4 @@ +body { + font-family: "helvetica", sans-serif; + text-align: center; +} diff --git a/appengine/standard_python37/building-an-app/building-an-app-4/templates/index.html b/appengine/standard_python37/building-an-app/building-an-app-4/templates/index.html new file mode 100644 index 00000000000..5045b1d5255 --- /dev/null +++ b/appengine/standard_python37/building-an-app/building-an-app-4/templates/index.html @@ -0,0 +1,54 @@ + + + + Datastore and Firebase Auth Example + + + + + + + + + + + + + + + + +

          Datastore and Firebase Auth Example

          + + +
          + + + + + + + diff --git a/appengine/standard_python37/cloudsql/app.yaml b/appengine/standard_python37/cloudsql/app.yaml new file mode 100644 index 00000000000..aed5d2f0e69 --- /dev/null +++ b/appengine/standard_python37/cloudsql/app.yaml @@ -0,0 +1,9 @@ +# [START gae_python37_cloudsql_config] +runtime: python37 + +env_variables: + CLOUD_SQL_USERNAME: YOUR-USERNAME + CLOUD_SQL_PASSWORD: YOUR-PASSWORD + CLOUD_SQL_DATABASE_NAME: YOUR-DATABASE + CLOUD_SQL_CONNECTION_NAME: YOUR-CONNECTION-NAME +# [END gae_python37_cloudsql_config] diff --git a/appengine/standard_python37/cloudsql/main_mysql.py b/appengine/standard_python37/cloudsql/main_mysql.py new file mode 100644 index 00000000000..e949fc98380 --- /dev/null +++ b/appengine/standard_python37/cloudsql/main_mysql.py @@ -0,0 +1,58 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START gae_python37_cloudsql_mysql] +import os + +from flask import Flask +import pymysql + +db_user = os.environ.get('CLOUD_SQL_USERNAME') +db_password = os.environ.get('CLOUD_SQL_PASSWORD') +db_name = os.environ.get('CLOUD_SQL_DATABASE_NAME') +db_connection_name = os.environ.get('CLOUD_SQL_CONNECTION_NAME') + +app = Flask(__name__) + + +@app.route('/') +def main(): + # When deployed to App Engine, the `GAE_ENV` environment variable will be + # set to `standard` + if os.environ.get('GAE_ENV') == 'standard': + # If deployed, use the local socket interface for accessing Cloud SQL + unix_socket = '/cloudsql/{}'.format(db_connection_name) + cnx = pymysql.connect(user=db_user, password=db_password, + unix_socket=unix_socket, db=db_name) + else: + # If running locally, use the TCP connections instead + # Set up Cloud SQL Proxy (cloud.google.com/sql/docs/mysql/sql-proxy) + # so that your application can use 127.0.0.1:3306 to connect to your + # Cloud SQL instance + host = '127.0.0.1' + cnx = pymysql.connect(user=db_user, password=db_password, + host=host, db=db_name) + + with cnx.cursor() as cursor: + cursor.execute('SELECT NOW() as now;') + result = cursor.fetchall() + current_time = result[0][0] + cnx.close() + + return str(current_time) +# [END gae_python37_cloudsql_mysql] + + +if __name__ == '__main__': + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/appengine/standard_python37/cloudsql/main_mysql_pooling.py b/appengine/standard_python37/cloudsql/main_mysql_pooling.py new file mode 100644 index 00000000000..2f85b675e6b --- /dev/null +++ b/appengine/standard_python37/cloudsql/main_mysql_pooling.py @@ -0,0 +1,65 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START gae_python37_cloudsql_mysql_pooling] +import os + +from flask import Flask +import sqlalchemy + +db_user = os.environ.get('CLOUD_SQL_USERNAME') +db_password = os.environ.get('CLOUD_SQL_PASSWORD') +db_name = os.environ.get('CLOUD_SQL_DATABASE_NAME') +db_connection_name = os.environ.get('CLOUD_SQL_CONNECTION_NAME') + +# When deployed to App Engine, the `GAE_ENV` environment variable will be +# set to `standard` +if os.environ.get('GAE_ENV') == 'standard': + # If deployed, use the local socket interface for accessing Cloud SQL + unix_socket = '/cloudsql/{}'.format(db_connection_name) + engine_url = 'mysql+pymysql://{}:{}@/{}?unix_socket={}'.format( + db_user, db_password, db_name, unix_socket) +else: + # If running locally, use the TCP connections instead + # Set up Cloud SQL Proxy (cloud.google.com/sql/docs/mysql/sql-proxy) + # so that your application can use 127.0.0.1:3306 to connect to your + # Cloud SQL instance + host = '127.0.0.1' + engine_url = 'mysql+pymysql://{}:{}@{}/{}'.format( + db_user, db_password, host, db_name) + +# The Engine object returned by create_engine() has a QueuePool integrated +# See https://docs.sqlalchemy.org/en/latest/core/pooling.html for more +# information +engine = sqlalchemy.create_engine(engine_url, pool_size=3) + +app = Flask(__name__) + + +@app.route('/') +def main(): + cnx = engine.connect() + cursor = cnx.execute('SELECT NOW() as now;') + result = cursor.fetchall() + current_time = result[0][0] + # If the connection comes from a pool, close() will send the connection + # back to the pool instead of closing it + cnx.close() + + return str(current_time) +# [END gae_python37_cloudsql_mysql_pooling] + + +if __name__ == '__main__': + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/appengine/standard_python37/cloudsql/main_postgres.py b/appengine/standard_python37/cloudsql/main_postgres.py new file mode 100644 index 00000000000..157a9551c63 --- /dev/null +++ b/appengine/standard_python37/cloudsql/main_postgres.py @@ -0,0 +1,57 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START gae_python37_cloudsql_psql] +import os + +from flask import Flask +import psycopg2 + +db_user = os.environ.get('CLOUD_SQL_USERNAME') +db_password = os.environ.get('CLOUD_SQL_PASSWORD') +db_name = os.environ.get('CLOUD_SQL_DATABASE_NAME') +db_connection_name = os.environ.get('CLOUD_SQL_CONNECTION_NAME') + +app = Flask(__name__) + + +@app.route('/') +def main(): + # When deployed to App Engine, the `GAE_ENV` environment variable will be + # set to `standard` + if os.environ.get('GAE_ENV') == 'standard': + # If deployed, use the local socket interface for accessing Cloud SQL + host = '/cloudsql/{}'.format(db_connection_name) + else: + # If running locally, use the TCP connections instead + # Set up Cloud SQL Proxy (cloud.google.com/sql/docs/mysql/sql-proxy) + # so that your application can use 127.0.0.1:3306 to connect to your + # Cloud SQL instance + host = '127.0.0.1' + + cnx = psycopg2.connect(dbname=db_name, user=db_user, + password=db_password, host=host) + with cnx.cursor() as cursor: + cursor.execute('SELECT NOW() as now;') + result = cursor.fetchall() + current_time = result[0][0] + cnx.commit() + cnx.close() + + return str(current_time) +# [END gae_python37_cloudsql_psql] + + +if __name__ == '__main__': + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/appengine/standard_python37/cloudsql/main_postgres_pooling.py b/appengine/standard_python37/cloudsql/main_postgres_pooling.py new file mode 100644 index 00000000000..78b3d428b53 --- /dev/null +++ b/appengine/standard_python37/cloudsql/main_postgres_pooling.py @@ -0,0 +1,66 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START gae_python37_cloudsql_psql_pooling] +import os + +from flask import Flask +import psycopg2.pool + +db_user = os.environ.get('CLOUD_SQL_USERNAME') +db_password = os.environ.get('CLOUD_SQL_PASSWORD') +db_name = os.environ.get('CLOUD_SQL_DATABASE_NAME') +db_connection_name = os.environ.get('CLOUD_SQL_CONNECTION_NAME') + +# When deployed to App Engine, the `GAE_ENV` environment variable will be +# set to `standard` +if os.environ.get('GAE_ENV') == 'standard': + # If deployed, use the local socket interface for accessing Cloud SQL + host = '/cloudsql/{}'.format(db_connection_name) +else: + # If running locally, use the TCP connections instead + # Set up Cloud SQL Proxy (cloud.google.com/sql/docs/mysql/sql-proxy) + # so that your application can use 127.0.0.1:3306 to connect to your + # Cloud SQL instance + host = '127.0.0.1' + +db_config = { + 'user': db_user, + 'password': db_password, + 'database': db_name, + 'host': host +} + +cnxpool = psycopg2.pool.ThreadedConnectionPool(minconn=1, maxconn=3, + **db_config) + +app = Flask(__name__) + + +@app.route('/') +def main(): + cnx = cnxpool.getconn() + with cnx.cursor() as cursor: + cursor.execute('SELECT NOW() as now;') + result = cursor.fetchall() + current_time = result[0][0] + cnx.commit() + cnxpool.putconn(cnx) + + return str(current_time) +# [END gae_python37_cloudsql_psql_pooling] + + +if __name__ == '__main__': + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/appengine/standard_python37/cloudsql/main_test.py b/appengine/standard_python37/cloudsql/main_test.py new file mode 100644 index 00000000000..e9496692e6a --- /dev/null +++ b/appengine/standard_python37/cloudsql/main_test.py @@ -0,0 +1,78 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import MagicMock + +import psycopg2.pool +import sqlalchemy + + +def test_main(): + import main_mysql + main_mysql.pymysql = MagicMock() + fetchall_mock = main_mysql.pymysql.connect().cursor().__enter__().fetchall + fetchall_mock.return_value = [['0']] + + main_mysql.app.testing = True + client = main_mysql.app.test_client() + + r = client.get('/') + assert r.status_code == 200 + assert '0' in r.data.decode('utf-8') + + +def test_main_pooling(): + sqlalchemy.create_engine = MagicMock() + + import main_mysql_pooling + + cnx_mock = main_mysql_pooling.sqlalchemy.create_engine().connect() + cnx_mock.execute().fetchall.return_value = [['0']] + + main_mysql_pooling.app.testing = True + client = main_mysql_pooling.app.test_client() + + r = client.get('/') + assert r.status_code == 200 + assert '0' in r.data.decode('utf-8') + + +def test_main_postgressql(): + import main_postgres + main_postgres.psycopg2.connect = MagicMock() + mock_cursor = main_postgres.psycopg2.connect().cursor() + mock_cursor.__enter__().fetchall.return_value = [['0']] + + main_postgres.app.testing = True + client = main_postgres.app.test_client() + + r = client.get('/') + assert r.status_code == 200 + assert '0' in r.data.decode('utf-8') + + +def test_main_postgressql_pooling(): + psycopg2.pool.ThreadedConnectionPool = MagicMock() + + import main_postgres_pooling + + mock_pool = main_postgres_pooling.psycopg2.pool.ThreadedConnectionPool() + mock_pool.getconn().cursor().__enter__().fetchall.return_value = [['0']] + + main_postgres_pooling.app.testing = True + client = main_postgres_pooling.app.test_client() + + r = client.get('/') + assert r.status_code == 200 + assert '0' in r.data.decode('utf-8') diff --git a/appengine/standard_python37/cloudsql/requirements.txt b/appengine/standard_python37/cloudsql/requirements.txt new file mode 100644 index 00000000000..c0bb6a061c1 --- /dev/null +++ b/appengine/standard_python37/cloudsql/requirements.txt @@ -0,0 +1,5 @@ +flask==1.0.2 +psycopg2==2.7.7 +psycopg2-binary==2.7.7 +PyMySQL==0.9.3 +SQLAlchemy==1.2.17 diff --git a/appengine/standard_python37/custom-server/app.yaml b/appengine/standard_python37/custom-server/app.yaml new file mode 100644 index 00000000000..2b602fa5a70 --- /dev/null +++ b/appengine/standard_python37/custom-server/app.yaml @@ -0,0 +1,4 @@ +# [START gae_python37_custom_runtime] +runtime: python37 +entrypoint: uwsgi --http-socket :8080 --wsgi-file main.py --callable app --master --processes 1 --threads 2 +# [END gae_python37_custom_runtime] diff --git a/appengine/standard_python37/custom-server/main.py b/appengine/standard_python37/custom-server/main.py new file mode 100644 index 00000000000..3c9a534ac54 --- /dev/null +++ b/appengine/standard_python37/custom-server/main.py @@ -0,0 +1,26 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from flask import Flask + +app = Flask(__name__) + + +@app.route('/') +def main(): + return 'Hello, World!' + + +if __name__ == '__main__': + app.run(debug=True) diff --git a/appengine/standard_python37/custom-server/requirements.txt b/appengine/standard_python37/custom-server/requirements.txt new file mode 100644 index 00000000000..e91eb8f3552 --- /dev/null +++ b/appengine/standard_python37/custom-server/requirements.txt @@ -0,0 +1,2 @@ +uwsgi==2.0.17.1 +flask==1.0.2 diff --git a/appengine/standard_python37/django/.gcloudignore b/appengine/standard_python37/django/.gcloudignore new file mode 100644 index 00000000000..c6be208a110 --- /dev/null +++ b/appengine/standard_python37/django/.gcloudignore @@ -0,0 +1,17 @@ +# This file specifies files that are *not* uploaded to Google Cloud Platform +# using gcloud. It follows the same syntax as .gitignore, with the addition of +# "#!include" directives (which insert the entries of the given .gitignore-style +# file at that point). +# +# For more information, run: +# $ gcloud topic gcloudignore +# +.gcloudignore +# If you would like to upload your .git directory, .gitignore file or files +# from your .gitignore file, remove the corresponding line +# below: +.git +.gitignore + +# Python pycache: +__pycache__/ diff --git a/appengine/standard_python37/django/README.md b/appengine/standard_python37/django/README.md new file mode 100644 index 00000000000..f34d5ee16e2 --- /dev/null +++ b/appengine/standard_python37/django/README.md @@ -0,0 +1,15 @@ +# Getting started with Django on Google Cloud Platform on App Engine Standard + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard_python37/django/README.md + +This repository is an example of how to run a [Django](https://www.djangoproject.com/) +app on Google App Engine Standard Environment. It uses the +[Writing your first Django app](https://docs.djangoproject.com/en/2.1/intro/tutorial01/) as the +example app to deploy. + + +# Tutorial +See our [Running Django in the App Engine Standard Environment](https://cloud.google.com/python/django/appengine) tutorial for instructions for setting up and deploying this sample application. diff --git a/appengine/standard_python37/django/app.yaml b/appengine/standard_python37/django/app.yaml new file mode 100644 index 00000000000..cd0367ab683 --- /dev/null +++ b/appengine/standard_python37/django/app.yaml @@ -0,0 +1,15 @@ +# [START django_app] +runtime: python37 + +handlers: +# This configures Google App Engine to serve the files in the app's static +# directory. +- url: /static + static_dir: static/ + +# This handler routes all requests not caught above to your main app. It is +# required when static routes are defined, but can be omitted (along with +# the entire handlers section) when there are no static files defined. +- url: /.* + script: auto +# [END django_app] diff --git a/appengine/standard_python37/django/main.py b/appengine/standard_python37/django/main.py new file mode 100644 index 00000000000..155b346dda9 --- /dev/null +++ b/appengine/standard_python37/django/main.py @@ -0,0 +1,10 @@ +from mysite.wsgi import application + +# App Engine by default looks for a main.py file at the root of the app +# directory with a WSGI-compatible object called app. +# This file imports the WSGI-compatible object of your Django app, +# application from mysite/wsgi.py and renames it app so it is discoverable by +# App Engine without additional configuration. +# Alternatively, you can add a custom entrypoint field in your app.yaml: +# entrypoint: gunicorn -b :$PORT mysite.wsgi +app = application diff --git a/appengine/standard_python37/django/manage.py b/appengine/standard_python37/django/manage.py new file mode 100755 index 00000000000..390c7673fc8 --- /dev/null +++ b/appengine/standard_python37/django/manage.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +import os +import sys + +if __name__ == '__main__': + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) diff --git a/appengine/standard_python37/django/mysite/__init__.py b/appengine/standard_python37/django/mysite/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/appengine/standard_python37/django/mysite/settings.py b/appengine/standard_python37/django/mysite/settings.py new file mode 100644 index 00000000000..cd0b2e8e4bf --- /dev/null +++ b/appengine/standard_python37/django/mysite/settings.py @@ -0,0 +1,158 @@ +""" +Django settings for mysite project. + +Generated by 'django-admin startproject' using Django 2.1.1. + +For more information on this file, see +https://docs.djangoproject.com/en/2.1/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/2.1/ref/settings/ +""" + +import os + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/2.1/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +# Update the secret key to a value of your own before deploying the app. +SECRET_KEY = 'lldtg$9(wi49j_hpv8nnqlh!cj7kmbwq0$rj7vy(b(b30vlyzj' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +# SECURITY WARNING: App Engine's security features ensure that it is safe to +# have ALLOWED_HOSTS = ['*'] when the app is deployed. If you deploy a Django +# app not on App Engine, make sure to set an appropriate host here. +# See https://docs.djangoproject.com/en/2.1/ref/settings/ +ALLOWED_HOSTS = ['*'] + + +# Application definition + +INSTALLED_APPS = [ + 'polls.apps.PollsConfig', + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'mysite.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'mysite.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/2.1/ref/settings/#databases + +# Install PyMySQL as mysqlclient/MySQLdb to use Django's mysqlclient adapter +# See https://docs.djangoproject.com/en/2.1/ref/databases/#mysql-db-api-drivers +# for more information +import pymysql # noqa: 402 +pymysql.install_as_MySQLdb() + +# [START db_setup] +if os.getenv('GAE_APPLICATION', None): + # Running on production App Engine, so connect to Google Cloud SQL using + # the unix socket at /cloudsql/ + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.mysql', + 'HOST': '/cloudsql/[YOUR-CONNECTION-NAME]', + 'USER': '[YOUR-USERNAME]', + 'PASSWORD': '[YOUR-PASSWORD]', + 'NAME': '[YOUR-DATABASE]', + } + } +else: + # Running locally so connect to either a local MySQL instance or connect to + # Cloud SQL via the proxy. To start the proxy via command line: + # + # $ cloud_sql_proxy -instances=[INSTANCE_CONNECTION_NAME]=tcp:3306 + # + # See https://cloud.google.com/sql/docs/mysql-connect-proxy + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.mysql', + 'HOST': '127.0.0.1', + 'PORT': '3306', + 'NAME': '[YOUR-DATABASE]', + 'USER': '[YOUR-USERNAME]', + 'PASSWORD': '[YOUR-PASSWORD]', + } + } +# [END db_setup] + + +# Password validation +# https://docs.djangoproject.com/en/2.1/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', # noqa: 501 + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', # noqa: 501 + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', # noqa: 501 + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', # noqa: 501 + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/2.1/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/2.1/howto/static-files/ + +STATIC_ROOT = 'static' +STATIC_URL = '/static/' diff --git a/appengine/standard_python37/django/mysite/urls.py b/appengine/standard_python37/django/mysite/urls.py new file mode 100644 index 00000000000..6fd4ec09211 --- /dev/null +++ b/appengine/standard_python37/django/mysite/urls.py @@ -0,0 +1,22 @@ +"""mysite URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/2.1/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import include, path + +urlpatterns = [ + path('', include('polls.urls')), + path('admin/', admin.site.urls), +] diff --git a/appengine/standard_python37/django/mysite/wsgi.py b/appengine/standard_python37/django/mysite/wsgi.py new file mode 100644 index 00000000000..77f71eb3330 --- /dev/null +++ b/appengine/standard_python37/django/mysite/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for mysite project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/2.1/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings') + +application = get_wsgi_application() diff --git a/appengine/standard_python37/django/polls/__init__.py b/appengine/standard_python37/django/polls/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/appengine/standard_python37/django/polls/admin.py b/appengine/standard_python37/django/polls/admin.py new file mode 100644 index 00000000000..66762c6c53d --- /dev/null +++ b/appengine/standard_python37/django/polls/admin.py @@ -0,0 +1,6 @@ +from django.contrib import admin + +from .models import Choice, Question + +admin.site.register(Question) +admin.site.register(Choice) diff --git a/appengine/standard_python37/django/polls/apps.py b/appengine/standard_python37/django/polls/apps.py new file mode 100644 index 00000000000..d0f109e60e4 --- /dev/null +++ b/appengine/standard_python37/django/polls/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class PollsConfig(AppConfig): + name = 'polls' diff --git a/appengine/standard_python37/django/polls/models.py b/appengine/standard_python37/django/polls/models.py new file mode 100644 index 00000000000..bfb969713c1 --- /dev/null +++ b/appengine/standard_python37/django/polls/models.py @@ -0,0 +1,24 @@ +import datetime + +from django.db import models +from django.utils import timezone + + +class Question(models.Model): + question_text = models.CharField(max_length=200) + pub_date = models.DateTimeField('date published') + + def __str__(self): + return self.question_text + + def was_published_recently(self): + return self.pub_date >= timezone.now() - datetime.timedelta(days=1) + + +class Choice(models.Model): + question = models.ForeignKey(Question, on_delete=models.CASCADE) + choice_text = models.CharField(max_length=200) + votes = models.IntegerField(default=0) + + def __str__(self): + return self.choice_text diff --git a/appengine/standard_python37/django/polls/templates/polls/detail.html b/appengine/standard_python37/django/polls/templates/polls/detail.html new file mode 100644 index 00000000000..3e555446d4a --- /dev/null +++ b/appengine/standard_python37/django/polls/templates/polls/detail.html @@ -0,0 +1,12 @@ +

          {{ question.question_text }}

          + +{% if error_message %}

          {{ error_message }}

          {% endif %} + +
          +{% csrf_token %} +{% for choice in question.choice_set.all %} + +
          +{% endfor %} + +
          diff --git a/appengine/standard_python37/django/polls/templates/polls/index.html b/appengine/standard_python37/django/polls/templates/polls/index.html new file mode 100644 index 00000000000..4560139bcb9 --- /dev/null +++ b/appengine/standard_python37/django/polls/templates/polls/index.html @@ -0,0 +1,9 @@ +{% if latest_question_list %} + +{% else %} +

          No polls are available.

          +{% endif %} diff --git a/appengine/standard_python37/django/polls/templates/polls/results.html b/appengine/standard_python37/django/polls/templates/polls/results.html new file mode 100644 index 00000000000..3b2c74f45ec --- /dev/null +++ b/appengine/standard_python37/django/polls/templates/polls/results.html @@ -0,0 +1,9 @@ +

          {{ question.question_text }}

          + +
            +{% for choice in question.choice_set.all %} +
          • {{ choice.choice_text }} -- {{ choice.votes }} vote{{ choice.votes|pluralize }}
            • +{% endfor %} +
          + +Vote again? diff --git a/appengine/standard_python37/django/polls/tests.py b/appengine/standard_python37/django/polls/tests.py new file mode 100644 index 00000000000..1848508267c --- /dev/null +++ b/appengine/standard_python37/django/polls/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase # noqa: 401 + +# Create your tests here. diff --git a/appengine/standard_python37/django/polls/urls.py b/appengine/standard_python37/django/polls/urls.py new file mode 100644 index 00000000000..eff2be0b214 --- /dev/null +++ b/appengine/standard_python37/django/polls/urls.py @@ -0,0 +1,11 @@ +from django.urls import path + +from . import views + +app_name = 'polls' +urlpatterns = [ + path('', views.IndexView.as_view(), name='index'), + path('/', views.DetailView.as_view(), name='detail'), + path('/results/', views.ResultsView.as_view(), name='results'), + path('/vote/', views.vote, name='vote'), +] diff --git a/appengine/standard_python37/django/polls/views.py b/appengine/standard_python37/django/polls/views.py new file mode 100644 index 00000000000..20adc951ff8 --- /dev/null +++ b/appengine/standard_python37/django/polls/views.py @@ -0,0 +1,46 @@ +from django.http import HttpResponse, HttpResponseRedirect # noqa: 401 +from django.shortcuts import get_object_or_404, render +from django.urls import reverse +from django.views import generic + +from .models import Choice, Question + + +class IndexView(generic.ListView): + template_name = 'polls/index.html' + context_object_name = 'latest_question_list' + + def get_queryset(self): + """Return the last five published questions.""" + return Question.objects.order_by('-pub_date')[:5] + + +class DetailView(generic.DetailView): + model = Question + template_name = 'polls/detail.html' + + +class ResultsView(generic.DetailView): + model = Question + template_name = 'polls/results.html' + + +def vote(request, question_id): + question = get_object_or_404(Question, pk=question_id) + try: + selected_choice = question.choice_set.get(pk=request.POST['choice']) + except (KeyError, Choice.DoesNotExist): + # Redisplay the question voting form. + return render(request, 'polls/detail.html', { + 'question': question, + 'error_message': "You didn't select a choice.", + }) + else: + selected_choice.votes += 1 + selected_choice.save() + # Always return an HttpResponseRedirect after successfully dealing + # with POST data. This prevents data from being posted twice if a + # user hits the Back button. + return HttpResponseRedirect( + reverse('polls:results', args=(question.id,)) + ) diff --git a/appengine/standard_python37/django/requirements.txt b/appengine/standard_python37/django/requirements.txt new file mode 100644 index 00000000000..6af98c09acc --- /dev/null +++ b/appengine/standard_python37/django/requirements.txt @@ -0,0 +1,2 @@ +Django==2.1.5 +PyMySQL==0.9.3 diff --git a/appengine/standard_python37/hello_world/app.yaml b/appengine/standard_python37/hello_world/app.yaml new file mode 100644 index 00000000000..a0b719d6dd4 --- /dev/null +++ b/appengine/standard_python37/hello_world/app.yaml @@ -0,0 +1 @@ +runtime: python37 diff --git a/appengine/standard_python37/hello_world/main.py b/appengine/standard_python37/hello_world/main.py new file mode 100644 index 00000000000..bb823989d59 --- /dev/null +++ b/appengine/standard_python37/hello_world/main.py @@ -0,0 +1,35 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START gae_python37_app] +from flask import Flask + + +# If `entrypoint` is not defined in app.yaml, App Engine will look for an app +# called `app` in `main.py`. +app = Flask(__name__) + + +@app.route('/') +def hello(): + """Return a friendly HTTP greeting.""" + return 'Hello World!' + + +if __name__ == '__main__': + # This is used when running locally only. When deploying to Google App + # Engine, a webserver process such as Gunicorn will serve the app. This + # can be configured by adding an `entrypoint` to app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) +# [END gae_python37_app] diff --git a/appengine/standard_python37/hello_world/main_test.py b/appengine/standard_python37/hello_world/main_test.py new file mode 100644 index 00000000000..ff6a2598f2c --- /dev/null +++ b/appengine/standard_python37/hello_world/main_test.py @@ -0,0 +1,24 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import main + + +def test_index(): + main.app.testing = True + client = main.app.test_client() + + r = client.get('/') + assert r.status_code == 200 + assert 'Hello World' in r.data.decode('utf-8') diff --git a/appengine/standard_python37/hello_world/requirements.txt b/appengine/standard_python37/hello_world/requirements.txt new file mode 100644 index 00000000000..f2e1e506599 --- /dev/null +++ b/appengine/standard_python37/hello_world/requirements.txt @@ -0,0 +1 @@ +Flask==1.0.2 diff --git a/appengine/standard_python37/pubsub/README.md b/appengine/standard_python37/pubsub/README.md new file mode 100644 index 00000000000..6cb534990e8 --- /dev/null +++ b/appengine/standard_python37/pubsub/README.md @@ -0,0 +1,79 @@ +# Python 3 Google Cloud Pub/Sub sample for Google App Engine Standard Environment + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/standard/pubsub/README.md + +This demonstrates how to send and receive messages using [Google Cloud Pub/Sub](https://cloud.google.com/pubsub) on [Google App Engine Standard Environment](https://cloud.google.com/appengine/docs/standard/). + +## Setup + +Before you can run or deploy the sample, you will need to do the following: + +1. Enable the Cloud Pub/Sub API in the [Google Developers Console](https://console.developers.google.com/project/_/apiui/apiview/pubsub/overview). + +2. Create a topic and subscription. The push auth service account must have Service Account Token Creator Role assigned, which can be done in the Cloud Console [IAM & admin](https://console.cloud.google.com/iam-admin/iam) UI. `--push-auth-token-audience` is optional. If set, remember to modify the audience field check in `main.py` (line 88). + + $ gcloud pubsub topics create [your-topic-name] + $ gcloud beta pubsub subscriptions create [your-subscription-name] \ + --topic=[your-topic-name] \ + --push-endpoint=\ + https://[your-app-id].appspot.com/_ah/push-handlers/receive_messages/token=[your-token] \ + --ack-deadline=30 \ + --push-auth-service-account=[your-service-account-email] \ + --push-auth-token-audience=example.com + +3. Update the environment variables in ``app.yaml``. + +## Running locally + +When running locally, you can use the [Google Cloud SDK](https://cloud.google.com/sdk) to provide authentication to use Google Cloud APIs: + + $ gcloud init + +Install dependencies, preferably with a virtualenv: + + $ virtualenv env + $ source env/bin/activate + $ pip install -r requirements.txt + +Then set environment variables before starting your application: + + $ export GOOGLE_CLOUD_PROJECT=[your-project-name] + $ export PUBSUB_VERIFICATION_TOKEN=[your-verification-token] + $ export PUBSUB_TOPIC=[your-topic] + $ python main.py + +### Simulating push notifications + +The application can send messages locally, but it is not able to receive push messages locally. You can, however, simulate a push message by making an HTTP request to the local push notification endpoint. There is an included ``sample_message.json``. You can use +``curl`` or [httpie](https://github.com/jkbrzt/httpie) to POST this: + + $ curl -i --data @sample_message.json "localhost:8080/_ah/push-handlers/receive_messages?token=[your-token]" + +Or + + $ http POST ":8080/_ah/push-handlers/receive_messages?token=[your-token]" < sample_message.json + +Response: + + HTTP/1.0 400 BAD REQUEST + Content-Type: text/html; charset=utf-8 + Content-Length: 58 + Server: Werkzeug/0.15.2 Python/3.7.3 + Date: Sat, 06 Apr 2019 04:56:12 GMT + + Invalid token: 'NoneType' object has no attribute 'split' + +The simulated push request fails because it does not have a Cloud Pub/Sub-generated JWT in the "Authorization" header. + +## Running on App Engine + +Note: Not all the files in the current directory are needed to run your code on App Engine. Specifically, `main_test.py` and the `data` directory, which contains a mocked private key file and a mocked public certs file, are for testing purposes only. They SHOULD NOT be included in when deploying your app. When your app is up and running, Cloud Pub/Sub creates tokens using a private key, then the Google Auth Python library takes care of verifying and decoding the token using Google's public certs, to confirm that the push requests indeed come from Cloud Pub/Sub. + +In the current directory, deploy using `gcloud`: + + $ gcloud app deploy app.yaml + +You can now access the application at `https://[your-app-id].appspot.com`. You can use the form to submit messages, but it's non-deterministic which instance of your application will receive the notification. You can send multiple messages and refresh the page to see the received message. diff --git a/appengine/standard_python37/pubsub/app.yaml b/appengine/standard_python37/pubsub/app.yaml new file mode 100644 index 00000000000..492a16878ec --- /dev/null +++ b/appengine/standard_python37/pubsub/app.yaml @@ -0,0 +1,9 @@ +runtime: python37 + +#[START env] +env_variables: + PUBSUB_TOPIC: your-topic + # This token is used to verify that requests originate from your + # application. It can be any sufficiently random string. + PUBSUB_VERIFICATION_TOKEN: 1234abc +#[END env] diff --git a/appengine/standard_python37/pubsub/data/privatekey.pem b/appengine/standard_python37/pubsub/data/privatekey.pem new file mode 100644 index 00000000000..57443540ad3 --- /dev/null +++ b/appengine/standard_python37/pubsub/data/privatekey.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA4ej0p7bQ7L/r4rVGUz9RN4VQWoej1Bg1mYWIDYslvKrk1gpj +7wZgkdmM7oVK2OfgrSj/FCTkInKPqaCR0gD7K80q+mLBrN3PUkDrJQZpvRZIff3/ +xmVU1WeruQLFJjnFb2dqu0s/FY/2kWiJtBCakXvXEOb7zfbINuayL+MSsCGSdVYs +SliS5qQpgyDap+8b5fpXZVJkq92hrcNtbkg7hCYUJczt8n9hcCTJCfUpApvaFQ18 +pe+zpyl4+WzkP66I28hniMQyUlA1hBiskT7qiouq0m8IOodhv2fagSZKjOTTU2xk +SBc//fy3ZpsL7WqgsZS7Q+0VRK8gKfqkxg5OYQIDAQABAoIBAQDGGHzQxGKX+ANk +nQi53v/c6632dJKYXVJC+PDAz4+bzU800Y+n/bOYsWf/kCp94XcG4Lgsdd0Gx+Zq +HD9CI1IcqqBRR2AFscsmmX6YzPLTuEKBGMW8twaYy3utlFxElMwoUEsrSWRcCA1y +nHSDzTt871c7nxCXHxuZ6Nm/XCL7Bg8uidRTSC1sQrQyKgTPhtQdYrPQ4WZ1A4J9 +IisyDYmZodSNZe5P+LTJ6M1SCgH8KH9ZGIxv3diMwzNNpk3kxJc9yCnja4mjiGE2 +YCNusSycU5IhZwVeCTlhQGcNeV/skfg64xkiJE34c2y2ttFbdwBTPixStGaF09nU +Z422D40BAoGBAPvVyRRsC3BF+qZdaSMFwI1yiXY7vQw5+JZh01tD28NuYdRFzjcJ +vzT2n8LFpj5ZfZFvSMLMVEFVMgQvWnN0O6xdXvGov6qlRUSGaH9u+TCPNnIldjMP +B8+xTwFMqI7uQr54wBB+Poq7dVRP+0oHb0NYAwUBXoEuvYo3c/nDoRcZAoGBAOWl +aLHjMv4CJbArzT8sPfic/8waSiLV9Ixs3Re5YREUTtnLq7LoymqB57UXJB3BNz/2 +eCueuW71avlWlRtE/wXASj5jx6y5mIrlV4nZbVuyYff0QlcG+fgb6pcJQuO9DxMI +aqFGrWP3zye+LK87a6iR76dS9vRU+bHZpSVvGMKJAoGAFGt3TIKeQtJJyqeUWNSk +klORNdcOMymYMIlqG+JatXQD1rR6ThgqOt8sgRyJqFCVT++YFMOAqXOBBLnaObZZ +CFbh1fJ66BlSjoXff0W+SuOx5HuJJAa5+WtFHrPajwxeuRcNa8jwxUsB7n41wADu +UqWWSRedVBg4Ijbw3nWwYDECgYB0pLew4z4bVuvdt+HgnJA9n0EuYowVdadpTEJg +soBjNHV4msLzdNqbjrAqgz6M/n8Ztg8D2PNHMNDNJPVHjJwcR7duSTA6w2p/4k28 +bvvk/45Ta3XmzlxZcZSOct3O31Cw0i2XDVc018IY5be8qendDYM08icNo7vQYkRH +504kQQKBgQDjx60zpz8ozvm1XAj0wVhi7GwXe+5lTxiLi9Fxq721WDxPMiHDW2XL +YXfFVy/9/GIMvEiGYdmarK1NW+VhWl1DC5xhDg0kvMfxplt4tynoq1uTsQTY31Mx +BeF5CT/JuNYk3bEBF0H/Q3VGO1/ggVS+YezdFbLWIRoMnLj6XCFEGg== +-----END RSA PRIVATE KEY----- diff --git a/appengine/standard_python37/pubsub/data/public_cert.pem b/appengine/standard_python37/pubsub/data/public_cert.pem new file mode 100644 index 00000000000..7af6ca3f931 --- /dev/null +++ b/appengine/standard_python37/pubsub/data/public_cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDIzCCAgugAwIBAgIJAMfISuBQ5m+5MA0GCSqGSIb3DQEBBQUAMBUxEzARBgNV +BAMTCnVuaXQtdGVzdHMwHhcNMTExMjA2MTYyNjAyWhcNMjExMjAzMTYyNjAyWjAV +MRMwEQYDVQQDEwp1bml0LXRlc3RzMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEA4ej0p7bQ7L/r4rVGUz9RN4VQWoej1Bg1mYWIDYslvKrk1gpj7wZgkdmM +7oVK2OfgrSj/FCTkInKPqaCR0gD7K80q+mLBrN3PUkDrJQZpvRZIff3/xmVU1Wer +uQLFJjnFb2dqu0s/FY/2kWiJtBCakXvXEOb7zfbINuayL+MSsCGSdVYsSliS5qQp +gyDap+8b5fpXZVJkq92hrcNtbkg7hCYUJczt8n9hcCTJCfUpApvaFQ18pe+zpyl4 ++WzkP66I28hniMQyUlA1hBiskT7qiouq0m8IOodhv2fagSZKjOTTU2xkSBc//fy3 +ZpsL7WqgsZS7Q+0VRK8gKfqkxg5OYQIDAQABo3YwdDAdBgNVHQ4EFgQU2RQ8yO+O +gN8oVW2SW7RLrfYd9jEwRQYDVR0jBD4wPIAU2RQ8yO+OgN8oVW2SW7RLrfYd9jGh +GaQXMBUxEzARBgNVBAMTCnVuaXQtdGVzdHOCCQDHyErgUOZvuTAMBgNVHRMEBTAD +AQH/MA0GCSqGSIb3DQEBBQUAA4IBAQBRv+M/6+FiVu7KXNjFI5pSN17OcW5QUtPr +odJMlWrJBtynn/TA1oJlYu3yV5clc/71Vr/AxuX5xGP+IXL32YDF9lTUJXG/uUGk ++JETpKmQviPbRsvzYhz4pf6ZIOZMc3/GIcNq92ECbseGO+yAgyWUVKMmZM0HqXC9 +ovNslqe0M8C1sLm1zAR5z/h/litE7/8O2ietija3Q/qtl2TOXJdCA6sgjJX2WUql +ybrC55ct18NKf3qhpcEkGQvFU40rVYApJpi98DiZPYFdx1oBDp/f4uZ3ojpxRVFT +cDwcJLfNRCPUhormsY7fDS9xSyThiHsW9mjJYdcaKQkwYZ0F11yB +-----END CERTIFICATE----- diff --git a/appengine/standard_python37/pubsub/main.py b/appengine/standard_python37/pubsub/main.py new file mode 100644 index 00000000000..7552dfaee35 --- /dev/null +++ b/appengine/standard_python37/pubsub/main.py @@ -0,0 +1,110 @@ +# Copyright 2019 Google, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START app] +import base64 +from flask import current_app, Flask, render_template, request +import json +import logging +import os + +from google.auth.transport import requests +from google.cloud import pubsub_v1 +from google.oauth2 import id_token + + +app = Flask(__name__) + +# Configure the following environment variables via app.yaml +# This is used in the push request handler to verify that the request came from +# pubsub and originated from a trusted source. +app.config['PUBSUB_VERIFICATION_TOKEN'] = \ + os.environ['PUBSUB_VERIFICATION_TOKEN'] +app.config['PUBSUB_TOPIC'] = os.environ['PUBSUB_TOPIC'] +app.config['GCLOUD_PROJECT'] = os.environ['GOOGLE_CLOUD_PROJECT'] + +# Global list to store messages, tokens, etc. received by this instance. +MESSAGES = [] +TOKENS = [] +CLAIMS = [] + +# [START index] +@app.route('/', methods=['GET', 'POST']) +def index(): + if request.method == 'GET': + return render_template('index.html', messages=MESSAGES, tokens=TOKENS, + claims=CLAIMS) + + data = request.form.get('payload', 'Example payload').encode('utf-8') + + publisher = pubsub_v1.PublisherClient() + topic_path = publisher.topic_path(app.config['GCLOUD_PROJECT'], + app.config['PUBSUB_TOPIC']) + future = publisher.publish(topic_path, data) + future.result() + return 'OK', 200 +# [END index] + + +# [START push] +@app.route('/_ah/push-handlers/receive_messages', methods=['POST']) +def receive_messages_handler(): + # Verify that the request originates from the application. + if (request.args.get('token', '') != + current_app.config['PUBSUB_VERIFICATION_TOKEN']): + return 'Invalid request', 400 + + # Verify that the push request originates from Cloud Pub/Sub. + try: + # Get the Cloud Pub/Sub-generated JWT in the "Authorization" header. + bearer_token = request.headers.get('Authorization') + token = bearer_token.split(' ')[1] + TOKENS.append(token) + + # Verify and decode the JWT. `verify_oauth2_token` verifies + # the JWT signature, the `aud` claim, and the `exp` claim. + claim = id_token.verify_oauth2_token(token, requests.Request(), + audience='example.com') + # Must also verify the `iss` claim. + if claim['iss'] not in [ + 'accounts.google.com', + 'https://accounts.google.com' + ]: + raise ValueError('Wrong issuer.') + CLAIMS.append(claim) + except Exception as e: + return 'Invalid token: {}\n'.format(e), 400 + + envelope = json.loads(request.data.decode('utf-8')) + payload = base64.b64decode(envelope['message']['data']) + MESSAGES.append(payload) + # Returning any 2xx status indicates successful receipt of the message. + return 'OK', 200 +# [END push] + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
          {}
          + See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) +# [END app] diff --git a/appengine/standard_python37/pubsub/main_test.py b/appengine/standard_python37/pubsub/main_test.py new file mode 100644 index 00000000000..66601db0e00 --- /dev/null +++ b/appengine/standard_python37/pubsub/main_test.py @@ -0,0 +1,123 @@ +# Copyright 2019 Google, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is for testing purposes only. You SHOULD NOT include it +# or the PEM files when deploying your app. + +import base64 +import calendar +import datetime +import json +import os +import pytest + +from google.auth import crypt +from google.auth import jwt +from google.oauth2 import id_token + +import main + + +DATA_DIR = os.path.join(os.path.dirname(__file__), 'data') + +with open(os.path.join(DATA_DIR, 'privatekey.pem'), 'rb') as fh: + PRIVATE_KEY_BYTES = fh.read() + +with open(os.path.join(DATA_DIR, 'public_cert.pem'), 'rb') as fh: + PUBLIC_CERT_BYTES = fh.read() + + +@pytest.fixture +def client(): + main.app.testing = True + return main.app.test_client() + + +@pytest.fixture +def signer(): + return crypt.RSASigner.from_string(PRIVATE_KEY_BYTES, '1') + + +@pytest.fixture +def fake_token(signer): + now = calendar.timegm(datetime.datetime.utcnow().utctimetuple()) + payload = { + 'aud': 'example.com', + 'azp': '1234567890', + 'email': 'pubsub@example.iam.gserviceaccount.com', + 'email_verified': True, + 'iat': now, + 'exp': now + 3600, + 'iss': 'https://accounts.google.com', + 'sub': '1234567890' + } + header = { + 'alg': 'RS256', + 'kid': signer.key_id, + 'typ': 'JWT' + } + yield jwt.encode(signer, payload, header=header) + + +def _verify_mocked_oauth2_token(token, request, audience): + claims = jwt.decode(token, certs=PUBLIC_CERT_BYTES, verify=True) + return claims + + +def test_index(client): + r = client.get('/') + assert r.status_code == 200 + + +def test_post_index(client): + r = client.post('/', data={'payload': 'Test payload'}) + assert r.status_code == 200 + + +def test_push_endpoint(monkeypatch, client, fake_token): + monkeypatch.setattr(id_token, 'verify_oauth2_token', + _verify_mocked_oauth2_token) + + url = '/_ah/push-handlers/receive_messages?token=' + \ + os.environ['PUBSUB_VERIFICATION_TOKEN'] + + r = client.post( + url, + data=json.dumps({ + "message": { + "data": base64.b64encode( + u'Test message'.encode('utf-8') + ).decode('utf-8') + } + }), + headers=dict( + Authorization="Bearer " + fake_token.decode('utf-8') + ) + ) + assert r.status_code == 200 + + # Make sure the message is visible on the home page. + r = client.get('/') + assert r.status_code == 200 + assert 'Test message' in r.data.decode('utf-8') + + +def test_push_endpoint_errors(client): + # no token + r = client.post('/_ah/push-handlers/receive_messages') + assert r.status_code == 400 + + # invalid token + r = client.post('/_ah/push-handlers/receive_messages?token=bad') + assert r.status_code == 400 diff --git a/appengine/standard_python37/pubsub/requirements.txt b/appengine/standard_python37/pubsub/requirements.txt new file mode 100644 index 00000000000..04d95eb5fbd --- /dev/null +++ b/appengine/standard_python37/pubsub/requirements.txt @@ -0,0 +1,4 @@ +Flask==1.0.2 +google-api-python-client==1.7.8 +google-auth==1.6.3 +google-cloud-pubsub==0.40.0 diff --git a/appengine/standard_python37/pubsub/sample_message.json b/appengine/standard_python37/pubsub/sample_message.json new file mode 100644 index 00000000000..8fe62d23fb9 --- /dev/null +++ b/appengine/standard_python37/pubsub/sample_message.json @@ -0,0 +1,5 @@ +{ + "message": { + "data": "SGVsbG8sIFdvcmxkIQ==" + } +} diff --git a/appengine/standard_python37/pubsub/templates/index.html b/appengine/standard_python37/pubsub/templates/index.html new file mode 100644 index 00000000000..9323b6b2f42 --- /dev/null +++ b/appengine/standard_python37/pubsub/templates/index.html @@ -0,0 +1,48 @@ +{# +# Copyright 2019 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + + + + Pub/Sub Python on Google App Engine Standard Environment + + +
          +

          Print BEARER TOKENS: + {% for token in tokens: %} +

        • {{token}}
          • + {% endfor %} +

          +

          Print CLAIMS: + {% for claim in claims: %} +

        • {{claim}}
          • + {% endfor %} +

          +

          Messages received by this instance:

          +
            + {% for message in messages: %} +
          • {{message}}
            • + {% endfor %} +
          +

          Note: because your application is likely running multiple instances, each instance will have a different list of messages.

          +
          + +
          + + +
          + + + diff --git a/appengine/standard_python37/redis/app.yaml b/appengine/standard_python37/redis/app.yaml new file mode 100644 index 00000000000..f4ba52c8352 --- /dev/null +++ b/appengine/standard_python37/redis/app.yaml @@ -0,0 +1,6 @@ +runtime: python37 + +env_variables: + REDIS_HOST: your-redis-host + REDIS_PORT: your-redis-port + REDIS_PASSWORD: your-redis-password diff --git a/appengine/standard_python37/redis/main.py b/appengine/standard_python37/redis/main.py new file mode 100644 index 00000000000..68faf431c44 --- /dev/null +++ b/appengine/standard_python37/redis/main.py @@ -0,0 +1,45 @@ +# Copyright 2018 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from flask import Flask +import redis + + +app = Flask(__name__) + + +# [START gae_py37_redis_client] +redis_host = os.environ.get('REDIS_HOST', 'localhost') +redis_port = int(os.environ.get('REDIS_PORT', 6379)) +redis_password = os.environ.get('REDIS_PASSWORD', None) +redis_client = redis.StrictRedis( + host=redis_host, port=redis_port, password=redis_password) +# [END gae_py37_redis_client] + + +# [START gae_py37_redis_example] +@app.route('/') +def index(): + value = redis_client.incr('counter', 1) + return 'Value is {}'.format(value) +# [END gae_py37_redis_example] + + +if __name__ == '__main__': + # This is used when running locally only. When deploying to Google App + # Engine, a webserver process such as Gunicorn will serve the app. This + # can be configured by adding an `entrypoint` to app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/appengine/standard_python37/redis/main_test.py b/appengine/standard_python37/redis/main_test.py new file mode 100644 index 00000000000..d52ea81e9a0 --- /dev/null +++ b/appengine/standard_python37/redis/main_test.py @@ -0,0 +1,35 @@ +# Copyright 2018 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +import main + + +def test_index(): + try: + main.redis_client.set('counter', 0) + except Exception: + pytest.skip('Redis is unavailable.') + + main.app.testing = True + client = main.app.test_client() + + r = client.get('/') + assert r.status_code == 200 + assert '1' in r.data.decode('utf-8') + + r = client.get('/') + assert r.status_code == 200 + assert '2' in r.data.decode('utf-8') diff --git a/appengine/standard_python37/redis/requirements.txt b/appengine/standard_python37/redis/requirements.txt new file mode 100644 index 00000000000..cdc6af1bf92 --- /dev/null +++ b/appengine/standard_python37/redis/requirements.txt @@ -0,0 +1,2 @@ +Flask==1.0.2 +redis==3.1.0 diff --git a/appengine/standard_python37/spanner/app.yaml b/appengine/standard_python37/spanner/app.yaml new file mode 100644 index 00000000000..7b6a30c8647 --- /dev/null +++ b/appengine/standard_python37/spanner/app.yaml @@ -0,0 +1,5 @@ +runtime: python37 + +env_variables: + SPANNER_INSTANCE: "YOUR-SPANNER-INSTANCE-ID" + SPANNER_DATABASE: "YOUR-SPANNER-DATABASE-ID" diff --git a/appengine/standard_python37/spanner/main.py b/appengine/standard_python37/spanner/main.py new file mode 100644 index 00000000000..3e420b0408b --- /dev/null +++ b/appengine/standard_python37/spanner/main.py @@ -0,0 +1,36 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START gae_python37_cloud_spanner] +import os + +from flask import Flask +from google.cloud import spanner + +app = Flask(__name__) +spanner_client = spanner.Client() + +instance_id = os.environ.get('SPANNER_INSTANCE') +database_id = os.environ.get('SPANNER_DATABASE') + + +@app.route('/') +def main(): + database = spanner_client.instance(instance_id).database(database_id) + with database.snapshot() as snapshot: + cursor = snapshot.execute_sql('SELECT 1') + results = list(cursor) + + return 'Query Result: {}'.format(results[0][0]) +# [END gae_python37_cloud_spanner] diff --git a/appengine/standard_python37/spanner/main_test.py b/appengine/standard_python37/spanner/main_test.py new file mode 100644 index 00000000000..b3557941dad --- /dev/null +++ b/appengine/standard_python37/spanner/main_test.py @@ -0,0 +1,26 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def test_main(): + import main + + main.database_id = 'example-db' + + main.app.testing = True + client = main.app.test_client() + + r = client.get('/') + assert r.status_code == 200 + assert 'Query Result: 1' in r.data.decode('utf-8') diff --git a/appengine/standard_python37/spanner/requirements.txt b/appengine/standard_python37/spanner/requirements.txt new file mode 100644 index 00000000000..2d556659c34 --- /dev/null +++ b/appengine/standard_python37/spanner/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-spanner==1.7.1 +Flask==1.0.2 diff --git a/appengine/standard_python37/warmup/app.yaml b/appengine/standard_python37/warmup/app.yaml new file mode 100644 index 00000000000..ba886b0ef8f --- /dev/null +++ b/appengine/standard_python37/warmup/app.yaml @@ -0,0 +1,4 @@ +runtime: python37 + +inbound_services: +- warmup diff --git a/appengine/standard_python37/warmup/main.py b/appengine/standard_python37/warmup/main.py new file mode 100644 index 00000000000..8d265189b77 --- /dev/null +++ b/appengine/standard_python37/warmup/main.py @@ -0,0 +1,38 @@ +# Copyright 2018 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START gae_python37_warmup_app] +from flask import Flask + + +app = Flask(__name__) + + +@app.route('/') +def main(): + return 'Hello World!' + + +@app.route('/_ah/warmup') +def warmup(): + # Handle your warmup logic here, e.g. set up a database connection pool + return '', 200, {} + + +if __name__ == '__main__': + # This is used when running locally only. When deploying to Google App + # Engine, a webserver process such as Gunicorn will serve the app. This + # can be configured by adding an `entrypoint` to app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) +# [END gae_python37_warmup_app] diff --git a/appengine/standard_python37/warmup/main_test.py b/appengine/standard_python37/warmup/main_test.py new file mode 100644 index 00000000000..3a6a6512ed7 --- /dev/null +++ b/appengine/standard_python37/warmup/main_test.py @@ -0,0 +1,32 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import main + + +def test_index(): + main.app.testing = True + client = main.app.test_client() + + r = client.get('/') + assert r.status_code == 200 + assert 'Hello World' in r.data.decode('utf-8') + + +def test_warmup(): + main.app.testing = True + client = main.app.test_client() + + r = client.get('/') + assert r.status_code == 200 diff --git a/appengine/standard_python37/warmup/requirements.txt b/appengine/standard_python37/warmup/requirements.txt new file mode 100644 index 00000000000..7d267af964d --- /dev/null +++ b/appengine/standard_python37/warmup/requirements.txt @@ -0,0 +1 @@ +flask==1.0.2 diff --git a/appengine/storage/README.md b/appengine/storage/README.md deleted file mode 100644 index b934b2f46ab..00000000000 --- a/appengine/storage/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# Cloud Storage & Google App Engine - -This sample demonstrates how to use the [Google Cloud Storage API](https://cloud.google.com/storage/docs/json_api/) from Google App Engine. - -Refer to the [App Engine Samples README](../README.md) for information on how to run and deploy this sample. - -## Setup - -Before running the sample: - -1. You need a Cloud Storage Bucket. You create one with [`gsutil`](https://cloud.google.com/storage/docs/gsutil): - - gsutil mb gs://your-bucket-name - -2. Update `main.py` and replace `` with your Cloud Storage bucket. diff --git a/appengine/storage/main.py b/appengine/storage/main.py deleted file mode 100644 index e952878ced8..00000000000 --- a/appengine/storage/main.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Sample Google App Engine application that lists the objects in a Google Cloud -Storage bucket. - -For more information about Cloud Storage, see README.md in /storage. -For more information about Google App Engine, see README.md in /appengine. -""" - -import json - -from googleapiclient import discovery -from oauth2client.client import GoogleCredentials -import webapp2 - - -# The bucket that will be used to list objects. -BUCKET_NAME = '' - -credentials = GoogleCredentials.get_application_default() -storage = discovery.build('storage', 'v1', credentials=credentials) - - -class MainPage(webapp2.RequestHandler): - def get(self): - response = storage.objects().list(bucket=BUCKET_NAME).execute() - - self.response.write( - '

          Objects.list raw response:

          ' - '
          {}
          '.format( - json.dumps(response, sort_keys=True, indent=2))) - - -app = webapp2.WSGIApplication([ - ('/', MainPage) -], debug=True) diff --git a/appengine/storage/main_test.py b/appengine/storage/main_test.py deleted file mode 100644 index e68fe66a16d..00000000000 --- a/appengine/storage/main_test.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import re - -import main -import webtest - - -def test_get(cloud_config): - main.BUCKET_NAME = cloud_config.project - app = webtest.TestApp(main.app) - - response = app.get('/') - - assert response.status_int == 200 - assert re.search( - re.compile(r'.*.*items.*etag.*', re.DOTALL), - response.body) diff --git a/appengine/storage/requirements.txt b/appengine/storage/requirements.txt deleted file mode 100644 index c3b2784ce87..00000000000 --- a/appengine/storage/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -google-api-python-client==1.5.0 diff --git a/asset/cloud-client/quickstart_batchgetassetshistory.py b/asset/cloud-client/quickstart_batchgetassetshistory.py new file mode 100644 index 00000000000..ca6a29ac92c --- /dev/null +++ b/asset/cloud-client/quickstart_batchgetassetshistory.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse + + +def batch_get_assets_history(project_id, asset_names): + # [START asset_quickstart_batch_get_assets_history] + from google.cloud import asset_v1 + from google.cloud.asset_v1.proto import assets_pb2 + from google.cloud.asset_v1 import enums + + # TODO project_id = 'Your Google Cloud Project ID' + # TODO asset_names = 'Your asset names list, e.g.: + # ["//storage.googleapis.com/[BUCKET_NAME]",]' + + client = asset_v1.AssetServiceClient() + parent = client.project_path(project_id) + content_type = enums.ContentType.RESOURCE + read_time_window = assets_pb2.TimeWindow() + response = client.batch_get_assets_history( + parent, content_type, read_time_window, asset_names) + print('assets: {}'.format(response.assets)) + # [END asset_quickstart_batch_get_assets_history] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument('project_id', help='Your Google Cloud project ID') + parser.add_argument( + 'asset_names', + help='The asset names for which history will be fetched, comma ' + 'delimited, e.g.: //storage.googleapis.com/[BUCKET_NAME]') + + args = parser.parse_args() + + asset_name_list = args.asset_names.split(',') + + batch_get_assets_history(args.project_id, asset_name_list) diff --git a/asset/cloud-client/quickstart_batchgetassetshistory_test.py b/asset/cloud-client/quickstart_batchgetassetshistory_test.py new file mode 100644 index 00000000000..9cdc30ffbce --- /dev/null +++ b/asset/cloud-client/quickstart_batchgetassetshistory_test.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time + +from google.cloud import storage +import pytest + +import quickstart_batchgetassetshistory + +PROJECT = os.environ['GCLOUD_PROJECT'] +BUCKET = 'assets-{}'.format(int(time.time())) + + +@pytest.fixture(scope='module') +def storage_client(): + yield storage.Client() + + +@pytest.fixture(scope='module') +def asset_bucket(storage_client): + bucket = storage_client.create_bucket(BUCKET) + + yield BUCKET + + try: + bucket.delete(force=True) + except Exception as e: + print('Failed to delete bucket{}'.format(BUCKET)) + raise e + + +def test_batch_get_assets_history(asset_bucket, capsys): + bucket_asset_name = '//storage.googleapis.com/{}'.format(BUCKET) + asset_names = [bucket_asset_name, ] + # There's some delay between bucket creation and when it's reflected in the + # backend. + time.sleep(15) + quickstart_batchgetassetshistory.batch_get_assets_history( + PROJECT, asset_names) + out, _ = capsys.readouterr() + + if not out: + assert bucket_asset_name in out diff --git a/asset/cloud-client/quickstart_exportassets.py b/asset/cloud-client/quickstart_exportassets.py new file mode 100644 index 00000000000..ddc05e5544e --- /dev/null +++ b/asset/cloud-client/quickstart_exportassets.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse + + +def export_assets(project_id, dump_file_path): + # [START asset_quickstart_export_assets] + from google.cloud import asset_v1 + from google.cloud.asset_v1.proto import asset_service_pb2 + + # TODO project_id = 'Your Google Cloud Project ID' + # TODO dump_file_path = 'Your asset dump file path' + + client = asset_v1.AssetServiceClient() + parent = client.project_path(project_id) + output_config = asset_service_pb2.OutputConfig() + output_config.gcs_destination.uri = dump_file_path + response = client.export_assets(parent, output_config) + print(response.result()) + # [END asset_quickstart_export_assets] + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument('project_id', help='Your Google Cloud project ID') + parser.add_argument( + 'dump_file_path', + help='The file ExportAssets API will dump assets to, ' + 'e.g.: gs:///asset_dump_file') + + args = parser.parse_args() + + export_assets(args.project_id, args.dump_file_path) diff --git a/asset/cloud-client/quickstart_exportassets_test.py b/asset/cloud-client/quickstart_exportassets_test.py new file mode 100644 index 00000000000..4e08e007aed --- /dev/null +++ b/asset/cloud-client/quickstart_exportassets_test.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time + +from google.cloud import storage +import pytest + +import quickstart_exportassets + +PROJECT = os.environ['GCLOUD_PROJECT'] +BUCKET = 'assets-{}'.format(int(time.time())) + + +@pytest.fixture(scope='module') +def storage_client(): + yield storage.Client() + + +@pytest.fixture(scope='module') +def asset_bucket(storage_client): + bucket = storage_client.create_bucket(BUCKET) + + yield BUCKET + + try: + bucket.delete(force=True) + except Exception as e: + print('Failed to delete bucket{}'.format(BUCKET)) + raise e + + +def test_export_assets(asset_bucket, capsys): + dump_file_path = 'gs://{}/assets-dump.txt'.format(asset_bucket) + quickstart_exportassets.export_assets(PROJECT, dump_file_path) + out, _ = capsys.readouterr() + + assert dump_file_path in out diff --git a/asset/cloud-client/requirements.txt b/asset/cloud-client/requirements.txt new file mode 100644 index 00000000000..0e87ed672ee --- /dev/null +++ b/asset/cloud-client/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-storage==1.13.2 +google-cloud-asset==0.2.0 diff --git a/auth/api-client/requirements.txt b/auth/api-client/requirements.txt new file mode 100644 index 00000000000..db583fb6f65 --- /dev/null +++ b/auth/api-client/requirements.txt @@ -0,0 +1,3 @@ +google-api-python-client==1.7.8 +google-auth-httplib2==0.0.3 +google-auth==1.6.2 diff --git a/auth/api-client/snippets.py b/auth/api-client/snippets.py new file mode 100644 index 00000000000..ff1f8804f40 --- /dev/null +++ b/auth/api-client/snippets.py @@ -0,0 +1,116 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Demonstrates how to authenticate to Google Cloud Platform APIs using +the Google API Client.""" + +import argparse + + +# [START auth_api_implicit] +def implicit(project): + import googleapiclient.discovery + + # If you don't specify credentials when constructing the client, the + # client library will look for credentials in the environment. + storage_client = googleapiclient.discovery.build('storage', 'v1') + + # Make an authenticated API request + buckets = storage_client.buckets().list(project=project).execute() + print(buckets) +# [END auth_api_implicit] + + +# [START auth_api_explicit] +def explicit(project): + from google.oauth2 import service_account + import googleapiclient.discovery + + # Construct service account credentials using the service account key + # file. + credentials = service_account.Credentials.from_service_account_file( + 'service_account.json') + + # Explicitly pass the credentials to the client library. + storage_client = googleapiclient.discovery.build( + 'storage', 'v1', credentials=credentials) + + # Make an authenticated API request + buckets = storage_client.buckets().list(project=project).execute() + print(buckets) +# [END auth_api_explicit] + + +# [START auth_api_explicit_compute_engine] +def explicit_compute_engine(project): + from google.auth import compute_engine + import googleapiclient.discovery + + # Explicitly use Compute Engine credentials. These credentials are + # available on Compute Engine, App Engine Flexible, and Container Engine. + credentials = compute_engine.Credentials() + + # Explicitly pass the credentials to the client library. + storage_client = googleapiclient.discovery.build( + 'storage', 'v1', credentials=credentials) + + # Make an authenticated API request + buckets = storage_client.buckets().list(project=project).execute() + print(buckets) +# [END auth_api_explicit_compute_engine] + + +# [START auth_api_explicit_app_engine] +def explicit_app_engine(project): + from google.auth import app_engine + import googleapiclient.discovery + + # Explicitly use App Engine credentials. These credentials are + # only available when running on App Engine Standard. + credentials = app_engine.Credentials() + + # Explicitly pass the credentials to the client library. + storage_client = googleapiclient.discovery.build( + 'storage', 'v1', credentials=credentials) + + # Make an authenticated API request + buckets = storage_client.buckets().list(project=project).execute() + print(buckets) +# [END auth_api_explicit_app_engine] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('project') + + subparsers = parser.add_subparsers(dest='command') + subparsers.add_parser('implicit', help=implicit.__doc__) + subparsers.add_parser('explicit', help=explicit.__doc__) + subparsers.add_parser( + 'explicit_compute_engine', help=explicit_compute_engine.__doc__) + subparsers.add_parser( + 'explicit_app_engine', help=explicit_app_engine.__doc__) + + args = parser.parse_args() + + if args.command == 'implicit': + implicit(args.project) + elif args.command == 'explicit': + explicit(args.project) + elif args.command == 'explicit_compute_engine': + explicit_compute_engine(args.project) + elif args.command == 'explicit_app_engine': + explicit_app_engine(args.project) diff --git a/auth/api-client/snippets_test.py b/auth/api-client/snippets_test.py new file mode 100644 index 00000000000..00161fc4d47 --- /dev/null +++ b/auth/api-client/snippets_test.py @@ -0,0 +1,52 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import google.auth +import mock + +import snippets + + +def test_implicit(): + snippets.implicit(os.environ['GCLOUD_PROJECT']) + + +def test_explicit(): + with open(os.environ['GOOGLE_APPLICATION_CREDENTIALS']) as creds_file: + creds_file_data = creds_file.read() + + open_mock = mock.mock_open(read_data=creds_file_data) + + with mock.patch('io.open', open_mock): + snippets.explicit(os.environ['GCLOUD_PROJECT']) + + +def test_explicit_compute_engine(): + adc, project = google.auth.default() + credentials_patch = mock.patch( + 'google.auth.compute_engine.Credentials', return_value=adc) + + with credentials_patch: + snippets.explicit_compute_engine(project) + + +def test_explicit_app_engine(): + adc, project = google.auth.default() + credentials_patch = mock.patch( + 'google.auth.app_engine.Credentials', return_value=adc) + + with credentials_patch: + snippets.explicit_app_engine(project) diff --git a/auth/cloud-client/requirements.txt b/auth/cloud-client/requirements.txt new file mode 100644 index 00000000000..8561c491b35 --- /dev/null +++ b/auth/cloud-client/requirements.txt @@ -0,0 +1 @@ +google-cloud-storage==1.13.2 diff --git a/auth/cloud-client/snippets.py b/auth/cloud-client/snippets.py new file mode 100644 index 00000000000..a7ae867aa22 --- /dev/null +++ b/auth/cloud-client/snippets.py @@ -0,0 +1,87 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Demonstrates how to authenticate to Google Cloud Platform APIs using +the Google Cloud Client Libraries.""" + +import argparse + + +# [START auth_cloud_implicit] +def implicit(): + from google.cloud import storage + + # If you don't specify credentials when constructing the client, the + # client library will look for credentials in the environment. + storage_client = storage.Client() + + # Make an authenticated API request + buckets = list(storage_client.list_buckets()) + print(buckets) +# [END auth_cloud_implicit] + + +# [START auth_cloud_explicit] +def explicit(): + from google.cloud import storage + + # Explicitly use service account credentials by specifying the private key + # file. + storage_client = storage.Client.from_service_account_json( + 'service_account.json') + + # Make an authenticated API request + buckets = list(storage_client.list_buckets()) + print(buckets) +# [END auth_cloud_explicit] + + +# [START auth_cloud_explicit_compute_engine] +def explicit_compute_engine(project): + from google.auth import compute_engine + from google.cloud import storage + + # Explicitly use Compute Engine credentials. These credentials are + # available on Compute Engine, App Engine Flexible, and Container Engine. + credentials = compute_engine.Credentials() + + # Create the client using the credentials and specifying a project ID. + storage_client = storage.Client(credentials=credentials, project=project) + + # Make an authenticated API request + buckets = list(storage_client.list_buckets()) + print(buckets) +# [END auth_cloud_explicit_compute_engine] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + + subparsers = parser.add_subparsers(dest='command') + subparsers.add_parser('implicit', help=implicit.__doc__) + subparsers.add_parser('explicit', help=explicit.__doc__) + explicit_gce_parser = subparsers.add_parser( + 'explicit_compute_engine', help=explicit_compute_engine.__doc__) + explicit_gce_parser.add_argument('project') + + args = parser.parse_args() + + if args.command == 'implicit': + implicit() + elif args.command == 'explicit': + explicit() + elif args.command == 'explicit_compute_engine': + explicit_compute_engine(args.project) diff --git a/auth/cloud-client/snippets_test.py b/auth/cloud-client/snippets_test.py new file mode 100644 index 00000000000..af8832b8aa1 --- /dev/null +++ b/auth/cloud-client/snippets_test.py @@ -0,0 +1,43 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import google.auth +import mock + +import snippets + + +def test_implicit(): + snippets.implicit() + + +def test_explicit(): + with open(os.environ['GOOGLE_APPLICATION_CREDENTIALS']) as creds_file: + creds_file_data = creds_file.read() + + open_mock = mock.mock_open(read_data=creds_file_data) + + with mock.patch('io.open', open_mock): + snippets.explicit() + + +def test_explicit_compute_engine(): + adc, project = google.auth.default() + credentials_patch = mock.patch( + 'google.auth.compute_engine.Credentials', return_value=adc) + + with credentials_patch: + snippets.explicit_compute_engine(project) diff --git a/auth/end-user/web/main.py b/auth/end-user/web/main.py new file mode 100644 index 00000000000..64785098488 --- /dev/null +++ b/auth/end-user/web/main.py @@ -0,0 +1,119 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""An example web application that obtains authorization and credentials from +an end user. + +This sample is used on +https://developers.google.com/identity/protocols/OAuth2WebServer. Please +refer to that page for instructions on using this sample. + +Notably, you'll need to obtain a OAuth2.0 client secrets file and set the +``GOOGLE_CLIENT_SECRETS`` environment variable to point to that file. +""" + +import os + +import flask +import google.oauth2.credentials +import google_auth_oauthlib.flow +import googleapiclient.discovery + +# The path to the client-secrets.json file obtained from the Google API +# Console. You must set this before running this application. +CLIENT_SECRETS_FILENAME = os.environ['GOOGLE_CLIENT_SECRETS'] +# The OAuth 2.0 scopes that this application will ask the user for. In this +# case the application will ask for basic profile information. +SCOPES = ['email', 'profile'] + +app = flask.Flask(__name__) +# TODO: A secret key is included in the sample so that it works but if you +# use this code in your application please replace this with a truly secret +# key. See http://flask.pocoo.org/docs/0.12/quickstart/#sessions. +app.secret_key = 'TODO: replace with a secret value' + + +@app.route('/') +def index(): + if 'credentials' not in flask.session: + return flask.redirect('authorize') + + # Load the credentials from the session. + credentials = google.oauth2.credentials.Credentials( + **flask.session['credentials']) + + # Get the basic user info from the Google OAuth2.0 API. + client = googleapiclient.discovery.build( + 'oauth2', 'v2', credentials=credentials) + + response = client.userinfo().v2().me().get().execute() + + return str(response) + + +@app.route('/authorize') +def authorize(): + # Create a flow instance to manage the OAuth 2.0 Authorization Grant Flow + # steps. + flow = google_auth_oauthlib.flow.Flow.from_client_secrets_file( + CLIENT_SECRETS_FILENAME, scopes=SCOPES) + flow.redirect_uri = flask.url_for('oauth2callback', _external=True) + authorization_url, state = flow.authorization_url( + # This parameter enables offline access which gives your application + # an access token and a refresh token for the user's credentials. + access_type='offline', + # This parameter enables incremental auth. + include_granted_scopes='true') + + # Store the state in the session so that the callback can verify the + # authorization server response. + flask.session['state'] = state + + return flask.redirect(authorization_url) + + +@app.route('/oauth2callback') +def oauth2callback(): + # Specify the state when creating the flow in the callback so that it can + # verify the authorization server response. + state = flask.session['state'] + flow = google_auth_oauthlib.flow.Flow.from_client_secrets_file( + CLIENT_SECRETS_FILENAME, scopes=SCOPES, state=state) + flow.redirect_uri = flask.url_for('oauth2callback', _external=True) + + # Use the authorization server's response to fetch the OAuth 2.0 tokens. + authorization_response = flask.request.url + flow.fetch_token(authorization_response=authorization_response) + + # Store the credentials in the session. + credentials = flow.credentials + flask.session['credentials'] = { + 'token': credentials.token, + 'refresh_token': credentials.refresh_token, + 'token_uri': credentials.token_uri, + 'client_id': credentials.client_id, + 'client_secret': credentials.client_secret, + 'scopes': credentials.scopes + } + + return flask.redirect(flask.url_for('index')) + + +if __name__ == '__main__': + # When running locally with Flask's development server this disables + # OAuthlib's HTTPs verification. When running in production with a WSGI + # server such as gunicorn this option will not be set and your application + # *must* use HTTPS. + os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1' + app.run('localhost', 8080, debug=True) diff --git a/auth/end-user/web/main_test.py b/auth/end-user/web/main_test.py new file mode 100644 index 00000000000..7663d8f354c --- /dev/null +++ b/auth/end-user/web/main_test.py @@ -0,0 +1,38 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +import main + +# Note: samples that do end-user auth are difficult to test in an automated +# way. These tests are basic sanity checks. + + +@pytest.fixture +def client(): + main.app.testing = True + return main.app.test_client() + + +def test_index_wo_credentials(client): + r = client.get('/') + assert r.status_code == 302 + assert r.headers['location'].endswith('/authorize') + + +def test_authorize(client): + r = client.get('/authorize') + assert r.status_code == 302 + assert r.headers['location'].startswith('https://accounts.google.com') diff --git a/auth/end-user/web/requirements.txt b/auth/end-user/web/requirements.txt new file mode 100644 index 00000000000..d63350dd214 --- /dev/null +++ b/auth/end-user/web/requirements.txt @@ -0,0 +1,6 @@ +google-auth==1.6.2 +google-auth-oauthlib==0.2.0 +google-auth-httplib2==0.0.3 +google-api-python-client==1.7.8 +flask==1.0.2 +requests==2.21.0 diff --git a/auth/http-client/requirements.txt b/auth/http-client/requirements.txt new file mode 100644 index 00000000000..39137024e5d --- /dev/null +++ b/auth/http-client/requirements.txt @@ -0,0 +1,2 @@ +requests==2.21.0 +google-auth==1.6.2 diff --git a/auth/http-client/snippets.py b/auth/http-client/snippets.py new file mode 100644 index 00000000000..9985160982c --- /dev/null +++ b/auth/http-client/snippets.py @@ -0,0 +1,83 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Demonstrates how to authenticate to Google Cloud Platform APIs using the +Requests HTTP library.""" + +import argparse + + +# [START auth_http_implicit] +def implicit(): + import google.auth + from google.auth.transport import requests + + # Get the credentials and project ID from the environment. + credentials, project = google.auth.default( + scopes=['https://www.googleapis.com/auth/cloud-platform']) + + # Create a requests Session object with the credentials. + session = requests.AuthorizedSession(credentials) + + # Make an authenticated API request + response = session.get( + 'https://www.googleapis.com/storage/v1/b'.format(project), + params={'project': project}) + response.raise_for_status() + buckets = response.json() + print(buckets) +# [END auth_http_implicit] + + +# [START auth_http_explicit] +def explicit(project): + from google.auth.transport import requests + from google.oauth2 import service_account + + # Construct service account credentials using the service account key + # file. + credentials = service_account.Credentials.from_service_account_file( + 'service_account.json') + credentials = credentials.with_scopes( + ['https://www.googleapis.com/auth/cloud-platform']) + + # Create a requests Session object with the credentials. + session = requests.AuthorizedSession(credentials) + + # Make an authenticated API request + response = session.get( + 'https://www.googleapis.com/storage/v1/b'.format(project), + params={'project': project}) + response.raise_for_status() + buckets = response.json() + print(buckets) +# [END auth_http_explicit] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + + subparsers = parser.add_subparsers(dest='command') + subparsers.add_parser('implicit', help=implicit.__doc__) + explicit_parser = subparsers.add_parser('explicit', help=explicit.__doc__) + explicit_parser.add_argument('project') + + args = parser.parse_args() + + if args.command == 'implicit': + implicit() + elif args.command == 'explicit': + explicit(args.project) diff --git a/auth/http-client/snippets_test.py b/auth/http-client/snippets_test.py new file mode 100644 index 00000000000..c1e2948e196 --- /dev/null +++ b/auth/http-client/snippets_test.py @@ -0,0 +1,33 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import mock + +import snippets + + +def test_implicit(): + snippets.implicit() + + +def test_explicit(): + with open(os.environ['GOOGLE_APPLICATION_CREDENTIALS']) as creds_file: + creds_file_data = creds_file.read() + + open_mock = mock.mock_open(read_data=creds_file_data) + + with mock.patch('io.open', open_mock): + snippets.explicit(os.environ['GCLOUD_PROJECT']) diff --git a/bigquery/README.md b/bigquery/README.md deleted file mode 100644 index 6b5977b6931..00000000000 --- a/bigquery/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# Google BigQuery Samples - -This section contains samples for [Google BigQuery](https://cloud.google.com/bigquery). - -## Running the samples - -1. Your environment must be setup with [authentication -information](https://developers.google.com/identity/protocols/application-default-credentials#howtheywork). If you're running in your local development environment and you have the [Google Cloud SDK](https://cloud.google.com/sdk/) installed, you can do this easily by running: - - $ gcloud init - -2. Install dependencies in `requirements.txt`: - - $ pip install -r requirements.txt - -3. Depending on the sample, you may also need to create resources on the [Google Developers Console](https://console.developers.google.com). Refer to the sample description and associated documentation page. - -## Additional resources - -For more information on BigQuery you can visit: - -> https://developers.google.com/bigquery - -For more information on the BigQuery API Python library surface you -can visit: - -> https://developers.google.com/resources/api-libraries/documentation/bigquery/v2/python/latest/ - -For information on the Python Client Library visit: - -> https://developers.google.com/api-client-library/python - -## Other Samples - -* [Using BigQuery from Google App Engine](../appengine/bigquery). diff --git a/bigquery/api/README.md b/bigquery/api/README.md deleted file mode 100644 index aa021b0919f..00000000000 --- a/bigquery/api/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# BigQuery API Samples - - -These samples are used on the following documentation pages: - -> -* https://cloud.google.com/bigquery/exporting-data-from-bigquery -* https://cloud.google.com/bigquery/authentication -* https://cloud.google.com/bigquery/bigquery-api-quickstart -* https://cloud.google.com/bigquery/docs/managing_jobs_datasets_projects -* https://cloud.google.com/bigquery/streaming-data-into-bigquery -* https://cloud.google.com/bigquery/docs/data -* https://cloud.google.com/bigquery/querying-data -* https://cloud.google.com/bigquery/loading-data-into-bigquery - - diff --git a/bigquery/api/async_query.py b/bigquery/api/async_query.py deleted file mode 100755 index 4df547159cd..00000000000 --- a/bigquery/api/async_query.py +++ /dev/null @@ -1,142 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Command-line application to perform an asynchronous query in BigQuery. - -This sample is used on this page: - - https://cloud.google.com/bigquery/querying-data#asyncqueries - -For more information, see the README.md under /bigquery. -""" - -import argparse -import json -import time -import uuid - -from googleapiclient import discovery -from oauth2client.client import GoogleCredentials - - -# [START async_query] -def async_query(bigquery, project_id, query, batch=False, num_retries=5): - # Generate a unique job_id so retries - # don't accidentally duplicate query - job_data = { - 'jobReference': { - 'projectId': project_id, - 'job_id': str(uuid.uuid4()) - }, - 'configuration': { - 'query': { - 'query': query, - 'priority': 'BATCH' if batch else 'INTERACTIVE' - } - } - } - return bigquery.jobs().insert( - projectId=project_id, - body=job_data).execute(num_retries=num_retries) -# [END async_query] - - -# [START poll_job] -def poll_job(bigquery, job): - """Waits for a job to complete.""" - - print('Waiting for job to finish...') - - request = bigquery.jobs().get( - projectId=job['jobReference']['projectId'], - jobId=job['jobReference']['jobId']) - - while True: - result = request.execute(num_retries=2) - - if result['status']['state'] == 'DONE': - if 'errorResult' in result['status']: - raise RuntimeError(result['status']['errorResult']) - print('Job complete.') - return - - time.sleep(1) -# [END poll_job] - - -# [START run] -def main(project_id, query_string, batch, num_retries, interval): - # [START build_service] - # Grab the application's default credentials from the environment. - credentials = GoogleCredentials.get_application_default() - - # Construct the service object for interacting with the BigQuery API. - bigquery = discovery.build('bigquery', 'v2', credentials=credentials) - # [END build_service] - - # Submit the job and wait for it to complete. - query_job = async_query( - bigquery, - project_id, - query_string, - batch, - num_retries) - - poll_job(bigquery, query_job) - - # Page through the result set and print all results. - page_token = None - while True: - page = bigquery.jobs().getQueryResults( - pageToken=page_token, - **query_job['jobReference']).execute(num_retries=2) - - print(json.dumps(page['rows'])) - - page_token = page.get('pageToken') - if not page_token: - break -# [END run] - - -# [START main] -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('project_id', help='Your Google Cloud project ID.') - parser.add_argument('query', help='BigQuery SQL Query.') - parser.add_argument( - '-b', '--batch', help='Run query in batch mode.', action='store_true') - parser.add_argument( - '-r', '--num_retries', - help='Number of times to retry in case of 500 error.', - type=int, - default=5) - parser.add_argument( - '-p', '--poll_interval', - help='How often to poll the query for completion (seconds).', - type=int, - default=1) - - args = parser.parse_args() - - main( - args.project_id, - args.query, - args.batch, - args.num_retries, - args.poll_interval) -# [END main] diff --git a/bigquery/api/async_query_test.py b/bigquery/api/async_query_test.py deleted file mode 100644 index f8e9f06891d..00000000000 --- a/bigquery/api/async_query_test.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json - -from async_query import main - - -def test_async_query(cloud_config, capsys): - query = ( - 'SELECT corpus FROM publicdata:samples.shakespeare ' - 'GROUP BY corpus;') - - main( - project_id=cloud_config.project, - query_string=query, - batch=False, - num_retries=5, - interval=1) - - out, _ = capsys.readouterr() - value = out.strip().split('\n').pop() - - assert json.loads(value) is not None diff --git a/bigquery/api/export_data_to_cloud_storage.py b/bigquery/api/export_data_to_cloud_storage.py deleted file mode 100755 index 2bbaced7aba..00000000000 --- a/bigquery/api/export_data_to_cloud_storage.py +++ /dev/null @@ -1,168 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Command-line application to export a table from BigQuery to Google Cloud -Storage. - -This sample is used on this page: - - https://cloud.google.com/bigquery/exporting-data-from-bigquery - -For more information, see the README.md under /bigquery. -""" - -import argparse -import time -import uuid - -from googleapiclient import discovery -from oauth2client.client import GoogleCredentials - - -# [START export_table] -def export_table(bigquery, cloud_storage_path, - project_id, dataset_id, table_id, - export_format="CSV", - num_retries=5, - compression="NONE"): - """ - Starts an export job - - Args: - bigquery: initialized and authorized bigquery - google-api-client object. - cloud_storage_path: fully qualified - path to a Google Cloud Storage location. - e.g. gs://mybucket/myfolder/ - export_format: format to export in; - "CSV", "NEWLINE_DELIMITED_JSON", or "AVRO". - compression: format to compress results with, - "NONE" (default) or "GZIP". - - Returns: an extract job resource representing the - job, see https://cloud.google.com/bigquery/docs/reference/v2/jobs - """ - # Generate a unique job_id so retries - # don't accidentally duplicate export - job_data = { - 'jobReference': { - 'projectId': project_id, - 'jobId': str(uuid.uuid4()) - }, - 'configuration': { - 'extract': { - 'sourceTable': { - 'projectId': project_id, - 'datasetId': dataset_id, - 'tableId': table_id, - }, - 'destinationUris': [cloud_storage_path], - 'destinationFormat': export_format, - 'compression': compression - } - } - } - return bigquery.jobs().insert( - projectId=project_id, - body=job_data).execute(num_retries=num_retries) -# [END export_table] - - -# [START poll_job] -def poll_job(bigquery, job): - """Waits for a job to complete.""" - - print('Waiting for job to finish...') - - request = bigquery.jobs().get( - projectId=job['jobReference']['projectId'], - jobId=job['jobReference']['jobId']) - - while True: - result = request.execute(num_retries=2) - - if result['status']['state'] == 'DONE': - if 'errorResult' in result['status']: - raise RuntimeError(result['status']['errorResult']) - print('Job complete.') - return - - time.sleep(1) -# [END poll_job] - - -# [START run] -def main(cloud_storage_path, project_id, dataset_id, table_id, - num_retries, interval, export_format="CSV", compression="NONE"): - # [START build_service] - # Grab the application's default credentials from the environment. - credentials = GoogleCredentials.get_application_default() - - # Construct the service object for interacting with the BigQuery API. - bigquery = discovery.build('bigquery', 'v2', credentials=credentials) - # [END build_service] - - job = export_table( - bigquery, - cloud_storage_path, - project_id, - dataset_id, - table_id, - num_retries=num_retries, - export_format=export_format, - compression=compression) - poll_job(bigquery, job) -# [END run] - - -# [START main] -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('project_id', help='Your Google Cloud project ID.') - parser.add_argument('dataset_id', help='BigQuery dataset to export.') - parser.add_argument('table_id', help='BigQuery table to export.') - parser.add_argument( - 'gcs_path', - help=('Google Cloud Storage path to store the exported data. For ' - 'example, gs://mybucket/mydata.csv')) - parser.add_argument( - '-p', '--poll_interval', - help='How often to poll the query for completion (seconds).', - type=int, - default=1) - parser.add_argument( - '-r', '--num_retries', - help='Number of times to retry in case of 500 error.', - type=int, - default=5) - parser.add_argument( - '-z', '--gzip', - help='compress resultset with gzip', - action='store_true', - default=False) - - args = parser.parse_args() - - main( - args.gcs_path, - args.project_id, - args.dataset_id, - args.table_id, - args.num_retries, - args.poll_interval, - compression="GZIP" if args.gzip else "NONE") -# [END main] diff --git a/bigquery/api/export_data_to_cloud_storage_test.py b/bigquery/api/export_data_to_cloud_storage_test.py deleted file mode 100644 index 72a41069199..00000000000 --- a/bigquery/api/export_data_to_cloud_storage_test.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from export_data_to_cloud_storage import main -from gcp.testing.flaky import flaky - -DATASET_ID = 'test_dataset' -TABLE_ID = 'test_table' - - -@flaky -def test_export_table_csv(cloud_config): - cloud_storage_output_uri = \ - 'gs://{}/output.csv'.format(cloud_config.storage_bucket) - main( - cloud_storage_output_uri, - cloud_config.project, - DATASET_ID, - TABLE_ID, - num_retries=5, - interval=1, - export_format="CSV") - - -@flaky -def test_export_table_json(cloud_config): - cloud_storage_output_uri = \ - 'gs://{}/output.json'.format(cloud_config.storage_bucket) - main( - cloud_storage_output_uri, - cloud_config.project, - DATASET_ID, - TABLE_ID, - num_retries=5, - interval=1, - export_format="NEWLINE_DELIMITED_JSON") - - -@flaky -def test_export_table_avro(cloud_config): - cloud_storage_output_uri = \ - 'gs://{}/output.avro'.format(cloud_config.storage_bucket) - main( - cloud_storage_output_uri, - cloud_config.project, - DATASET_ID, - TABLE_ID, - num_retries=5, - interval=1, - export_format="AVRO") diff --git a/bigquery/api/getting_started.py b/bigquery/api/getting_started.py deleted file mode 100755 index 1ed8fd9dd76..00000000000 --- a/bigquery/api/getting_started.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Command-line application that demonstrates basic BigQuery API usage. - -This sample queries a public shakespeare dataset and displays the 10 of -Shakespeare's works with the greatest number of distinct words. - -This sample is used on this page: - - https://cloud.google.com/bigquery/bigquery-api-quickstart - -For more information, see the README.md under /bigquery. -""" -# [START all] -import argparse - -from googleapiclient.discovery import build -from googleapiclient.errors import HttpError -from oauth2client.client import GoogleCredentials - - -def main(project_id): - # [START build_service] - # Grab the application's default credentials from the environment. - credentials = GoogleCredentials.get_application_default() - # Construct the service object for interacting with the BigQuery API. - bigquery_service = build('bigquery', 'v2', credentials=credentials) - # [END build_service] - - try: - # [START run_query] - query_request = bigquery_service.jobs() - query_data = { - 'query': ( - 'SELECT TOP(corpus, 10) as title, ' - 'COUNT(*) as unique_words ' - 'FROM [publicdata:samples.shakespeare];') - } - - query_response = query_request.query( - projectId=project_id, - body=query_data).execute() - # [END run_query] - - # [START print_results] - print('Query Results:') - for row in query_response['rows']: - print('\t'.join(field['v'] for field in row['f'])) - # [END print_results] - - except HttpError as err: - print('Error: {}'.format(err.content)) - raise err - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('project_id', help='Your Google Cloud Project ID.') - - args = parser.parse_args() - - main(args.project_id) -# [END all] diff --git a/bigquery/api/getting_started_test.py b/bigquery/api/getting_started_test.py deleted file mode 100644 index 8f0866f46d1..00000000000 --- a/bigquery/api/getting_started_test.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from getting_started import main - - -def test_main(cloud_config, capsys): - main(cloud_config.project) - - out, _ = capsys.readouterr() - - assert re.search(re.compile( - r'Query Results:.hamlet', re.DOTALL), out) diff --git a/bigquery/api/installed_app.py b/bigquery/api/installed_app.py deleted file mode 100644 index 4bbc13a0010..00000000000 --- a/bigquery/api/installed_app.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Command-line application that demonstrates using BigQuery with credentials -obtained from an installed app. - -This sample is used on this page: - - https://cloud.google.com/bigquery/authentication - -For more information, see the README.md under /bigquery. -""" -# [START all] - -import argparse -import pprint - -from googleapiclient import discovery -from googleapiclient.errors import HttpError -from oauth2client import tools -from oauth2client.client import AccessTokenRefreshError -from oauth2client.client import flow_from_clientsecrets -from oauth2client.file import Storage - -SCOPES = ['https://www.googleapis.com/auth/bigquery'] -# Update with the full path to your client secrets json file. -CLIENT_SECRETS = 'client_secrets.json' - - -def main(args): - storage = Storage('credentials.dat') - credentials = storage.get() - - if credentials is None or credentials.invalid: - flow = flow_from_clientsecrets( - CLIENT_SECRETS, scope=SCOPES) - # run_flow will prompt the user to authorize the application's - # access to BigQuery and return the credentials. - credentials = tools.run_flow(flow, storage, args) - - # Create a BigQuery client using the credentials. - bigquery_service = discovery.build( - 'bigquery', 'v2', credentials=credentials) - - # List all datasets in BigQuery - try: - datasets = bigquery_service.datasets() - listReply = datasets.list(projectId=args.project_id).execute() - print('Dataset list:') - pprint.pprint(listReply) - - except HttpError as err: - print('Error in listDatasets:') - pprint.pprint(err.content) - - except AccessTokenRefreshError: - print('Credentials have been revoked or expired, please re-run' - 'the application to re-authorize') - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, - # Use oauth2client's argparse as a base, so that the flags needed - # for run_flow are available. - parents=[tools.argparser]) - parser.add_argument( - 'project_id', help='Your Google Cloud Project ID.') - args = parser.parse_args() - main(args) -# [END all] diff --git a/bigquery/api/installed_app_test.py b/bigquery/api/installed_app_test.py deleted file mode 100644 index e8265f4a400..00000000000 --- a/bigquery/api/installed_app_test.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -import installed_app -from oauth2client.client import GoogleCredentials - - -class Namespace(object): - def __init__(self, **kwargs): - self.__dict__.update(kwargs) - - -def test_main(cloud_config, monkeypatch, capsys): - installed_app.CLIENT_SECRETS = cloud_config.client_secrets - - # Replace the user credentials flow with Application Default Credentials. - # Unfortunately, there's no easy way to fully test the user flow. - def mock_run_flow(flow, storage, args): - return GoogleCredentials.get_application_default() - - monkeypatch.setattr(installed_app.tools, 'run_flow', mock_run_flow) - - args = Namespace( - project_id=cloud_config.project, - logging_level='INFO', - noauth_local_webserver=True) - - installed_app.main(args) - - out, _ = capsys.readouterr() - - assert re.search(re.compile( - r'bigquery#datasetList', re.DOTALL), out) diff --git a/bigquery/api/list_datasets_projects.py b/bigquery/api/list_datasets_projects.py deleted file mode 100755 index a4a6c00e4b5..00000000000 --- a/bigquery/api/list_datasets_projects.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Command-line application to list all projects and datasets in BigQuery. - -This sample is used on this page: - - https://cloud.google.com/bigquery/docs/managing_jobs_datasets_projects - -For more information, see the README.md under /bigquery. -""" - -import argparse -from pprint import pprint - -from googleapiclient import discovery -from oauth2client.client import GoogleCredentials -from six.moves.urllib.error import HTTPError - - -# [START list_datasets] -def list_datasets(bigquery, project): - try: - datasets = bigquery.datasets() - list_reply = datasets.list(projectId=project).execute() - print('Dataset list:') - pprint(list_reply) - - except HTTPError as err: - print('Error in list_datasets: %s' % err.content) - raise err -# [END list_datasets] - - -# [START list_projects] -def list_projects(bigquery): - try: - # Start training on a data set - projects = bigquery.projects() - list_reply = projects.list().execute() - - print('Project list:') - pprint(list_reply) - - except HTTPError as err: - print('Error in list_projects: %s' % err.content) - raise err -# [END list_projects] - - -def main(project_id): - credentials = GoogleCredentials.get_application_default() - # Construct the service object for interacting with the BigQuery API. - bigquery = discovery.build('bigquery', 'v2', credentials=credentials) - - list_datasets(bigquery, project_id) - list_projects(bigquery) - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('project_id', help='the project id to list.') - - args = parser.parse_args() - - main(args.project_id) diff --git a/bigquery/api/list_datasets_projects_test.py b/bigquery/api/list_datasets_projects_test.py deleted file mode 100644 index e096a0342b2..00000000000 --- a/bigquery/api/list_datasets_projects_test.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from list_datasets_projects import main - - -def test_main(cloud_config, capsys): - main(cloud_config.project) - - out, _ = capsys.readouterr() - - assert re.search(re.compile( - r'Project list:.*bigquery#projectList.*projects', re.DOTALL), out) - assert re.search(re.compile( - r'Dataset list:.*datasets.*datasetId', re.DOTALL), out) diff --git a/bigquery/api/load_data_by_post.py b/bigquery/api/load_data_by_post.py deleted file mode 100755 index 63e49819e07..00000000000 --- a/bigquery/api/load_data_by_post.py +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Command-line application that loads data into BigQuery via HTTP POST. - -This sample is used on this page: - - https://cloud.google.com/bigquery/loading-data-into-bigquery - -For more information, see the README.md under /bigquery. -""" - -import argparse -import json -import time - -from googleapiclient import discovery -from googleapiclient.http import MediaFileUpload -from oauth2client.client import GoogleCredentials - - -# [START make_post] -def load_data(schema_path, data_path, project_id, dataset_id, table_id): - """Loads the given data file into BigQuery. - - Args: - schema_path: the path to a file containing a valid bigquery schema. - see https://cloud.google.com/bigquery/docs/reference/v2/tables - data_path: the name of the file to insert into the table. - project_id: The project id that the table exists under. This is also - assumed to be the project id this request is to be made under. - dataset_id: The dataset id of the destination table. - table_id: The table id to load data into. - """ - # Create a bigquery service object, using the application's default auth - credentials = GoogleCredentials.get_application_default() - bigquery = discovery.build('bigquery', 'v2', credentials=credentials) - - # Infer the data format from the name of the data file. - source_format = 'CSV' - if data_path[-5:].lower() == '.json': - source_format = 'NEWLINE_DELIMITED_JSON' - - # Post to the jobs resource using the client's media upload interface. See: - # http://developers.google.com/api-client-library/python/guide/media_upload - insert_request = bigquery.jobs().insert( - projectId=project_id, - # Provide a configuration object. See: - # https://cloud.google.com/bigquery/docs/reference/v2/jobs#resource - body={ - 'configuration': { - 'load': { - 'schema': { - 'fields': json.load(open(schema_path, 'r')) - }, - 'destinationTable': { - 'projectId': project_id, - 'datasetId': dataset_id, - 'tableId': table_id - }, - 'sourceFormat': source_format, - } - } - }, - media_body=MediaFileUpload( - data_path, - mimetype='application/octet-stream')) - job = insert_request.execute() - - print('Waiting for job to finish...') - - status_request = bigquery.jobs().get( - projectId=job['jobReference']['projectId'], - jobId=job['jobReference']['jobId']) - - # Poll the job until it finishes. - while True: - result = status_request.execute(num_retries=2) - - if result['status']['state'] == 'DONE': - if result['status'].get('errors'): - raise RuntimeError('\n'.join( - e['message'] for e in result['status']['errors'])) - print('Job complete.') - return - - time.sleep(1) -# [END make_post] - - -# [START main] -def main(project_id, dataset_id, table_name, schema_path, data_path): - load_data( - schema_path, - data_path, - project_id, - dataset_id, - table_name) -# [END main] - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('project_id', help='Your Google Cloud project ID.') - parser.add_argument('dataset_id', help='A BigQuery dataset ID.') - parser.add_argument( - 'table_name', help='Name of the table to load data into.') - parser.add_argument( - 'schema_file', - help='Path to a schema file describing the table schema.') - parser.add_argument( - 'data_file', - help='Path to the data file.') - - args = parser.parse_args() - - main( - args.project_id, - args.dataset_id, - args.table_name, - args.schema_file, - args.data_file) diff --git a/bigquery/api/load_data_by_post_test.py b/bigquery/api/load_data_by_post_test.py deleted file mode 100644 index ca4d1ea38e9..00000000000 --- a/bigquery/api/load_data_by_post_test.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from gcp.testing.flaky import flaky -from load_data_by_post import load_data - -DATASET_ID = 'ephemeral_test_dataset' -TABLE_ID = 'load_data_by_post' - - -@flaky -def test_load_csv_data(cloud_config, resource, capsys): - schema_path = resource('schema.json') - data_path = resource('data.csv') - - load_data( - schema_path, - data_path, - cloud_config.project, - DATASET_ID, - TABLE_ID - ) - - out, _ = capsys.readouterr() - - assert re.search(re.compile( - r'Waiting for job to finish.*Job complete.', re.DOTALL), out) - - -@flaky -def test_load_json_data(cloud_config, resource, capsys): - schema_path = resource('schema.json') - data_path = resource('data.json') - - load_data( - schema_path, - data_path, - cloud_config.project, - DATASET_ID, - TABLE_ID - ) - - out, _ = capsys.readouterr() - - assert re.search(re.compile( - r'Waiting for job to finish.*Job complete.', re.DOTALL), out) diff --git a/bigquery/api/load_data_from_csv.py b/bigquery/api/load_data_from_csv.py deleted file mode 100755 index 4668be10b3a..00000000000 --- a/bigquery/api/load_data_from_csv.py +++ /dev/null @@ -1,168 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Command-line application that loads data into BigQuery from a CSV file in -Google Cloud Storage. - -This sample is used on this page: - - https://cloud.google.com/bigquery/loading-data-into-bigquery#loaddatagcs - -For more information, see the README.md under /bigquery. -""" - -import argparse -import json -import time -import uuid - -from googleapiclient import discovery -from oauth2client.client import GoogleCredentials - - -# [START load_table] -def load_table(bigquery, project_id, dataset_id, table_name, source_schema, - source_path, num_retries=5): - """ - Starts a job to load a bigquery table from CSV - - Args: - bigquery: an initialized and authorized bigquery client - google-api-client object - source_schema: a valid bigquery schema, - see https://cloud.google.com/bigquery/docs/reference/v2/tables - source_path: the fully qualified Google Cloud Storage location of - the data to load into your table - - Returns: a bigquery load job, see - https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.load - """ - - # Generate a unique job_id so retries - # don't accidentally duplicate query - job_data = { - 'jobReference': { - 'projectId': project_id, - 'job_id': str(uuid.uuid4()) - }, - 'configuration': { - 'load': { - 'sourceUris': [source_path], - 'schema': { - 'fields': source_schema - }, - 'destinationTable': { - 'projectId': project_id, - 'datasetId': dataset_id, - 'tableId': table_name - } - } - } - } - - return bigquery.jobs().insert( - projectId=project_id, - body=job_data).execute(num_retries=num_retries) -# [END load_table] - - -# [START poll_job] -def poll_job(bigquery, job): - """Waits for a job to complete.""" - - print('Waiting for job to finish...') - - request = bigquery.jobs().get( - projectId=job['jobReference']['projectId'], - jobId=job['jobReference']['jobId']) - - while True: - result = request.execute(num_retries=2) - - if result['status']['state'] == 'DONE': - if 'errorResult' in result['status']: - raise RuntimeError(result['status']['errorResult']) - print('Job complete.') - return - - time.sleep(1) -# [END poll_job] - - -# [START run] -def main(project_id, dataset_id, table_name, schema_file, data_path, - poll_interval, num_retries): - # [START build_service] - # Grab the application's default credentials from the environment. - credentials = GoogleCredentials.get_application_default() - - # Construct the service object for interacting with the BigQuery API. - bigquery = discovery.build('bigquery', 'v2', credentials=credentials) - # [END build_service] - - with open(schema_file, 'r') as f: - schema = json.load(f) - - job = load_table( - bigquery, - project_id, - dataset_id, - table_name, - schema, - data_path, - num_retries) - - poll_job(bigquery, job) -# [END run] - - -# [START main] -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('project_id', help='Your Google Cloud project ID.') - parser.add_argument('dataset_id', help='A BigQuery dataset ID.') - parser.add_argument( - 'table_name', help='Name of the table to load data into.') - parser.add_argument( - 'schema_file', - help='Path to a schema file describing the table schema.') - parser.add_argument( - 'data_path', - help='Google Cloud Storage path to the CSV data, for example: ' - 'gs://mybucket/in.csv') - parser.add_argument( - '-p', '--poll_interval', - help='How often to poll the query for completion (seconds).', - type=int, - default=1) - parser.add_argument( - '-r', '--num_retries', - help='Number of times to retry in case of 500 error.', - type=int, - default=5) - - args = parser.parse_args() - - main( - args.project_id, - args.dataset_id, - args.table_name, - args.schema_file, - args.data_path, - args.poll_interval, - args.num_retries) -# [END main] diff --git a/bigquery/api/load_data_from_csv_test.py b/bigquery/api/load_data_from_csv_test.py deleted file mode 100644 index 6e7fc4323db..00000000000 --- a/bigquery/api/load_data_from_csv_test.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from gcp.testing.flaky import flaky -from load_data_from_csv import main - -DATASET_ID = 'test_dataset' -TABLE_ID = 'test_import_table' - - -@flaky -def test_load_table(cloud_config, resource): - cloud_storage_input_uri = 'gs://{}/data.csv'.format( - cloud_config.storage_bucket) - schema_file = resource('schema.json') - - main( - cloud_config.project, - DATASET_ID, - TABLE_ID, - schema_file=schema_file, - data_path=cloud_storage_input_uri, - poll_interval=1, - num_retries=5 - ) diff --git a/bigquery/api/requirements.txt b/bigquery/api/requirements.txt deleted file mode 100644 index c3b2784ce87..00000000000 --- a/bigquery/api/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -google-api-python-client==1.5.0 diff --git a/bigquery/api/resources/data.csv b/bigquery/api/resources/data.csv deleted file mode 100644 index 230a96b559d..00000000000 --- a/bigquery/api/resources/data.csv +++ /dev/null @@ -1 +0,0 @@ -Gandalf, 2000, 140.0, 1 diff --git a/bigquery/api/resources/data.json b/bigquery/api/resources/data.json deleted file mode 100644 index b8eef90c591..00000000000 --- a/bigquery/api/resources/data.json +++ /dev/null @@ -1 +0,0 @@ -{"Name": "Gandalf", "Age": 2000, "Weight": 140.0, "IsMagic": true} diff --git a/bigquery/api/resources/schema.json b/bigquery/api/resources/schema.json deleted file mode 100644 index a48971ef857..00000000000 --- a/bigquery/api/resources/schema.json +++ /dev/null @@ -1 +0,0 @@ -[{"type": "STRING", "name": "Name"}, {"type": "INTEGER", "name": "Age"}, {"type": "FLOAT", "name": "Weight"}, {"type": "BOOLEAN", "name": "IsMagic"}] \ No newline at end of file diff --git a/bigquery/api/resources/streamrows.json b/bigquery/api/resources/streamrows.json deleted file mode 100644 index fbc7a392971..00000000000 --- a/bigquery/api/resources/streamrows.json +++ /dev/null @@ -1,7 +0,0 @@ -[ - {"Name": "test", "Age": 0, "Weight": 100.0, "IsMagic": false}, - {"Name": "test", "Age": 1, "Weight": 100.0, "IsMagic": false}, - {"Name": "test", "Age": 2, "Weight": 100.0, "IsMagic": false}, - {"Name": "test", "Age": 3, "Weight": 100.0, "IsMagic": false}, - {"Name": "test", "Age": 0, "Weight": 100.0, "IsMagic": false} -] diff --git a/bigquery/api/streaming.py b/bigquery/api/streaming.py deleted file mode 100755 index c0ad599d1e2..00000000000 --- a/bigquery/api/streaming.py +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Command-line application that streams data into BigQuery. - -This sample is used on this page: - - https://cloud.google.com/bigquery/streaming-data-into-bigquery - -For more information, see the README.md under /bigquery. -""" - -import argparse -import ast -import json -import uuid - -from googleapiclient import discovery -from oauth2client.client import GoogleCredentials -from six.moves import input - - -# [START stream_row_to_bigquery] -def stream_row_to_bigquery(bigquery, project_id, dataset_id, table_name, row, - num_retries=5): - insert_all_data = { - 'rows': [{ - 'json': row, - # Generate a unique id for each row so retries don't accidentally - # duplicate insert - 'insertId': str(uuid.uuid4()), - }] - } - return bigquery.tabledata().insertAll( - projectId=project_id, - datasetId=dataset_id, - tableId=table_name, - body=insert_all_data).execute(num_retries=num_retries) - # [END stream_row_to_bigquery] - - -# [START run] -def main(project_id, dataset_id, table_name, num_retries): - # [START build_service] - # Grab the application's default credentials from the environment. - credentials = GoogleCredentials.get_application_default() - - # Construct the service object for interacting with the BigQuery API. - bigquery = discovery.build('bigquery', 'v2', credentials=credentials) - # [END build_service] - - for row in get_rows(): - response = stream_row_to_bigquery( - bigquery, project_id, dataset_id, table_name, row, num_retries) - print(json.dumps(response)) - - -def get_rows(): - line = input("Enter a row (python dict) into the table: ") - while line: - yield ast.literal_eval(line) - line = input("Enter another row into the table \n" + - "[hit enter to stop]: ") -# [END run] - - -# [START main] -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('project_id', help='Your Google Cloud project ID.') - parser.add_argument('dataset_id', help='A BigQuery dataset ID.') - parser.add_argument( - 'table_name', help='Name of the table to load data into.') - parser.add_argument( - '-p', '--poll_interval', - help='How often to poll the query for completion (seconds).', - type=int, - default=1) - parser.add_argument( - '-r', '--num_retries', - help='Number of times to retry in case of 500 error.', - type=int, - default=5) - - args = parser.parse_args() - - main( - args.project_id, - args.dataset_id, - args.table_name, - args.num_retries) -# [END main] diff --git a/bigquery/api/streaming_test.py b/bigquery/api/streaming_test.py deleted file mode 100644 index 66347da7cc0..00000000000 --- a/bigquery/api/streaming_test.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json - -import streaming - - -DATASET_ID = 'test_dataset' -TABLE_ID = 'test_table' - - -def test_stream_row_to_bigquery(cloud_config, resource, capsys): - with open(resource('streamrows.json'), 'r') as rows_file: - rows = json.load(rows_file) - - streaming.get_rows = lambda: rows - - streaming.main( - cloud_config.project, - DATASET_ID, - TABLE_ID, - num_retries=5) - - out, _ = capsys.readouterr() - results = out.split('\n') - - assert json.loads(results[0]) is not None diff --git a/bigquery/api/sync_query.py b/bigquery/api/sync_query.py deleted file mode 100755 index 5a8668c2b66..00000000000 --- a/bigquery/api/sync_query.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Command-line application to perform an synchronous query in BigQuery. - -This sample is used on this page: - - https://cloud.google.com/bigquery/querying-data#syncqueries - -For more information, see the README.md under /bigquery. -""" - -import argparse -import json - -from googleapiclient import discovery -from oauth2client.client import GoogleCredentials - - -# [START sync_query] -def sync_query(bigquery, project_id, query, timeout=10000, num_retries=5): - query_data = { - 'query': query, - 'timeoutMs': timeout, - } - return bigquery.jobs().query( - projectId=project_id, - body=query_data).execute(num_retries=num_retries) -# [END sync_query] - - -# [START run] -def main(project_id, query, timeout, num_retries): - # [START build_service] - # Grab the application's default credentials from the environment. - credentials = GoogleCredentials.get_application_default() - - # Construct the service object for interacting with the BigQuery API. - bigquery = discovery.build('bigquery', 'v2', credentials=credentials) - # [END build_service] - - query_job = sync_query( - bigquery, - project_id, - query, - timeout, - num_retries) - - # [START paging] - # Page through the result set and print all results. - results = [] - page_token = None - - while True: - page = bigquery.jobs().getQueryResults( - pageToken=page_token, - **query_job['jobReference']).execute(num_retries=2) - - results.extend(page.get('rows', [])) - - page_token = page.get('pageToken') - if not page_token: - break - - print(json.dumps(results)) - # [END paging] -# [END run] - - -# [START main] -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('project_id', help='Your Google Cloud project ID.') - parser.add_argument('query', help='BigQuery SQL Query.') - parser.add_argument( - '-t', '--timeout', - help='Number seconds to wait for a result', - type=int, - default=30) - parser.add_argument( - '-r', '--num_retries', - help='Number of times to retry in case of 500 error.', - type=int, - default=5) - - args = parser.parse_args() - - main( - args.project_id, - args.query, - args.timeout, - args.num_retries) - -# [END main] diff --git a/bigquery/api/sync_query_test.py b/bigquery/api/sync_query_test.py deleted file mode 100644 index 02d84f7c9c1..00000000000 --- a/bigquery/api/sync_query_test.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json - -from sync_query import main - - -def test_sync_query(cloud_config, capsys): - query = ( - 'SELECT corpus FROM publicdata:samples.shakespeare ' - 'GROUP BY corpus;') - - main( - project_id=cloud_config.project, - query=query, - timeout=30, - num_retries=5) - - out, _ = capsys.readouterr() - result = out.split('\n')[0] - - assert json.loads(result) is not None diff --git a/bigquery/bqml/data_scientist_tutorial_test.py b/bigquery/bqml/data_scientist_tutorial_test.py new file mode 100644 index 00000000000..532835294d1 --- /dev/null +++ b/bigquery/bqml/data_scientist_tutorial_test.py @@ -0,0 +1,134 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START bqml_data_scientist_tutorial_import_and_client] +from google.cloud import bigquery +# [END bqml_data_scientist_tutorial_import_and_client] +import pytest + +# [START bqml_data_scientist_tutorial_import_and_client] +client = bigquery.Client() +# [END bqml_data_scientist_tutorial_import_and_client] + + +@pytest.fixture +def delete_dataset(): + yield + client.delete_dataset( + client.dataset('bqml_tutorial'), delete_contents=True) + + +def test_data_scientist_tutorial(delete_dataset): + # [START bqml_data_scientist_tutorial_create_dataset] + dataset = bigquery.Dataset(client.dataset('bqml_tutorial')) + dataset.location = 'US' + client.create_dataset(dataset) + # [END bqml_data_scientist_tutorial_create_dataset] + + # [START bqml_data_scientist_tutorial_create_model] + sql = """ + CREATE OR REPLACE MODEL `bqml_tutorial.sample_model` + OPTIONS(model_type='logistic_reg') AS + SELECT + IF(totals.transactions IS NULL, 0, 1) AS label, + IFNULL(device.operatingSystem, "") AS os, + device.isMobile AS is_mobile, + IFNULL(geoNetwork.country, "") AS country, + IFNULL(totals.pageviews, 0) AS pageviews + FROM + `bigquery-public-data.google_analytics_sample.ga_sessions_*` + WHERE + _TABLE_SUFFIX BETWEEN '20160801' AND '20170630' + """ + df = client.query(sql).to_dataframe() + print(df) + # [END bqml_data_scientist_tutorial_create_model] + + # [START bqml_data_scientist_tutorial_get_training_statistics] + sql = """ + SELECT + * + FROM + ML.TRAINING_INFO(MODEL `bqml_tutorial.sample_model`) + """ + df = client.query(sql).to_dataframe() + print(df) + # [END bqml_data_scientist_tutorial_get_training_statistics] + + # [START bqml_data_scientist_tutorial_evaluate_model] + sql = """ + SELECT + * + FROM ML.EVALUATE(MODEL `bqml_tutorial.sample_model`, ( + SELECT + IF(totals.transactions IS NULL, 0, 1) AS label, + IFNULL(device.operatingSystem, "") AS os, + device.isMobile AS is_mobile, + IFNULL(geoNetwork.country, "") AS country, + IFNULL(totals.pageviews, 0) AS pageviews + FROM + `bigquery-public-data.google_analytics_sample.ga_sessions_*` + WHERE + _TABLE_SUFFIX BETWEEN '20170701' AND '20170801')) + """ + df = client.query(sql).to_dataframe() + print(df) + # [END bqml_data_scientist_tutorial_evaluate_model] + + # [START bqml_data_scientist_tutorial_predict_transactions] + sql = """ + SELECT + country, + SUM(predicted_label) as total_predicted_purchases + FROM ML.PREDICT(MODEL `bqml_tutorial.sample_model`, ( + SELECT + IFNULL(device.operatingSystem, "") AS os, + device.isMobile AS is_mobile, + IFNULL(totals.pageviews, 0) AS pageviews, + IFNULL(geoNetwork.country, "") AS country + FROM + `bigquery-public-data.google_analytics_sample.ga_sessions_*` + WHERE + _TABLE_SUFFIX BETWEEN '20170701' AND '20170801')) + GROUP BY country + ORDER BY total_predicted_purchases DESC + LIMIT 10 + """ + df = client.query(sql).to_dataframe() + print(df) + # [END bqml_data_scientist_tutorial_predict_transactions] + + # [START bqml_data_scientist_tutorial_predict_purchases] + sql = """ + SELECT + fullVisitorId, + SUM(predicted_label) as total_predicted_purchases + FROM ML.PREDICT(MODEL `bqml_tutorial.sample_model`, ( + SELECT + IFNULL(device.operatingSystem, "") AS os, + device.isMobile AS is_mobile, + IFNULL(totals.pageviews, 0) AS pageviews, + IFNULL(geoNetwork.country, "") AS country, + fullVisitorId + FROM + `bigquery-public-data.google_analytics_sample.ga_sessions_*` + WHERE + _TABLE_SUFFIX BETWEEN '20170701' AND '20170801')) + GROUP BY fullVisitorId + ORDER BY total_predicted_purchases DESC + LIMIT 10 + """ + df = client.query(sql).to_dataframe() + print(df) + # [END bqml_data_scientist_tutorial_predict_purchases] diff --git a/bigquery/bqml/ncaa_tutorial_test.py b/bigquery/bqml/ncaa_tutorial_test.py new file mode 100644 index 00000000000..5fd96a3961f --- /dev/null +++ b/bigquery/bqml/ncaa_tutorial_test.py @@ -0,0 +1,141 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import io +import os + +# [START bqml_ncaa_tutorial_import_and_client] +from google.cloud import bigquery +# [END bqml_ncaa_tutorial_import_and_client] +import pytest + +# [START bqml_ncaa_tutorial_import_and_client] +client = bigquery.Client() +# [END bqml_ncaa_tutorial_import_and_client] + + +@pytest.fixture +def delete_dataset(): + yield + client.delete_dataset( + client.dataset('bqml_tutorial'), delete_contents=True) + + +def test_ncaa_tutorial(delete_dataset): + # [START bqml_ncaa_tutorial_create_dataset] + dataset = bigquery.Dataset(client.dataset('bqml_tutorial')) + dataset.location = 'US' + client.create_dataset(dataset) + # [END bqml_ncaa_tutorial_create_dataset] + + # Create the tables used by the tutorial + # Note: the queries are saved to a file. This should be updated to use the + # saved queries once the library supports running saved queries. + query_filepath_to_table_name = { + 'feature_input_query.sql': 'cume_games', + 'training_data_query.sql': 'wide_games' + } + resources_directory = os.path.join(os.path.dirname(__file__), 'resources') + for query_filepath, table_name in query_filepath_to_table_name.items(): + table_ref = dataset.table(table_name) + job_config = bigquery.QueryJobConfig() + job_config.destination = table_ref + query_filepath = os.path.join( + resources_directory, query_filepath) + sql = io.open(query_filepath, 'r', encoding='utf-8').read() + client.query(sql, job_config=job_config).result() + + # [START bqml_ncaa_tutorial_create_model] + sql = """ + CREATE OR REPLACE MODEL `bqml_tutorial.ncaa_model` + OPTIONS ( + model_type='linear_reg', + max_iteration=50 ) AS + SELECT + * EXCEPT ( + game_id, season, scheduled_date, + total_three_points_made, + total_three_points_att), + total_three_points_att as label + FROM + `bqml_tutorial.wide_games` + WHERE + # remove the game to predict + game_id != 'f1063e80-23c7-486b-9a5e-faa52beb2d83' + """ + df = client.query(sql).to_dataframe() + print(df) + # [END bqml_ncaa_tutorial_create_model] + + # [START bqml_ncaa_tutorial_get_training_statistics] + sql = """ + SELECT + * + FROM + ML.TRAINING_INFO(MODEL `bqml_tutorial.ncaa_model`) + """ + df = client.query(sql).to_dataframe() + print(df) + # [END bqml_ncaa_tutorial_get_training_statistics] + + # [START bqml_ncaa_tutorial_evaluate_model] + sql = """ + WITH eval_table AS ( + SELECT + *, + total_three_points_att AS label + FROM + `bqml_tutorial.wide_games` ) + SELECT + * + FROM + ML.EVALUATE(MODEL `bqml_tutorial.ncaa_model`, + TABLE eval_table) + """ + df = client.query(sql).to_dataframe() + print(df) + # [END bqml_ncaa_tutorial_evaluate_model] + + # [START bqml_ncaa_tutorial_predict_outcomes] + sql = """ + WITH game_to_predict AS ( + SELECT + * + FROM + `bqml_tutorial.wide_games` + WHERE + game_id='f1063e80-23c7-486b-9a5e-faa52beb2d83' ) + SELECT + truth.game_id AS game_id, + total_three_points_att, + predicted_total_three_points_att + FROM ( + SELECT + game_id, + predicted_label AS predicted_total_three_points_att + FROM + ML.PREDICT(MODEL `bqml_tutorial.ncaa_model`, + table game_to_predict) ) AS predict + JOIN ( + SELECT + game_id, + total_three_points_att AS total_three_points_att + FROM + game_to_predict) AS truth + ON + predict.game_id = truth.game_id + """ + df = client.query(sql).to_dataframe() + print(df) + # [END bqml_ncaa_tutorial_predict_outcomes] diff --git a/bigquery/bqml/requirements.txt b/bigquery/bqml/requirements.txt new file mode 100644 index 00000000000..a4265ef90d1 --- /dev/null +++ b/bigquery/bqml/requirements.txt @@ -0,0 +1,4 @@ +google-cloud-bigquery[pandas]==1.9.0 +flaky==3.5.3 +mock==2.0.0 +pytest==4.2.0 diff --git a/bigquery/bqml/resources/feature_input_query.sql b/bigquery/bqml/resources/feature_input_query.sql new file mode 100644 index 00000000000..d54f003425d --- /dev/null +++ b/bigquery/bqml/resources/feature_input_query.sql @@ -0,0 +1,475 @@ +#standardSQL +SELECT + game_id, + season, + scheduled_date, + home_team, + market as school_name, + CONCAT(CAST(season AS STRING), ":", team_id) AS team_id, + conf_name, + division_name, + minutes, + points, + fast_break_pts, + second_chance_pts, + field_goals_made, + field_goals_att, + field_goals_pct, + three_points_made, + three_points_att, + three_points_pct, + two_points_made, + two_points_att, + two_points_pct, + free_throws_made, + free_throws_att, + free_throws_pct, + ts_pct, + efg_pct, + rebounds, + offensive_rebounds, + defensive_rebounds, + dreb_pct, + oreb_pct, + steals, + blocks, + blocked_att, + assists, + turnovers, + team_turnovers, + points_off_turnovers, + assists_turnover_ratio, + ast_fgm_pct, + personal_fouls, + flagrant_fouls, + player_tech_fouls, + team_tech_fouls, + coach_tech_fouls, + ejections, + foulouts, + score_delta, + opp_score_delta, + possessions, + ROW_NUMBER() OVER(partition by season, team_id order by scheduled_date ASC) AS game_number, + SUM(is_win) OVER(partition by season, team_id order by scheduled_date ASC ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) AS wins, + SUM(is_loss) OVER(partition by season, team_id order by scheduled_date ASC ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) AS losses, + ROUND(AVG(points) OVER w1, 2) AS points_avg_last_1, + ROUND(AVG(points) OVER w5, 2) AS points_avg_last_5, + ROUND(AVG(points) OVER w10, 2) AS points_avg_last_10, + ROUND(AVG(fast_break_pts) OVER w1, 2) AS fast_break_pts_avg_last_1, + ROUND(AVG(fast_break_pts) OVER w5, 2) AS fast_break_pts_avg_last_5, + ROUND(AVG(fast_break_pts) OVER w10, 2) AS fast_break_pts_avg_last_10, + ROUND(AVG(second_chance_pts) OVER w1, 2) AS second_chance_pts_avg_last_1, + ROUND(AVG(second_chance_pts) OVER w5, 2) AS second_chance_pts_avg_last_5, + ROUND(AVG(second_chance_pts) OVER w10, 2) AS second_chance_pts_avg_last_10, + ROUND(AVG(field_goals_made) OVER w1, 2) AS field_goals_made_avg_last_1, + ROUND(AVG(field_goals_made) OVER w5, 2) AS field_goals_made_avg_last_5, + ROUND(AVG(field_goals_made) OVER w10, 2) AS field_goals_made_avg_last_10, + ROUND(AVG(field_goals_att) OVER w1, 2) AS field_goals_att_avg_last_1, + ROUND(AVG(field_goals_att) OVER w5, 2) AS field_goals_att_avg_last_5, + ROUND(AVG(field_goals_att) OVER w10, 2) AS field_goals_att_avg_last_10, + ROUND(AVG(field_goals_pct) OVER w1, 2) AS field_goals_pct_avg_last_1, + ROUND(AVG(field_goals_pct) OVER w5, 2) AS field_goals_pct_avg_last_5, + ROUND(AVG(field_goals_pct) OVER w10, 2) AS field_goals_pct_avg_last_10, + ROUND(AVG(three_points_made) OVER w1, 2) AS three_points_made_avg_last_1, + ROUND(AVG(three_points_made) OVER w5, 2) AS three_points_made_avg_last_5, + ROUND(AVG(three_points_made) OVER w10, 2) AS three_points_made_avg_last_10, + ROUND(AVG(three_points_att) OVER w1, 2) AS three_points_att_avg_last_1, + ROUND(AVG(three_points_att) OVER w5, 2) AS three_points_att_avg_last_5, + ROUND(AVG(three_points_att) OVER w10, 2) AS three_points_att_avg_last_10, + ROUND(AVG(three_points_pct) OVER w1, 2) AS three_points_pct_avg_last_1, + ROUND(AVG(three_points_pct) OVER w5, 2) AS three_points_pct_avg_last_5, + ROUND(AVG(three_points_pct) OVER w10, 2) AS three_points_pct_avg_last_10, + ROUND(AVG(two_points_made) OVER w1, 2) AS two_points_made_avg_last_1, + ROUND(AVG(two_points_made) OVER w5, 2) AS two_points_made_avg_last_5, + ROUND(AVG(two_points_made) OVER w10, 2) AS two_points_made_avg_last_10, + ROUND(AVG(two_points_att) OVER w1, 2) AS two_points_att_avg_last_1, + ROUND(AVG(two_points_att) OVER w5, 2) AS two_points_att_avg_last_5, + ROUND(AVG(two_points_att) OVER w10, 2) AS two_points_att_avg_last_10, + ROUND(AVG(two_points_pct) OVER w1, 2) AS two_points_pct_avg_last_1, + ROUND(AVG(two_points_pct) OVER w5, 2) AS two_points_pct_avg_last_5, + ROUND(AVG(two_points_pct) OVER w10, 2) AS two_points_pct_avg_last_10, + ROUND(AVG(free_throws_made) OVER w1, 2) AS free_throws_made_avg_last_1, + ROUND(AVG(free_throws_made) OVER w5, 2) AS free_throws_made_avg_last_5, + ROUND(AVG(free_throws_made) OVER w10, 2) AS free_throws_made_avg_last_10, + ROUND(AVG(free_throws_att) OVER w1, 2) AS free_throws_att_avg_last_1, + ROUND(AVG(free_throws_att) OVER w5, 2) AS free_throws_att_avg_last_5, + ROUND(AVG(free_throws_att) OVER w10, 2) AS free_throws_att_avg_last_10, + ROUND(AVG(free_throws_pct) OVER w1, 2) AS free_throws_pct_avg_last_1, + ROUND(AVG(free_throws_pct) OVER w5, 2) AS free_throws_pct_avg_last_5, + ROUND(AVG(free_throws_pct) OVER w10, 2) AS free_throws_pct_avg_last_10, + ROUND(AVG(ts_pct) OVER w1, 2) AS ts_pct_avg_last_1, + ROUND(AVG(ts_pct) OVER w5, 2) AS ts_pct_avg_last_5, + ROUND(AVG(ts_pct) OVER w10, 2) AS ts_pct_avg_last_10, + ROUND(AVG(efg_pct) OVER w1, 2) AS efg_pct_avg_last_1, + ROUND(AVG(efg_pct) OVER w5, 2) AS efg_pct_avg_last_5, + ROUND(AVG(efg_pct) OVER w10, 2) AS efg_pct_avg_last_10, + ROUND(AVG(rebounds) OVER w1, 2) AS rebounds_avg_last_1, + ROUND(AVG(rebounds) OVER w5, 2) AS rebounds_avg_last_5, + ROUND(AVG(rebounds) OVER w10, 2) AS rebounds_avg_last_10, + ROUND(AVG(offensive_rebounds) OVER w1, 2) AS offensive_rebounds_avg_last_1, + ROUND(AVG(offensive_rebounds) OVER w5, 2) AS offensive_rebounds_avg_last_5, + ROUND(AVG(offensive_rebounds) OVER w10, 2) AS offensive_rebounds_avg_last_10, + ROUND(AVG(defensive_rebounds) OVER w1, 2) AS defensive_rebounds_avg_last_1, + ROUND(AVG(defensive_rebounds) OVER w5, 2) AS defensive_rebounds_avg_last_5, + ROUND(AVG(defensive_rebounds) OVER w10, 2) AS defensive_rebounds_avg_last_10, + ROUND(AVG(dreb_pct) OVER w1, 2) AS dreb_pct_avg_last_1, + ROUND(AVG(dreb_pct) OVER w5, 2) AS dreb_pct_avg_last_5, + ROUND(AVG(dreb_pct) OVER w10, 2) AS dreb_pct_avg_last_10, + ROUND(AVG(oreb_pct) OVER w1, 2) AS oreb_pct_avg_last_1, + ROUND(AVG(oreb_pct) OVER w5, 2) AS oreb_pct_avg_last_5, + ROUND(AVG(oreb_pct) OVER w10, 2) AS oreb_pct_avg_last_10, + ROUND(AVG(steals) OVER w1, 2) AS steals_avg_last_1, + ROUND(AVG(steals) OVER w5, 2) AS steals_avg_last_5, + ROUND(AVG(steals) OVER w10, 2) AS steals_avg_last_10, + ROUND(AVG(blocks) OVER w1, 2) AS blocks_avg_last_1, + ROUND(AVG(blocks) OVER w5, 2) AS blocks_avg_last_5, + ROUND(AVG(blocks) OVER w10, 2) AS blocks_avg_last_10, + ROUND(AVG(assists) OVER w1, 2) AS assists_avg_last_1, + ROUND(AVG(assists) OVER w5, 2) AS assists_avg_last_5, + ROUND(AVG(assists) OVER w10, 2) AS assists_avg_last_10, + ROUND(AVG(turnovers) OVER w1, 2) AS turnovers_avg_last_1, + ROUND(AVG(turnovers) OVER w5, 2) AS turnovers_avg_last_5, + ROUND(AVG(turnovers) OVER w10, 2) AS turnovers_avg_last_10, + ROUND(AVG(team_turnovers) OVER w1, 2) AS team_turnovers_avg_last_1, + ROUND(AVG(team_turnovers) OVER w5, 2) AS team_turnovers_avg_last_5, + ROUND(AVG(team_turnovers) OVER w10, 2) AS team_turnovers_avg_last_10, + ROUND(AVG(points_off_turnovers) OVER w1, 2) AS points_off_turnovers_avg_last_1, + ROUND(AVG(points_off_turnovers) OVER w5, 2) AS points_off_turnovers_avg_last_5, + ROUND(AVG(points_off_turnovers) OVER w10, 2) AS points_off_turnovers_avg_last_10, + ROUND(AVG(assists_turnover_ratio) OVER w1, 2) AS assists_turnover_ratio_avg_last_1, + ROUND(AVG(assists_turnover_ratio) OVER w5, 2) AS assists_turnover_ratio_avg_last_5, + ROUND(AVG(assists_turnover_ratio) OVER w10, 2) AS assists_turnover_ratio_avg_last_10, + ROUND(AVG(ast_fgm_pct) OVER w1, 2) AS ast_fgm_pct_avg_last_1, + ROUND(AVG(ast_fgm_pct) OVER w5, 2) AS ast_fgm_pct_avg_last_5, + ROUND(AVG(ast_fgm_pct) OVER w10, 2) AS ast_fgm_pct_avg_last_10, + ROUND(AVG(personal_fouls) OVER w1, 2) AS personal_fouls_avg_last_1, + ROUND(AVG(personal_fouls) OVER w5, 2) AS personal_fouls_avg_last_5, + ROUND(AVG(personal_fouls) OVER w10, 2) AS personal_fouls_avg_last_10, + ROUND(AVG(flagrant_fouls) OVER w1, 2) AS flagrant_fouls_avg_last_1, + ROUND(AVG(flagrant_fouls) OVER w5, 2) AS flagrant_fouls_avg_last_5, + ROUND(AVG(flagrant_fouls) OVER w10, 2) AS flagrant_fouls_avg_last_10, + ROUND(AVG(player_tech_fouls) OVER w1, 2) AS player_tech_fouls_avg_last_1, + ROUND(AVG(player_tech_fouls) OVER w5, 2) AS player_tech_fouls_avg_last_5, + ROUND(AVG(player_tech_fouls) OVER w10, 2) AS player_tech_fouls_avg_last_10, + ROUND(AVG(team_tech_fouls) OVER w1, 2) AS team_tech_fouls_avg_last_1, + ROUND(AVG(team_tech_fouls) OVER w5, 2) AS team_tech_fouls_avg_last_5, + ROUND(AVG(team_tech_fouls) OVER w10, 2) AS team_tech_fouls_avg_last_10, + ROUND(AVG(coach_tech_fouls) OVER w1, 2) AS coach_tech_fouls_avg_last_1, + ROUND(AVG(coach_tech_fouls) OVER w5, 2) AS coach_tech_fouls_avg_last_5, + ROUND(AVG(coach_tech_fouls) OVER w10, 2) AS coach_tech_fouls_avg_last_10, + ROUND(AVG(ejections) OVER w1, 2) AS ejections_avg_last_1, + ROUND(AVG(ejections) OVER w5, 2) AS ejections_avg_last_5, + ROUND(AVG(ejections) OVER w10, 2) AS ejections_avg_last_10, + ROUND(AVG(foulouts) OVER w1, 2) AS foulouts_avg_last_1, + ROUND(AVG(foulouts) OVER w5, 2) AS foulouts_avg_last_5, + ROUND(AVG(foulouts) OVER w10, 2) AS foulouts_avg_last_10, + ROUND(AVG(score_delta) OVER w1, 2) AS score_delta_avg_last_1, + ROUND(AVG(score_delta) OVER w5, 2) AS score_delta_avg_last_5, + ROUND(AVG(score_delta) OVER w10, 2) AS score_delta_avg_last_10, + ROUND(AVG(possessions) OVER w1, 2) AS possessions_avg_last_1, + ROUND(AVG(possessions) OVER w5, 2) AS possessions_avg_last_5, + ROUND(AVG(possessions) OVER w10, 2) AS possessions_avg_last_10, + ROUND(AVG(opp_points) OVER w1, 2) AS opp_points_avg_last_1, + ROUND(AVG(opp_points) OVER w5, 2) AS opp_points_avg_last_5, + ROUND(AVG(opp_points) OVER w10, 2) AS opp_points_avg_last_10, + ROUND(AVG(opp_fast_break_pts) OVER w1, 2) AS opp_fast_break_pts_avg_last_1, + ROUND(AVG(opp_fast_break_pts) OVER w5, 2) AS opp_fast_break_pts_avg_last_5, + ROUND(AVG(opp_fast_break_pts) OVER w10, 2) AS opp_fast_break_pts_avg_last_10, + ROUND(AVG(opp_second_chance_pts) OVER w1, 2) AS opp_second_chance_pts_avg_last_1, + ROUND(AVG(opp_second_chance_pts) OVER w5, 2) AS opp_second_chance_pts_avg_last_5, + ROUND(AVG(opp_second_chance_pts) OVER w10, 2) AS opp_second_chance_pts_avg_last_10, + ROUND(AVG(opp_field_goals_made) OVER w1, 2) AS opp_field_goals_made_avg_last_1, + ROUND(AVG(opp_field_goals_made) OVER w5, 2) AS opp_field_goals_made_avg_last_5, + ROUND(AVG(opp_field_goals_made) OVER w10, 2) AS opp_field_goals_made_avg_last_10, + ROUND(AVG(opp_field_goals_att) OVER w1, 2) AS opp_field_goals_att_avg_last_1, + ROUND(AVG(opp_field_goals_att) OVER w5, 2) AS opp_field_goals_att_avg_last_5, + ROUND(AVG(opp_field_goals_att) OVER w10, 2) AS opp_field_goals_att_avg_last_10, + ROUND(AVG(opp_field_goals_pct) OVER w1, 2) AS opp_field_goals_pct_avg_last_1, + ROUND(AVG(opp_field_goals_pct) OVER w5, 2) AS opp_field_goals_pct_avg_last_5, + ROUND(AVG(opp_field_goals_pct) OVER w10, 2) AS opp_field_goals_pct_avg_last_10, + ROUND(AVG(opp_three_points_made) OVER w1, 2) AS opp_three_points_made_avg_last_1, + ROUND(AVG(opp_three_points_made) OVER w5, 2) AS opp_three_points_made_avg_last_5, + ROUND(AVG(opp_three_points_made) OVER w10, 2) AS opp_three_points_made_avg_last_10, + ROUND(AVG(opp_three_points_att) OVER w1, 2) AS opp_three_points_att_avg_last_1, + ROUND(AVG(opp_three_points_att) OVER w5, 2) AS opp_three_points_att_avg_last_5, + ROUND(AVG(opp_three_points_att) OVER w10, 2) AS opp_three_points_att_avg_last_10, + ROUND(AVG(opp_three_points_pct) OVER w1, 2) AS opp_three_points_pct_avg_last_1, + ROUND(AVG(opp_three_points_pct) OVER w5, 2) AS opp_three_points_pct_avg_last_5, + ROUND(AVG(opp_three_points_pct) OVER w10, 2) AS opp_three_points_pct_avg_last_10, + ROUND(AVG(opp_two_points_made) OVER w1, 2) AS opp_two_points_made_avg_last_1, + ROUND(AVG(opp_two_points_made) OVER w5, 2) AS opp_two_points_made_avg_last_5, + ROUND(AVG(opp_two_points_made) OVER w10, 2) AS opp_two_points_made_avg_last_10, + ROUND(AVG(opp_two_points_att) OVER w1, 2) AS opp_two_points_att_avg_last_1, + ROUND(AVG(opp_two_points_att) OVER w5, 2) AS opp_two_points_att_avg_last_5, + ROUND(AVG(opp_two_points_att) OVER w10, 2) AS opp_two_points_att_avg_last_10, + ROUND(AVG(opp_two_points_pct) OVER w1, 2) AS opp_two_points_pct_avg_last_1, + ROUND(AVG(opp_two_points_pct) OVER w5, 2) AS opp_two_points_pct_avg_last_5, + ROUND(AVG(opp_two_points_pct) OVER w10, 2) AS opp_two_points_pct_avg_last_10, + ROUND(AVG(opp_free_throws_made) OVER w1, 2) AS opp_free_throws_made_avg_last_1, + ROUND(AVG(opp_free_throws_made) OVER w5, 2) AS opp_free_throws_made_avg_last_5, + ROUND(AVG(opp_free_throws_made) OVER w10, 2) AS opp_free_throws_made_avg_last_10, + ROUND(AVG(opp_free_throws_att) OVER w1, 2) AS opp_free_throws_att_avg_last_1, + ROUND(AVG(opp_free_throws_att) OVER w5, 2) AS opp_free_throws_att_avg_last_5, + ROUND(AVG(opp_free_throws_att) OVER w10, 2) AS opp_free_throws_att_avg_last_10, + ROUND(AVG(opp_free_throws_pct) OVER w1, 2) AS opp_free_throws_pct_avg_last_1, + ROUND(AVG(opp_free_throws_pct) OVER w5, 2) AS opp_free_throws_pct_avg_last_5, + ROUND(AVG(opp_free_throws_pct) OVER w10, 2) AS opp_free_throws_pct_avg_last_10, + ROUND(AVG(opp_ts_pct) OVER w1, 2) AS opp_ts_pct_avg_last_1, + ROUND(AVG(opp_ts_pct) OVER w5, 2) AS opp_ts_pct_avg_last_5, + ROUND(AVG(opp_ts_pct) OVER w10, 2) AS opp_ts_pct_avg_last_10, + ROUND(AVG(opp_efg_pct) OVER w1, 2) AS opp_efg_pct_avg_last_1, + ROUND(AVG(opp_efg_pct) OVER w5, 2) AS opp_efg_pct_avg_last_5, + ROUND(AVG(opp_efg_pct) OVER w10, 2) AS opp_efg_pct_avg_last_10, + ROUND(AVG(opp_rebounds) OVER w1, 2) AS opp_rebounds_avg_last_1, + ROUND(AVG(opp_rebounds) OVER w5, 2) AS opp_rebounds_avg_last_5, + ROUND(AVG(opp_rebounds) OVER w10, 2) AS opp_rebounds_avg_last_10, + ROUND(AVG(opp_offensive_rebounds) OVER w1, 2) AS opp_offensive_rebounds_avg_last_1, + ROUND(AVG(opp_offensive_rebounds) OVER w5, 2) AS opp_offensive_rebounds_avg_last_5, + ROUND(AVG(opp_offensive_rebounds) OVER w10, 2) AS opp_offensive_rebounds_avg_last_10, + ROUND(AVG(opp_defensive_rebounds) OVER w1, 2) AS opp_defensive_rebounds_avg_last_1, + ROUND(AVG(opp_defensive_rebounds) OVER w5, 2) AS opp_defensive_rebounds_avg_last_5, + ROUND(AVG(opp_defensive_rebounds) OVER w10, 2) AS opp_defensive_rebounds_avg_last_10, + ROUND(AVG(opp_dreb_pct) OVER w1, 2) AS opp_dreb_pct_avg_last_1, + ROUND(AVG(opp_dreb_pct) OVER w5, 2) AS opp_dreb_pct_avg_last_5, + ROUND(AVG(opp_dreb_pct) OVER w10, 2) AS opp_dreb_pct_avg_last_10, + ROUND(AVG(opp_oreb_pct) OVER w1, 2) AS opp_oreb_pct_avg_last_1, + ROUND(AVG(opp_oreb_pct) OVER w5, 2) AS opp_oreb_pct_avg_last_5, + ROUND(AVG(opp_oreb_pct) OVER w10, 2) AS opp_oreb_pct_avg_last_10, + ROUND(AVG(opp_steals) OVER w1, 2) AS opp_steals_avg_last_1, + ROUND(AVG(opp_steals) OVER w5, 2) AS opp_steals_avg_last_5, + ROUND(AVG(opp_steals) OVER w10, 2) AS opp_steals_avg_last_10, + ROUND(AVG(opp_blocks) OVER w1, 2) AS opp_blocks_avg_last_1, + ROUND(AVG(opp_blocks) OVER w5, 2) AS opp_blocks_avg_last_5, + ROUND(AVG(opp_blocks) OVER w10, 2) AS opp_blocks_avg_last_10, + ROUND(AVG(opp_assists) OVER w1, 2) AS opp_assists_avg_last_1, + ROUND(AVG(opp_assists) OVER w5, 2) AS opp_assists_avg_last_5, + ROUND(AVG(opp_assists) OVER w10, 2) AS opp_assists_avg_last_10, + ROUND(AVG(opp_turnovers) OVER w1, 2) AS opp_turnovers_avg_last_1, + ROUND(AVG(opp_turnovers) OVER w5, 2) AS opp_turnovers_avg_last_5, + ROUND(AVG(opp_turnovers) OVER w10, 2) AS opp_turnovers_avg_last_10, + ROUND(AVG(opp_team_turnovers) OVER w1, 2) AS opp_team_turnovers_avg_last_1, + ROUND(AVG(opp_team_turnovers) OVER w5, 2) AS opp_team_turnovers_avg_last_5, + ROUND(AVG(opp_team_turnovers) OVER w10, 2) AS opp_team_turnovers_avg_last_10, + ROUND(AVG(opp_points_off_turnovers) OVER w1, 2) AS opp_points_off_turnovers_avg_last_1, + ROUND(AVG(opp_points_off_turnovers) OVER w5, 2) AS opp_points_off_turnovers_avg_last_5, + ROUND(AVG(opp_points_off_turnovers) OVER w10, 2) AS opp_points_off_turnovers_avg_last_10, + ROUND(AVG(opp_assists_turnover_ratio) OVER w1, 2) AS opp_assists_turnover_ratio_avg_last_1, + ROUND(AVG(opp_assists_turnover_ratio) OVER w5, 2) AS opp_assists_turnover_ratio_avg_last_5, + ROUND(AVG(opp_assists_turnover_ratio) OVER w10, 2) AS opp_assists_turnover_ratio_avg_last_10, + ROUND(AVG(opp_ast_fgm_pct) OVER w1, 2) AS opp_ast_fgm_pct_avg_last_1, + ROUND(AVG(opp_ast_fgm_pct) OVER w5, 2) AS opp_ast_fgm_pct_avg_last_5, + ROUND(AVG(opp_ast_fgm_pct) OVER w10, 2) AS opp_ast_fgm_pct_avg_last_10, + ROUND(AVG(opp_personal_fouls) OVER w1, 2) AS opp_personal_fouls_avg_last_1, + ROUND(AVG(opp_personal_fouls) OVER w5, 2) AS opp_personal_fouls_avg_last_5, + ROUND(AVG(opp_personal_fouls) OVER w10, 2) AS opp_personal_fouls_avg_last_10, + ROUND(AVG(opp_flagrant_fouls) OVER w1, 2) AS opp_flagrant_fouls_avg_last_1, + ROUND(AVG(opp_flagrant_fouls) OVER w5, 2) AS opp_flagrant_fouls_avg_last_5, + ROUND(AVG(opp_flagrant_fouls) OVER w10, 2) AS opp_flagrant_fouls_avg_last_10, + ROUND(AVG(opp_player_tech_fouls) OVER w1, 2) AS opp_player_tech_fouls_avg_last_1, + ROUND(AVG(opp_player_tech_fouls) OVER w5, 2) AS opp_player_tech_fouls_avg_last_5, + ROUND(AVG(opp_player_tech_fouls) OVER w10, 2) AS opp_player_tech_fouls_avg_last_10, + ROUND(AVG(opp_team_tech_fouls) OVER w1, 2) AS opp_team_tech_fouls_avg_last_1, + ROUND(AVG(opp_team_tech_fouls) OVER w5, 2) AS opp_team_tech_fouls_avg_last_5, + ROUND(AVG(opp_team_tech_fouls) OVER w10, 2) AS opp_team_tech_fouls_avg_last_10, + ROUND(AVG(opp_coach_tech_fouls) OVER w1, 2) AS opp_coach_tech_fouls_avg_last_1, + ROUND(AVG(opp_coach_tech_fouls) OVER w5, 2) AS opp_coach_tech_fouls_avg_last_5, + ROUND(AVG(opp_coach_tech_fouls) OVER w10, 2) AS opp_coach_tech_fouls_avg_last_10, + ROUND(AVG(opp_ejections) OVER w1, 2) AS opp_ejections_avg_last_1, + ROUND(AVG(opp_ejections) OVER w5, 2) AS opp_ejections_avg_last_5, + ROUND(AVG(opp_ejections) OVER w10, 2) AS opp_ejections_avg_last_10, + ROUND(AVG(opp_foulouts) OVER w1, 2) AS opp_foulouts_avg_last_1, + ROUND(AVG(opp_foulouts) OVER w5, 2) AS opp_foulouts_avg_last_5, + ROUND(AVG(opp_foulouts) OVER w10, 2) AS opp_foulouts_avg_last_10, + ROUND(AVG(opp_score_delta) OVER w1, 2) AS opp_score_delta_avg_last_1, + ROUND(AVG(opp_score_delta) OVER w5, 2) AS opp_score_delta_avg_last_5, + ROUND(AVG(opp_score_delta) OVER w10, 2) AS opp_score_delta_avg_last_10, + ROUND(AVG(opp_possessions) OVER w1, 2) AS opp_possessions_avg_last_1, + ROUND(AVG(opp_possessions) OVER w5, 2) AS opp_possessions_avg_last_5, + ROUND(AVG(opp_possessions) OVER w10, 2) AS opp_possessions_avg_last_10, + ROUND(STDDEV_POP(points) OVER w5, 2) AS points_std_last_5, + ROUND(STDDEV_POP(points) OVER w10, 2) AS points_std_last_10, + ROUND(STDDEV_POP(fast_break_pts) OVER w5, 2) AS fast_break_pts_std_last_5, + ROUND(STDDEV_POP(fast_break_pts) OVER w10, 2) AS fast_break_pts_std_last_10, + ROUND(STDDEV_POP(second_chance_pts) OVER w5, 2) AS second_chance_pts_std_last_5, + ROUND(STDDEV_POP(second_chance_pts) OVER w10, 2) AS second_chance_pts_std_last_10, + ROUND(STDDEV_POP(field_goals_made) OVER w5, 2) AS field_goals_made_std_last_5, + ROUND(STDDEV_POP(field_goals_made) OVER w10, 2) AS field_goals_made_std_last_10, + ROUND(STDDEV_POP(field_goals_att) OVER w5, 2) AS field_goals_att_std_last_5, + ROUND(STDDEV_POP(field_goals_att) OVER w10, 2) AS field_goals_att_std_last_10, + ROUND(STDDEV_POP(field_goals_pct) OVER w5, 2) AS field_goals_pct_std_last_5, + ROUND(STDDEV_POP(field_goals_pct) OVER w10, 2) AS field_goals_pct_std_last_10, + ROUND(STDDEV_POP(three_points_made) OVER w5, 2) AS three_points_made_std_last_5, + ROUND(STDDEV_POP(three_points_made) OVER w10, 2) AS three_points_made_std_last_10, + ROUND(STDDEV_POP(three_points_att) OVER w5, 2) AS three_points_att_std_last_5, + ROUND(STDDEV_POP(three_points_att) OVER w10, 2) AS three_points_att_std_last_10, + ROUND(STDDEV_POP(three_points_pct) OVER w5, 2) AS three_points_pct_std_last_5, + ROUND(STDDEV_POP(three_points_pct) OVER w10, 2) AS three_points_pct_std_last_10, + ROUND(STDDEV_POP(two_points_made) OVER w5, 2) AS two_points_made_std_last_5, + ROUND(STDDEV_POP(two_points_made) OVER w10, 2) AS two_points_made_std_last_10, + ROUND(STDDEV_POP(two_points_att) OVER w5, 2) AS two_points_att_std_last_5, + ROUND(STDDEV_POP(two_points_att) OVER w10, 2) AS two_points_att_std_last_10, + ROUND(STDDEV_POP(two_points_pct) OVER w5, 2) AS two_points_pct_std_last_5, + ROUND(STDDEV_POP(two_points_pct) OVER w10, 2) AS two_points_pct_std_last_10, + ROUND(STDDEV_POP(free_throws_made) OVER w5, 2) AS free_throws_made_std_last_5, + ROUND(STDDEV_POP(free_throws_made) OVER w10, 2) AS free_throws_made_std_last_10, + ROUND(STDDEV_POP(free_throws_att) OVER w5, 2) AS free_throws_att_std_last_5, + ROUND(STDDEV_POP(free_throws_att) OVER w10, 2) AS free_throws_att_std_last_10, + ROUND(STDDEV_POP(free_throws_pct) OVER w5, 2) AS free_throws_pct_std_last_5, + ROUND(STDDEV_POP(free_throws_pct) OVER w10, 2) AS free_throws_pct_std_last_10, + ROUND(STDDEV_POP(ts_pct) OVER w5, 2) AS ts_pct_std_last_5, + ROUND(STDDEV_POP(ts_pct) OVER w10, 2) AS ts_pct_std_last_10, + ROUND(STDDEV_POP(efg_pct) OVER w5, 2) AS efg_pct_std_last_5, + ROUND(STDDEV_POP(efg_pct) OVER w10, 2) AS efg_pct_std_last_10, + ROUND(STDDEV_POP(rebounds) OVER w5, 2) AS rebounds_std_last_5, + ROUND(STDDEV_POP(rebounds) OVER w10, 2) AS rebounds_std_last_10, + ROUND(STDDEV_POP(offensive_rebounds) OVER w5, 2) AS offensive_rebounds_std_last_5, + ROUND(STDDEV_POP(offensive_rebounds) OVER w10, 2) AS offensive_rebounds_std_last_10, + ROUND(STDDEV_POP(defensive_rebounds) OVER w5, 2) AS defensive_rebounds_std_last_5, + ROUND(STDDEV_POP(defensive_rebounds) OVER w10, 2) AS defensive_rebounds_std_last_10, + ROUND(STDDEV_POP(dreb_pct) OVER w5, 2) AS dreb_pct_std_last_5, + ROUND(STDDEV_POP(dreb_pct) OVER w10, 2) AS dreb_pct_std_last_10, + ROUND(STDDEV_POP(oreb_pct) OVER w5, 2) AS oreb_pct_std_last_5, + ROUND(STDDEV_POP(oreb_pct) OVER w10, 2) AS oreb_pct_std_last_10, + ROUND(STDDEV_POP(steals) OVER w5, 2) AS steals_std_last_5, + ROUND(STDDEV_POP(steals) OVER w10, 2) AS steals_std_last_10, + ROUND(STDDEV_POP(blocks) OVER w5, 2) AS blocks_std_last_5, + ROUND(STDDEV_POP(blocks) OVER w10, 2) AS blocks_std_last_10, + ROUND(STDDEV_POP(assists) OVER w5, 2) AS assists_std_last_5, + ROUND(STDDEV_POP(assists) OVER w10, 2) AS assists_std_last_10, + ROUND(STDDEV_POP(turnovers) OVER w5, 2) AS turnovers_std_last_5, + ROUND(STDDEV_POP(turnovers) OVER w10, 2) AS turnovers_std_last_10, + ROUND(STDDEV_POP(team_turnovers) OVER w5, 2) AS team_turnovers_std_last_5, + ROUND(STDDEV_POP(team_turnovers) OVER w10, 2) AS team_turnovers_std_last_10, + ROUND(STDDEV_POP(points_off_turnovers) OVER w5, 2) AS points_off_turnovers_std_last_5, + ROUND(STDDEV_POP(points_off_turnovers) OVER w10, 2) AS points_off_turnovers_std_last_10, + ROUND(STDDEV_POP(assists_turnover_ratio) OVER w5, 2) AS assists_turnover_ratio_std_last_5, + ROUND(STDDEV_POP(assists_turnover_ratio) OVER w10, 2) AS assists_turnover_ratio_std_last_10, + ROUND(STDDEV_POP(ast_fgm_pct) OVER w5, 2) AS ast_fgm_pct_std_last_5, + ROUND(STDDEV_POP(ast_fgm_pct) OVER w10, 2) AS ast_fgm_pct_std_last_10, + ROUND(STDDEV_POP(personal_fouls) OVER w5, 2) AS personal_fouls_std_last_5, + ROUND(STDDEV_POP(personal_fouls) OVER w10, 2) AS personal_fouls_std_last_10, + ROUND(STDDEV_POP(flagrant_fouls) OVER w5, 2) AS flagrant_fouls_std_last_5, + ROUND(STDDEV_POP(flagrant_fouls) OVER w10, 2) AS flagrant_fouls_std_last_10, + ROUND(STDDEV_POP(player_tech_fouls) OVER w5, 2) AS player_tech_fouls_std_last_5, + ROUND(STDDEV_POP(player_tech_fouls) OVER w10, 2) AS player_tech_fouls_std_last_10, + ROUND(STDDEV_POP(team_tech_fouls) OVER w5, 2) AS team_tech_fouls_std_last_5, + ROUND(STDDEV_POP(team_tech_fouls) OVER w10, 2) AS team_tech_fouls_std_last_10, + ROUND(STDDEV_POP(coach_tech_fouls) OVER w5, 2) AS coach_tech_fouls_std_last_5, + ROUND(STDDEV_POP(coach_tech_fouls) OVER w10, 2) AS coach_tech_fouls_std_last_10, + ROUND(STDDEV_POP(ejections) OVER w5, 2) AS ejections_std_last_5, + ROUND(STDDEV_POP(ejections) OVER w10, 2) AS ejections_std_last_10, + ROUND(STDDEV_POP(foulouts) OVER w5, 2) AS foulouts_std_last_5, + ROUND(STDDEV_POP(foulouts) OVER w10, 2) AS foulouts_std_last_10, + ROUND(STDDEV_POP(score_delta) OVER w5, 2) AS score_delta_std_last_5, + ROUND(STDDEV_POP(score_delta) OVER w10, 2) AS score_delta_std_last_10, + ROUND(STDDEV_POP(possessions) OVER w5, 2) AS possessions_std_last_5, + ROUND(STDDEV_POP(possessions) OVER w10, 2) AS possessions_std_last_10, + ROUND(STDDEV_POP(opp_points) OVER w5, 2) AS opp_points_std_last_5, + ROUND(STDDEV_POP(opp_points) OVER w10, 2) AS opp_points_std_last_10, + ROUND(STDDEV_POP(opp_fast_break_pts) OVER w5, 2) AS opp_fast_break_pts_std_last_5, + ROUND(STDDEV_POP(opp_fast_break_pts) OVER w10, 2) AS opp_fast_break_pts_std_last_10, + ROUND(STDDEV_POP(opp_second_chance_pts) OVER w5, 2) AS opp_second_chance_pts_std_last_5, + ROUND(STDDEV_POP(opp_second_chance_pts) OVER w10, 2) AS opp_second_chance_pts_std_last_10, + ROUND(STDDEV_POP(opp_field_goals_made) OVER w5, 2) AS opp_field_goals_made_std_last_5, + ROUND(STDDEV_POP(opp_field_goals_made) OVER w10, 2) AS opp_field_goals_made_std_last_10, + ROUND(STDDEV_POP(opp_field_goals_att) OVER w5, 2) AS opp_field_goals_att_std_last_5, + ROUND(STDDEV_POP(opp_field_goals_att) OVER w10, 2) AS opp_field_goals_att_std_last_10, + ROUND(STDDEV_POP(opp_field_goals_pct) OVER w5, 2) AS opp_field_goals_pct_std_last_5, + ROUND(STDDEV_POP(opp_field_goals_pct) OVER w10, 2) AS opp_field_goals_pct_std_last_10, + ROUND(STDDEV_POP(opp_three_points_made) OVER w5, 2) AS opp_three_points_made_std_last_5, + ROUND(STDDEV_POP(opp_three_points_made) OVER w10, 2) AS opp_three_points_made_std_last_10, + ROUND(STDDEV_POP(opp_three_points_att) OVER w5, 2) AS opp_three_points_att_std_last_5, + ROUND(STDDEV_POP(opp_three_points_att) OVER w10, 2) AS opp_three_points_att_std_last_10, + ROUND(STDDEV_POP(opp_three_points_pct) OVER w5, 2) AS opp_three_points_pct_std_last_5, + ROUND(STDDEV_POP(opp_three_points_pct) OVER w10, 2) AS opp_three_points_pct_std_last_10, + ROUND(STDDEV_POP(opp_two_points_made) OVER w5, 2) AS opp_two_points_made_std_last_5, + ROUND(STDDEV_POP(opp_two_points_made) OVER w10, 2) AS opp_two_points_made_std_last_10, + ROUND(STDDEV_POP(opp_two_points_att) OVER w5, 2) AS opp_two_points_att_std_last_5, + ROUND(STDDEV_POP(opp_two_points_att) OVER w10, 2) AS opp_two_points_att_std_last_10, + ROUND(STDDEV_POP(opp_two_points_pct) OVER w5, 2) AS opp_two_points_pct_std_last_5, + ROUND(STDDEV_POP(opp_two_points_pct) OVER w10, 2) AS opp_two_points_pct_std_last_10, + ROUND(STDDEV_POP(opp_free_throws_made) OVER w5, 2) AS opp_free_throws_made_std_last_5, + ROUND(STDDEV_POP(opp_free_throws_made) OVER w10, 2) AS opp_free_throws_made_std_last_10, + ROUND(STDDEV_POP(opp_free_throws_att) OVER w5, 2) AS opp_free_throws_att_std_last_5, + ROUND(STDDEV_POP(opp_free_throws_att) OVER w10, 2) AS opp_free_throws_att_std_last_10, + ROUND(STDDEV_POP(opp_free_throws_pct) OVER w5, 2) AS opp_free_throws_pct_std_last_5, + ROUND(STDDEV_POP(opp_free_throws_pct) OVER w10, 2) AS opp_free_throws_pct_std_last_10, + ROUND(STDDEV_POP(opp_ts_pct) OVER w5, 2) AS opp_ts_pct_std_last_5, + ROUND(STDDEV_POP(opp_ts_pct) OVER w10, 2) AS opp_ts_pct_std_last_10, + ROUND(STDDEV_POP(opp_efg_pct) OVER w5, 2) AS opp_efg_pct_std_last_5, + ROUND(STDDEV_POP(opp_efg_pct) OVER w10, 2) AS opp_efg_pct_std_last_10, + ROUND(STDDEV_POP(opp_rebounds) OVER w5, 2) AS opp_rebounds_std_last_5, + ROUND(STDDEV_POP(opp_rebounds) OVER w10, 2) AS opp_rebounds_std_last_10, + ROUND(STDDEV_POP(opp_offensive_rebounds) OVER w5, 2) AS opp_offensive_rebounds_std_last_5, + ROUND(STDDEV_POP(opp_offensive_rebounds) OVER w10, 2) AS opp_offensive_rebounds_std_last_10, + ROUND(STDDEV_POP(opp_defensive_rebounds) OVER w5, 2) AS opp_defensive_rebounds_std_last_5, + ROUND(STDDEV_POP(opp_defensive_rebounds) OVER w10, 2) AS opp_defensive_rebounds_std_last_10, + ROUND(STDDEV_POP(opp_dreb_pct) OVER w5, 2) AS opp_dreb_pct_std_last_5, + ROUND(STDDEV_POP(opp_dreb_pct) OVER w10, 2) AS opp_dreb_pct_std_last_10, + ROUND(STDDEV_POP(opp_oreb_pct) OVER w5, 2) AS opp_oreb_pct_std_last_5, + ROUND(STDDEV_POP(opp_oreb_pct) OVER w10, 2) AS opp_oreb_pct_std_last_10, + ROUND(STDDEV_POP(opp_steals) OVER w5, 2) AS opp_steals_std_last_5, + ROUND(STDDEV_POP(opp_steals) OVER w10, 2) AS opp_steals_std_last_10, + ROUND(STDDEV_POP(opp_blocks) OVER w5, 2) AS opp_blocks_std_last_5, + ROUND(STDDEV_POP(opp_blocks) OVER w10, 2) AS opp_blocks_std_last_10, + ROUND(STDDEV_POP(opp_assists) OVER w5, 2) AS opp_assists_std_last_5, + ROUND(STDDEV_POP(opp_assists) OVER w10, 2) AS opp_assists_std_last_10, + ROUND(STDDEV_POP(opp_turnovers) OVER w5, 2) AS opp_turnovers_std_last_5, + ROUND(STDDEV_POP(opp_turnovers) OVER w10, 2) AS opp_turnovers_std_last_10, + ROUND(STDDEV_POP(opp_team_turnovers) OVER w5, 2) AS opp_team_turnovers_std_last_5, + ROUND(STDDEV_POP(opp_team_turnovers) OVER w10, 2) AS opp_team_turnovers_std_last_10, + ROUND(STDDEV_POP(opp_points_off_turnovers) OVER w5, 2) AS opp_points_off_turnovers_std_last_5, + ROUND(STDDEV_POP(opp_points_off_turnovers) OVER w10, 2) AS opp_points_off_turnovers_std_last_10, + ROUND(STDDEV_POP(opp_assists_turnover_ratio) OVER w5, 2) AS opp_assists_turnover_ratio_std_last_5, + ROUND(STDDEV_POP(opp_assists_turnover_ratio) OVER w10, 2) AS opp_assists_turnover_ratio_std_last_10, + ROUND(STDDEV_POP(opp_ast_fgm_pct) OVER w5, 2) AS opp_ast_fgm_pct_std_last_5, + ROUND(STDDEV_POP(opp_ast_fgm_pct) OVER w10, 2) AS opp_ast_fgm_pct_std_last_10, + ROUND(STDDEV_POP(opp_personal_fouls) OVER w5, 2) AS opp_personal_fouls_std_last_5, + ROUND(STDDEV_POP(opp_personal_fouls) OVER w10, 2) AS opp_personal_fouls_std_last_10, + ROUND(STDDEV_POP(opp_flagrant_fouls) OVER w5, 2) AS opp_flagrant_fouls_std_last_5, + ROUND(STDDEV_POP(opp_flagrant_fouls) OVER w10, 2) AS opp_flagrant_fouls_std_last_10, + ROUND(STDDEV_POP(opp_player_tech_fouls) OVER w5, 2) AS opp_player_tech_fouls_std_last_5, + ROUND(STDDEV_POP(opp_player_tech_fouls) OVER w10, 2) AS opp_player_tech_fouls_std_last_10, + ROUND(STDDEV_POP(opp_team_tech_fouls) OVER w5, 2) AS opp_team_tech_fouls_std_last_5, + ROUND(STDDEV_POP(opp_team_tech_fouls) OVER w10, 2) AS opp_team_tech_fouls_std_last_10, + ROUND(STDDEV_POP(opp_coach_tech_fouls) OVER w5, 2) AS opp_coach_tech_fouls_std_last_5, + ROUND(STDDEV_POP(opp_coach_tech_fouls) OVER w10, 2) AS opp_coach_tech_fouls_std_last_10, + ROUND(STDDEV_POP(opp_ejections) OVER w5, 2) AS opp_ejections_std_last_5, + ROUND(STDDEV_POP(opp_ejections) OVER w10, 2) AS opp_ejections_std_last_10, + ROUND(STDDEV_POP(opp_foulouts) OVER w5, 2) AS opp_foulouts_std_last_5, + ROUND(STDDEV_POP(opp_foulouts) OVER w10, 2) AS opp_foulouts_std_last_10, + ROUND(STDDEV_POP(opp_score_delta) OVER w5, 2) AS opp_score_delta_std_last_5, + ROUND(STDDEV_POP(opp_score_delta) OVER w10, 2) AS opp_score_delta_std_last_10, + ROUND(STDDEV_POP(opp_possessions) OVER w5, 2) AS opp_possessions_std_last_5, + ROUND(STDDEV_POP(opp_possessions) OVER w10, 2) AS opp_possessions_std_last_10 +FROM ( + SELECT + *, + IF(win = true, 1, 0) as is_win, + IF(win = false, 0, 1) as is_loss, + ROUND(field_goals_att - offensive_rebounds + turnovers + 0.475 * free_throws_att, 2) as possessions, + ROUND(opp_field_goals_att - opp_offensive_rebounds + opp_turnovers + 0.475 * opp_free_throws_att, 2) as opp_possessions, + ROUND(points / (2 * (field_goals_att) + 0.475 * free_throws_att), 3) AS ts_pct, + ROUND(opp_points / (2 * (opp_field_goals_att) + 0.475 * opp_free_throws_att), 3) AS opp_ts_pct, + ROUND(((field_goals_made) + 0.5 * three_points_att) / (field_goals_att), 3) AS efg_pct, + ROUND(((opp_field_goals_made) + 0.5 * opp_three_points_att) / (opp_field_goals_att), 3) AS opp_efg_pct, + ROUND(defensive_rebounds / rebounds, 3) AS dreb_pct, + ROUND(opp_defensive_rebounds / opp_rebounds, 3) AS opp_dreb_pct, + ROUND(offensive_rebounds / rebounds, 3) AS oreb_pct, + ROUND(opp_offensive_rebounds / opp_rebounds, 3) AS opp_oreb_pct, + ROUND(assists / field_goals_made, 3) AS ast_fgm_pct, + ROUND(opp_assists / opp_field_goals_made, 3) AS opp_ast_fgm_pct, + (points_game - opp_points_game) as score_delta, + (opp_points_game - points_game) as opp_score_delta + FROM + `bigquery-public-data.ncaa_basketball.mbb_teams_games_sr` + WHERE field_goals_att > 0 AND opp_field_goals_att > 0 AND rebounds > 0 AND opp_rebounds > 0 AND field_goals_made > 0 AND opp_field_goals_made > 0 + AND division_name = "NCAA Division I" + AND opp_division_name = "NCAA Division I" + AND scheduled_date > '2009-05-01' +) +WINDOW w1 AS (partition by season, team_id order by scheduled_date ASC ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING), + w5 AS (partition by season, team_id order by scheduled_date ASC ROWS BETWEEN 5 PRECEDING AND 1 PRECEDING), + w10 AS (partition by season, team_id order by scheduled_date ASC ROWS BETWEEN 10 PRECEDING AND 1 PRECEDING) +ORDER BY season, team_id diff --git a/bigquery/bqml/resources/training_data_query.sql b/bigquery/bqml/resources/training_data_query.sql new file mode 100644 index 00000000000..74f39e9f0aa --- /dev/null +++ b/bigquery/bqml/resources/training_data_query.sql @@ -0,0 +1,783 @@ +#standardSQL +SELECT + team.game_id AS game_id, + team.season AS season, + team.scheduled_date AS scheduled_date, + team.team_id AS team_id, + opponent.team_id AS opponent_id, + team.three_points_made + opponent.three_points_made as total_three_points_made, + team.three_points_att + opponent.three_points_att as total_three_points_att, + team.points_avg_last_1 AS team_points_avg_last_1, + team.points_avg_last_5 AS team_points_avg_last_5, + team.points_avg_last_10 AS team_points_avg_last_10, + team.fast_break_pts_avg_last_1 AS team_fast_break_pts_avg_last_1, + team.fast_break_pts_avg_last_5 AS team_fast_break_pts_avg_last_5, + team.fast_break_pts_avg_last_10 AS team_fast_break_pts_avg_last_10, + team.second_chance_pts_avg_last_1 AS team_second_chance_pts_avg_last_1, + team.second_chance_pts_avg_last_5 AS team_second_chance_pts_avg_last_5, + team.second_chance_pts_avg_last_10 AS team_second_chance_pts_avg_last_10, + team.field_goals_made_avg_last_1 AS team_field_goals_made_avg_last_1, + team.field_goals_made_avg_last_5 AS team_field_goals_made_avg_last_5, + team.field_goals_made_avg_last_10 AS team_field_goals_made_avg_last_10, + team.field_goals_att_avg_last_1 AS team_field_goals_att_avg_last_1, + team.field_goals_att_avg_last_5 AS team_field_goals_att_avg_last_5, + team.field_goals_att_avg_last_10 AS team_field_goals_att_avg_last_10, + team.field_goals_pct_avg_last_1 AS team_field_goals_pct_avg_last_1, + team.field_goals_pct_avg_last_5 AS team_field_goals_pct_avg_last_5, + team.field_goals_pct_avg_last_10 AS team_field_goals_pct_avg_last_10, + team.three_points_made_avg_last_1 AS team_three_points_made_avg_last_1, + team.three_points_made_avg_last_5 AS team_three_points_made_avg_last_5, + team.three_points_made_avg_last_10 AS team_three_points_made_avg_last_10, + team.three_points_att_avg_last_1 AS team_three_points_att_avg_last_1, + team.three_points_att_avg_last_5 AS team_three_points_att_avg_last_5, + team.three_points_att_avg_last_10 AS team_three_points_att_avg_last_10, + team.three_points_pct_avg_last_1 AS team_three_points_pct_avg_last_1, + team.three_points_pct_avg_last_5 AS team_three_points_pct_avg_last_5, + team.three_points_pct_avg_last_10 AS team_three_points_pct_avg_last_10, + team.two_points_made_avg_last_1 AS team_two_points_made_avg_last_1, + team.two_points_made_avg_last_5 AS team_two_points_made_avg_last_5, + team.two_points_made_avg_last_10 AS team_two_points_made_avg_last_10, + team.two_points_att_avg_last_1 AS team_two_points_att_avg_last_1, + team.two_points_att_avg_last_5 AS team_two_points_att_avg_last_5, + team.two_points_att_avg_last_10 AS team_two_points_att_avg_last_10, + team.two_points_pct_avg_last_1 AS team_two_points_pct_avg_last_1, + team.two_points_pct_avg_last_5 AS team_two_points_pct_avg_last_5, + team.two_points_pct_avg_last_10 AS team_two_points_pct_avg_last_10, + team.free_throws_made_avg_last_1 AS team_free_throws_made_avg_last_1, + team.free_throws_made_avg_last_5 AS team_free_throws_made_avg_last_5, + team.free_throws_made_avg_last_10 AS team_free_throws_made_avg_last_10, + team.free_throws_att_avg_last_1 AS team_free_throws_att_avg_last_1, + team.free_throws_att_avg_last_5 AS team_free_throws_att_avg_last_5, + team.free_throws_att_avg_last_10 AS team_free_throws_att_avg_last_10, + team.free_throws_pct_avg_last_1 AS team_free_throws_pct_avg_last_1, + team.free_throws_pct_avg_last_5 AS team_free_throws_pct_avg_last_5, + team.free_throws_pct_avg_last_10 AS team_free_throws_pct_avg_last_10, + team.ts_pct_avg_last_1 AS team_ts_pct_avg_last_1, + team.ts_pct_avg_last_5 AS team_ts_pct_avg_last_5, + team.ts_pct_avg_last_10 AS team_ts_pct_avg_last_10, + team.efg_pct_avg_last_1 AS team_efg_pct_avg_last_1, + team.efg_pct_avg_last_5 AS team_efg_pct_avg_last_5, + team.efg_pct_avg_last_10 AS team_efg_pct_avg_last_10, + team.rebounds_avg_last_1 AS team_rebounds_avg_last_1, + team.rebounds_avg_last_5 AS team_rebounds_avg_last_5, + team.rebounds_avg_last_10 AS team_rebounds_avg_last_10, + team.offensive_rebounds_avg_last_1 AS team_offensive_rebounds_avg_last_1, + team.offensive_rebounds_avg_last_5 AS team_offensive_rebounds_avg_last_5, + team.offensive_rebounds_avg_last_10 AS team_offensive_rebounds_avg_last_10, + team.defensive_rebounds_avg_last_1 AS team_defensive_rebounds_avg_last_1, + team.defensive_rebounds_avg_last_5 AS team_defensive_rebounds_avg_last_5, + team.defensive_rebounds_avg_last_10 AS team_defensive_rebounds_avg_last_10, + team.dreb_pct_avg_last_1 AS team_dreb_pct_avg_last_1, + team.dreb_pct_avg_last_5 AS team_dreb_pct_avg_last_5, + team.dreb_pct_avg_last_10 AS team_dreb_pct_avg_last_10, + team.oreb_pct_avg_last_1 AS team_oreb_pct_avg_last_1, + team.oreb_pct_avg_last_5 AS team_oreb_pct_avg_last_5, + team.oreb_pct_avg_last_10 AS team_oreb_pct_avg_last_10, + team.steals_avg_last_1 AS team_steals_avg_last_1, + team.steals_avg_last_5 AS team_steals_avg_last_5, + team.steals_avg_last_10 AS team_steals_avg_last_10, + team.blocks_avg_last_1 AS team_blocks_avg_last_1, + team.blocks_avg_last_5 AS team_blocks_avg_last_5, + team.blocks_avg_last_10 AS team_blocks_avg_last_10, + team.assists_avg_last_1 AS team_assists_avg_last_1, + team.assists_avg_last_5 AS team_assists_avg_last_5, + team.assists_avg_last_10 AS team_assists_avg_last_10, + team.turnovers_avg_last_1 AS team_turnovers_avg_last_1, + team.turnovers_avg_last_5 AS team_turnovers_avg_last_5, + team.turnovers_avg_last_10 AS team_turnovers_avg_last_10, + team.team_turnovers_avg_last_1 AS team_team_turnovers_avg_last_1, + team.team_turnovers_avg_last_5 AS team_team_turnovers_avg_last_5, + team.team_turnovers_avg_last_10 AS team_team_turnovers_avg_last_10, + team.points_off_turnovers_avg_last_1 AS team_points_off_turnovers_avg_last_1, + team.points_off_turnovers_avg_last_5 AS team_points_off_turnovers_avg_last_5, + team.points_off_turnovers_avg_last_10 AS team_points_off_turnovers_avg_last_10, + team.assists_turnover_ratio_avg_last_1 AS team_assists_turnover_ratio_avg_last_1, + team.assists_turnover_ratio_avg_last_5 AS team_assists_turnover_ratio_avg_last_5, + team.assists_turnover_ratio_avg_last_10 AS team_assists_turnover_ratio_avg_last_10, + team.ast_fgm_pct_avg_last_1 AS team_ast_fgm_pct_avg_last_1, + team.ast_fgm_pct_avg_last_5 AS team_ast_fgm_pct_avg_last_5, + team.ast_fgm_pct_avg_last_10 AS team_ast_fgm_pct_avg_last_10, + team.personal_fouls_avg_last_1 AS team_personal_fouls_avg_last_1, + team.personal_fouls_avg_last_5 AS team_personal_fouls_avg_last_5, + team.personal_fouls_avg_last_10 AS team_personal_fouls_avg_last_10, + team.flagrant_fouls_avg_last_1 AS team_flagrant_fouls_avg_last_1, + team.flagrant_fouls_avg_last_5 AS team_flagrant_fouls_avg_last_5, + team.flagrant_fouls_avg_last_10 AS team_flagrant_fouls_avg_last_10, + team.player_tech_fouls_avg_last_1 AS team_player_tech_fouls_avg_last_1, + team.player_tech_fouls_avg_last_5 AS team_player_tech_fouls_avg_last_5, + team.player_tech_fouls_avg_last_10 AS team_player_tech_fouls_avg_last_10, + team.team_tech_fouls_avg_last_1 AS team_team_tech_fouls_avg_last_1, + team.team_tech_fouls_avg_last_5 AS team_team_tech_fouls_avg_last_5, + team.team_tech_fouls_avg_last_10 AS team_team_tech_fouls_avg_last_10, + team.coach_tech_fouls_avg_last_1 AS team_coach_tech_fouls_avg_last_1, + team.coach_tech_fouls_avg_last_5 AS team_coach_tech_fouls_avg_last_5, + team.coach_tech_fouls_avg_last_10 AS team_coach_tech_fouls_avg_last_10, + team.foulouts_avg_last_1 AS team_foulouts_avg_last_1, + team.foulouts_avg_last_5 AS team_foulouts_avg_last_5, + team.foulouts_avg_last_10 AS team_foulouts_avg_last_10, + team.score_delta_avg_last_1 AS team_score_delta_avg_last_1, + team.score_delta_avg_last_5 AS team_score_delta_avg_last_5, + team.score_delta_avg_last_10 AS team_score_delta_avg_last_10, + team.possessions_avg_last_1 AS team_possessions_avg_last_1, + team.possessions_avg_last_5 AS team_possessions_avg_last_5, + team.possessions_avg_last_10 AS team_possessions_avg_last_10, + team.opp_points_avg_last_1 AS team_opp_points_avg_last_1, + team.opp_points_avg_last_5 AS team_opp_points_avg_last_5, + team.opp_points_avg_last_10 AS team_opp_points_avg_last_10, + team.opp_fast_break_pts_avg_last_1 AS team_opp_fast_break_pts_avg_last_1, + team.opp_fast_break_pts_avg_last_5 AS team_opp_fast_break_pts_avg_last_5, + team.opp_fast_break_pts_avg_last_10 AS team_opp_fast_break_pts_avg_last_10, + team.opp_second_chance_pts_avg_last_1 AS team_opp_second_chance_pts_avg_last_1, + team.opp_second_chance_pts_avg_last_5 AS team_opp_second_chance_pts_avg_last_5, + team.opp_second_chance_pts_avg_last_10 AS team_opp_second_chance_pts_avg_last_10, + team.opp_field_goals_made_avg_last_1 AS team_opp_field_goals_made_avg_last_1, + team.opp_field_goals_made_avg_last_5 AS team_opp_field_goals_made_avg_last_5, + team.opp_field_goals_made_avg_last_10 AS team_opp_field_goals_made_avg_last_10, + team.opp_field_goals_att_avg_last_1 AS team_opp_field_goals_att_avg_last_1, + team.opp_field_goals_att_avg_last_5 AS team_opp_field_goals_att_avg_last_5, + team.opp_field_goals_att_avg_last_10 AS team_opp_field_goals_att_avg_last_10, + team.opp_field_goals_pct_avg_last_1 AS team_opp_field_goals_pct_avg_last_1, + team.opp_field_goals_pct_avg_last_5 AS team_opp_field_goals_pct_avg_last_5, + team.opp_field_goals_pct_avg_last_10 AS team_opp_field_goals_pct_avg_last_10, + team.opp_three_points_made_avg_last_1 AS team_opp_three_points_made_avg_last_1, + team.opp_three_points_made_avg_last_5 AS team_opp_three_points_made_avg_last_5, + team.opp_three_points_made_avg_last_10 AS team_opp_three_points_made_avg_last_10, + team.opp_three_points_att_avg_last_1 AS team_opp_three_points_att_avg_last_1, + team.opp_three_points_att_avg_last_5 AS team_opp_three_points_att_avg_last_5, + team.opp_three_points_att_avg_last_10 AS team_opp_three_points_att_avg_last_10, + team.opp_three_points_pct_avg_last_1 AS team_opp_three_points_pct_avg_last_1, + team.opp_three_points_pct_avg_last_5 AS team_opp_three_points_pct_avg_last_5, + team.opp_three_points_pct_avg_last_10 AS team_opp_three_points_pct_avg_last_10, + team.opp_two_points_made_avg_last_1 AS team_opp_two_points_made_avg_last_1, + team.opp_two_points_made_avg_last_5 AS team_opp_two_points_made_avg_last_5, + team.opp_two_points_made_avg_last_10 AS team_opp_two_points_made_avg_last_10, + team.opp_two_points_att_avg_last_1 AS team_opp_two_points_att_avg_last_1, + team.opp_two_points_att_avg_last_5 AS team_opp_two_points_att_avg_last_5, + team.opp_two_points_att_avg_last_10 AS team_opp_two_points_att_avg_last_10, + team.opp_two_points_pct_avg_last_1 AS team_opp_two_points_pct_avg_last_1, + team.opp_two_points_pct_avg_last_5 AS team_opp_two_points_pct_avg_last_5, + team.opp_two_points_pct_avg_last_10 AS team_opp_two_points_pct_avg_last_10, + team.opp_free_throws_made_avg_last_1 AS team_opp_free_throws_made_avg_last_1, + team.opp_free_throws_made_avg_last_5 AS team_opp_free_throws_made_avg_last_5, + team.opp_free_throws_made_avg_last_10 AS team_opp_free_throws_made_avg_last_10, + team.opp_free_throws_att_avg_last_1 AS team_opp_free_throws_att_avg_last_1, + team.opp_free_throws_att_avg_last_5 AS team_opp_free_throws_att_avg_last_5, + team.opp_free_throws_att_avg_last_10 AS team_opp_free_throws_att_avg_last_10, + team.opp_free_throws_pct_avg_last_1 AS team_opp_free_throws_pct_avg_last_1, + team.opp_free_throws_pct_avg_last_5 AS team_opp_free_throws_pct_avg_last_5, + team.opp_free_throws_pct_avg_last_10 AS team_opp_free_throws_pct_avg_last_10, + team.opp_ts_pct_avg_last_1 AS team_opp_ts_pct_avg_last_1, + team.opp_ts_pct_avg_last_5 AS team_opp_ts_pct_avg_last_5, + team.opp_ts_pct_avg_last_10 AS team_opp_ts_pct_avg_last_10, + team.opp_efg_pct_avg_last_1 AS team_opp_efg_pct_avg_last_1, + team.opp_efg_pct_avg_last_5 AS team_opp_efg_pct_avg_last_5, + team.opp_efg_pct_avg_last_10 AS team_opp_efg_pct_avg_last_10, + team.opp_rebounds_avg_last_1 AS team_opp_rebounds_avg_last_1, + team.opp_rebounds_avg_last_5 AS team_opp_rebounds_avg_last_5, + team.opp_rebounds_avg_last_10 AS team_opp_rebounds_avg_last_10, + team.opp_offensive_rebounds_avg_last_1 AS team_opp_offensive_rebounds_avg_last_1, + team.opp_offensive_rebounds_avg_last_5 AS team_opp_offensive_rebounds_avg_last_5, + team.opp_offensive_rebounds_avg_last_10 AS team_opp_offensive_rebounds_avg_last_10, + team.opp_defensive_rebounds_avg_last_1 AS team_opp_defensive_rebounds_avg_last_1, + team.opp_defensive_rebounds_avg_last_5 AS team_opp_defensive_rebounds_avg_last_5, + team.opp_defensive_rebounds_avg_last_10 AS team_opp_defensive_rebounds_avg_last_10, + team.opp_dreb_pct_avg_last_1 AS team_opp_dreb_pct_avg_last_1, + team.opp_dreb_pct_avg_last_5 AS team_opp_dreb_pct_avg_last_5, + team.opp_dreb_pct_avg_last_10 AS team_opp_dreb_pct_avg_last_10, + team.opp_oreb_pct_avg_last_1 AS team_opp_oreb_pct_avg_last_1, + team.opp_oreb_pct_avg_last_5 AS team_opp_oreb_pct_avg_last_5, + team.opp_oreb_pct_avg_last_10 AS team_opp_oreb_pct_avg_last_10, + team.opp_steals_avg_last_1 AS team_opp_steals_avg_last_1, + team.opp_steals_avg_last_5 AS team_opp_steals_avg_last_5, + team.opp_steals_avg_last_10 AS team_opp_steals_avg_last_10, + team.opp_blocks_avg_last_1 AS team_opp_blocks_avg_last_1, + team.opp_blocks_avg_last_5 AS team_opp_blocks_avg_last_5, + team.opp_blocks_avg_last_10 AS team_opp_blocks_avg_last_10, + team.opp_assists_avg_last_1 AS team_opp_assists_avg_last_1, + team.opp_assists_avg_last_5 AS team_opp_assists_avg_last_5, + team.opp_assists_avg_last_10 AS team_opp_assists_avg_last_10, + team.opp_turnovers_avg_last_1 AS team_opp_turnovers_avg_last_1, + team.opp_turnovers_avg_last_5 AS team_opp_turnovers_avg_last_5, + team.opp_turnovers_avg_last_10 AS team_opp_turnovers_avg_last_10, + team.opp_team_turnovers_avg_last_1 AS team_opp_team_turnovers_avg_last_1, + team.opp_team_turnovers_avg_last_5 AS team_opp_team_turnovers_avg_last_5, + team.opp_team_turnovers_avg_last_10 AS team_opp_team_turnovers_avg_last_10, + team.opp_points_off_turnovers_avg_last_1 AS team_opp_points_off_turnovers_avg_last_1, + team.opp_points_off_turnovers_avg_last_5 AS team_opp_points_off_turnovers_avg_last_5, + team.opp_points_off_turnovers_avg_last_10 AS team_opp_points_off_turnovers_avg_last_10, + team.opp_assists_turnover_ratio_avg_last_1 AS team_opp_assists_turnover_ratio_avg_last_1, + team.opp_assists_turnover_ratio_avg_last_5 AS team_opp_assists_turnover_ratio_avg_last_5, + team.opp_assists_turnover_ratio_avg_last_10 AS team_opp_assists_turnover_ratio_avg_last_10, + team.opp_ast_fgm_pct_avg_last_1 AS team_opp_ast_fgm_pct_avg_last_1, + team.opp_ast_fgm_pct_avg_last_5 AS team_opp_ast_fgm_pct_avg_last_5, + team.opp_ast_fgm_pct_avg_last_10 AS team_opp_ast_fgm_pct_avg_last_10, + team.opp_personal_fouls_avg_last_1 AS team_opp_personal_fouls_avg_last_1, + team.opp_personal_fouls_avg_last_5 AS team_opp_personal_fouls_avg_last_5, + team.opp_personal_fouls_avg_last_10 AS team_opp_personal_fouls_avg_last_10, + team.opp_flagrant_fouls_avg_last_1 AS team_opp_flagrant_fouls_avg_last_1, + team.opp_flagrant_fouls_avg_last_5 AS team_opp_flagrant_fouls_avg_last_5, + team.opp_flagrant_fouls_avg_last_10 AS team_opp_flagrant_fouls_avg_last_10, + team.opp_player_tech_fouls_avg_last_1 AS team_opp_player_tech_fouls_avg_last_1, + team.opp_player_tech_fouls_avg_last_5 AS team_opp_player_tech_fouls_avg_last_5, + team.opp_player_tech_fouls_avg_last_10 AS team_opp_player_tech_fouls_avg_last_10, + team.opp_team_tech_fouls_avg_last_1 AS team_opp_team_tech_fouls_avg_last_1, + team.opp_team_tech_fouls_avg_last_5 AS team_opp_team_tech_fouls_avg_last_5, + team.opp_team_tech_fouls_avg_last_10 AS team_opp_team_tech_fouls_avg_last_10, + team.opp_coach_tech_fouls_avg_last_1 AS team_opp_coach_tech_fouls_avg_last_1, + team.opp_coach_tech_fouls_avg_last_5 AS team_opp_coach_tech_fouls_avg_last_5, + team.opp_coach_tech_fouls_avg_last_10 AS team_opp_coach_tech_fouls_avg_last_10, + team.opp_foulouts_avg_last_1 AS team_opp_foulouts_avg_last_1, + team.opp_foulouts_avg_last_5 AS team_opp_foulouts_avg_last_5, + team.opp_foulouts_avg_last_10 AS team_opp_foulouts_avg_last_10, + team.opp_score_delta_avg_last_1 AS team_opp_score_delta_avg_last_1, + team.opp_score_delta_avg_last_5 AS team_opp_score_delta_avg_last_5, + team.opp_score_delta_avg_last_10 AS team_opp_score_delta_avg_last_10, + team.opp_possessions_avg_last_1 AS team_opp_possessions_avg_last_1, + team.opp_possessions_avg_last_5 AS team_opp_possessions_avg_last_5, + team.opp_possessions_avg_last_10 AS team_opp_possessions_avg_last_10, + team.points_std_last_5 AS team_points_std_last_5, + team.points_std_last_10 AS team_points_std_last_10, + team.fast_break_pts_std_last_5 AS team_fast_break_pts_std_last_5, + team.fast_break_pts_std_last_10 AS team_fast_break_pts_std_last_10, + team.second_chance_pts_std_last_5 AS team_second_chance_pts_std_last_5, + team.second_chance_pts_std_last_10 AS team_second_chance_pts_std_last_10, + team.field_goals_made_std_last_5 AS team_field_goals_made_std_last_5, + team.field_goals_made_std_last_10 AS team_field_goals_made_std_last_10, + team.field_goals_att_std_last_5 AS team_field_goals_att_std_last_5, + team.field_goals_att_std_last_10 AS team_field_goals_att_std_last_10, + team.field_goals_pct_std_last_5 AS team_field_goals_pct_std_last_5, + team.field_goals_pct_std_last_10 AS team_field_goals_pct_std_last_10, + team.three_points_made_std_last_5 AS team_three_points_made_std_last_5, + team.three_points_made_std_last_10 AS team_three_points_made_std_last_10, + team.three_points_att_std_last_5 AS team_three_points_att_std_last_5, + team.three_points_att_std_last_10 AS team_three_points_att_std_last_10, + team.three_points_pct_std_last_5 AS team_three_points_pct_std_last_5, + team.three_points_pct_std_last_10 AS team_three_points_pct_std_last_10, + team.two_points_made_std_last_5 AS team_two_points_made_std_last_5, + team.two_points_made_std_last_10 AS team_two_points_made_std_last_10, + team.two_points_att_std_last_5 AS team_two_points_att_std_last_5, + team.two_points_att_std_last_10 AS team_two_points_att_std_last_10, + team.two_points_pct_std_last_5 AS team_two_points_pct_std_last_5, + team.two_points_pct_std_last_10 AS team_two_points_pct_std_last_10, + team.free_throws_made_std_last_5 AS team_free_throws_made_std_last_5, + team.free_throws_made_std_last_10 AS team_free_throws_made_std_last_10, + team.free_throws_att_std_last_5 AS team_free_throws_att_std_last_5, + team.free_throws_att_std_last_10 AS team_free_throws_att_std_last_10, + team.free_throws_pct_std_last_5 AS team_free_throws_pct_std_last_5, + team.free_throws_pct_std_last_10 AS team_free_throws_pct_std_last_10, + team.ts_pct_std_last_5 AS team_ts_pct_std_last_5, + team.ts_pct_std_last_10 AS team_ts_pct_std_last_10, + team.efg_pct_std_last_5 AS team_efg_pct_std_last_5, + team.efg_pct_std_last_10 AS team_efg_pct_std_last_10, + team.rebounds_std_last_5 AS team_rebounds_std_last_5, + team.rebounds_std_last_10 AS team_rebounds_std_last_10, + team.offensive_rebounds_std_last_5 AS team_offensive_rebounds_std_last_5, + team.offensive_rebounds_std_last_10 AS team_offensive_rebounds_std_last_10, + team.defensive_rebounds_std_last_5 AS team_defensive_rebounds_std_last_5, + team.defensive_rebounds_std_last_10 AS team_defensive_rebounds_std_last_10, + team.dreb_pct_std_last_5 AS team_dreb_pct_std_last_5, + team.dreb_pct_std_last_10 AS team_dreb_pct_std_last_10, + team.oreb_pct_std_last_5 AS team_oreb_pct_std_last_5, + team.oreb_pct_std_last_10 AS team_oreb_pct_std_last_10, + team.steals_std_last_5 AS team_steals_std_last_5, + team.steals_std_last_10 AS team_steals_std_last_10, + team.blocks_std_last_5 AS team_blocks_std_last_5, + team.blocks_std_last_10 AS team_blocks_std_last_10, + team.assists_std_last_5 AS team_assists_std_last_5, + team.assists_std_last_10 AS team_assists_std_last_10, + team.turnovers_std_last_5 AS team_turnovers_std_last_5, + team.turnovers_std_last_10 AS team_turnovers_std_last_10, + team.team_turnovers_std_last_5 AS team_team_turnovers_std_last_5, + team.team_turnovers_std_last_10 AS team_team_turnovers_std_last_10, + team.points_off_turnovers_std_last_5 AS team_points_off_turnovers_std_last_5, + team.points_off_turnovers_std_last_10 AS team_points_off_turnovers_std_last_10, + team.assists_turnover_ratio_std_last_5 AS team_assists_turnover_ratio_std_last_5, + team.assists_turnover_ratio_std_last_10 AS team_assists_turnover_ratio_std_last_10, + team.ast_fgm_pct_std_last_5 AS team_ast_fgm_pct_std_last_5, + team.ast_fgm_pct_std_last_10 AS team_ast_fgm_pct_std_last_10, + team.personal_fouls_std_last_5 AS team_personal_fouls_std_last_5, + team.personal_fouls_std_last_10 AS team_personal_fouls_std_last_10, + team.flagrant_fouls_std_last_5 AS team_flagrant_fouls_std_last_5, + team.flagrant_fouls_std_last_10 AS team_flagrant_fouls_std_last_10, + team.player_tech_fouls_std_last_5 AS team_player_tech_fouls_std_last_5, + team.player_tech_fouls_std_last_10 AS team_player_tech_fouls_std_last_10, + team.team_tech_fouls_std_last_5 AS team_team_tech_fouls_std_last_5, + team.team_tech_fouls_std_last_10 AS team_team_tech_fouls_std_last_10, + team.coach_tech_fouls_std_last_5 AS team_coach_tech_fouls_std_last_5, + team.coach_tech_fouls_std_last_10 AS team_coach_tech_fouls_std_last_10, + team.foulouts_std_last_5 AS team_foulouts_std_last_5, + team.foulouts_std_last_10 AS team_foulouts_std_last_10, + team.score_delta_std_last_5 AS team_score_delta_std_last_5, + team.score_delta_std_last_10 AS team_score_delta_std_last_10, + team.possessions_std_last_5 AS team_possessions_std_last_5, + team.possessions_std_last_10 AS team_possessions_std_last_10, + team.opp_points_std_last_5 AS team_opp_points_std_last_5, + team.opp_points_std_last_10 AS team_opp_points_std_last_10, + team.opp_fast_break_pts_std_last_5 AS team_opp_fast_break_pts_std_last_5, + team.opp_fast_break_pts_std_last_10 AS team_opp_fast_break_pts_std_last_10, + team.opp_second_chance_pts_std_last_5 AS team_opp_second_chance_pts_std_last_5, + team.opp_second_chance_pts_std_last_10 AS team_opp_second_chance_pts_std_last_10, + team.opp_field_goals_made_std_last_5 AS team_opp_field_goals_made_std_last_5, + team.opp_field_goals_made_std_last_10 AS team_opp_field_goals_made_std_last_10, + team.opp_field_goals_att_std_last_5 AS team_opp_field_goals_att_std_last_5, + team.opp_field_goals_att_std_last_10 AS team_opp_field_goals_att_std_last_10, + team.opp_field_goals_pct_std_last_5 AS team_opp_field_goals_pct_std_last_5, + team.opp_field_goals_pct_std_last_10 AS team_opp_field_goals_pct_std_last_10, + team.opp_three_points_made_std_last_5 AS team_opp_three_points_made_std_last_5, + team.opp_three_points_made_std_last_10 AS team_opp_three_points_made_std_last_10, + team.opp_three_points_att_std_last_5 AS team_opp_three_points_att_std_last_5, + team.opp_three_points_att_std_last_10 AS team_opp_three_points_att_std_last_10, + team.opp_three_points_pct_std_last_5 AS team_opp_three_points_pct_std_last_5, + team.opp_three_points_pct_std_last_10 AS team_opp_three_points_pct_std_last_10, + team.opp_two_points_made_std_last_5 AS team_opp_two_points_made_std_last_5, + team.opp_two_points_made_std_last_10 AS team_opp_two_points_made_std_last_10, + team.opp_two_points_att_std_last_5 AS team_opp_two_points_att_std_last_5, + team.opp_two_points_att_std_last_10 AS team_opp_two_points_att_std_last_10, + team.opp_two_points_pct_std_last_5 AS team_opp_two_points_pct_std_last_5, + team.opp_two_points_pct_std_last_10 AS team_opp_two_points_pct_std_last_10, + team.opp_free_throws_made_std_last_5 AS team_opp_free_throws_made_std_last_5, + team.opp_free_throws_made_std_last_10 AS team_opp_free_throws_made_std_last_10, + team.opp_free_throws_att_std_last_5 AS team_opp_free_throws_att_std_last_5, + team.opp_free_throws_att_std_last_10 AS team_opp_free_throws_att_std_last_10, + team.opp_free_throws_pct_std_last_5 AS team_opp_free_throws_pct_std_last_5, + team.opp_free_throws_pct_std_last_10 AS team_opp_free_throws_pct_std_last_10, + team.opp_ts_pct_std_last_5 AS team_opp_ts_pct_std_last_5, + team.opp_ts_pct_std_last_10 AS team_opp_ts_pct_std_last_10, + team.opp_efg_pct_std_last_5 AS team_opp_efg_pct_std_last_5, + team.opp_efg_pct_std_last_10 AS team_opp_efg_pct_std_last_10, + team.opp_rebounds_std_last_5 AS team_opp_rebounds_std_last_5, + team.opp_rebounds_std_last_10 AS team_opp_rebounds_std_last_10, + team.opp_offensive_rebounds_std_last_5 AS team_opp_offensive_rebounds_std_last_5, + team.opp_offensive_rebounds_std_last_10 AS team_opp_offensive_rebounds_std_last_10, + team.opp_defensive_rebounds_std_last_5 AS team_opp_defensive_rebounds_std_last_5, + team.opp_defensive_rebounds_std_last_10 AS team_opp_defensive_rebounds_std_last_10, + team.opp_dreb_pct_std_last_5 AS team_opp_dreb_pct_std_last_5, + team.opp_dreb_pct_std_last_10 AS team_opp_dreb_pct_std_last_10, + team.opp_oreb_pct_std_last_5 AS team_opp_oreb_pct_std_last_5, + team.opp_oreb_pct_std_last_10 AS team_opp_oreb_pct_std_last_10, + team.opp_steals_std_last_5 AS team_opp_steals_std_last_5, + team.opp_steals_std_last_10 AS team_opp_steals_std_last_10, + team.opp_blocks_std_last_5 AS team_opp_blocks_std_last_5, + team.opp_blocks_std_last_10 AS team_opp_blocks_std_last_10, + team.opp_assists_std_last_5 AS team_opp_assists_std_last_5, + team.opp_assists_std_last_10 AS team_opp_assists_std_last_10, + team.opp_turnovers_std_last_5 AS team_opp_turnovers_std_last_5, + team.opp_turnovers_std_last_10 AS team_opp_turnovers_std_last_10, + team.opp_team_turnovers_std_last_5 AS team_opp_team_turnovers_std_last_5, + team.opp_team_turnovers_std_last_10 AS team_opp_team_turnovers_std_last_10, + team.opp_points_off_turnovers_std_last_5 AS team_opp_points_off_turnovers_std_last_5, + team.opp_points_off_turnovers_std_last_10 AS team_opp_points_off_turnovers_std_last_10, + team.opp_assists_turnover_ratio_std_last_5 AS team_opp_assists_turnover_ratio_std_last_5, + team.opp_assists_turnover_ratio_std_last_10 AS team_opp_assists_turnover_ratio_std_last_10, + team.opp_ast_fgm_pct_std_last_5 AS team_opp_ast_fgm_pct_std_last_5, + team.opp_ast_fgm_pct_std_last_10 AS team_opp_ast_fgm_pct_std_last_10, + team.opp_personal_fouls_std_last_5 AS team_opp_personal_fouls_std_last_5, + team.opp_personal_fouls_std_last_10 AS team_opp_personal_fouls_std_last_10, + team.opp_flagrant_fouls_std_last_5 AS team_opp_flagrant_fouls_std_last_5, + team.opp_flagrant_fouls_std_last_10 AS team_opp_flagrant_fouls_std_last_10, + team.opp_player_tech_fouls_std_last_5 AS team_opp_player_tech_fouls_std_last_5, + team.opp_player_tech_fouls_std_last_10 AS team_opp_player_tech_fouls_std_last_10, + team.opp_team_tech_fouls_std_last_5 AS team_opp_team_tech_fouls_std_last_5, + team.opp_team_tech_fouls_std_last_10 AS team_opp_team_tech_fouls_std_last_10, + team.opp_coach_tech_fouls_std_last_5 AS team_opp_coach_tech_fouls_std_last_5, + team.opp_coach_tech_fouls_std_last_10 AS team_opp_coach_tech_fouls_std_last_10, + team.opp_foulouts_std_last_5 AS team_opp_foulouts_std_last_5, + team.opp_foulouts_std_last_10 AS team_opp_foulouts_std_last_10, + team.opp_score_delta_std_last_5 AS team_opp_score_delta_std_last_5, + team.opp_score_delta_std_last_10 AS team_opp_score_delta_std_last_10, + team.opp_possessions_std_last_5 AS team_opp_possessions_std_last_5, + team.opp_possessions_std_last_10 AS team_opp_possessions_std_last_10, + opponent.points_avg_last_1 AS opponent_points_avg_last_1, + opponent.points_avg_last_5 AS opponent_points_avg_last_5, + opponent.points_avg_last_10 AS opponent_points_avg_last_10, + opponent.fast_break_pts_avg_last_1 AS opponent_fast_break_pts_avg_last_1, + opponent.fast_break_pts_avg_last_5 AS opponent_fast_break_pts_avg_last_5, + opponent.fast_break_pts_avg_last_10 AS opponent_fast_break_pts_avg_last_10, + opponent.second_chance_pts_avg_last_1 AS opponent_second_chance_pts_avg_last_1, + opponent.second_chance_pts_avg_last_5 AS opponent_second_chance_pts_avg_last_5, + opponent.second_chance_pts_avg_last_10 AS opponent_second_chance_pts_avg_last_10, + opponent.field_goals_made_avg_last_1 AS opponent_field_goals_made_avg_last_1, + opponent.field_goals_made_avg_last_5 AS opponent_field_goals_made_avg_last_5, + opponent.field_goals_made_avg_last_10 AS opponent_field_goals_made_avg_last_10, + opponent.field_goals_att_avg_last_1 AS opponent_field_goals_att_avg_last_1, + opponent.field_goals_att_avg_last_5 AS opponent_field_goals_att_avg_last_5, + opponent.field_goals_att_avg_last_10 AS opponent_field_goals_att_avg_last_10, + opponent.field_goals_pct_avg_last_1 AS opponent_field_goals_pct_avg_last_1, + opponent.field_goals_pct_avg_last_5 AS opponent_field_goals_pct_avg_last_5, + opponent.field_goals_pct_avg_last_10 AS opponent_field_goals_pct_avg_last_10, + opponent.three_points_made_avg_last_1 AS opponent_three_points_made_avg_last_1, + opponent.three_points_made_avg_last_5 AS opponent_three_points_made_avg_last_5, + opponent.three_points_made_avg_last_10 AS opponent_three_points_made_avg_last_10, + opponent.three_points_att_avg_last_1 AS opponent_three_points_att_avg_last_1, + opponent.three_points_att_avg_last_5 AS opponent_three_points_att_avg_last_5, + opponent.three_points_att_avg_last_10 AS opponent_three_points_att_avg_last_10, + opponent.three_points_pct_avg_last_1 AS opponent_three_points_pct_avg_last_1, + opponent.three_points_pct_avg_last_5 AS opponent_three_points_pct_avg_last_5, + opponent.three_points_pct_avg_last_10 AS opponent_three_points_pct_avg_last_10, + opponent.two_points_made_avg_last_1 AS opponent_two_points_made_avg_last_1, + opponent.two_points_made_avg_last_5 AS opponent_two_points_made_avg_last_5, + opponent.two_points_made_avg_last_10 AS opponent_two_points_made_avg_last_10, + opponent.two_points_att_avg_last_1 AS opponent_two_points_att_avg_last_1, + opponent.two_points_att_avg_last_5 AS opponent_two_points_att_avg_last_5, + opponent.two_points_att_avg_last_10 AS opponent_two_points_att_avg_last_10, + opponent.two_points_pct_avg_last_1 AS opponent_two_points_pct_avg_last_1, + opponent.two_points_pct_avg_last_5 AS opponent_two_points_pct_avg_last_5, + opponent.two_points_pct_avg_last_10 AS opponent_two_points_pct_avg_last_10, + opponent.free_throws_made_avg_last_1 AS opponent_free_throws_made_avg_last_1, + opponent.free_throws_made_avg_last_5 AS opponent_free_throws_made_avg_last_5, + opponent.free_throws_made_avg_last_10 AS opponent_free_throws_made_avg_last_10, + opponent.free_throws_att_avg_last_1 AS opponent_free_throws_att_avg_last_1, + opponent.free_throws_att_avg_last_5 AS opponent_free_throws_att_avg_last_5, + opponent.free_throws_att_avg_last_10 AS opponent_free_throws_att_avg_last_10, + opponent.free_throws_pct_avg_last_1 AS opponent_free_throws_pct_avg_last_1, + opponent.free_throws_pct_avg_last_5 AS opponent_free_throws_pct_avg_last_5, + opponent.free_throws_pct_avg_last_10 AS opponent_free_throws_pct_avg_last_10, + opponent.ts_pct_avg_last_1 AS opponent_ts_pct_avg_last_1, + opponent.ts_pct_avg_last_5 AS opponent_ts_pct_avg_last_5, + opponent.ts_pct_avg_last_10 AS opponent_ts_pct_avg_last_10, + opponent.efg_pct_avg_last_1 AS opponent_efg_pct_avg_last_1, + opponent.efg_pct_avg_last_5 AS opponent_efg_pct_avg_last_5, + opponent.efg_pct_avg_last_10 AS opponent_efg_pct_avg_last_10, + opponent.rebounds_avg_last_1 AS opponent_rebounds_avg_last_1, + opponent.rebounds_avg_last_5 AS opponent_rebounds_avg_last_5, + opponent.rebounds_avg_last_10 AS opponent_rebounds_avg_last_10, + opponent.offensive_rebounds_avg_last_1 AS opponent_offensive_rebounds_avg_last_1, + opponent.offensive_rebounds_avg_last_5 AS opponent_offensive_rebounds_avg_last_5, + opponent.offensive_rebounds_avg_last_10 AS opponent_offensive_rebounds_avg_last_10, + opponent.defensive_rebounds_avg_last_1 AS opponent_defensive_rebounds_avg_last_1, + opponent.defensive_rebounds_avg_last_5 AS opponent_defensive_rebounds_avg_last_5, + opponent.defensive_rebounds_avg_last_10 AS opponent_defensive_rebounds_avg_last_10, + opponent.dreb_pct_avg_last_1 AS opponent_dreb_pct_avg_last_1, + opponent.dreb_pct_avg_last_5 AS opponent_dreb_pct_avg_last_5, + opponent.dreb_pct_avg_last_10 AS opponent_dreb_pct_avg_last_10, + opponent.oreb_pct_avg_last_1 AS opponent_oreb_pct_avg_last_1, + opponent.oreb_pct_avg_last_5 AS opponent_oreb_pct_avg_last_5, + opponent.oreb_pct_avg_last_10 AS opponent_oreb_pct_avg_last_10, + opponent.steals_avg_last_1 AS opponent_steals_avg_last_1, + opponent.steals_avg_last_5 AS opponent_steals_avg_last_5, + opponent.steals_avg_last_10 AS opponent_steals_avg_last_10, + opponent.blocks_avg_last_1 AS opponent_blocks_avg_last_1, + opponent.blocks_avg_last_5 AS opponent_blocks_avg_last_5, + opponent.blocks_avg_last_10 AS opponent_blocks_avg_last_10, + opponent.assists_avg_last_1 AS opponent_assists_avg_last_1, + opponent.assists_avg_last_5 AS opponent_assists_avg_last_5, + opponent.assists_avg_last_10 AS opponent_assists_avg_last_10, + opponent.turnovers_avg_last_1 AS opponent_turnovers_avg_last_1, + opponent.turnovers_avg_last_5 AS opponent_turnovers_avg_last_5, + opponent.turnovers_avg_last_10 AS opponent_turnovers_avg_last_10, + opponent.team_turnovers_avg_last_1 AS opponent_team_turnovers_avg_last_1, + opponent.team_turnovers_avg_last_5 AS opponent_team_turnovers_avg_last_5, + opponent.team_turnovers_avg_last_10 AS opponent_team_turnovers_avg_last_10, + opponent.points_off_turnovers_avg_last_1 AS opponent_points_off_turnovers_avg_last_1, + opponent.points_off_turnovers_avg_last_5 AS opponent_points_off_turnovers_avg_last_5, + opponent.points_off_turnovers_avg_last_10 AS opponent_points_off_turnovers_avg_last_10, + opponent.assists_turnover_ratio_avg_last_1 AS opponent_assists_turnover_ratio_avg_last_1, + opponent.assists_turnover_ratio_avg_last_5 AS opponent_assists_turnover_ratio_avg_last_5, + opponent.assists_turnover_ratio_avg_last_10 AS opponent_assists_turnover_ratio_avg_last_10, + opponent.ast_fgm_pct_avg_last_1 AS opponent_ast_fgm_pct_avg_last_1, + opponent.ast_fgm_pct_avg_last_5 AS opponent_ast_fgm_pct_avg_last_5, + opponent.ast_fgm_pct_avg_last_10 AS opponent_ast_fgm_pct_avg_last_10, + opponent.personal_fouls_avg_last_1 AS opponent_personal_fouls_avg_last_1, + opponent.personal_fouls_avg_last_5 AS opponent_personal_fouls_avg_last_5, + opponent.personal_fouls_avg_last_10 AS opponent_personal_fouls_avg_last_10, + opponent.flagrant_fouls_avg_last_1 AS opponent_flagrant_fouls_avg_last_1, + opponent.flagrant_fouls_avg_last_5 AS opponent_flagrant_fouls_avg_last_5, + opponent.flagrant_fouls_avg_last_10 AS opponent_flagrant_fouls_avg_last_10, + opponent.player_tech_fouls_avg_last_1 AS opponent_player_tech_fouls_avg_last_1, + opponent.player_tech_fouls_avg_last_5 AS opponent_player_tech_fouls_avg_last_5, + opponent.player_tech_fouls_avg_last_10 AS opponent_player_tech_fouls_avg_last_10, + opponent.team_tech_fouls_avg_last_1 AS opponent_team_tech_fouls_avg_last_1, + opponent.team_tech_fouls_avg_last_5 AS opponent_team_tech_fouls_avg_last_5, + opponent.team_tech_fouls_avg_last_10 AS opponent_team_tech_fouls_avg_last_10, + opponent.coach_tech_fouls_avg_last_1 AS opponent_coach_tech_fouls_avg_last_1, + opponent.coach_tech_fouls_avg_last_5 AS opponent_coach_tech_fouls_avg_last_5, + opponent.coach_tech_fouls_avg_last_10 AS opponent_coach_tech_fouls_avg_last_10, + opponent.foulouts_avg_last_1 AS opponent_foulouts_avg_last_1, + opponent.foulouts_avg_last_5 AS opponent_foulouts_avg_last_5, + opponent.foulouts_avg_last_10 AS opponent_foulouts_avg_last_10, + opponent.score_delta_avg_last_1 AS opponent_score_delta_avg_last_1, + opponent.score_delta_avg_last_5 AS opponent_score_delta_avg_last_5, + opponent.score_delta_avg_last_10 AS opponent_score_delta_avg_last_10, + opponent.possessions_avg_last_1 AS opponent_possessions_avg_last_1, + opponent.possessions_avg_last_5 AS opponent_possessions_avg_last_5, + opponent.possessions_avg_last_10 AS opponent_possessions_avg_last_10, + opponent.opp_points_avg_last_1 AS opponent_opp_points_avg_last_1, + opponent.opp_points_avg_last_5 AS opponent_opp_points_avg_last_5, + opponent.opp_points_avg_last_10 AS opponent_opp_points_avg_last_10, + opponent.opp_fast_break_pts_avg_last_1 AS opponent_opp_fast_break_pts_avg_last_1, + opponent.opp_fast_break_pts_avg_last_5 AS opponent_opp_fast_break_pts_avg_last_5, + opponent.opp_fast_break_pts_avg_last_10 AS opponent_opp_fast_break_pts_avg_last_10, + opponent.opp_second_chance_pts_avg_last_1 AS opponent_opp_second_chance_pts_avg_last_1, + opponent.opp_second_chance_pts_avg_last_5 AS opponent_opp_second_chance_pts_avg_last_5, + opponent.opp_second_chance_pts_avg_last_10 AS opponent_opp_second_chance_pts_avg_last_10, + opponent.opp_field_goals_made_avg_last_1 AS opponent_opp_field_goals_made_avg_last_1, + opponent.opp_field_goals_made_avg_last_5 AS opponent_opp_field_goals_made_avg_last_5, + opponent.opp_field_goals_made_avg_last_10 AS opponent_opp_field_goals_made_avg_last_10, + opponent.opp_field_goals_att_avg_last_1 AS opponent_opp_field_goals_att_avg_last_1, + opponent.opp_field_goals_att_avg_last_5 AS opponent_opp_field_goals_att_avg_last_5, + opponent.opp_field_goals_att_avg_last_10 AS opponent_opp_field_goals_att_avg_last_10, + opponent.opp_field_goals_pct_avg_last_1 AS opponent_opp_field_goals_pct_avg_last_1, + opponent.opp_field_goals_pct_avg_last_5 AS opponent_opp_field_goals_pct_avg_last_5, + opponent.opp_field_goals_pct_avg_last_10 AS opponent_opp_field_goals_pct_avg_last_10, + opponent.opp_three_points_made_avg_last_1 AS opponent_opp_three_points_made_avg_last_1, + opponent.opp_three_points_made_avg_last_5 AS opponent_opp_three_points_made_avg_last_5, + opponent.opp_three_points_made_avg_last_10 AS opponent_opp_three_points_made_avg_last_10, + opponent.opp_three_points_att_avg_last_1 AS opponent_opp_three_points_att_avg_last_1, + opponent.opp_three_points_att_avg_last_5 AS opponent_opp_three_points_att_avg_last_5, + opponent.opp_three_points_att_avg_last_10 AS opponent_opp_three_points_att_avg_last_10, + opponent.opp_three_points_pct_avg_last_1 AS opponent_opp_three_points_pct_avg_last_1, + opponent.opp_three_points_pct_avg_last_5 AS opponent_opp_three_points_pct_avg_last_5, + opponent.opp_three_points_pct_avg_last_10 AS opponent_opp_three_points_pct_avg_last_10, + opponent.opp_two_points_made_avg_last_1 AS opponent_opp_two_points_made_avg_last_1, + opponent.opp_two_points_made_avg_last_5 AS opponent_opp_two_points_made_avg_last_5, + opponent.opp_two_points_made_avg_last_10 AS opponent_opp_two_points_made_avg_last_10, + opponent.opp_two_points_att_avg_last_1 AS opponent_opp_two_points_att_avg_last_1, + opponent.opp_two_points_att_avg_last_5 AS opponent_opp_two_points_att_avg_last_5, + opponent.opp_two_points_att_avg_last_10 AS opponent_opp_two_points_att_avg_last_10, + opponent.opp_two_points_pct_avg_last_1 AS opponent_opp_two_points_pct_avg_last_1, + opponent.opp_two_points_pct_avg_last_5 AS opponent_opp_two_points_pct_avg_last_5, + opponent.opp_two_points_pct_avg_last_10 AS opponent_opp_two_points_pct_avg_last_10, + opponent.opp_free_throws_made_avg_last_1 AS opponent_opp_free_throws_made_avg_last_1, + opponent.opp_free_throws_made_avg_last_5 AS opponent_opp_free_throws_made_avg_last_5, + opponent.opp_free_throws_made_avg_last_10 AS opponent_opp_free_throws_made_avg_last_10, + opponent.opp_free_throws_att_avg_last_1 AS opponent_opp_free_throws_att_avg_last_1, + opponent.opp_free_throws_att_avg_last_5 AS opponent_opp_free_throws_att_avg_last_5, + opponent.opp_free_throws_att_avg_last_10 AS opponent_opp_free_throws_att_avg_last_10, + opponent.opp_free_throws_pct_avg_last_1 AS opponent_opp_free_throws_pct_avg_last_1, + opponent.opp_free_throws_pct_avg_last_5 AS opponent_opp_free_throws_pct_avg_last_5, + opponent.opp_free_throws_pct_avg_last_10 AS opponent_opp_free_throws_pct_avg_last_10, + opponent.opp_ts_pct_avg_last_1 AS opponent_opp_ts_pct_avg_last_1, + opponent.opp_ts_pct_avg_last_5 AS opponent_opp_ts_pct_avg_last_5, + opponent.opp_ts_pct_avg_last_10 AS opponent_opp_ts_pct_avg_last_10, + opponent.opp_efg_pct_avg_last_1 AS opponent_opp_efg_pct_avg_last_1, + opponent.opp_efg_pct_avg_last_5 AS opponent_opp_efg_pct_avg_last_5, + opponent.opp_efg_pct_avg_last_10 AS opponent_opp_efg_pct_avg_last_10, + opponent.opp_rebounds_avg_last_1 AS opponent_opp_rebounds_avg_last_1, + opponent.opp_rebounds_avg_last_5 AS opponent_opp_rebounds_avg_last_5, + opponent.opp_rebounds_avg_last_10 AS opponent_opp_rebounds_avg_last_10, + opponent.opp_offensive_rebounds_avg_last_1 AS opponent_opp_offensive_rebounds_avg_last_1, + opponent.opp_offensive_rebounds_avg_last_5 AS opponent_opp_offensive_rebounds_avg_last_5, + opponent.opp_offensive_rebounds_avg_last_10 AS opponent_opp_offensive_rebounds_avg_last_10, + opponent.opp_defensive_rebounds_avg_last_1 AS opponent_opp_defensive_rebounds_avg_last_1, + opponent.opp_defensive_rebounds_avg_last_5 AS opponent_opp_defensive_rebounds_avg_last_5, + opponent.opp_defensive_rebounds_avg_last_10 AS opponent_opp_defensive_rebounds_avg_last_10, + opponent.opp_dreb_pct_avg_last_1 AS opponent_opp_dreb_pct_avg_last_1, + opponent.opp_dreb_pct_avg_last_5 AS opponent_opp_dreb_pct_avg_last_5, + opponent.opp_dreb_pct_avg_last_10 AS opponent_opp_dreb_pct_avg_last_10, + opponent.opp_oreb_pct_avg_last_1 AS opponent_opp_oreb_pct_avg_last_1, + opponent.opp_oreb_pct_avg_last_5 AS opponent_opp_oreb_pct_avg_last_5, + opponent.opp_oreb_pct_avg_last_10 AS opponent_opp_oreb_pct_avg_last_10, + opponent.opp_steals_avg_last_1 AS opponent_opp_steals_avg_last_1, + opponent.opp_steals_avg_last_5 AS opponent_opp_steals_avg_last_5, + opponent.opp_steals_avg_last_10 AS opponent_opp_steals_avg_last_10, + opponent.opp_blocks_avg_last_1 AS opponent_opp_blocks_avg_last_1, + opponent.opp_blocks_avg_last_5 AS opponent_opp_blocks_avg_last_5, + opponent.opp_blocks_avg_last_10 AS opponent_opp_blocks_avg_last_10, + opponent.opp_assists_avg_last_1 AS opponent_opp_assists_avg_last_1, + opponent.opp_assists_avg_last_5 AS opponent_opp_assists_avg_last_5, + opponent.opp_assists_avg_last_10 AS opponent_opp_assists_avg_last_10, + opponent.opp_turnovers_avg_last_1 AS opponent_opp_turnovers_avg_last_1, + opponent.opp_turnovers_avg_last_5 AS opponent_opp_turnovers_avg_last_5, + opponent.opp_turnovers_avg_last_10 AS opponent_opp_turnovers_avg_last_10, + opponent.opp_team_turnovers_avg_last_1 AS opponent_opp_team_turnovers_avg_last_1, + opponent.opp_team_turnovers_avg_last_5 AS opponent_opp_team_turnovers_avg_last_5, + opponent.opp_team_turnovers_avg_last_10 AS opponent_opp_team_turnovers_avg_last_10, + opponent.opp_points_off_turnovers_avg_last_1 AS opponent_opp_points_off_turnovers_avg_last_1, + opponent.opp_points_off_turnovers_avg_last_5 AS opponent_opp_points_off_turnovers_avg_last_5, + opponent.opp_points_off_turnovers_avg_last_10 AS opponent_opp_points_off_turnovers_avg_last_10, + opponent.opp_assists_turnover_ratio_avg_last_1 AS opponent_opp_assists_turnover_ratio_avg_last_1, + opponent.opp_assists_turnover_ratio_avg_last_5 AS opponent_opp_assists_turnover_ratio_avg_last_5, + opponent.opp_assists_turnover_ratio_avg_last_10 AS opponent_opp_assists_turnover_ratio_avg_last_10, + opponent.opp_ast_fgm_pct_avg_last_1 AS opponent_opp_ast_fgm_pct_avg_last_1, + opponent.opp_ast_fgm_pct_avg_last_5 AS opponent_opp_ast_fgm_pct_avg_last_5, + opponent.opp_ast_fgm_pct_avg_last_10 AS opponent_opp_ast_fgm_pct_avg_last_10, + opponent.opp_personal_fouls_avg_last_1 AS opponent_opp_personal_fouls_avg_last_1, + opponent.opp_personal_fouls_avg_last_5 AS opponent_opp_personal_fouls_avg_last_5, + opponent.opp_personal_fouls_avg_last_10 AS opponent_opp_personal_fouls_avg_last_10, + opponent.opp_flagrant_fouls_avg_last_1 AS opponent_opp_flagrant_fouls_avg_last_1, + opponent.opp_flagrant_fouls_avg_last_5 AS opponent_opp_flagrant_fouls_avg_last_5, + opponent.opp_flagrant_fouls_avg_last_10 AS opponent_opp_flagrant_fouls_avg_last_10, + opponent.opp_player_tech_fouls_avg_last_1 AS opponent_opp_player_tech_fouls_avg_last_1, + opponent.opp_player_tech_fouls_avg_last_5 AS opponent_opp_player_tech_fouls_avg_last_5, + opponent.opp_player_tech_fouls_avg_last_10 AS opponent_opp_player_tech_fouls_avg_last_10, + opponent.opp_team_tech_fouls_avg_last_1 AS opponent_opp_team_tech_fouls_avg_last_1, + opponent.opp_team_tech_fouls_avg_last_5 AS opponent_opp_team_tech_fouls_avg_last_5, + opponent.opp_team_tech_fouls_avg_last_10 AS opponent_opp_team_tech_fouls_avg_last_10, + opponent.opp_coach_tech_fouls_avg_last_1 AS opponent_opp_coach_tech_fouls_avg_last_1, + opponent.opp_coach_tech_fouls_avg_last_5 AS opponent_opp_coach_tech_fouls_avg_last_5, + opponent.opp_coach_tech_fouls_avg_last_10 AS opponent_opp_coach_tech_fouls_avg_last_10, + opponent.opp_foulouts_avg_last_1 AS opponent_opp_foulouts_avg_last_1, + opponent.opp_foulouts_avg_last_5 AS opponent_opp_foulouts_avg_last_5, + opponent.opp_foulouts_avg_last_10 AS opponent_opp_foulouts_avg_last_10, + opponent.opp_score_delta_avg_last_1 AS opponent_opp_score_delta_avg_last_1, + opponent.opp_score_delta_avg_last_5 AS opponent_opp_score_delta_avg_last_5, + opponent.opp_score_delta_avg_last_10 AS opponent_opp_score_delta_avg_last_10, + opponent.opp_possessions_avg_last_1 AS opponent_opp_possessions_avg_last_1, + opponent.opp_possessions_avg_last_5 AS opponent_opp_possessions_avg_last_5, + opponent.opp_possessions_avg_last_10 AS opponent_opp_possessions_avg_last_10, + opponent.points_std_last_5 AS opponent_points_std_last_5, + opponent.points_std_last_10 AS opponent_points_std_last_10, + opponent.fast_break_pts_std_last_5 AS opponent_fast_break_pts_std_last_5, + opponent.fast_break_pts_std_last_10 AS opponent_fast_break_pts_std_last_10, + opponent.second_chance_pts_std_last_5 AS opponent_second_chance_pts_std_last_5, + opponent.second_chance_pts_std_last_10 AS opponent_second_chance_pts_std_last_10, + opponent.field_goals_made_std_last_5 AS opponent_field_goals_made_std_last_5, + opponent.field_goals_made_std_last_10 AS opponent_field_goals_made_std_last_10, + opponent.field_goals_att_std_last_5 AS opponent_field_goals_att_std_last_5, + opponent.field_goals_att_std_last_10 AS opponent_field_goals_att_std_last_10, + opponent.field_goals_pct_std_last_5 AS opponent_field_goals_pct_std_last_5, + opponent.field_goals_pct_std_last_10 AS opponent_field_goals_pct_std_last_10, + opponent.three_points_made_std_last_5 AS opponent_three_points_made_std_last_5, + opponent.three_points_made_std_last_10 AS opponent_three_points_made_std_last_10, + opponent.three_points_att_std_last_5 AS opponent_three_points_att_std_last_5, + opponent.three_points_att_std_last_10 AS opponent_three_points_att_std_last_10, + opponent.three_points_pct_std_last_5 AS opponent_three_points_pct_std_last_5, + opponent.three_points_pct_std_last_10 AS opponent_three_points_pct_std_last_10, + opponent.two_points_made_std_last_5 AS opponent_two_points_made_std_last_5, + opponent.two_points_made_std_last_10 AS opponent_two_points_made_std_last_10, + opponent.two_points_att_std_last_5 AS opponent_two_points_att_std_last_5, + opponent.two_points_att_std_last_10 AS opponent_two_points_att_std_last_10, + opponent.two_points_pct_std_last_5 AS opponent_two_points_pct_std_last_5, + opponent.two_points_pct_std_last_10 AS opponent_two_points_pct_std_last_10, + opponent.free_throws_made_std_last_5 AS opponent_free_throws_made_std_last_5, + opponent.free_throws_made_std_last_10 AS opponent_free_throws_made_std_last_10, + opponent.free_throws_att_std_last_5 AS opponent_free_throws_att_std_last_5, + opponent.free_throws_att_std_last_10 AS opponent_free_throws_att_std_last_10, + opponent.free_throws_pct_std_last_5 AS opponent_free_throws_pct_std_last_5, + opponent.free_throws_pct_std_last_10 AS opponent_free_throws_pct_std_last_10, + opponent.ts_pct_std_last_5 AS opponent_ts_pct_std_last_5, + opponent.ts_pct_std_last_10 AS opponent_ts_pct_std_last_10, + opponent.efg_pct_std_last_5 AS opponent_efg_pct_std_last_5, + opponent.efg_pct_std_last_10 AS opponent_efg_pct_std_last_10, + opponent.rebounds_std_last_5 AS opponent_rebounds_std_last_5, + opponent.rebounds_std_last_10 AS opponent_rebounds_std_last_10, + opponent.offensive_rebounds_std_last_5 AS opponent_offensive_rebounds_std_last_5, + opponent.offensive_rebounds_std_last_10 AS opponent_offensive_rebounds_std_last_10, + opponent.defensive_rebounds_std_last_5 AS opponent_defensive_rebounds_std_last_5, + opponent.defensive_rebounds_std_last_10 AS opponent_defensive_rebounds_std_last_10, + opponent.dreb_pct_std_last_5 AS opponent_dreb_pct_std_last_5, + opponent.dreb_pct_std_last_10 AS opponent_dreb_pct_std_last_10, + opponent.oreb_pct_std_last_5 AS opponent_oreb_pct_std_last_5, + opponent.oreb_pct_std_last_10 AS opponent_oreb_pct_std_last_10, + opponent.steals_std_last_5 AS opponent_steals_std_last_5, + opponent.steals_std_last_10 AS opponent_steals_std_last_10, + opponent.blocks_std_last_5 AS opponent_blocks_std_last_5, + opponent.blocks_std_last_10 AS opponent_blocks_std_last_10, + opponent.assists_std_last_5 AS opponent_assists_std_last_5, + opponent.assists_std_last_10 AS opponent_assists_std_last_10, + opponent.turnovers_std_last_5 AS opponent_turnovers_std_last_5, + opponent.turnovers_std_last_10 AS opponent_turnovers_std_last_10, + opponent.team_turnovers_std_last_5 AS opponent_team_turnovers_std_last_5, + opponent.team_turnovers_std_last_10 AS opponent_team_turnovers_std_last_10, + opponent.points_off_turnovers_std_last_5 AS opponent_points_off_turnovers_std_last_5, + opponent.points_off_turnovers_std_last_10 AS opponent_points_off_turnovers_std_last_10, + opponent.assists_turnover_ratio_std_last_5 AS opponent_assists_turnover_ratio_std_last_5, + opponent.assists_turnover_ratio_std_last_10 AS opponent_assists_turnover_ratio_std_last_10, + opponent.ast_fgm_pct_std_last_5 AS opponent_ast_fgm_pct_std_last_5, + opponent.ast_fgm_pct_std_last_10 AS opponent_ast_fgm_pct_std_last_10, + opponent.personal_fouls_std_last_5 AS opponent_personal_fouls_std_last_5, + opponent.personal_fouls_std_last_10 AS opponent_personal_fouls_std_last_10, + opponent.flagrant_fouls_std_last_5 AS opponent_flagrant_fouls_std_last_5, + opponent.flagrant_fouls_std_last_10 AS opponent_flagrant_fouls_std_last_10, + opponent.player_tech_fouls_std_last_5 AS opponent_player_tech_fouls_std_last_5, + opponent.player_tech_fouls_std_last_10 AS opponent_player_tech_fouls_std_last_10, + opponent.team_tech_fouls_std_last_5 AS opponent_team_tech_fouls_std_last_5, + opponent.team_tech_fouls_std_last_10 AS opponent_team_tech_fouls_std_last_10, + opponent.coach_tech_fouls_std_last_5 AS opponent_coach_tech_fouls_std_last_5, + opponent.coach_tech_fouls_std_last_10 AS opponent_coach_tech_fouls_std_last_10, + opponent.foulouts_std_last_5 AS opponent_foulouts_std_last_5, + opponent.foulouts_std_last_10 AS opponent_foulouts_std_last_10, + opponent.score_delta_std_last_5 AS opponent_score_delta_std_last_5, + opponent.score_delta_std_last_10 AS opponent_score_delta_std_last_10, + opponent.possessions_std_last_5 AS opponent_possessions_std_last_5, + opponent.possessions_std_last_10 AS opponent_possessions_std_last_10, + opponent.opp_points_std_last_5 AS opponent_opp_points_std_last_5, + opponent.opp_points_std_last_10 AS opponent_opp_points_std_last_10, + opponent.opp_fast_break_pts_std_last_5 AS opponent_opp_fast_break_pts_std_last_5, + opponent.opp_fast_break_pts_std_last_10 AS opponent_opp_fast_break_pts_std_last_10, + opponent.opp_second_chance_pts_std_last_5 AS opponent_opp_second_chance_pts_std_last_5, + opponent.opp_second_chance_pts_std_last_10 AS opponent_opp_second_chance_pts_std_last_10, + opponent.opp_field_goals_made_std_last_5 AS opponent_opp_field_goals_made_std_last_5, + opponent.opp_field_goals_made_std_last_10 AS opponent_opp_field_goals_made_std_last_10, + opponent.opp_field_goals_att_std_last_5 AS opponent_opp_field_goals_att_std_last_5, + opponent.opp_field_goals_att_std_last_10 AS opponent_opp_field_goals_att_std_last_10, + opponent.opp_field_goals_pct_std_last_5 AS opponent_opp_field_goals_pct_std_last_5, + opponent.opp_field_goals_pct_std_last_10 AS opponent_opp_field_goals_pct_std_last_10, + opponent.opp_three_points_made_std_last_5 AS opponent_opp_three_points_made_std_last_5, + opponent.opp_three_points_made_std_last_10 AS opponent_opp_three_points_made_std_last_10, + opponent.opp_three_points_att_std_last_5 AS opponent_opp_three_points_att_std_last_5, + opponent.opp_three_points_att_std_last_10 AS opponent_opp_three_points_att_std_last_10, + opponent.opp_three_points_pct_std_last_5 AS opponent_opp_three_points_pct_std_last_5, + opponent.opp_three_points_pct_std_last_10 AS opponent_opp_three_points_pct_std_last_10, + opponent.opp_two_points_made_std_last_5 AS opponent_opp_two_points_made_std_last_5, + opponent.opp_two_points_made_std_last_10 AS opponent_opp_two_points_made_std_last_10, + opponent.opp_two_points_att_std_last_5 AS opponent_opp_two_points_att_std_last_5, + opponent.opp_two_points_att_std_last_10 AS opponent_opp_two_points_att_std_last_10, + opponent.opp_two_points_pct_std_last_5 AS opponent_opp_two_points_pct_std_last_5, + opponent.opp_two_points_pct_std_last_10 AS opponent_opp_two_points_pct_std_last_10, + opponent.opp_free_throws_made_std_last_5 AS opponent_opp_free_throws_made_std_last_5, + opponent.opp_free_throws_made_std_last_10 AS opponent_opp_free_throws_made_std_last_10, + opponent.opp_free_throws_att_std_last_5 AS opponent_opp_free_throws_att_std_last_5, + opponent.opp_free_throws_att_std_last_10 AS opponent_opp_free_throws_att_std_last_10, + opponent.opp_free_throws_pct_std_last_5 AS opponent_opp_free_throws_pct_std_last_5, + opponent.opp_free_throws_pct_std_last_10 AS opponent_opp_free_throws_pct_std_last_10, + opponent.opp_ts_pct_std_last_5 AS opponent_opp_ts_pct_std_last_5, + opponent.opp_ts_pct_std_last_10 AS opponent_opp_ts_pct_std_last_10, + opponent.opp_efg_pct_std_last_5 AS opponent_opp_efg_pct_std_last_5, + opponent.opp_efg_pct_std_last_10 AS opponent_opp_efg_pct_std_last_10, + opponent.opp_rebounds_std_last_5 AS opponent_opp_rebounds_std_last_5, + opponent.opp_rebounds_std_last_10 AS opponent_opp_rebounds_std_last_10, + opponent.opp_offensive_rebounds_std_last_5 AS opponent_opp_offensive_rebounds_std_last_5, + opponent.opp_offensive_rebounds_std_last_10 AS opponent_opp_offensive_rebounds_std_last_10, + opponent.opp_defensive_rebounds_std_last_5 AS opponent_opp_defensive_rebounds_std_last_5, + opponent.opp_defensive_rebounds_std_last_10 AS opponent_opp_defensive_rebounds_std_last_10, + opponent.opp_dreb_pct_std_last_5 AS opponent_opp_dreb_pct_std_last_5, + opponent.opp_dreb_pct_std_last_10 AS opponent_opp_dreb_pct_std_last_10, + opponent.opp_oreb_pct_std_last_5 AS opponent_opp_oreb_pct_std_last_5, + opponent.opp_oreb_pct_std_last_10 AS opponent_opp_oreb_pct_std_last_10, + opponent.opp_steals_std_last_5 AS opponent_opp_steals_std_last_5, + opponent.opp_steals_std_last_10 AS opponent_opp_steals_std_last_10, + opponent.opp_blocks_std_last_5 AS opponent_opp_blocks_std_last_5, + opponent.opp_blocks_std_last_10 AS opponent_opp_blocks_std_last_10, + opponent.opp_assists_std_last_5 AS opponent_opp_assists_std_last_5, + opponent.opp_assists_std_last_10 AS opponent_opp_assists_std_last_10, + opponent.opp_turnovers_std_last_5 AS opponent_opp_turnovers_std_last_5, + opponent.opp_turnovers_std_last_10 AS opponent_opp_turnovers_std_last_10, + opponent.opp_team_turnovers_std_last_5 AS opponent_opp_team_turnovers_std_last_5, + opponent.opp_team_turnovers_std_last_10 AS opponent_opp_team_turnovers_std_last_10, + opponent.opp_points_off_turnovers_std_last_5 AS opponent_opp_points_off_turnovers_std_last_5, + opponent.opp_points_off_turnovers_std_last_10 AS opponent_opp_points_off_turnovers_std_last_10, + opponent.opp_assists_turnover_ratio_std_last_5 AS opponent_opp_assists_turnover_ratio_std_last_5, + opponent.opp_assists_turnover_ratio_std_last_10 AS opponent_opp_assists_turnover_ratio_std_last_10, + opponent.opp_ast_fgm_pct_std_last_5 AS opponent_opp_ast_fgm_pct_std_last_5, + opponent.opp_ast_fgm_pct_std_last_10 AS opponent_opp_ast_fgm_pct_std_last_10, + opponent.opp_personal_fouls_std_last_5 AS opponent_opp_personal_fouls_std_last_5, + opponent.opp_personal_fouls_std_last_10 AS opponent_opp_personal_fouls_std_last_10, + opponent.opp_flagrant_fouls_std_last_5 AS opponent_opp_flagrant_fouls_std_last_5, + opponent.opp_flagrant_fouls_std_last_10 AS opponent_opp_flagrant_fouls_std_last_10, + opponent.opp_player_tech_fouls_std_last_5 AS opponent_opp_player_tech_fouls_std_last_5, + opponent.opp_player_tech_fouls_std_last_10 AS opponent_opp_player_tech_fouls_std_last_10, + opponent.opp_team_tech_fouls_std_last_5 AS opponent_opp_team_tech_fouls_std_last_5, + opponent.opp_team_tech_fouls_std_last_10 AS opponent_opp_team_tech_fouls_std_last_10, + opponent.opp_coach_tech_fouls_std_last_5 AS opponent_opp_coach_tech_fouls_std_last_5, + opponent.opp_coach_tech_fouls_std_last_10 AS opponent_opp_coach_tech_fouls_std_last_10, + opponent.opp_foulouts_std_last_5 AS opponent_opp_foulouts_std_last_5, + opponent.opp_foulouts_std_last_10 AS opponent_opp_foulouts_std_last_10, + opponent.opp_score_delta_std_last_5 AS opponent_opp_score_delta_std_last_5, + opponent.opp_score_delta_std_last_10 AS opponent_opp_score_delta_std_last_10, + opponent.opp_possessions_std_last_5 AS opponent_opp_possessions_std_last_5, + opponent.opp_possessions_std_last_10 AS opponent_opp_possessions_std_last_10 +FROM + `bqml_tutorial.cume_games` AS team +JOIN + `bqml_tutorial.cume_games` AS opponent +ON + team.game_id = opponent.game_id AND team.team_id != opponent.team_id +WHERE + team.home_team = true AND team.game_number > 10 AND opponent.game_number > 10 + AND ((team.scheduled_date >= DATE('2014-03-18') AND team.scheduled_date <= DATE('2014-04-07')) + OR (team.scheduled_date >= DATE('2015-03-17') AND team.scheduled_date <= DATE('2015-04-06')) + OR (team.scheduled_date >= DATE('2016-03-15') AND team.scheduled_date <= DATE('2016-04-04')) + OR (team.scheduled_date >= DATE('2017-03-14') AND team.scheduled_date <= DATE('2017-04-03')) + OR (team.scheduled_date >= DATE('2017-05-01'))) +ORDER BY season, scheduled_date diff --git a/bigquery/cloud-client/.gitignore b/bigquery/cloud-client/.gitignore new file mode 100644 index 00000000000..0dc05ffadec --- /dev/null +++ b/bigquery/cloud-client/.gitignore @@ -0,0 +1,2 @@ +client_secrets.json +service_account.json diff --git a/bigquery/cloud-client/README.rst b/bigquery/cloud-client/README.rst new file mode 100644 index 00000000000..02bc856f978 --- /dev/null +++ b/bigquery/cloud-client/README.rst @@ -0,0 +1,145 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google BigQuery Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/README.rst + + +This directory contains samples for Google BigQuery. `Google BigQuery`_ is Google's fully managed, petabyte scale, low cost analytics data warehouse. BigQuery is NoOps—there is no infrastructure to manage and you don't need a database administrator—so you can focus on analyzing data to find meaningful insights, use familiar SQL, and take advantage of our pay-as-you-go model. + + + + +.. _Google BigQuery: https://cloud.google.com/bigquery/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Quickstart ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/quickstart.py,bigquery/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python quickstart.py + + +Simple Application ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/simple_app.py,bigquery/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python simple_app.py + + +User Credentials ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigquery/cloud-client/user_credentials.py,bigquery/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python user_credentials.py + + usage: user_credentials.py [-h] [--launch-browser] project query + + Command-line application to run a query using user credentials. + + You must supply a client secrets file, which would normally be bundled with + your application. + + positional arguments: + project Project to use for BigQuery billing. + query BigQuery SQL Query. + + optional arguments: + -h, --help show this help message and exit + --launch-browser Use a local server flow to authenticate. + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/bigquery/cloud-client/README.rst.in b/bigquery/cloud-client/README.rst.in new file mode 100644 index 00000000000..008b5179565 --- /dev/null +++ b/bigquery/cloud-client/README.rst.in @@ -0,0 +1,29 @@ +# This file is used to generate README.rst + +product: + name: Google BigQuery + short_name: BigQuery + url: https://cloud.google.com/bigquery/docs + description: > + `Google BigQuery`_ is Google's fully managed, petabyte scale, low cost + analytics data warehouse. BigQuery is NoOps—there is no infrastructure to + manage and you don't need a database administrator—so you can focus on + analyzing data to find meaningful insights, use familiar SQL, and take + advantage of our pay-as-you-go model. + +setup: +- auth +- install_deps + +samples: +- name: Quickstart + file: quickstart.py +- name: Simple Application + file: simple_app.py +- name: User Credentials + file: user_credentials.py + show_help: true + +cloud_client_library: true + +folder: bigquery/cloud-client \ No newline at end of file diff --git a/bigquery/cloud-client/authorized_view_tutorial.py b/bigquery/cloud-client/authorized_view_tutorial.py new file mode 100644 index 00000000000..abe4e0cb131 --- /dev/null +++ b/bigquery/cloud-client/authorized_view_tutorial.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python + +# Copyright 2018 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def run_authorized_view_tutorial(): + # Note to user: This is a group email for testing purposes. Replace with + # your own group email address when running this code. + analyst_group_email = 'example-analyst-group@google.com' + + # [START bigquery_authorized_view_tutorial] + # Create a source dataset + # [START bigquery_avt_create_source_dataset] + from google.cloud import bigquery + + client = bigquery.Client() + source_dataset_id = 'github_source_data' + + source_dataset = bigquery.Dataset(client.dataset(source_dataset_id)) + # Specify the geographic location where the dataset should reside. + source_dataset.location = 'US' + source_dataset = client.create_dataset(source_dataset) # API request + # [END bigquery_avt_create_source_dataset] + + # Populate a source table + # [START bigquery_avt_create_source_table] + source_table_id = 'github_contributors' + job_config = bigquery.QueryJobConfig() + job_config.destination = source_dataset.table(source_table_id) + sql = """ + SELECT commit, author, committer, repo_name + FROM `bigquery-public-data.github_repos.commits` + LIMIT 1000 + """ + query_job = client.query( + sql, + # Location must match that of the dataset(s) referenced in the query + # and of the destination table. + location='US', + job_config=job_config) # API request - starts the query + + query_job.result() # Waits for the query to finish + # [END bigquery_avt_create_source_table] + + # Create a separate dataset to store your view + # [START bigquery_avt_create_shared_dataset] + shared_dataset_id = 'shared_views' + shared_dataset = bigquery.Dataset(client.dataset(shared_dataset_id)) + shared_dataset.location = 'US' + shared_dataset = client.create_dataset(shared_dataset) # API request + # [END bigquery_avt_create_shared_dataset] + + # Create the view in the new dataset + # [START bigquery_avt_create_view] + shared_view_id = 'github_analyst_view' + view = bigquery.Table(shared_dataset.table(shared_view_id)) + sql_template = """ + SELECT + commit, author.name as author, + committer.name as committer, repo_name + FROM + `{}.{}.{}` + """ + view.view_query = sql_template.format( + client.project, source_dataset_id, source_table_id) + view = client.create_table(view) # API request + # [END bigquery_avt_create_view] + + # Assign access controls to the dataset containing the view + # [START bigquery_avt_shared_dataset_access] + # analyst_group_email = 'data_analysts@example.com' + access_entries = shared_dataset.access_entries + access_entries.append( + bigquery.AccessEntry('READER', 'groupByEmail', analyst_group_email) + ) + shared_dataset.access_entries = access_entries + shared_dataset = client.update_dataset( + shared_dataset, ['access_entries']) # API request + # [END bigquery_avt_shared_dataset_access] + + # Authorize the view to access the source dataset + # [START bigquery_avt_source_dataset_access] + access_entries = source_dataset.access_entries + access_entries.append( + bigquery.AccessEntry(None, 'view', view.reference.to_api_repr()) + ) + source_dataset.access_entries = access_entries + source_dataset = client.update_dataset( + source_dataset, ['access_entries']) # API request + # [END bigquery_avt_source_dataset_access] + # [END bigquery_authorized_view_tutorial] + + +if __name__ == '__main__': + run_authorized_view_tutorial() diff --git a/bigquery/cloud-client/authorized_view_tutorial_test.py b/bigquery/cloud-client/authorized_view_tutorial_test.py new file mode 100644 index 00000000000..954c47072c3 --- /dev/null +++ b/bigquery/cloud-client/authorized_view_tutorial_test.py @@ -0,0 +1,62 @@ +# Copyright 2018 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.cloud import bigquery +import pytest + +import authorized_view_tutorial + + +@pytest.fixture(scope='module') +def client(): + return bigquery.Client() + + +@pytest.fixture +def to_delete(client): + doomed = [] + yield doomed + for item in doomed: + if isinstance(item, (bigquery.Dataset, bigquery.DatasetReference)): + client.delete_dataset(item, delete_contents=True) + elif isinstance(item, (bigquery.Table, bigquery.TableReference)): + client.delete_table(item) + else: + item.delete() + + +def test_authorized_view_tutorial(client, to_delete): + source_dataset_ref = client.dataset('github_source_data') + shared_dataset_ref = client.dataset('shared_views') + to_delete.extend([source_dataset_ref, shared_dataset_ref]) + + authorized_view_tutorial.run_authorized_view_tutorial() + + source_dataset = client.get_dataset(source_dataset_ref) + shared_dataset = client.get_dataset(shared_dataset_ref) + analyst_email = 'example-analyst-group@google.com' + analyst_entries = [entry for entry in shared_dataset.access_entries + if entry.entity_id == analyst_email] + assert len(analyst_entries) == 1 + assert analyst_entries[0].role == 'READER' + + authorized_view_entries = [entry for entry in source_dataset.access_entries + if entry.entity_type == 'view'] + expected_view_ref = { + 'projectId': client.project, + 'datasetId': 'shared_views', + 'tableId': 'github_analyst_view', + } + assert len(authorized_view_entries) == 1 + assert authorized_view_entries[0].entity_id == expected_view_ref diff --git a/bigquery/cloud-client/jupyter_tutorial_test.py b/bigquery/cloud-client/jupyter_tutorial_test.py new file mode 100644 index 00000000000..0affbabcb03 --- /dev/null +++ b/bigquery/cloud-client/jupyter_tutorial_test.py @@ -0,0 +1,164 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import IPython +from IPython.terminal import interactiveshell +from IPython.testing import tools +import matplotlib +import pytest + + +# Ignore semicolon lint warning because semicolons are used in notebooks +# flake8: noqa E703 + + +@pytest.fixture(scope='session') +def ipython(): + config = tools.default_config() + config.TerminalInteractiveShell.simple_prompt = True + shell = interactiveshell.TerminalInteractiveShell.instance(config=config) + return shell + + +@pytest.fixture() +def ipython_interactive(request, ipython): + """Activate IPython's builtin hooks + + for the duration of the test scope. + """ + with ipython.builtin_trap: + yield ipython + + +def _strip_region_tags(sample_text): + """Remove blank lines and region tags from sample text""" + magic_lines = [line for line in sample_text.split('\n') + if len(line) > 0 and '# [' not in line] + return '\n'.join(magic_lines) + + +def test_jupyter_tutorial(ipython): + matplotlib.use('agg') + ip = IPython.get_ipython() + ip.extension_manager.load_extension('google.cloud.bigquery') + + sample = """ + # [START bigquery_jupyter_magic_gender_by_year] + %%bigquery + SELECT + source_year AS year, + COUNT(is_male) AS birth_count + FROM `bigquery-public-data.samples.natality` + GROUP BY year + ORDER BY year DESC + LIMIT 15 + # [END bigquery_jupyter_magic_gender_by_year] + """ + result = ip.run_cell(_strip_region_tags(sample)) + result.raise_error() # Throws an exception if the cell failed. + + sample = """ + # [START bigquery_jupyter_magic_gender_by_year_var] + %%bigquery total_births + SELECT + source_year AS year, + COUNT(is_male) AS birth_count + FROM `bigquery-public-data.samples.natality` + GROUP BY year + ORDER BY year DESC + LIMIT 15 + # [END bigquery_jupyter_magic_gender_by_year_var] + """ + result = ip.run_cell(_strip_region_tags(sample)) + result.raise_error() # Throws an exception if the cell failed. + + assert 'total_births' in ip.user_ns # verify that variable exists + total_births = ip.user_ns['total_births'] + # [START bigquery_jupyter_plot_births_by_year] + total_births.plot(kind='bar', x='year', y='birth_count'); + # [END bigquery_jupyter_plot_births_by_year] + + sample = """ + # [START bigquery_jupyter_magic_gender_by_weekday] + %%bigquery births_by_weekday + SELECT + wday, + SUM(CASE WHEN is_male THEN 1 ELSE 0 END) AS male_births, + SUM(CASE WHEN is_male THEN 0 ELSE 1 END) AS female_births + FROM `bigquery-public-data.samples.natality` + WHERE wday IS NOT NULL + GROUP BY wday + ORDER BY wday ASC + # [END bigquery_jupyter_magic_gender_by_weekday] + """ + result = ip.run_cell(_strip_region_tags(sample)) + result.raise_error() # Throws an exception if the cell failed. + + assert 'births_by_weekday' in ip.user_ns # verify that variable exists + births_by_weekday = ip.user_ns['births_by_weekday'] + # [START bigquery_jupyter_plot_births_by_weekday] + births_by_weekday.plot(x='wday'); + # [END bigquery_jupyter_plot_births_by_weekday] + + # [START bigquery_jupyter_import_and_client] + from google.cloud import bigquery + client = bigquery.Client() + # [END bigquery_jupyter_import_and_client] + + # [START bigquery_jupyter_query_plurality_by_year] + sql = """ + SELECT + plurality, + COUNT(1) AS count, + year + FROM + `bigquery-public-data.samples.natality` + WHERE + NOT IS_NAN(plurality) AND plurality > 1 + GROUP BY + plurality, year + ORDER BY + count DESC + """ + df = client.query(sql).to_dataframe() + df.head() + # [END bigquery_jupyter_query_plurality_by_year] + + # [START bigquery_jupyter_plot_plurality_by_year] + pivot_table = df.pivot(index='year', columns='plurality', values='count') + pivot_table.plot(kind='bar', stacked=True, figsize=(15, 7)); + # [END bigquery_jupyter_plot_plurality_by_year] + + # [START bigquery_jupyter_query_births_by_gestation] + sql = """ + SELECT + gestation_weeks, + COUNT(1) AS count + FROM + `bigquery-public-data.samples.natality` + WHERE + NOT IS_NAN(gestation_weeks) AND gestation_weeks <> 99 + GROUP BY + gestation_weeks + ORDER BY + gestation_weeks + """ + df = client.query(sql).to_dataframe() + # [END bigquery_jupyter_query_births_by_gestation] + + # [START bigquery_jupyter_plot_births_by_gestation] + ax = df.plot(kind='bar', x='gestation_weeks', y='count', figsize=(15,7)) + ax.set_title('Count of Births by Gestation Weeks') + ax.set_xlabel('Gestation Weeks') + ax.set_ylabel('Count'); + # [END bigquery_jupyter_plot_births_by_gestation] diff --git a/bigquery/cloud-client/natality_tutorial.py b/bigquery/cloud-client/natality_tutorial.py new file mode 100644 index 00000000000..5bfa8f1d27a --- /dev/null +++ b/bigquery/cloud-client/natality_tutorial.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python + +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def run_natality_tutorial(): + # [START bigquery_query_natality_tutorial] + """Create a Google BigQuery linear regression input table. + + In the code below, the following actions are taken: + * A new dataset is created "natality_regression." + * A query is run against the public dataset, + bigquery-public-data.samples.natality, selecting only the data of + interest to the regression, the output of which is stored in a new + "regression_input" table. + * The output table is moved over the wire to the user's default project via + the built-in BigQuery Connector for Spark that bridges BigQuery and + Cloud Dataproc. + """ + + from google.cloud import bigquery + + # Create a new Google BigQuery client using Google Cloud Platform project + # defaults. + client = bigquery.Client() + + # Prepare a reference to a new dataset for storing the query results. + dataset_ref = client.dataset('natality_regression') + dataset = bigquery.Dataset(dataset_ref) + + # Create the new BigQuery dataset. + dataset = client.create_dataset(dataset) + + # In the new BigQuery dataset, create a reference to a new table for + # storing the query results. + table_ref = dataset.table('regression_input') + + # Configure the query job. + job_config = bigquery.QueryJobConfig() + + # Set the destination table to the table reference created above. + job_config.destination = table_ref + + # Set up a query in Standard SQL, which is the default for the BigQuery + # Python client library. + # The query selects the fields of interest. + query = """ + SELECT + weight_pounds, mother_age, father_age, gestation_weeks, + weight_gain_pounds, apgar_5min + FROM + `bigquery-public-data.samples.natality` + WHERE + weight_pounds IS NOT NULL + AND mother_age IS NOT NULL + AND father_age IS NOT NULL + AND gestation_weeks IS NOT NULL + AND weight_gain_pounds IS NOT NULL + AND apgar_5min IS NOT NULL + """ + + # Run the query. + query_job = client.query(query, job_config=job_config) + query_job.result() # Waits for the query to finish + # [END bigquery_query_natality_tutorial] + + +if __name__ == '__main__': + run_natality_tutorial() diff --git a/bigquery/cloud-client/natality_tutorial_test.py b/bigquery/cloud-client/natality_tutorial_test.py new file mode 100644 index 00000000000..5165f7244f1 --- /dev/null +++ b/bigquery/cloud-client/natality_tutorial_test.py @@ -0,0 +1,42 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.cloud import bigquery +from google.cloud import exceptions + +import natality_tutorial + + +def dataset_exists(dataset, client): + try: + client.get_dataset(dataset) + return True + except exceptions.NotFound: + return False + + +def test_natality_tutorial(): + client = bigquery.Client() + dataset_ref = client.dataset('natality_regression') + assert not dataset_exists(dataset_ref, client) + + natality_tutorial.run_natality_tutorial() + + assert dataset_exists(dataset_ref, client) + + table = client.get_table( + bigquery.Table(dataset_ref.table('regression_input'))) + assert table.num_rows > 0 + + client.delete_dataset(dataset_ref, delete_contents=True) diff --git a/bigquery/cloud-client/quickstart.py b/bigquery/cloud-client/quickstart.py new file mode 100644 index 00000000000..10ae58e84ca --- /dev/null +++ b/bigquery/cloud-client/quickstart.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def run_quickstart(): + # [START bigquery_quickstart] + # Imports the Google Cloud client library + from google.cloud import bigquery + + # Instantiates a client + bigquery_client = bigquery.Client() + + # The name for the new dataset + dataset_id = 'my_new_dataset' + + # Prepares a reference to the new dataset + dataset_ref = bigquery_client.dataset(dataset_id) + dataset = bigquery.Dataset(dataset_ref) + + # Creates the new dataset + dataset = bigquery_client.create_dataset(dataset) + + print('Dataset {} created.'.format(dataset.dataset_id)) + # [END bigquery_quickstart] + + +if __name__ == '__main__': + run_quickstart() diff --git a/bigquery/cloud-client/quickstart_test.py b/bigquery/cloud-client/quickstart_test.py new file mode 100644 index 00000000000..02931086a11 --- /dev/null +++ b/bigquery/cloud-client/quickstart_test.py @@ -0,0 +1,54 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.cloud import bigquery +from google.cloud.exceptions import NotFound +import pytest + +import quickstart + + +# Must match the dataset listed in quickstart.py (there's no easy way to +# extract this). +DATASET_ID = 'my_new_dataset' + + +@pytest.fixture +def temporary_dataset(): + """Fixture that ensures the test dataset does not exist before or + after a test.""" + bigquery_client = bigquery.Client() + dataset_ref = bigquery_client.dataset(DATASET_ID) + + if dataset_exists(dataset_ref, bigquery_client): + bigquery_client.delete_dataset(dataset_ref) + + yield + + if dataset_exists(dataset_ref, bigquery_client): + bigquery_client.delete_dataset(dataset_ref) + + +def dataset_exists(dataset, client): + try: + client.get_dataset(dataset) + return True + except NotFound: + return False + + +def test_quickstart(capsys, temporary_dataset): + quickstart.run_quickstart() + out, _ = capsys.readouterr() + assert DATASET_ID in out diff --git a/bigquery/cloud-client/requirements.txt b/bigquery/cloud-client/requirements.txt new file mode 100644 index 00000000000..bfdc27e7d82 --- /dev/null +++ b/bigquery/cloud-client/requirements.txt @@ -0,0 +1,5 @@ +google-cloud-bigquery[pandas]==1.9.0 +google-auth-oauthlib==0.2.0 +ipython==7.2.0 +matplotlib==3.0.2 +pytz==2018.9 diff --git a/bigquery/cloud-client/simple_app.py b/bigquery/cloud-client/simple_app.py new file mode 100644 index 00000000000..a09e97f1246 --- /dev/null +++ b/bigquery/cloud-client/simple_app.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Simple application that performs a query with BigQuery.""" +# [START bigquery_simple_app_all] +# [START bigquery_simple_app_deps] +from google.cloud import bigquery +# [END bigquery_simple_app_deps] + + +def query_stackoverflow(): + # [START bigquery_simple_app_client] + client = bigquery.Client() + # [END bigquery_simple_app_client] + # [START bigquery_simple_app_query] + query_job = client.query(""" + SELECT + CONCAT( + 'https://stackoverflow.com/questions/', + CAST(id as STRING)) as url, + view_count + FROM `bigquery-public-data.stackoverflow.posts_questions` + WHERE tags like '%google-bigquery%' + ORDER BY view_count DESC + LIMIT 10""") + + results = query_job.result() # Waits for job to complete. + # [END bigquery_simple_app_query] + + # [START bigquery_simple_app_print] + for row in results: + print("{} : {} views".format(row.url, row.view_count)) + # [END bigquery_simple_app_print] + + +if __name__ == '__main__': + query_stackoverflow() +# [END bigquery_simple_app_all] diff --git a/bigquery/cloud-client/simple_app_test.py b/bigquery/cloud-client/simple_app_test.py new file mode 100644 index 00000000000..33f9f1adf69 --- /dev/null +++ b/bigquery/cloud-client/simple_app_test.py @@ -0,0 +1,21 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import simple_app + + +def test_query_stackoverflow(capsys): + simple_app.query_stackoverflow() + out, _ = capsys.readouterr() + assert 'views' in out diff --git a/bigquery/cloud-client/user_credentials.py b/bigquery/cloud-client/user_credentials.py new file mode 100644 index 00000000000..4917fdd3a41 --- /dev/null +++ b/bigquery/cloud-client/user_credentials.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python + +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Command-line application to run a query using user credentials. + +You must supply a client secrets file, which would normally be bundled with +your application. +""" + +import argparse + + +def main(project, launch_browser=True): + # [START bigquery_auth_user_flow] + from google_auth_oauthlib import flow + + # TODO: Uncomment the line below to set the `launch_browser` variable. + # launch_browser = True + # + # The `launch_browser` boolean variable indicates if a local server is used + # as the callback URL in the auth flow. A value of `True` is recommended, + # but a local server does not work if accessing the application remotely, + # such as over SSH or from a remote Jupyter notebook. + + appflow = flow.InstalledAppFlow.from_client_secrets_file( + 'client_secrets.json', + scopes=['https://www.googleapis.com/auth/bigquery']) + + if launch_browser: + appflow.run_local_server() + else: + appflow.run_console() + + credentials = appflow.credentials + # [END bigquery_auth_user_flow] + + # [START bigquery_auth_user_query] + from google.cloud import bigquery + + # TODO: Uncomment the line below to set the `project` variable. + # project = 'user-project-id' + # + # The `project` variable defines the project to be billed for query + # processing. The user must have the bigquery.jobs.create permission on + # this project to run a query. See: + # https://cloud.google.com/bigquery/docs/access-control#permissions + + client = bigquery.Client(project=project, credentials=credentials) + + query_string = """SELECT name, SUM(number) as total + FROM `bigquery-public-data.usa_names.usa_1910_current` + WHERE name = 'William' + GROUP BY name; + """ + query_job = client.query(query_string) + + # Print the results. + for row in query_job.result(): # Wait for the job to complete. + print("{}: {}".format(row['name'], row['total'])) + # [END bigquery_auth_user_query] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--launch-browser', + help='Use a local server flow to authenticate. ', + action='store_true') + parser.add_argument('project', help='Project to use for BigQuery billing.') + + args = parser.parse_args() + + main( + args.project, launch_browser=args.launch_browser) diff --git a/bigquery/cloud-client/user_credentials_test.py b/bigquery/cloud-client/user_credentials_test.py new file mode 100644 index 00000000000..009b9be7f95 --- /dev/null +++ b/bigquery/cloud-client/user_credentials_test.py @@ -0,0 +1,42 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import google.auth +import mock +import pytest + +from user_credentials import main + + +PROJECT = os.environ['GCLOUD_PROJECT'] + + +@pytest.fixture +def mock_flow(): + flow_patch = mock.patch( + 'google_auth_oauthlib.flow.InstalledAppFlow', autospec=True) + + with flow_patch as flow_mock: + flow_mock.from_client_secrets_file.return_value = flow_mock + flow_mock.credentials = google.auth.default()[0] + yield flow_mock + + +def test_auth_query_console(mock_flow, capsys): + main(PROJECT, launch_browser=False) + out, _ = capsys.readouterr() + # Fun fact: William P. Wood was the 1st director of the US Secret Service. + assert 'William' in out diff --git a/bigquery/datalab-migration/README.md b/bigquery/datalab-migration/README.md new file mode 100644 index 00000000000..bfe697e4f18 --- /dev/null +++ b/bigquery/datalab-migration/README.md @@ -0,0 +1,4 @@ +# Datalab Migration Guide + +This directory contains samples used in the `datalab` to +`google-cloud-bigquery` migration guide. diff --git a/bigquery/datalab-migration/requirements.txt b/bigquery/datalab-migration/requirements.txt new file mode 100644 index 00000000000..4a946ff2724 --- /dev/null +++ b/bigquery/datalab-migration/requirements.txt @@ -0,0 +1,5 @@ +google-cloud-bigquery[pandas,pyarrow]==1.8.1 +datalab==1.1.4 +ipython==7.2.0; python_version > "2.7" +ipython<=5.5; python_version == "2.7" +google-cloud-monitoring<=0.28.1 diff --git a/bigquery/datalab-migration/samples_test.py b/bigquery/datalab-migration/samples_test.py new file mode 100644 index 00000000000..04cef49ddb0 --- /dev/null +++ b/bigquery/datalab-migration/samples_test.py @@ -0,0 +1,414 @@ +# Copyright 2018 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +import google.auth +import google.datalab +import pytest +import IPython +from IPython.testing import tools +from IPython.terminal import interactiveshell + + +# Get default project +_, PROJECT_ID = google.auth.default() +# Set Datalab project ID +context = google.datalab.Context.default() +context.set_project_id(PROJECT_ID) + + +@pytest.fixture(scope='session') +def ipython_interactive(): + config = tools.default_config() + config.TerminalInteractiveShell.simple_prompt = True + shell = interactiveshell.TerminalInteractiveShell.instance(config=config) + return shell + + +@pytest.fixture +def to_delete(): + from google.cloud import bigquery + client = bigquery.Client() + doomed = [] + yield doomed + for dataset_id in doomed: + dataset = client.get_dataset(dataset_id) + client.delete_dataset(dataset, delete_contents=True) + + +def _set_up_ipython(extension): + ip = IPython.get_ipython() + ip.extension_manager.load_extension(extension) + return ip + + +def _strip_region_tags(sample_text): + """Remove blank lines and region tags from sample text""" + magic_lines = [line for line in sample_text.split('\n') + if len(line) > 0 and '# [' not in line] + return '\n'.join(magic_lines) + + +def test_datalab_query_magic(ipython_interactive): + import google.datalab.bigquery as bq + + ip = _set_up_ipython('google.datalab.kernel') + + sample = """ + # [START bigquery_migration_datalab_query_magic] + %%bq query + SELECT word, SUM(word_count) as count + FROM `bigquery-public-data.samples.shakespeare` + GROUP BY word + ORDER BY count ASC + LIMIT 100 + # [END bigquery_migration_datalab_query_magic] + """ + ip.run_cell(_strip_region_tags(sample)) + + results = ip.user_ns["_"] # Last returned object in notebook session + assert isinstance(results, bq.QueryResultsTable) + df = results.to_dataframe() + assert len(df) == 100 + + +def test_client_library_query_magic(ipython_interactive): + import pandas + + ip = _set_up_ipython('google.cloud.bigquery') + + sample = """ + # [START bigquery_migration_client_library_query_magic] + %%bigquery + SELECT word, SUM(word_count) as count + FROM `bigquery-public-data.samples.shakespeare` + GROUP BY word + ORDER BY count ASC + LIMIT 100 + # [END bigquery_migration_client_library_query_magic] + """ + ip.run_cell(_strip_region_tags(sample)) + + df = ip.user_ns["_"] # Last returned object in notebook session + assert isinstance(df, pandas.DataFrame) + assert len(df) == 100 + + +def test_datalab_query_magic_results_variable(ipython_interactive): + ip = _set_up_ipython('google.datalab.kernel') + + sample = """ + # [START bigquery_migration_datalab_query_magic_define_query] + %%bq query -n my_query + SELECT name FROM `bigquery-public-data.usa_names.usa_1910_current` + WHERE state = "TX" + LIMIT 100 + # [END bigquery_migration_datalab_query_magic_define_query] + """ + ip.run_cell(_strip_region_tags(sample)) + + sample = """ + # [START bigquery_migration_datalab_execute_query] + import google.datalab.bigquery as bq + + my_variable = my_query.execute().result().to_dataframe() + # [END bigquery_migration_datalab_execute_query] + """ + ip.run_cell(_strip_region_tags(sample)) + + variable_name = "my_variable" + assert variable_name in ip.user_ns # verify that variable exists + my_variable = ip.user_ns[variable_name] + assert len(my_variable) == 100 + ip.user_ns.pop(variable_name) # clean up variable + + +def test_client_library_query_magic_results_variable(ipython_interactive): + ip = _set_up_ipython('google.cloud.bigquery') + + sample = """ + # [START bigquery_migration_client_library_query_magic_results_variable] + %%bigquery my_variable + SELECT name FROM `bigquery-public-data.usa_names.usa_1910_current` + WHERE state = "TX" + LIMIT 100 + # [END bigquery_migration_client_library_query_magic_results_variable] + """ + ip.run_cell(_strip_region_tags(sample)) + + variable_name = "my_variable" + assert variable_name in ip.user_ns # verify that variable exists + my_variable = ip.user_ns[variable_name] + assert len(my_variable) == 100 + ip.user_ns.pop(variable_name) # clean up variable + + +def test_datalab_magic_parameterized_query(ipython_interactive): + import pandas + + ip = _set_up_ipython('google.datalab.kernel') + + sample = """ + # [START bigquery_migration_datalab_magic_define_parameterized_query] + %%bq query -n my_query + SELECT word, SUM(word_count) as count + FROM `bigquery-public-data.samples.shakespeare` + WHERE corpus = @corpus_name + GROUP BY word + ORDER BY count ASC + LIMIT @limit + # [END bigquery_migration_datalab_magic_define_parameterized_query] + """ + ip.run_cell(_strip_region_tags(sample)) + + sample = """ + # [START bigquery_migration_datalab_magic_query_params] + corpus_name = "hamlet" + limit = 10 + # [END bigquery_migration_datalab_magic_query_params] + """ + ip.run_cell(_strip_region_tags(sample)) + + sample = """ + # [START bigquery_migration_datalab_magic_execute_parameterized_query] + %%bq execute -q my_query --to-dataframe + parameters: + - name: corpus_name + type: STRING + value: $corpus_name + - name: limit + type: INTEGER + value: $limit + # [END bigquery_migration_datalab_magic_execute_parameterized_query] + """ + ip.run_cell(_strip_region_tags(sample)) + df = ip.user_ns["_"] # Retrieves last returned object in notebook session + assert isinstance(df, pandas.DataFrame) + assert len(df) == 10 + + +def test_client_library_magic_parameterized_query(ipython_interactive): + import pandas + + ip = _set_up_ipython('google.cloud.bigquery') + + sample = """ + # [START bigquery_migration_client_library_magic_query_params] + params = {"corpus_name": "hamlet", "limit": 10} + # [END bigquery_migration_client_library_magic_query_params] + """ + ip.run_cell(_strip_region_tags(sample)) + + sample = """ + # [START bigquery_migration_client_library_magic_parameterized_query] + %%bigquery --params $params + SELECT word, SUM(word_count) as count + FROM `bigquery-public-data.samples.shakespeare` + WHERE corpus = @corpus_name + GROUP BY word + ORDER BY count ASC + LIMIT @limit + # [END bigquery_migration_client_library_magic_parameterized_query] + """ + ip.run_cell(_strip_region_tags(sample)) + + df = ip.user_ns["_"] # Retrieves last returned object in notebook session + assert isinstance(df, pandas.DataFrame) + assert len(df) == 10 + + +def test_datalab_list_tables_magic(ipython_interactive): + ip = _set_up_ipython('google.datalab.kernel') + + sample = """ + # [START bigquery_migration_datalab_list_tables_magic] + %bq tables list --dataset bigquery-public-data.samples + # [END bigquery_migration_datalab_list_tables_magic] + """ + ip.run_cell(_strip_region_tags(sample)) + + # Retrieves last returned object in notebook session + html_element = ip.user_ns["_"] + assert "shakespeare" in html_element.data + + +def test_datalab_query(): + # [START bigquery_migration_datalab_query] + import google.datalab.bigquery as bq + + sql = """ + SELECT name FROM `bigquery-public-data.usa_names.usa_1910_current` + WHERE state = "TX" + LIMIT 100 + """ + df = bq.Query(sql).execute().result().to_dataframe() + # [END bigquery_migration_datalab_query] + + assert len(df) == 100 + + +def test_client_library_query(): + # [START bigquery_migration_client_library_query] + from google.cloud import bigquery + + client = bigquery.Client() + sql = """ + SELECT name FROM `bigquery-public-data.usa_names.usa_1910_current` + WHERE state = "TX" + LIMIT 100 + """ + df = client.query(sql).to_dataframe() + # [END bigquery_migration_client_library_query] + + assert len(df) == 100 + + +def test_datalab_load_table_from_gcs_csv(to_delete): + # [START bigquery_migration_datalab_load_table_from_gcs_csv] + import google.datalab.bigquery as bq + + # Create the dataset + dataset_id = 'import_sample' + # [END bigquery_migration_datalab_load_table_from_gcs_csv] + # Use unique dataset ID to avoid collisions when running tests + dataset_id = 'test_dataset_{}'.format(int(time.time() * 1000)) + to_delete.append(dataset_id) + # [START bigquery_migration_datalab_load_table_from_gcs_csv] + bq.Dataset(dataset_id).create() + + # Create the table + schema = [ + {'name': 'name', 'type': 'STRING'}, + {'name': 'post_abbr', 'type': 'STRING'}, + ] + table = bq.Table( + '{}.us_states'.format(dataset_id)).create(schema=schema) + table.load( + 'gs://cloud-samples-data/bigquery/us-states/us-states.csv', + mode='append', + source_format='csv', + csv_options=bq.CSVOptions(skip_leading_rows=1) + ) # Waits for the job to complete + # [END bigquery_migration_datalab_load_table_from_gcs_csv] + + assert table.length == 50 + + +def test_client_library_load_table_from_gcs_csv(to_delete): + # [START bigquery_migration_client_library_load_table_from_gcs_csv] + from google.cloud import bigquery + + client = bigquery.Client(location='US') + + # Create the dataset + dataset_id = 'import_sample' + # [END bigquery_migration_client_library_load_table_from_gcs_csv] + # Use unique dataset ID to avoid collisions when running tests + dataset_id = 'test_dataset_{}'.format(int(time.time() * 1000)) + to_delete.append(dataset_id) + # [START bigquery_migration_client_library_load_table_from_gcs_csv] + dataset = client.create_dataset(dataset_id) + + # Create the table + job_config = bigquery.LoadJobConfig( + schema=[ + bigquery.SchemaField('name', 'STRING'), + bigquery.SchemaField('post_abbr', 'STRING') + ], + skip_leading_rows=1, + # The source format defaults to CSV, so the line below is optional. + source_format=bigquery.SourceFormat.CSV + ) + load_job = client.load_table_from_uri( + 'gs://cloud-samples-data/bigquery/us-states/us-states.csv', + dataset.table('us_states'), + job_config=job_config + ) + load_job.result() # Waits for table load to complete. + # [END bigquery_migration_client_library_load_table_from_gcs_csv] + + table = client.get_table(dataset.table('us_states')) + assert table.num_rows == 50 + + +def test_datalab_load_table_from_dataframe(to_delete): + # [START bigquery_migration_datalab_load_table_from_dataframe] + import google.datalab.bigquery as bq + import pandas + + # Create the dataset + dataset_id = 'import_sample' + # [END bigquery_migration_datalab_load_table_from_dataframe] + # Use unique dataset ID to avoid collisions when running tests + dataset_id = 'test_dataset_{}'.format(int(time.time() * 1000)) + to_delete.append(dataset_id) + # [START bigquery_migration_datalab_load_table_from_dataframe] + bq.Dataset(dataset_id).create() + + # Create the table and load the data + dataframe = pandas.DataFrame([ + {'title': 'The Meaning of Life', 'release_year': 1983}, + {'title': 'Monty Python and the Holy Grail', 'release_year': 1975}, + {'title': 'Life of Brian', 'release_year': 1979}, + { + 'title': 'And Now for Something Completely Different', + 'release_year': 1971 + }, + ]) + schema = bq.Schema.from_data(dataframe) + table = bq.Table( + '{}.monty_python'.format(dataset_id)).create(schema=schema) + table.insert(dataframe) # Starts steaming insert of data + # [END bigquery_migration_datalab_load_table_from_dataframe] + # The Datalab library uses tabledata().insertAll() to load data from + # pandas DataFrames to tables. Because it can take a long time for the rows + # to be available in the table, this test does not assert on the number of + # rows in the destination table after the job is run. If errors are + # encountered during the insertion, this test will fail. + # See https://cloud.google.com/bigquery/streaming-data-into-bigquery + + +def test_client_library_load_table_from_dataframe(to_delete): + # [START bigquery_migration_client_library_load_table_from_dataframe] + from google.cloud import bigquery + import pandas + + client = bigquery.Client(location='US') + + dataset_id = 'import_sample' + # [END bigquery_migration_client_library_load_table_from_dataframe] + # Use unique dataset ID to avoid collisions when running tests + dataset_id = 'test_dataset_{}'.format(int(time.time() * 1000)) + to_delete.append(dataset_id) + # [START bigquery_migration_client_library_load_table_from_dataframe] + dataset = client.create_dataset(dataset_id) + + # Create the table and load the data + dataframe = pandas.DataFrame([ + {'title': 'The Meaning of Life', 'release_year': 1983}, + {'title': 'Monty Python and the Holy Grail', 'release_year': 1975}, + {'title': 'Life of Brian', 'release_year': 1979}, + { + 'title': 'And Now for Something Completely Different', + 'release_year': 1971 + }, + ]) + table_ref = dataset.table('monty_python') + load_job = client.load_table_from_dataframe(dataframe, table_ref) + load_job.result() # Waits for table load to complete. + # [END bigquery_migration_client_library_load_table_from_dataframe] + + table = client.get_table(table_ref) + assert table.num_rows == 4 diff --git a/bigquery/pandas-gbq-migration/README.md b/bigquery/pandas-gbq-migration/README.md new file mode 100644 index 00000000000..5d5ff00e530 --- /dev/null +++ b/bigquery/pandas-gbq-migration/README.md @@ -0,0 +1,4 @@ +# pandas-gbq Migration Guide + +This directory contains samples used in the `pandas-gbq` to +`google-cloud-bigquery` migration guide. diff --git a/bigquery/pandas-gbq-migration/requirements.txt b/bigquery/pandas-gbq-migration/requirements.txt new file mode 100644 index 00000000000..4886f85fa0a --- /dev/null +++ b/bigquery/pandas-gbq-migration/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-bigquery[pandas,pyarrow]==1.9.0 +pandas-gbq==0.9.0 diff --git a/bigquery/pandas-gbq-migration/samples_test.py b/bigquery/pandas-gbq-migration/samples_test.py new file mode 100644 index 00000000000..b7e982f60fa --- /dev/null +++ b/bigquery/pandas-gbq-migration/samples_test.py @@ -0,0 +1,224 @@ +# Copyright 2018 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time + +import pytest + + +@pytest.fixture +def temp_dataset(): + from google.cloud import bigquery + + client = bigquery.Client() + dataset_id = "temp_dataset_{}".format(int(time.time() * 1000)) + dataset_ref = bigquery.DatasetReference(client.project, dataset_id) + dataset = client.create_dataset(bigquery.Dataset(dataset_ref)) + yield dataset + client.delete_dataset(dataset, delete_contents=True) + + +def test_client_library_query(): + # [START bigquery_migration_client_library_query] + from google.cloud import bigquery + + client = bigquery.Client() + sql = """ + SELECT name + FROM `bigquery-public-data.usa_names.usa_1910_current` + WHERE state = 'TX' + LIMIT 100 + """ + + # Run a Standard SQL query using the environment's default project + df = client.query(sql).to_dataframe() + + # Run a Standard SQL query with the project set explicitly + project_id = 'your-project-id' + # [END bigquery_migration_client_library_query] + assert len(df) > 0 + project_id = os.environ['GCLOUD_PROJECT'] + # [START bigquery_migration_client_library_query] + df = client.query(sql, project=project_id).to_dataframe() + # [END bigquery_migration_client_library_query] + assert len(df) > 0 + + +def test_pandas_gbq_query(): + # [START bigquery_migration_pandas_gbq_query] + import pandas + + sql = """ + SELECT name + FROM `bigquery-public-data.usa_names.usa_1910_current` + WHERE state = 'TX' + LIMIT 100 + """ + + # Run a Standard SQL query using the environment's default project + df = pandas.read_gbq(sql, dialect='standard') + + # Run a Standard SQL query with the project set explicitly + project_id = 'your-project-id' + # [END bigquery_migration_pandas_gbq_query] + assert len(df) > 0 + project_id = os.environ['GCLOUD_PROJECT'] + # [START bigquery_migration_pandas_gbq_query] + df = pandas.read_gbq(sql, project_id=project_id, dialect='standard') + # [END bigquery_migration_pandas_gbq_query] + assert len(df) > 0 + + +def test_client_library_legacy_query(): + # [START bigquery_migration_client_library_query_legacy] + from google.cloud import bigquery + + client = bigquery.Client() + sql = """ + SELECT name + FROM [bigquery-public-data:usa_names.usa_1910_current] + WHERE state = 'TX' + LIMIT 100 + """ + query_config = bigquery.QueryJobConfig(use_legacy_sql=True) + + df = client.query(sql, job_config=query_config).to_dataframe() + # [END bigquery_migration_client_library_query_legacy] + assert len(df) > 0 + + +def test_pandas_gbq_legacy_query(): + # [START bigquery_migration_pandas_gbq_query_legacy] + import pandas + + sql = """ + SELECT name + FROM [bigquery-public-data:usa_names.usa_1910_current] + WHERE state = 'TX' + LIMIT 100 + """ + + df = pandas.read_gbq(sql, dialect='legacy') + # [END bigquery_migration_pandas_gbq_query_legacy] + assert len(df) > 0 + + +def test_client_library_query_with_parameters(): + # [START bigquery_migration_client_library_query_parameters] + from google.cloud import bigquery + + client = bigquery.Client() + sql = """ + SELECT name + FROM `bigquery-public-data.usa_names.usa_1910_current` + WHERE state = @state + LIMIT @limit + """ + query_config = bigquery.QueryJobConfig( + query_parameters=[ + bigquery.ScalarQueryParameter('state', 'STRING', 'TX'), + bigquery.ScalarQueryParameter('limit', 'INTEGER', 100) + ] + ) + + df = client.query(sql, job_config=query_config).to_dataframe() + # [END bigquery_migration_client_library_query_parameters] + assert len(df) > 0 + + +def test_pandas_gbq_query_with_parameters(): + # [START bigquery_migration_pandas_gbq_query_parameters] + import pandas + + sql = """ + SELECT name + FROM `bigquery-public-data.usa_names.usa_1910_current` + WHERE state = @state + LIMIT @limit + """ + query_config = { + 'query': { + 'parameterMode': 'NAMED', + 'queryParameters': [ + { + 'name': 'state', + 'parameterType': {'type': 'STRING'}, + 'parameterValue': {'value': 'TX'} + }, + { + 'name': 'limit', + 'parameterType': {'type': 'INTEGER'}, + 'parameterValue': {'value': 100} + } + ] + } + } + + df = pandas.read_gbq(sql, configuration=query_config) + # [END bigquery_migration_pandas_gbq_query_parameters] + assert len(df) > 0 + + +def test_client_library_upload_from_dataframe(temp_dataset): + # [START bigquery_migration_client_library_upload_from_dataframe] + from google.cloud import bigquery + import pandas + + df = pandas.DataFrame( + { + 'my_string': ['a', 'b', 'c'], + 'my_int64': [1, 2, 3], + 'my_float64': [4.0, 5.0, 6.0], + } + ) + client = bigquery.Client() + dataset_ref = client.dataset('my_dataset') + # [END bigquery_migration_client_library_upload_from_dataframe] + dataset_ref = client.dataset(temp_dataset.dataset_id) + # [START bigquery_migration_client_library_upload_from_dataframe] + table_ref = dataset_ref.table('new_table') + + client.load_table_from_dataframe(df, table_ref).result() + # [END bigquery_migration_client_library_upload_from_dataframe] + client = bigquery.Client() + table = client.get_table(table_ref) + assert table.num_rows == 3 + + +def test_pandas_gbq_upload_from_dataframe(temp_dataset): + from google.cloud import bigquery + # [START bigquery_migration_pandas_gbq_upload_from_dataframe] + import pandas + + df = pandas.DataFrame( + { + 'my_string': ['a', 'b', 'c'], + 'my_int64': [1, 2, 3], + 'my_float64': [4.0, 5.0, 6.0], + } + ) + full_table_id = 'my_dataset.new_table' + project_id = 'my-project-id' + # [END bigquery_migration_pandas_gbq_upload_from_dataframe] + table_id = 'new_table' + full_table_id = '{}.{}'.format(temp_dataset.dataset_id, table_id) + project_id = os.environ['GCLOUD_PROJECT'] + # [START bigquery_migration_pandas_gbq_upload_from_dataframe] + + df.to_gbq(full_table_id, project_id=project_id) + # [END bigquery_migration_pandas_gbq_upload_from_dataframe] + client = bigquery.Client() + table = client.get_table(temp_dataset.table(table_id)) + assert table.num_rows == 3 diff --git a/bigquery/transfer/cloud-client/quickstart.py b/bigquery/transfer/cloud-client/quickstart.py new file mode 100644 index 00000000000..a16a0c9cb62 --- /dev/null +++ b/bigquery/transfer/cloud-client/quickstart.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def run_quickstart(): + # [START bigquerydatatransfer_quickstart] + from google.cloud import bigquery_datatransfer + + client = bigquery_datatransfer.DataTransferServiceClient() + + project = 'my-project' # TODO: Update to your project ID. + + # Get the full path to your project. + parent = client.project_path(project) + + print('Supported Data Sources:') + + # Iterate over all possible data sources. + for data_source in client.list_data_sources(parent): + print('{}:'.format(data_source.display_name)) + print('\tID: {}'.format(data_source.data_source_id)) + print('\tFull path: {}'.format(data_source.name)) + print('\tDescription: {}'.format(data_source.description)) + # [END bigquerydatatransfer_quickstart] + + +if __name__ == '__main__': + run_quickstart() diff --git a/bigquery/transfer/cloud-client/quickstart_test.py b/bigquery/transfer/cloud-client/quickstart_test.py new file mode 100644 index 00000000000..0bb5ddd6811 --- /dev/null +++ b/bigquery/transfer/cloud-client/quickstart_test.py @@ -0,0 +1,41 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import mock +import pytest + +import quickstart + + +PROJECT = os.environ['GCLOUD_PROJECT'] + + +@pytest.fixture +def mock_project_path(): + """Mock out project and replace with project from environment.""" + project_patch = mock.patch( + 'google.cloud.bigquery_datatransfer.DataTransferServiceClient.' + 'project_path') + + with project_patch as project_mock: + project_mock.return_value = 'projects/{}'.format(PROJECT) + yield project_mock + + +def test_quickstart(capsys, mock_project_path): + quickstart.run_quickstart() + out, _ = capsys.readouterr() + assert 'Supported Data Sources:' in out diff --git a/bigquery/transfer/cloud-client/requirements.txt b/bigquery/transfer/cloud-client/requirements.txt new file mode 100644 index 00000000000..3e3a59d19ce --- /dev/null +++ b/bigquery/transfer/cloud-client/requirements.txt @@ -0,0 +1 @@ +google-cloud-bigquery-datatransfer==0.3.0 diff --git a/bigquery_storage/to_dataframe/__init__.py b/bigquery_storage/to_dataframe/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/bigquery_storage/to_dataframe/jupyter_test.py b/bigquery_storage/to_dataframe/jupyter_test.py new file mode 100644 index 00000000000..ef1b0ddb74f --- /dev/null +++ b/bigquery_storage/to_dataframe/jupyter_test.py @@ -0,0 +1,148 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import IPython +from IPython.terminal import interactiveshell +from IPython.testing import tools +import pytest + +# Ignore semicolon lint warning because semicolons are used in notebooks +# flake8: noqa E703 + + +@pytest.fixture(scope="session") +def ipython(): + config = tools.default_config() + config.TerminalInteractiveShell.simple_prompt = True + shell = interactiveshell.TerminalInteractiveShell.instance(config=config) + return shell + + +@pytest.fixture() +def ipython_interactive(request, ipython): + """Activate IPython's builtin hooks + + for the duration of the test scope. + """ + with ipython.builtin_trap: + yield ipython + + +def _strip_region_tags(sample_text): + """Remove blank lines and region tags from sample text""" + magic_lines = [ + line for line in sample_text.split("\n") if len(line) > 0 and "# [" not in line + ] + return "\n".join(magic_lines) + + +def test_jupyter_small_query(ipython): + ip = IPython.get_ipython() + ip.extension_manager.load_extension("google.cloud.bigquery") + + # Include a small query to demonstrate that it falls back to the + # tabledata.list API when the BQ Storage API cannot be used. + sample = """ + # [START bigquerystorage_jupyter_tutorial_fallback] + %%bigquery stackoverflow --use_bqstorage_api + SELECT + CONCAT( + 'https://stackoverflow.com/questions/', + CAST(id as STRING)) as url, + view_count + FROM `bigquery-public-data.stackoverflow.posts_questions` + WHERE tags like '%google-bigquery%' + ORDER BY view_count DESC + LIMIT 10 + # [END bigquerystorage_jupyter_tutorial_fallback] + """ + + result = ip.run_cell(_strip_region_tags(sample)) + result.raise_error() # Throws an exception if the cell failed. + assert "stackoverflow" in ip.user_ns # verify that variable exists + + +@pytest.mark.skipif( + "TRAVIS" in os.environ, reason="Not running long-running queries on Travis" +) +def test_jupyter_tutorial(ipython): + ip = IPython.get_ipython() + ip.extension_manager.load_extension("google.cloud.bigquery") + + # This code sample intentionally queries a lot of data to demonstrate the + # speed-up of using the BigQuery Storage API to download the results. + sample = """ + # [START bigquerystorage_jupyter_tutorial_query] + %%bigquery nodejs_deps --use_bqstorage_api + SELECT + dependency_name, + dependency_platform, + project_name, + project_id, + version_number, + version_id, + dependency_kind, + optional_dependency, + dependency_requirements, + dependency_project_id + FROM + `bigquery-public-data.libraries_io.dependencies` + WHERE + LOWER(dependency_platform) = 'npm' + LIMIT 2500000 + # [END bigquerystorage_jupyter_tutorial_query] + """ + result = ip.run_cell(_strip_region_tags(sample)) + result.raise_error() # Throws an exception if the cell failed. + + assert "nodejs_deps" in ip.user_ns # verify that variable exists + nodejs_deps = ip.user_ns["nodejs_deps"] + + # [START bigquerystorage_jupyter_tutorial_results] + nodejs_deps.head() + # [END bigquerystorage_jupyter_tutorial_results] + + # [START bigquerystorage_jupyter_tutorial_context] + import google.cloud.bigquery.magics + + google.cloud.bigquery.magics.context.use_bqstorage_api = True + # [END bigquerystorage_jupyter_tutorial_context] + + sample = """ + # [START bigquerystorage_jupyter_tutorial_query_default] + %%bigquery java_deps + SELECT + dependency_name, + dependency_platform, + project_name, + project_id, + version_number, + version_id, + dependency_kind, + optional_dependency, + dependency_requirements, + dependency_project_id + FROM + `bigquery-public-data.libraries_io.dependencies` + WHERE + LOWER(dependency_platform) = 'maven' + LIMIT 2500000 + # [END bigquerystorage_jupyter_tutorial_query_default] + """ + result = ip.run_cell(_strip_region_tags(sample)) + result.raise_error() # Throws an exception if the cell failed. + + assert "java_deps" in ip.user_ns # verify that variable exists diff --git a/bigquery_storage/to_dataframe/main_test.py b/bigquery_storage/to_dataframe/main_test.py new file mode 100644 index 00000000000..586ab3f94e9 --- /dev/null +++ b/bigquery_storage/to_dataframe/main_test.py @@ -0,0 +1,147 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + + +@pytest.fixture +def clients(): + # [START bigquerystorage_pandas_tutorial_all] + # [START bigquerystorage_pandas_tutorial_create_client] + import google.auth + from google.cloud import bigquery + from google.cloud import bigquery_storage_v1beta1 + + # Explicitly create a credentials object. This allows you to use the same + # credentials for both the BigQuery and BigQuery Storage clients, avoiding + # unnecessary API calls to fetch duplicate authentication tokens. + credentials, your_project_id = google.auth.default( + scopes=["https://www.googleapis.com/auth/cloud-platform"] + ) + + # Make clients. + bqclient = bigquery.Client( + credentials=credentials, + project=your_project_id + ) + bqstorageclient = bigquery_storage_v1beta1.BigQueryStorageClient( + credentials=credentials + ) + # [END bigquerystorage_pandas_tutorial_create_client] + # [END bigquerystorage_pandas_tutorial_all] + return bqclient, bqstorageclient + + +def test_table_to_dataframe(capsys, clients): + from google.cloud import bigquery + + bqclient, bqstorageclient = clients + + # [START bigquerystorage_pandas_tutorial_all] + # [START bigquerystorage_pandas_tutorial_read_table] + # Download a table. + table = bigquery.TableReference.from_string( + "bigquery-public-data.utility_us.country_code_iso" + ) + rows = bqclient.list_rows( + table, + selected_fields=[ + bigquery.SchemaField("country_name", "STRING"), + bigquery.SchemaField("fips_code", "STRING"), + ], + ) + dataframe = rows.to_dataframe(bqstorage_client=bqstorageclient) + print(dataframe.head()) + # [END bigquerystorage_pandas_tutorial_read_table] + # [END bigquerystorage_pandas_tutorial_all] + + out, _ = capsys.readouterr() + assert "country_name" in out + + +def test_query_to_dataframe(capsys, clients): + bqclient, bqstorageclient = clients + + # [START bigquerystorage_pandas_tutorial_all] + # [START bigquerystorage_pandas_tutorial_read_query_results] + # Download query results. + query_string = """ + SELECT + CONCAT( + 'https://stackoverflow.com/questions/', + CAST(id as STRING)) as url, + view_count + FROM `bigquery-public-data.stackoverflow.posts_questions` + WHERE tags like '%google-bigquery%' + ORDER BY view_count DESC + """ + + dataframe = ( + bqclient.query(query_string) + .result() + + # Note: The BigQuery Storage API cannot be used to download small query + # results, but as of google-cloud-bigquery version 1.11.1, the + # to_dataframe method will fallback to the tabledata.list API when the + # BigQuery Storage API fails to read the query results. + .to_dataframe(bqstorage_client=bqstorageclient) + ) + print(dataframe.head()) + # [END bigquerystorage_pandas_tutorial_read_query_results] + # [END bigquerystorage_pandas_tutorial_all] + + out, _ = capsys.readouterr() + assert "stackoverflow" in out + + +def test_session_to_dataframe(capsys, clients): + from google.cloud import bigquery_storage_v1beta1 + + bqclient, bqstorageclient = clients + your_project_id = bqclient.project + + # [START bigquerystorage_pandas_tutorial_all] + # [START bigquerystorage_pandas_tutorial_read_session] + table = bigquery_storage_v1beta1.types.TableReference() + table.project_id = "bigquery-public-data" + table.dataset_id = "new_york_trees" + table.table_id = "tree_species" + + # Select columns to read with read options. If no read options are + # specified, the whole table is read. + read_options = bigquery_storage_v1beta1.types.TableReadOptions() + read_options.selected_fields.append("species_common_name") + read_options.selected_fields.append("fall_color") + + parent = "projects/{}".format(your_project_id) + session = bqstorageclient.create_read_session( + table, parent, read_options=read_options + ) + + # This example reads from only a single stream. Read from multiple streams + # to fetch data faster. Note that the session may not contain any streams + # if there are no rows to read. + stream = session.streams[0] + position = bigquery_storage_v1beta1.types.StreamPosition(stream=stream) + reader = bqstorageclient.read_rows(position) + + # Parse all Avro blocks and create a dataframe. This call requires a + # session, because the session contains the schema for the row blocks. + dataframe = reader.to_dataframe(session) + print(dataframe.head()) + # [END bigquerystorage_pandas_tutorial_read_session] + # [END bigquerystorage_pandas_tutorial_all] + + out, _ = capsys.readouterr() + assert "species_common_name" in out diff --git a/bigquery_storage/to_dataframe/requirements.txt b/bigquery_storage/to_dataframe/requirements.txt new file mode 100644 index 00000000000..2fab885032f --- /dev/null +++ b/bigquery_storage/to_dataframe/requirements.txt @@ -0,0 +1,6 @@ +google-auth==1.6.2 +google-cloud-bigquery-storage==0.3.0 +google-cloud-bigquery==1.11.1 +fastavro==0.21.17 +ipython==7.2.0 +pandas==0.24.0 \ No newline at end of file diff --git a/bigtable/README.rst b/bigtable/README.rst new file mode 100644 index 00000000000..3f958d7e2a5 --- /dev/null +++ b/bigtable/README.rst @@ -0,0 +1,84 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Bigtable Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=/README.rst + + +This directory contains samples for Google Cloud Bigtable. `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's the same database that powers many core Google services, including Search, Analytics, Maps, and Gmail. + + +This directory contains samples that demonstrate using the Google Cloud Client Library for +Python as well as the `Google Cloud Client Library HappyBase package`_ to connect to and +interact with Cloud Bigtable. + +.. _Google Cloud Client Library HappyBase package: + https://github.com/GoogleCloudPlatform/google-cloud-python-happybase + + +.. _Google Cloud Bigtable: https://cloud.google.com/bigtable/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/bigtable/README.rst.in b/bigtable/README.rst.in new file mode 100644 index 00000000000..b8f68e21bc6 --- /dev/null +++ b/bigtable/README.rst.in @@ -0,0 +1,25 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Bigtable + short_name: Cloud Bigtable + url: https://cloud.google.com/bigtable/docs + description: > + `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's + the same database that powers many core Google services, including Search, + Analytics, Maps, and Gmail. + +description: | + This directory contains samples that demonstrate using the Google Cloud Client Library for + Python as well as the `Google Cloud Client Library HappyBase package`_ to connect to and + interact with Cloud Bigtable. + + .. _Google Cloud Client Library HappyBase package: + https://github.com/GoogleCloudPlatform/google-cloud-python-happybase + +setup: +- auth +- install_deps + + +cloud_client_library: true diff --git a/bigtable/hello/README.rst b/bigtable/hello/README.rst new file mode 100644 index 00000000000..893932ad5e7 --- /dev/null +++ b/bigtable/hello/README.rst @@ -0,0 +1,115 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Bigtable Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/hello/README.rst + + +This directory contains samples for Google Cloud Bigtable. `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's the same database that powers many core Google services, including Search, Analytics, Maps, and Gmail. + + + + +.. _Google Cloud Bigtable: https://cloud.google.com/bigtable/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Basic example ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/hello/main.py,bigtable/hello/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python main.py + + usage: main.py [-h] [--table TABLE] project_id instance_id + + Demonstrates how to connect to Cloud Bigtable and run some basic operations. + Prerequisites: - Create a Cloud Bigtable cluster. + https://cloud.google.com/bigtable/docs/creating-cluster - Set your Google + Application Default Credentials. + https://developers.google.com/identity/protocols/application-default- + credentials + + positional arguments: + project_id Your Cloud Platform project ID. + instance_id ID of the Cloud Bigtable instance to connect to. + + optional arguments: + -h, --help show this help message and exit + --table TABLE Table to create and destroy. (default: Hello-Bigtable) + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/bigtable/hello/README.rst.in b/bigtable/hello/README.rst.in new file mode 100644 index 00000000000..ed9253c115a --- /dev/null +++ b/bigtable/hello/README.rst.in @@ -0,0 +1,23 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Bigtable + short_name: Cloud Bigtable + url: https://cloud.google.com/bigtable/docs + description: > + `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's + the same database that powers many core Google services, including Search, + Analytics, Maps, and Gmail. + +setup: +- auth +- install_deps + +samples: +- name: Basic example + file: main.py + show_help: true + +cloud_client_library: true + +folder: bigtable/hello \ No newline at end of file diff --git a/bigtable/hello/main.py b/bigtable/hello/main.py new file mode 100644 index 00000000000..6020cf99222 --- /dev/null +++ b/bigtable/hello/main.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Demonstrates how to connect to Cloud Bigtable and run some basic operations. + +Prerequisites: + +- Create a Cloud Bigtable cluster. + https://cloud.google.com/bigtable/docs/creating-cluster +- Set your Google Application Default Credentials. + https://developers.google.com/identity/protocols/application-default-credentials +""" + +import argparse +# [START bigtable_hw_imports] +import datetime + +from google.cloud import bigtable +from google.cloud.bigtable import column_family +from google.cloud.bigtable import row_filters +# [END bigtable_hw_imports] + + +def main(project_id, instance_id, table_id): + # [START bigtable_hw_connect] + # The client must be created with admin=True because it will create a + # table. + client = bigtable.Client(project=project_id, admin=True) + instance = client.instance(instance_id) + # [END bigtable_hw_connect] + + # [START bigtable_hw_create_table] + print('Creating the {} table.'.format(table_id)) + table = instance.table(table_id) + + print('Creating column family cf1 with Max Version GC rule...') + # Create a column family with GC policy : most recent N versions + # Define the GC policy to retain only the most recent 2 versions + max_versions_rule = column_family.MaxVersionsGCRule(2) + column_family_id = 'cf1' + column_families = {column_family_id: max_versions_rule} + if not table.exists(): + table.create(column_families=column_families) + else: + print("Table {} already exists.".format(table_id)) + # [END bigtable_hw_create_table] + + # [START bigtable_hw_write_rows] + print('Writing some greetings to the table.') + greetings = ['Hello World!', 'Hello Cloud Bigtable!', 'Hello Python!'] + rows = [] + column = 'greeting'.encode() + for i, value in enumerate(greetings): + # Note: This example uses sequential numeric IDs for simplicity, + # but this can result in poor performance in a production + # application. Since rows are stored in sorted order by key, + # sequential keys can result in poor distribution of operations + # across nodes. + # + # For more information about how to design a Bigtable schema for + # the best performance, see the documentation: + # + # https://cloud.google.com/bigtable/docs/schema-design + row_key = 'greeting{}'.format(i).encode() + row = table.row(row_key) + row.set_cell(column_family_id, + column, + value, + timestamp=datetime.datetime.utcnow()) + rows.append(row) + table.mutate_rows(rows) + # [END bigtable_hw_write_rows] + + # [START bigtable_hw_create_filter] + # Create a filter to only retrieve the most recent version of the cell + # for each column accross entire row. + row_filter = row_filters.CellsColumnLimitFilter(1) + # [END bigtable_hw_create_filter] + + # [START bigtable_hw_get_with_filter] + print('Getting a single greeting by row key.') + key = 'greeting0'.encode() + + row = table.read_row(key, row_filter) + cell = row.cells[column_family_id][column][0] + print(cell.value.decode('utf-8')) + # [END bigtable_hw_get_with_filter] + + # [START bigtable_hw_scan_with_filter] + print('Scanning for all greetings:') + partial_rows = table.read_rows(filter_=row_filter) + + for row in partial_rows: + cell = row.cells[column_family_id][column][0] + print(cell.value.decode('utf-8')) + # [END bigtable_hw_scan_with_filter] + + # [START bigtable_hw_delete_table] + print('Deleting the {} table.'.format(table_id)) + table.delete() + # [END bigtable_hw_delete_table] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('project_id', help='Your Cloud Platform project ID.') + parser.add_argument( + 'instance_id', help='ID of the Cloud Bigtable instance to connect to.') + parser.add_argument( + '--table', + help='Table to create and destroy.', + default='Hello-Bigtable') + + args = parser.parse_args() + main(args.project_id, args.instance_id, args.table) diff --git a/bigtable/hello/main_test.py b/bigtable/hello/main_test.py new file mode 100644 index 00000000000..4080d7ee273 --- /dev/null +++ b/bigtable/hello/main_test.py @@ -0,0 +1,39 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import random + +from main import main + +PROJECT = os.environ['GCLOUD_PROJECT'] +BIGTABLE_CLUSTER = os.environ['BIGTABLE_CLUSTER'] +TABLE_NAME_FORMAT = 'hello-bigtable-system-tests-{}' +TABLE_NAME_RANGE = 10000 + + +def test_main(capsys): + table_name = TABLE_NAME_FORMAT.format( + random.randrange(TABLE_NAME_RANGE)) + + main(PROJECT, BIGTABLE_CLUSTER, table_name) + + out, _ = capsys.readouterr() + assert 'Creating the {} table.'.format(table_name) in out + assert 'Writing some greetings to the table.' in out + assert 'Getting a single greeting by row key.' in out + assert 'Hello World!' in out + assert 'Scanning for all greetings' in out + assert 'Hello Cloud Bigtable!' in out + assert 'Deleting the {} table.'.format(table_name) in out diff --git a/bigtable/hello/requirements.txt b/bigtable/hello/requirements.txt new file mode 100644 index 00000000000..f0aed1e60d3 --- /dev/null +++ b/bigtable/hello/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-bigtable==0.32.1 +google-cloud-core==0.29.1 diff --git a/bigtable/hello_happybase/README.rst b/bigtable/hello_happybase/README.rst new file mode 100644 index 00000000000..82a37653537 --- /dev/null +++ b/bigtable/hello_happybase/README.rst @@ -0,0 +1,122 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Bigtable Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/hello_happybase/README.rst + + +This directory contains samples for Google Cloud Bigtable. `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's the same database that powers many core Google services, including Search, Analytics, Maps, and Gmail. + + +This sample demonstrates using the `Google Cloud Client Library HappyBase +package`_, an implementation of the `HappyBase API`_ to connect to and +interact with Cloud Bigtable. + +.. _Google Cloud Client Library HappyBase package: + https://github.com/GoogleCloudPlatform/google-cloud-python-happybase +.. _HappyBase API: http://happybase.readthedocs.io/en/stable/ + + +.. _Google Cloud Bigtable: https://cloud.google.com/bigtable/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Basic example ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/hello_happybase/main.py,bigtable/hello_happybase/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python main.py + + usage: main.py [-h] [--table TABLE] project_id instance_id + + Demonstrates how to connect to Cloud Bigtable and run some basic operations. + Prerequisites: - Create a Cloud Bigtable cluster. + https://cloud.google.com/bigtable/docs/creating-cluster - Set your Google + Application Default Credentials. + https://developers.google.com/identity/protocols/application-default- + credentials + + positional arguments: + project_id Your Cloud Platform project ID. + instance_id ID of the Cloud Bigtable instance to connect to. + + optional arguments: + -h, --help show this help message and exit + --table TABLE Table to create and destroy. (default: Hello-Bigtable) + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/bigtable/hello_happybase/README.rst.in b/bigtable/hello_happybase/README.rst.in new file mode 100644 index 00000000000..8ef6a956b5e --- /dev/null +++ b/bigtable/hello_happybase/README.rst.in @@ -0,0 +1,32 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Bigtable + short_name: Cloud Bigtable + url: https://cloud.google.com/bigtable/docs + description: > + `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's + the same database that powers many core Google services, including Search, + Analytics, Maps, and Gmail. + +description: | + This sample demonstrates using the `Google Cloud Client Library HappyBase + package`_, an implementation of the `HappyBase API`_ to connect to and + interact with Cloud Bigtable. + + .. _Google Cloud Client Library HappyBase package: + https://github.com/GoogleCloudPlatform/google-cloud-python-happybase + .. _HappyBase API: http://happybase.readthedocs.io/en/stable/ + +setup: +- auth +- install_deps + +samples: +- name: Basic example + file: main.py + show_help: true + +cloud_client_library: true + +folder: bigtable/hello_happybase \ No newline at end of file diff --git a/bigtable/hello_happybase/main.py b/bigtable/hello_happybase/main.py new file mode 100644 index 00000000000..e4e68493448 --- /dev/null +++ b/bigtable/hello_happybase/main.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Demonstrates how to connect to Cloud Bigtable and run some basic operations. + +Prerequisites: + +- Create a Cloud Bigtable cluster. + https://cloud.google.com/bigtable/docs/creating-cluster +- Set your Google Application Default Credentials. + https://developers.google.com/identity/protocols/application-default-credentials +""" + +import argparse +# [START bigtable_hw_imports_happybase] +from google.cloud import bigtable +from google.cloud import happybase +# [END bigtable_hw_imports_happybase] + + +def main(project_id, instance_id, table_name): + # [START bigtable_hw_connect_happybase] + # The client must be created with admin=True because it will create a + # table. + client = bigtable.Client(project=project_id, admin=True) + instance = client.instance(instance_id) + connection = happybase.Connection(instance=instance) + # [END bigtable_hw_connect_happybase] + + try: + # [START bigtable_hw_create_table_happybase] + print('Creating the {} table.'.format(table_name)) + column_family_name = 'cf1' + connection.create_table( + table_name, + { + column_family_name: dict() # Use default options. + }) + # [END bigtable_hw_create_table_happybase] + + # [START bigtable_hw_write_rows_happybase] + print('Writing some greetings to the table.') + table = connection.table(table_name) + column_name = '{fam}:greeting'.format(fam=column_family_name) + greetings = [ + 'Hello World!', + 'Hello Cloud Bigtable!', + 'Hello HappyBase!', + ] + + for i, value in enumerate(greetings): + # Note: This example uses sequential numeric IDs for simplicity, + # but this can result in poor performance in a production + # application. Since rows are stored in sorted order by key, + # sequential keys can result in poor distribution of operations + # across nodes. + # + # For more information about how to design a Bigtable schema for + # the best performance, see the documentation: + # + # https://cloud.google.com/bigtable/docs/schema-design + row_key = 'greeting{}'.format(i) + table.put( + row_key, {column_name.encode('utf-8'): value.encode('utf-8')} + ) + # [END bigtable_hw_write_rows_happybase] + + # [START bigtable_hw_get_by_key_happybase] + print('Getting a single greeting by row key.') + key = 'greeting0'.encode('utf-8') + row = table.row(key) + print('\t{}: {}'.format(key, row[column_name.encode('utf-8')])) + # [END bigtable_hw_get_by_key_happybase] + + # [START bigtable_hw_scan_all_happybase] + print('Scanning for all greetings:') + + for key, row in table.scan(): + print('\t{}: {}'.format(key, row[column_name.encode('utf-8')])) + # [END bigtable_hw_scan_all_happybase] + + # [START bigtable_hw_delete_table_happybase] + print('Deleting the {} table.'.format(table_name)) + connection.delete_table(table_name) + # [END bigtable_hw_delete_table_happybase] + + finally: + connection.close() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('project_id', help='Your Cloud Platform project ID.') + parser.add_argument( + 'instance_id', help='ID of the Cloud Bigtable instance to connect to.') + parser.add_argument( + '--table', + help='Table to create and destroy.', + default='Hello-Bigtable') + + args = parser.parse_args() + main(args.project_id, args.instance_id, args.table) diff --git a/bigtable/hello_happybase/main_test.py b/bigtable/hello_happybase/main_test.py new file mode 100644 index 00000000000..3fc4ad13474 --- /dev/null +++ b/bigtable/hello_happybase/main_test.py @@ -0,0 +1,41 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import random + +from main import main + +PROJECT = os.environ['GCLOUD_PROJECT'] +BIGTABLE_CLUSTER = os.environ['BIGTABLE_CLUSTER'] +TABLE_NAME_FORMAT = 'hello_happybase-system-tests-{}' +TABLE_NAME_RANGE = 10000 + + +def test_main(capsys): + table_name = TABLE_NAME_FORMAT.format( + random.randrange(TABLE_NAME_RANGE)) + main( + PROJECT, + BIGTABLE_CLUSTER, + table_name) + + out, _ = capsys.readouterr() + assert 'Creating the {} table.'.format(table_name) in out + assert 'Writing some greetings to the table.' in out + assert 'Getting a single greeting by row key.' in out + assert 'Hello World!' in out + assert 'Scanning for all greetings' in out + assert 'Hello Cloud Bigtable!' in out + assert 'Deleting the {} table.'.format(table_name) in out diff --git a/bigtable/hello_happybase/requirements.txt b/bigtable/hello_happybase/requirements.txt new file mode 100644 index 00000000000..a144f03e1bc --- /dev/null +++ b/bigtable/hello_happybase/requirements.txt @@ -0,0 +1 @@ +google-cloud-happybase==0.33.0 diff --git a/bigtable/instanceadmin/README.rst b/bigtable/instanceadmin/README.rst new file mode 100644 index 00000000000..16f176a6099 --- /dev/null +++ b/bigtable/instanceadmin/README.rst @@ -0,0 +1,120 @@ +.. This file is automatically generated. Do not edit this file directly. + + +Google Cloud Bigtable table creation +=============================================================================== + +https://cloud.google.com/bigtable/docs/quickstart-cbt + +This page explains how to use the cbt command to connect to a Cloud Bigtable instance, perform basic administrative tasks, and read and write data in a table. + +Google Cloud Bigtable Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/hello/README.rst + + +This directory contains samples for Google Cloud Bigtable. `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's the same database that powers many core Google services, including Search, Analytics, Maps, and Gmail. + + + + +.. _Google Cloud Bigtable: https://cloud.google.com/bigtable/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Basic example ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/instanceadmin.py,bigtable/instanceadmin/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python instanceadmin.py + + usage: instanceadmin.py [-h] [run] [dev-instance] [del-instance] [add-cluster] [del-cluster] project_id instance_id cluster_id + + Demonstrates how to connect to Cloud Bigtable and run some basic operations + to create instance, create cluster, delete instance and delete cluster. + Prerequisites: - Create a Cloud Bigtable cluster. + https://cloud.google.com/bigtable/docs/creating-cluster - Set your Google + Application Default Credentials. + https://developers.google.com/identity/protocols/application-default- + credentials + + positional arguments: + project_id Your Cloud Platform project ID. + instance_id ID of the Cloud Bigtable instance to connect to. + cluster_id ID of the Cloud Bigtable cluster to connect to. + + optional arguments: + -h, --help show this help message and exit + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/bigtable/instanceadmin/README.rst.in b/bigtable/instanceadmin/README.rst.in new file mode 100644 index 00000000000..c085e40a627 --- /dev/null +++ b/bigtable/instanceadmin/README.rst.in @@ -0,0 +1,23 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Bigtable and run some basic operations. + short_name: Cloud Bigtable + url: https://cloud.google.com/bigtable/docs + description: > + `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's + the same database that powers many core Google services, including Search, + Analytics, Maps, and Gmail. + +setup: +- auth +- install_deps + +samples: +- name: Basic example with Bigtable Column family and GC rules. + file: instanceadmin.py + show_help: true + +cloud_client_library: true + +folder: bigtable/instanceadmin \ No newline at end of file diff --git a/bigtable/instanceadmin/instanceadmin.py b/bigtable/instanceadmin/instanceadmin.py new file mode 100644 index 00000000000..32120eb6375 --- /dev/null +++ b/bigtable/instanceadmin/instanceadmin.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python + +# Copyright 2018, Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Demonstrates how to connect to Cloud Bigtable and run some basic operations. +# http://www.apache.org/licenses/LICENSE-2.0 +Prerequisites: +- Create a Cloud Bigtable project. + https://cloud.google.com/bigtable/docs/ +- Set your Google Application Default Credentials. + https://developers.google.com/identity/protocols/application-default-credentials + +Operations performed: +- Create a Cloud Bigtable Instance. +- List Instance for a Cloud Bigtable. +- Delete a Cloud Bigtable Instance. +- Create a Cloud Bigtable Cluster. +- List Cloud Bigtable Clusters. +- Delete a Cloud Bigtable Cluster. +""" + +import argparse + +from google.cloud import bigtable +from google.cloud.bigtable import enums + + +def run_instance_operations(project_id, instance_id): + ''' Check Instance exists. + Creates a Production instance with default Cluster. + List instances in a project. + List clusters in an instance. + + :type project_id: str + :param project_id: Project id of the client. + + :type instance_id: str + :param instance_id: Instance of the client. + ''' + client = bigtable.Client(project=project_id, admin=True) + location_id = 'us-central1-f' + serve_nodes = 3 + storage_type = enums.StorageType.SSD + production = enums.Instance.Type.PRODUCTION + labels = {'prod-label': 'prod-label'} + instance = client.instance(instance_id, instance_type=production, + labels=labels) + + # [START bigtable_check_instance_exists] + if not instance.exists(): + print('Instance {} does not exists.'.format(instance_id)) + else: + print('Instance {} already exists.'.format(instance_id)) + # [END bigtable_check_instance_exists] + + # [START bigtable_create_prod_instance] + cluster = instance.cluster("ssd-cluster1", location_id=location_id, + serve_nodes=serve_nodes, + default_storage_type=storage_type) + if not instance.exists(): + print('\nCreating an Instance') + # Create instance with given options + instance.create(clusters=[cluster]) + print('\nCreated instance: {}'.format(instance_id)) + # [END bigtable_create_prod_instance] + + # [START bigtable_list_instances] + print('\nListing Instances:') + for instance_local in client.list_instances()[0]: + print(instance_local.instance_id) + # [END bigtable_list_instances] + + # [START bigtable_get_instance] + print('\nName of instance:{}\nLabels:{}'.format(instance.display_name, + instance.labels)) + # [END bigtable_get_instance] + + # [START bigtable_get_clusters] + print('\nListing Clusters...') + for cluster in instance.list_clusters()[0]: + print(cluster.cluster_id) + # [END bigtable_get_clusters] + + +def create_dev_instance(project_id, instance_id, cluster_id): + ''' Creates a Development instance with the name "hdd-instance" + location us-central1-f + Cluster nodes should not be set while creating Development + Instance + + :type project_id: str + :param project_id: Project id of the client. + + :type instance_id: str + :param instance_id: Instance of the client. + ''' + + client = bigtable.Client(project=project_id, admin=True) + + # [START bigtable_create_dev_instance] + print('\nCreating a DEVELOPMENT Instance') + # Set options to create an Instance + location_id = 'us-central1-f' + development = enums.Instance.Type.DEVELOPMENT + storage_type = enums.StorageType.HDD + labels = {'dev-label': 'dev-label'} + + # Create instance with given options + instance = client.instance(instance_id, instance_type=development, + labels=labels) + cluster = instance.cluster(cluster_id, location_id=location_id, + default_storage_type=storage_type) + + # Create development instance with given options + if not instance.exists(): + instance.create(clusters=[cluster]) + print('Created development instance: {}'.format(instance_id)) + else: + print('Instance {} already exists.'.format(instance_id)) + + # [END bigtable_create_dev_instance] + + +def delete_instance(project_id, instance_id): + ''' Delete the Instance + + :type project_id: str + :param project_id: Project id of the client. + + :type instance_id: str + :param instance_id: Instance of the client. + ''' + + client = bigtable.Client(project=project_id, admin=True) + instance = client.instance(instance_id) + # [START bigtable_delete_instance] + print('\nDeleting Instance') + if not instance.exists(): + print('Instance {} does not exists.'.format(instance_id)) + else: + instance.delete() + print('Deleted Instance: {}'.format(instance_id)) + # [END bigtable_delete_instance] + + +def add_cluster(project_id, instance_id, cluster_id): + ''' Add Cluster + + :type project_id: str + :param project_id: Project id of the client. + + :type instance_id: str + :param instance_id: Instance of the client. + + :type cluster_id: str + :param cluster_id: Cluster id. + ''' + client = bigtable.Client(project=project_id, admin=True) + instance = client.instance(instance_id) + + location_id = 'us-central1-a' + serve_nodes = 3 + storage_type = enums.StorageType.SSD + + if not instance.exists(): + print('Instance {} does not exists.'.format(instance_id)) + else: + print('\nAdding Cluster to Instance {}'.format(instance_id)) + # [START bigtable_create_cluster] + print('\nListing Clusters...') + for cluster in instance.list_clusters()[0]: + print(cluster.cluster_id) + cluster = instance.cluster(cluster_id, location_id=location_id, + serve_nodes=serve_nodes, + default_storage_type=storage_type) + if cluster.exists(): + print( + '\nCluster not created, as {} already exists.'. + format(cluster_id) + ) + else: + cluster.create() + print('\nCluster created: {}'.format(cluster_id)) + # [END bigtable_create_cluster] + + +def delete_cluster(project_id, instance_id, cluster_id): + ''' Delete the cluster + + :type project_id: str + :param project_id: Project id of the client. + + :type instance_id: str + :param instance_id: Instance of the client. + + :type cluster_id: str + :param cluster_id: Cluster id. + ''' + + client = bigtable.Client(project=project_id, admin=True) + instance = client.instance(instance_id) + cluster = instance.cluster(cluster_id) + + # [START bigtable_delete_cluster] + print('\nDeleting Cluster') + if cluster.exists(): + cluster.delete() + print('Cluster deleted: {}'.format(cluster_id)) + else: + print('\nCluster {} does not exist.'.format(cluster_id)) + + # [END bigtable_delete_cluster] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + parser.add_argument('command', + help='run, dev-instance, del-instance, \ + add-cluster or del-cluster. \ + Operation to perform on Instance.') + parser.add_argument('project_id', + help='Your Cloud Platform project ID.') + parser.add_argument('instance_id', + help='ID of the Cloud Bigtable instance to \ + connect to.') + parser.add_argument('cluster_id', + help='ID of the Cloud Bigtable cluster to \ + connect to.') + + args = parser.parse_args() + + if args.command.lower() == 'run': + run_instance_operations(args.project_id, args.instance_id) + elif args.command.lower() == 'dev-instance': + create_dev_instance(args.project_id, args.instance_id, + args.cluster_id) + elif args.command.lower() == 'del-instance': + delete_instance(args.project_id, args.instance_id) + elif args.command.lower() == 'add-cluster': + add_cluster(args.project_id, args.instance_id, args.cluster_id) + elif args.command.lower() == 'del-cluster': + delete_cluster(args.project_id, args.instance_id, args.cluster_id) + else: + print('Command should be either run \n Use argument -h, \ + --help to show help and exit.') diff --git a/bigtable/instanceadmin/requirements.txt b/bigtable/instanceadmin/requirements.txt new file mode 100755 index 00000000000..e70d7d99e00 --- /dev/null +++ b/bigtable/instanceadmin/requirements.txt @@ -0,0 +1 @@ +google-cloud-bigtable==0.32.1 diff --git a/bigtable/metricscaler/README.rst b/bigtable/metricscaler/README.rst new file mode 100644 index 00000000000..c64bbff1d8a --- /dev/null +++ b/bigtable/metricscaler/README.rst @@ -0,0 +1,128 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Bigtable Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/metricscaler/README.rst + + +This directory contains samples for Google Cloud Bigtable. `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's the same database that powers many core Google services, including Search, Analytics, Maps, and Gmail. + + +This sample demonstrates how to use `Stackdriver Monitoring`_ +to scale Cloud Bigtable based on CPU usage. + +.. _Stackdriver Monitoring: http://cloud.google.com/monitoring/docs/ + + +.. _Google Cloud Bigtable: https://cloud.google.com/bigtable/docs/ + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Metricscaling example ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/metricscaler/metricscaler.py,bigtable/metricscaler/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python metricscaler.py + + usage: metricscaler.py [-h] [--high_cpu_threshold HIGH_CPU_THRESHOLD] + [--low_cpu_threshold LOW_CPU_THRESHOLD] + [--short_sleep SHORT_SLEEP] [--long_sleep LONG_SLEEP] + bigtable_instance bigtable_cluster + + Scales Cloud Bigtable clusters based on CPU usage. + + positional arguments: + bigtable_instance ID of the Cloud Bigtable instance to connect to. + bigtable_cluster ID of the Cloud Bigtable cluster to connect to. + + optional arguments: + -h, --help show this help message and exit + --high_cpu_threshold HIGH_CPU_THRESHOLD + If Cloud Bigtable CPU usage is above this threshold, + scale up + --low_cpu_threshold LOW_CPU_THRESHOLD + If Cloud Bigtable CPU usage is below this threshold, + scale down + --short_sleep SHORT_SLEEP + How long to sleep in seconds between checking metrics + after no scale operation + --long_sleep LONG_SLEEP + How long to sleep in seconds between checking metrics + after a scaling operation + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/bigtable/metricscaler/README.rst.in b/bigtable/metricscaler/README.rst.in new file mode 100644 index 00000000000..44a548e4c1f --- /dev/null +++ b/bigtable/metricscaler/README.rst.in @@ -0,0 +1,29 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Bigtable + short_name: Cloud Bigtable + url: https://cloud.google.com/bigtable/docs/ + description: > + `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's + the same database that powers many core Google services, including Search, + Analytics, Maps, and Gmail. + +description: | + This sample demonstrates how to use `Stackdriver Monitoring`_ + to scale Cloud Bigtable based on CPU usage. + + .. _Stackdriver Monitoring: http://cloud.google.com/monitoring/docs/ + +setup: +- auth +- install_deps + +samples: +- name: Metricscaling example + file: metricscaler.py + show_help: true + +cloud_client_library: true + +folder: bigtable/metricscaler \ No newline at end of file diff --git a/bigtable/metricscaler/metricscaler.py b/bigtable/metricscaler/metricscaler.py new file mode 100644 index 00000000000..211a1e035b4 --- /dev/null +++ b/bigtable/metricscaler/metricscaler.py @@ -0,0 +1,172 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sample that demonstrates how to use Stackdriver Monitoring metrics to +programmatically scale a Google Cloud Bigtable cluster.""" + +import argparse +import os +import time + +from google.cloud import bigtable +from google.cloud import monitoring_v3 +from google.cloud.monitoring_v3 import query + +PROJECT = os.environ['GCLOUD_PROJECT'] + + +def get_cpu_load(): + """Returns the most recent Cloud Bigtable CPU load measurement. + + Returns: + float: The most recent Cloud Bigtable CPU usage metric + """ + # [START bigtable_cpu] + client = monitoring_v3.MetricServiceClient() + cpu_query = query.Query(client, + project=PROJECT, + metric_type='bigtable.googleapis.com/' + 'cluster/cpu_load', + minutes=5) + time_series = list(cpu_query) + recent_time_series = time_series[0] + return recent_time_series.points[0].value.double_value + # [END bigtable_cpu] + + +def scale_bigtable(bigtable_instance, bigtable_cluster, scale_up): + """Scales the number of Cloud Bigtable nodes up or down. + + Edits the number of nodes in the Cloud Bigtable cluster to be increased + or decreased, depending on the `scale_up` boolean argument. Currently + the `incremental` strategy from `strategies.py` is used. + + + Args: + bigtable_instance (str): Cloud Bigtable instance ID to scale + bigtable_cluster (str): Cloud Bigtable cluster ID to scale + scale_up (bool): If true, scale up, otherwise scale down + """ + + # The minimum number of nodes to use. The default minimum is 3. If you have + # a lot of data, the rule of thumb is to not go below 2.5 TB per node for + # SSD lusters, and 8 TB for HDD. The + # "bigtable.googleapis.com/disk/bytes_used" metric is useful in figuring + # out the minimum number of nodes. + min_node_count = 3 + + # The maximum number of nodes to use. The default maximum is 30 nodes per + # zone. If you need more quota, you can request more by following the + # instructions at https://cloud.google.com/bigtable/quota. + max_node_count = 30 + + # The number of nodes to change the cluster by. + size_change_step = 3 + + # [START bigtable_scale] + bigtable_client = bigtable.Client(admin=True) + instance = bigtable_client.instance(bigtable_instance) + instance.reload() + + cluster = instance.cluster(bigtable_cluster) + cluster.reload() + + current_node_count = cluster.serve_nodes + + if scale_up: + if current_node_count < max_node_count: + new_node_count = min( + current_node_count + size_change_step, max_node_count) + cluster.serve_nodes = new_node_count + cluster.update() + print('Scaled up from {} to {} nodes.'.format( + current_node_count, new_node_count)) + else: + if current_node_count > min_node_count: + new_node_count = max( + current_node_count - size_change_step, min_node_count) + cluster.serve_nodes = new_node_count + cluster.update() + print('Scaled down from {} to {} nodes.'.format( + current_node_count, new_node_count)) + # [END bigtable_scale] + + +def main( + bigtable_instance, + bigtable_cluster, + high_cpu_threshold, + low_cpu_threshold, + short_sleep, + long_sleep): + """Main loop runner that autoscales Cloud Bigtable. + + Args: + bigtable_instance (str): Cloud Bigtable instance ID to autoscale + high_cpu_threshold (float): If CPU is higher than this, scale up. + low_cpu_threshold (float): If CPU is lower than this, scale down. + short_sleep (int): How long to sleep after no operation + long_sleep (int): How long to sleep after the number of nodes is + changed + """ + cluster_cpu = get_cpu_load() + print('Detected cpu of {}'.format(cluster_cpu)) + if cluster_cpu > high_cpu_threshold: + scale_bigtable(bigtable_instance, bigtable_cluster, True) + time.sleep(long_sleep) + elif cluster_cpu < low_cpu_threshold: + scale_bigtable(bigtable_instance, bigtable_cluster, False) + time.sleep(long_sleep) + else: + print('CPU within threshold, sleeping.') + time.sleep(short_sleep) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Scales Cloud Bigtable clusters based on CPU usage.') + parser.add_argument( + 'bigtable_instance', + help='ID of the Cloud Bigtable instance to connect to.') + parser.add_argument( + 'bigtable_cluster', + help='ID of the Cloud Bigtable cluster to connect to.') + parser.add_argument( + '--high_cpu_threshold', + help='If Cloud Bigtable CPU usage is above this threshold, scale up', + default=0.6) + parser.add_argument( + '--low_cpu_threshold', + help='If Cloud Bigtable CPU usage is below this threshold, scale down', + default=0.2) + parser.add_argument( + '--short_sleep', + help='How long to sleep in seconds between checking metrics after no ' + 'scale operation', + default=60) + parser.add_argument( + '--long_sleep', + help='How long to sleep in seconds between checking metrics after a ' + 'scaling operation', + default=60 * 10) + args = parser.parse_args() + + while True: + main( + args.bigtable_instance, + args.bigtable_cluster, + float(args.high_cpu_threshold), + float(args.low_cpu_threshold), + int(args.short_sleep), + int(args.long_sleep)) diff --git a/bigtable/metricscaler/metricscaler_test.py b/bigtable/metricscaler/metricscaler_test.py new file mode 100644 index 00000000000..f4a18b8418b --- /dev/null +++ b/bigtable/metricscaler/metricscaler_test.py @@ -0,0 +1,89 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit and system tests for metricscaler.py""" + +import os +import time + +from google.cloud import bigtable +from mock import patch + +from metricscaler import get_cpu_load +from metricscaler import main +from metricscaler import scale_bigtable + +# tests assume instance and cluster have the same ID +BIGTABLE_INSTANCE = os.environ['BIGTABLE_CLUSTER'] +SIZE_CHANGE_STEP = 3 + +# System tests to verify API calls succeed + + +def test_get_cpu_load(): + assert float(get_cpu_load()) > 0.0 + + +def test_scale_bigtable(): + bigtable_client = bigtable.Client(admin=True) + instance = bigtable_client.instance(BIGTABLE_INSTANCE) + instance.reload() + + cluster = instance.cluster(BIGTABLE_INSTANCE) + cluster.reload() + original_node_count = cluster.serve_nodes + + scale_bigtable(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, True) + + time.sleep(10) + cluster.reload() + + new_node_count = cluster.serve_nodes + assert (new_node_count == (original_node_count + SIZE_CHANGE_STEP)) + + scale_bigtable(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, False) + time.sleep(10) + cluster.reload() + final_node_count = cluster.serve_nodes + assert final_node_count == original_node_count + + +# Unit test for logic +@patch('time.sleep') +@patch('metricscaler.get_cpu_load') +@patch('metricscaler.scale_bigtable') +def test_main(scale_bigtable, get_cpu_load, sleep): + SHORT_SLEEP = 5 + LONG_SLEEP = 10 + get_cpu_load.return_value = 0.5 + + main(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, 0.6, 0.3, SHORT_SLEEP, + LONG_SLEEP) + scale_bigtable.assert_not_called() + scale_bigtable.reset_mock() + + get_cpu_load.return_value = 0.7 + main(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, 0.6, 0.3, SHORT_SLEEP, + LONG_SLEEP) + scale_bigtable.assert_called_once_with(BIGTABLE_INSTANCE, + BIGTABLE_INSTANCE, True) + scale_bigtable.reset_mock() + + get_cpu_load.return_value = 0.2 + main(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, 0.6, 0.3, SHORT_SLEEP, + LONG_SLEEP) + scale_bigtable.assert_called_once_with(BIGTABLE_INSTANCE, + BIGTABLE_INSTANCE, False) + + scale_bigtable.reset_mock() diff --git a/bigtable/metricscaler/requirements.txt b/bigtable/metricscaler/requirements.txt new file mode 100644 index 00000000000..e0e4b1ecda6 --- /dev/null +++ b/bigtable/metricscaler/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-bigtable==0.32.1 +google-cloud-monitoring==0.31.1 diff --git a/bigtable/quickstart/README.rst b/bigtable/quickstart/README.rst new file mode 100644 index 00000000000..c7ffabf8c3b --- /dev/null +++ b/bigtable/quickstart/README.rst @@ -0,0 +1,108 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Bigtable Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/quickstart/README.rst + + +This directory contains samples for Google Cloud Bigtable. `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's the same database that powers many core Google services, including Search, Analytics, Maps, and Gmail. + + + + +.. _Google Cloud Bigtable: https://cloud.google.com/bigtable/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Quickstart ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/quickstart/main.py,bigtable/quickstart/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python main.py + + usage: main.py [-h] [--table TABLE] project_id instance_id + + positional arguments: + project_id Your Cloud Platform project ID. + instance_id ID of the Cloud Bigtable instance to connect to. + + optional arguments: + -h, --help show this help message and exit + --table TABLE Existing table used in the quickstart. (default: my-table) + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/bigtable/quickstart/README.rst.in b/bigtable/quickstart/README.rst.in new file mode 100644 index 00000000000..94f070a7c88 --- /dev/null +++ b/bigtable/quickstart/README.rst.in @@ -0,0 +1,23 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Bigtable + short_name: Cloud Bigtable + url: https://cloud.google.com/bigtable/docs + description: > + `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's + the same database that powers many core Google services, including Search, + Analytics, Maps, and Gmail. + +setup: +- auth +- install_deps + +samples: +- name: Quickstart + file: main.py + show_help: true + +cloud_client_library: true + +folder: bigtable/quickstart \ No newline at end of file diff --git a/bigtable/quickstart/main.py b/bigtable/quickstart/main.py new file mode 100644 index 00000000000..3763296f1e4 --- /dev/null +++ b/bigtable/quickstart/main.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python + +# Copyright 2018 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START bigtable_quickstart] +import argparse + +from google.cloud import bigtable + + +def main(project_id="project-id", instance_id="instance-id", + table_id="my-table"): + # Create a Cloud Bigtable client. + client = bigtable.Client(project=project_id) + + # Connect to an existing Cloud Bigtable instance. + instance = client.instance(instance_id) + + # Open an existing table. + table = instance.table(table_id) + + row_key = 'r1' + row = table.read_row(row_key.encode('utf-8')) + + column_family_id = 'cf1' + column_id = 'c1'.encode('utf-8') + value = row.cells[column_family_id][column_id][0].value.decode('utf-8') + + print('Row key: {}\nData: {}'.format(row_key, value)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('project_id', help='Your Cloud Platform project ID.') + parser.add_argument( + 'instance_id', help='ID of the Cloud Bigtable instance to connect to.') + parser.add_argument( + '--table', + help='Existing table used in the quickstart.', + default='my-table') + + args = parser.parse_args() + main(args.project_id, args.instance_id, args.table) +# [END bigtable_quickstart] diff --git a/bigtable/quickstart/main_test.py b/bigtable/quickstart/main_test.py new file mode 100644 index 00000000000..0c745ea8cd0 --- /dev/null +++ b/bigtable/quickstart/main_test.py @@ -0,0 +1,28 @@ +# Copyright 2018 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from main import main + +PROJECT = os.environ['GCLOUD_PROJECT'] +BIGTABLE_CLUSTER = os.environ['BIGTABLE_CLUSTER'] +TABLE_NAME = 'my-table' + + +def test_main(capsys): + main(PROJECT, BIGTABLE_CLUSTER, TABLE_NAME) + + out, _ = capsys.readouterr() + assert 'Row key: r1\nData: test-value\n' in out diff --git a/bigtable/quickstart/requirements.txt b/bigtable/quickstart/requirements.txt new file mode 100644 index 00000000000..f0aed1e60d3 --- /dev/null +++ b/bigtable/quickstart/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-bigtable==0.32.1 +google-cloud-core==0.29.1 diff --git a/bigtable/quickstart_happybase/README.rst b/bigtable/quickstart_happybase/README.rst new file mode 100644 index 00000000000..e2d1c45a272 --- /dev/null +++ b/bigtable/quickstart_happybase/README.rst @@ -0,0 +1,108 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Bigtable Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/quickstart_happybase/README.rst + + +This directory contains samples for Google Cloud Bigtable. `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's the same database that powers many core Google services, including Search, Analytics, Maps, and Gmail. + + + + +.. _Google Cloud Bigtable: https://cloud.google.com/bigtable/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Quickstart ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/quickstart_happybase/main.py,bigtable/quickstart_happybase/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python main.py + + usage: main.py [-h] [--table TABLE] project_id instance_id + + positional arguments: + project_id Your Cloud Platform project ID. + instance_id ID of the Cloud Bigtable instance to connect to. + + optional arguments: + -h, --help show this help message and exit + --table TABLE Existing table used in the quickstart. (default: my-table) + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/bigtable/quickstart_happybase/README.rst.in b/bigtable/quickstart_happybase/README.rst.in new file mode 100644 index 00000000000..811a0b868fb --- /dev/null +++ b/bigtable/quickstart_happybase/README.rst.in @@ -0,0 +1,23 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Bigtable + short_name: Cloud Bigtable + url: https://cloud.google.com/bigtable/docs + description: > + `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's + the same database that powers many core Google services, including Search, + Analytics, Maps, and Gmail. + +setup: +- auth +- install_deps + +samples: +- name: Quickstart + file: main.py + show_help: true + +cloud_client_library: true + +folder: bigtable/quickstart_happybase \ No newline at end of file diff --git a/bigtable/quickstart_happybase/main.py b/bigtable/quickstart_happybase/main.py new file mode 100644 index 00000000000..cdfc7720503 --- /dev/null +++ b/bigtable/quickstart_happybase/main.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python + +# Copyright 2018 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# [START bigtable_quickstart_happybase] +import argparse +import json + +from google.cloud import bigtable +from google.cloud import happybase + + +def main(project_id="project-id", instance_id="instance-id", + table_id="my-table"): + # Creates a Bigtable client + client = bigtable.Client(project=project_id) + + # Connect to an existing instance:my-bigtable-instance + instance = client.instance(instance_id) + + connection = happybase.Connection(instance=instance) + + try: + # Connect to an existing table:my-table + table = connection.table(table_id) + + key = 'r1' + row = table.row(key.encode('utf-8')) + value = {k.decode("utf-8"): v.decode("utf-8") for k, v in row.items()} + print('Row key: {}\nData: {}'.format(key, json.dumps(value, indent=4, + sort_keys=True))) + + finally: + connection.close() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('project_id', help='Your Cloud Platform project ID.') + parser.add_argument( + 'instance_id', help='ID of the Cloud Bigtable instance to connect to.') + parser.add_argument( + '--table', + help='Existing table used in the quickstart.', + default='my-table') + + args = parser.parse_args() + main(args.project_id, args.instance_id, args.table) +# [END bigtable_quickstart_happybase] diff --git a/bigtable/quickstart_happybase/main_test.py b/bigtable/quickstart_happybase/main_test.py new file mode 100644 index 00000000000..5f08c30b8b7 --- /dev/null +++ b/bigtable/quickstart_happybase/main_test.py @@ -0,0 +1,28 @@ +# Copyright 2018 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from main import main + +PROJECT = os.environ['GCLOUD_PROJECT'] +BIGTABLE_CLUSTER = os.environ['BIGTABLE_CLUSTER'] +TABLE_NAME = 'my-table' + + +def test_main(capsys): + main(PROJECT, BIGTABLE_CLUSTER, TABLE_NAME) + + out, _ = capsys.readouterr() + assert '"cf1:c1": "test-value"' in out diff --git a/bigtable/quickstart_happybase/requirements.txt b/bigtable/quickstart_happybase/requirements.txt new file mode 100644 index 00000000000..a667ebb828b --- /dev/null +++ b/bigtable/quickstart_happybase/requirements.txt @@ -0,0 +1 @@ +google-cloud-happybase==0.32.1 diff --git a/bigtable/tableadmin/README.rst b/bigtable/tableadmin/README.rst new file mode 100644 index 00000000000..f7f83d6d2a1 --- /dev/null +++ b/bigtable/tableadmin/README.rst @@ -0,0 +1,115 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Bigtable Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/hello/README.rst + + +This directory contains samples for Google Cloud Bigtable. `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's the same database that powers many core Google services, including Search, Analytics, Maps, and Gmail. + + + + +.. _Google Cloud Bigtable: https://cloud.google.com/bigtable/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Basic example ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=bigtable/hello/tableadmin.py,bigtable/hello/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python tableadmin.py + + usage: tableadmin.py [-h] [run] [delete] [--table TABLE] project_id instance_id + + Demonstrates how to connect to Cloud Bigtable and run some basic operations. + Prerequisites: - Create a Cloud Bigtable cluster. + https://cloud.google.com/bigtable/docs/creating-cluster - Set your Google + Application Default Credentials. + https://developers.google.com/identity/protocols/application-default- + credentials + + positional arguments: + project_id Your Cloud Platform project ID. + instance_id ID of the Cloud Bigtable instance to connect to. + + optional arguments: + -h, --help show this help message and exit + --table TABLE Table to create and destroy. (default: Hello-Bigtable) + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/bigtable/tableadmin/README.rst.in b/bigtable/tableadmin/README.rst.in new file mode 100644 index 00000000000..7fd37641969 --- /dev/null +++ b/bigtable/tableadmin/README.rst.in @@ -0,0 +1,23 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Bigtable and run some basic operations. + short_name: Cloud Bigtable + url: https://cloud.google.com/bigtable/docs + description: > + `Google Cloud Bigtable`_ is Google's NoSQL Big Data database service. It's + the same database that powers many core Google services, including Search, + Analytics, Maps, and Gmail. + +setup: +- auth +- install_deps + +samples: +- name: Basic example with Bigtable Column family and GC rules. + file: tableadmin.py + show_help: true + +cloud_client_library: true + +folder: bigtable/tableadmin \ No newline at end of file diff --git a/bigtable/tableadmin/requirements.txt b/bigtable/tableadmin/requirements.txt new file mode 100755 index 00000000000..e70d7d99e00 --- /dev/null +++ b/bigtable/tableadmin/requirements.txt @@ -0,0 +1 @@ +google-cloud-bigtable==0.32.1 diff --git a/bigtable/tableadmin/tableadmin.py b/bigtable/tableadmin/tableadmin.py new file mode 100644 index 00000000000..29551a7f390 --- /dev/null +++ b/bigtable/tableadmin/tableadmin.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python + +# Copyright 2018, Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Demonstrates how to connect to Cloud Bigtable and run some basic operations. +# http://www.apache.org/licenses/LICENSE-2.0 +Prerequisites: +- Create a Cloud Bigtable cluster. + https://cloud.google.com/bigtable/docs/creating-cluster +- Set your Google Application Default Credentials. + https://developers.google.com/identity/protocols/application-default-credentials + +Operations performed: +- Create a Cloud Bigtable table. +- List tables for a Cloud Bigtable instance. +- Print metadata of the newly created table. +- Create Column Families with different GC rules. + - GC Rules like: MaxAge, MaxVersions, Union, Intersection and Nested. +- Delete a Bigtable table. +""" + +import argparse +import datetime + +from google.cloud import bigtable +from google.cloud.bigtable import column_family + + +def create_table(project_id, instance_id, table_id): + ''' Create a Bigtable table + + :type project_id: str + :param project_id: Project id of the client. + + :type instance_id: str + :param instance_id: Instance of the client. + + :type table_id: str + :param table_id: Table id to create table. + ''' + + client = bigtable.Client(project=project_id, admin=True) + instance = client.instance(instance_id) + table = instance.table(table_id) + + # Check whether table exists in an instance. + # Create table if it does not exists. + print('Checking if table {} exists...'.format(table_id)) + if table.exists(): + print('Table {} already exists.'.format(table_id)) + else: + print('Creating the {} table.'.format(table_id)) + table.create() + print('Created table {}.'.format(table_id)) + + return client, instance, table + + +def run_table_operations(project_id, instance_id, table_id): + ''' Create a Bigtable table and perform basic operations on it + + :type project_id: str + :param project_id: Project id of the client. + + :type instance_id: str + :param instance_id: Instance of the client. + + :type table_id: str + :param table_id: Table id to create table. + ''' + + client, instance, table = create_table(project_id, instance_id, table_id) + + # [START bigtable_list_tables] + tables = instance.list_tables() + print('Listing tables in current project...') + if tables != []: + for tbl in tables: + print(tbl.table_id) + else: + print('No table exists in current project...') + # [END bigtable_list_tables] + + # [START bigtable_create_family_gc_max_age] + print('Creating column family cf1 with with MaxAge GC Rule...') + # Create a column family with GC policy : maximum age + # where age = current time minus cell timestamp + + # Define the GC rule to retain data with max age of 5 days + max_age_rule = column_family.MaxAgeGCRule(datetime.timedelta(days=5)) + + column_family1 = table.column_family('cf1', max_age_rule) + column_family1.create() + print('Created column family cf1 with MaxAge GC Rule.') + # [END bigtable_create_family_gc_max_age] + + # [START bigtable_create_family_gc_max_versions] + print('Creating column family cf2 with max versions GC rule...') + # Create a column family with GC policy : most recent N versions + # where 1 = most recent version + + # Define the GC policy to retain only the most recent 2 versions + max_versions_rule = column_family.MaxVersionsGCRule(2) + + column_family2 = table.column_family('cf2', max_versions_rule) + column_family2.create() + print('Created column family cf2 with Max Versions GC Rule.') + # [END bigtable_create_family_gc_max_versions] + + # [START bigtable_create_family_gc_union] + print('Creating column family cf3 with union GC rule...') + # Create a column family with GC policy to drop data that matches + # at least one condition. + # Define a GC rule to drop cells older than 5 days or not the + # most recent version + union_rule = column_family.GCRuleUnion([ + column_family.MaxAgeGCRule(datetime.timedelta(days=5)), + column_family.MaxVersionsGCRule(2)]) + + column_family3 = table.column_family('cf3', union_rule) + column_family3.create() + print('Created column family cf3 with Union GC rule') + # [END bigtable_create_family_gc_union] + + # [START bigtable_create_family_gc_intersection] + print('Creating column family cf4 with Intersection GC rule...') + # Create a column family with GC policy to drop data that matches + # all conditions + # GC rule: Drop cells older than 5 days AND older than the most + # recent 2 versions + intersection_rule = column_family.GCRuleIntersection([ + column_family.MaxAgeGCRule(datetime.timedelta(days=5)), + column_family.MaxVersionsGCRule(2)]) + + column_family4 = table.column_family('cf4', intersection_rule) + column_family4.create() + print('Created column family cf4 with Intersection GC rule.') + # [END bigtable_create_family_gc_intersection] + + # [START bigtable_create_family_gc_nested] + print('Creating column family cf5 with a Nested GC rule...') + # Create a column family with nested GC policies. + # Create a nested GC rule: + # Drop cells that are either older than the 10 recent versions + # OR + # Drop cells that are older than a month AND older than the + # 2 recent versions + rule1 = column_family.MaxVersionsGCRule(10) + rule2 = column_family.GCRuleIntersection([ + column_family.MaxAgeGCRule(datetime.timedelta(days=30)), + column_family.MaxVersionsGCRule(2)]) + + nested_rule = column_family.GCRuleUnion([rule1, rule2]) + + column_family5 = table.column_family('cf5', nested_rule) + column_family5.create() + print('Created column family cf5 with a Nested GC rule.') + # [END bigtable_create_family_gc_nested] + + # [START bigtable_list_column_families] + print('Printing Column Family and GC Rule for all column families...') + column_families = table.list_column_families() + for column_family_name, gc_rule in sorted(column_families.items()): + print('Column Family:', column_family_name) + print('GC Rule:') + print(gc_rule.to_pb()) + # Sample output: + # Column Family: cf4 + # GC Rule: + # gc_rule { + # intersection { + # rules { + # max_age { + # seconds: 432000 + # } + # } + # rules { + # max_num_versions: 2 + # } + # } + # } + # [END bigtable_list_column_families] + + print('Print column family cf1 GC rule before update...') + print('Column Family: cf1') + print(column_family1.to_pb()) + + # [START bigtable_update_gc_rule] + print('Updating column family cf1 GC rule...') + # Update the column family cf1 to update the GC rule + column_family1 = table.column_family( + 'cf1', + column_family.MaxVersionsGCRule(1)) + column_family1.update() + print('Updated column family cf1 GC rule\n') + # [END bigtable_update_gc_rule] + + print('Print column family cf1 GC rule after update...') + print('Column Family: cf1') + print(column_family1.to_pb()) + + # [START bigtable_delete_family] + print('Delete a column family cf2...') + # Delete a column family + column_family2.delete() + print('Column family cf2 deleted successfully.') + # [END bigtable_delete_family] + + print('execute command "python tableadmin.py delete [project_id] \ + [instance_id] --table [tableName]" to delete the table.') + + +def delete_table(project_id, instance_id, table_id): + ''' Delete bigtable. + + :type project_id: str + :param project_id: Project id of the client. + + :type instance_id: str + :param instance_id: Instance of the client. + + :type table_id: str + :param table_id: Table id to create table. + ''' + + client = bigtable.Client(project=project_id, admin=True) + instance = client.instance(instance_id) + table = instance.table(table_id) + + # [START bigtable_delete_table] + # Delete the entire table + + print('Checking if table {} exists...'.format(table_id)) + if table.exists(): + print('Table {} exists.'.format(table_id)) + print('Deleting {} table.'.format(table_id)) + table.delete() + print('Deleted {} table.'.format(table_id)) + else: + print('Table {} does not exists.'.format(table_id)) + # [END bigtable_delete_table] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + parser.add_argument('command', + help='run or delete. \ + Operation to perform on table.') + parser.add_argument( + '--table', + help='Cloud Bigtable Table name.', + default='Hello-Bigtable') + + parser.add_argument('project_id', + help='Your Cloud Platform project ID.') + parser.add_argument( + 'instance_id', + help='ID of the Cloud Bigtable instance to connect to.') + + args = parser.parse_args() + + if args.command.lower() == 'run': + run_table_operations(args.project_id, args.instance_id, + args.table) + elif args.command.lower() == 'delete': + delete_table(args.project_id, args.instance_id, args.table) + else: + print('Command should be either run or delete.\n Use argument -h,\ + --help to show help and exit.') diff --git a/bigtable/tableadmin/tableadmin_test.py b/bigtable/tableadmin/tableadmin_test.py new file mode 100755 index 00000000000..c5852ed77f2 --- /dev/null +++ b/bigtable/tableadmin/tableadmin_test.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python + +# Copyright 2018, Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import random + +from tableadmin import create_table +from tableadmin import delete_table +from tableadmin import run_table_operations + +PROJECT = os.environ['GCLOUD_PROJECT'] +BIGTABLE_CLUSTER = os.environ['BIGTABLE_CLUSTER'] +TABLE_NAME_FORMAT = 'hello-bigtable-system-tests-{}' +TABLE_NAME_RANGE = 10000 + + +def test_run_table_operations(capsys): + table_name = TABLE_NAME_FORMAT.format( + random.randrange(TABLE_NAME_RANGE)) + + run_table_operations(PROJECT, BIGTABLE_CLUSTER, table_name) + out, _ = capsys.readouterr() + + assert 'Creating the ' + table_name + ' table.' in out + assert 'Listing tables in current project.' in out + assert 'Creating column family cf1 with with MaxAge GC Rule' in out + assert 'Created column family cf1 with MaxAge GC Rule.' in out + assert 'Created column family cf2 with Max Versions GC Rule.' in out + assert 'Created column family cf3 with Union GC rule' in out + assert 'Created column family cf4 with Intersection GC rule.' in out + assert 'Created column family cf5 with a Nested GC rule.' in out + assert 'Printing Column Family and GC Rule for all column families.' in out + assert 'Updating column family cf1 GC rule...' in out + assert 'Updated column family cf1 GC rule' in out + assert 'Print column family cf1 GC rule after update...' in out + assert 'Column Family: cf1' in out + assert 'max_num_versions: 1' in out + assert 'Delete a column family cf2...' in out + assert 'Column family cf2 deleted successfully.' in out + + +def test_delete_table(capsys): + table_name = TABLE_NAME_FORMAT.format( + random.randrange(TABLE_NAME_RANGE)) + create_table(PROJECT, BIGTABLE_CLUSTER, table_name) + + delete_table(PROJECT, BIGTABLE_CLUSTER, table_name) + out, _ = capsys.readouterr() + + assert 'Table ' + table_name + ' exists.' in out + assert 'Deleting ' + table_name + ' table.' in out + assert 'Deleted ' + table_name + ' table.' in out diff --git a/blog/README.md b/blog/README.md index 90eb34d1922..f049214024c 100644 --- a/blog/README.md +++ b/blog/README.md @@ -1,5 +1,10 @@ # Blog Sample Code +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=blog/README.md + This directory contains samples used in the [Cloud Platform Blog](http://cloud.google.com/blog). Each sample should have a readme with instructions and a link to its respective blog post. diff --git a/blog/introduction_to_data_models_in_cloud_datastore/README.md b/blog/introduction_to_data_models_in_cloud_datastore/README.md index 1a05cc43036..e21c418ad71 100644 --- a/blog/introduction_to_data_models_in_cloud_datastore/README.md +++ b/blog/introduction_to_data_models_in_cloud_datastore/README.md @@ -1,5 +1,10 @@ # Introduction to data models in Cloud Datastore +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=blog/introduction_to_data_models_in_cloud_datastore/README.md + This sample code is used in [this blog post](http://googlecloudplatform.blogspot.com/2015/08/Introduction-to-data-models-in-Cloud-Datastore.html). It demonstrates two data models using [Google Cloud Datastore](https://cloud.google.com/datastore). diff --git a/blog/introduction_to_data_models_in_cloud_datastore/blog.py b/blog/introduction_to_data_models_in_cloud_datastore/blog.py index f3b0d54c0b8..bb64ebbd0eb 100644 --- a/blog/introduction_to_data_models_in_cloud_datastore/blog.py +++ b/blog/introduction_to_data_models_in_cloud_datastore/blog.py @@ -14,7 +14,7 @@ import argparse import datetime -from gcloud import datastore +from google.cloud import datastore def path_to_key(datastore, path): diff --git a/blog/introduction_to_data_models_in_cloud_datastore/blog_test.py b/blog/introduction_to_data_models_in_cloud_datastore/blog_test.py index 16f41c883ec..3ed52eada50 100644 --- a/blog/introduction_to_data_models_in_cloud_datastore/blog_test.py +++ b/blog/introduction_to_data_models_in_cloud_datastore/blog_test.py @@ -11,10 +11,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os + +from gcp_devrel.testing.flaky import flaky + from blog import main -from gcp.testing.flaky import flaky + +PROJECT = os.environ['GCLOUD_PROJECT'] @flaky -def test_main(cloud_config): - main(cloud_config.project) +def test_main(): + main(PROJECT) diff --git a/blog/introduction_to_data_models_in_cloud_datastore/requirements.txt b/blog/introduction_to_data_models_in_cloud_datastore/requirements.txt index b40295d1af9..ff31c9011c4 100644 --- a/blog/introduction_to_data_models_in_cloud_datastore/requirements.txt +++ b/blog/introduction_to_data_models_in_cloud_datastore/requirements.txt @@ -1 +1 @@ -gcloud==0.10.1 +google-cloud-datastore==1.7.3 diff --git a/blog/introduction_to_data_models_in_cloud_datastore/wiki.py b/blog/introduction_to_data_models_in_cloud_datastore/wiki.py index 28f12e4a352..160b58d16a8 100644 --- a/blog/introduction_to_data_models_in_cloud_datastore/wiki.py +++ b/blog/introduction_to_data_models_in_cloud_datastore/wiki.py @@ -14,7 +14,7 @@ import argparse import datetime -from gcloud import datastore +from google.cloud import datastore def path_to_key(datastore, path): diff --git a/blog/introduction_to_data_models_in_cloud_datastore/wiki_test.py b/blog/introduction_to_data_models_in_cloud_datastore/wiki_test.py index 7db45420ebe..3f70e5c5be9 100644 --- a/blog/introduction_to_data_models_in_cloud_datastore/wiki_test.py +++ b/blog/introduction_to_data_models_in_cloud_datastore/wiki_test.py @@ -11,10 +11,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -from gcp.testing.flaky import flaky +import os + +from gcp_devrel.testing.flaky import flaky + from wiki import main +PROJECT = os.environ['GCLOUD_PROJECT'] + @flaky -def test_main(cloud_config): - main(cloud_config.project) +def test_main(): + main(PROJECT) diff --git a/cdn/README.rst b/cdn/README.rst new file mode 100644 index 00000000000..0bb6537d883 --- /dev/null +++ b/cdn/README.rst @@ -0,0 +1,55 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud CDN Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=cdn/README.rst + + +This directory contains samples for Google Cloud CDN. 'Google Cloud CDN'_ enables low-latency, low-cost content delivery using Google's global network + + + + +.. _Google Cloud CDN: https://cloud.google.com/cdn/docs + + +Samples +------------------------------------------------------------------------------- + +Snippets ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=cdn/snippets.py,cdn/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python snippets.py + + usage: snippets.py [-h] {sign-url} ... + + This application demonstrates how to perform operations on data (content) + when using Google Cloud CDN (Content Delivery Network). + + For more information, see the README.md under /cdn and the documentation + at https://cloud.google.com/cdn/docs. + + positional arguments: + {sign-url} + sign-url Sign a URL to grant temporary authorized access. + + optional arguments: + -h, --help show this help message and exit + + + + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/cdn/README.rst.in b/cdn/README.rst.in new file mode 100644 index 00000000000..f9c47171edc --- /dev/null +++ b/cdn/README.rst.in @@ -0,0 +1,16 @@ +# Used to generate README.rst + +product: + name: Google Cloud CDN + short_name: Cloud CDN + url: https://cloud.google.com/cdn/docs + description: > + 'Google Cloud CDN'_ enables low-latency, low-cost content delivery using + Google's global network + +samples: + - name: Snippets + file: snippets.py + show_help: true + +folder: cdn diff --git a/cdn/requirements.txt b/cdn/requirements.txt new file mode 100644 index 00000000000..b1013c19345 --- /dev/null +++ b/cdn/requirements.txt @@ -0,0 +1 @@ +six==1.12.0 diff --git a/cdn/snippets.py b/cdn/snippets.py new file mode 100644 index 00000000000..027b5cb04f9 --- /dev/null +++ b/cdn/snippets.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# +# Copyright 2017 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to perform operations on data (content) +when using Google Cloud CDN (Content Delivery Network). + +For more information, see the README.md under /cdn and the documentation +at https://cloud.google.com/cdn/docs. +""" + +import argparse +import base64 +import datetime +import hashlib +import hmac + +from six.moves import urllib + + +# [START sign_url] +def sign_url(url, key_name, base64_key, expiration_time): + """Gets the Signed URL string for the specified URL and configuration. + + Args: + url: URL to sign as a string. + key_name: name of the signing key as a string. + base64_key: signing key as a base64 encoded string. + expiration_time: expiration time as a UTC datetime object. + + Returns: + Returns the Signed URL appended with the query parameters based on the + specified configuration. + """ + stripped_url = url.strip() + parsed_url = urllib.parse.urlsplit(stripped_url) + query_params = urllib.parse.parse_qs( + parsed_url.query, keep_blank_values=True) + epoch = datetime.datetime.utcfromtimestamp(0) + expiration_timestamp = int((expiration_time - epoch).total_seconds()) + decoded_key = base64.urlsafe_b64decode(base64_key) + + url_pattern = u'{url}{separator}Expires={expires}&KeyName={key_name}' + + url_to_sign = url_pattern.format( + url=stripped_url, + separator='&' if query_params else '?', + expires=expiration_timestamp, + key_name=key_name) + + digest = hmac.new( + decoded_key, url_to_sign.encode('utf-8'), hashlib.sha1).digest() + signature = base64.urlsafe_b64encode(digest).decode('utf-8') + + signed_url = u'{url}&Signature={signature}'.format( + url=url_to_sign, signature=signature) + + print(signed_url) +# [END sign_url] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + + subparsers = parser.add_subparsers(dest='command') + + sign_url_parser = subparsers.add_parser( + 'sign-url', + help="Sign a URL to grant temporary authorized access.") + sign_url_parser.add_argument( + 'url', help='The URL to sign.') + sign_url_parser.add_argument( + 'key_name', + help='Key name for the signing key.') + sign_url_parser.add_argument( + 'base64_key', + help='The base64 encoded signing key.') + sign_url_parser.add_argument( + 'expiration_time', + type=lambda d: datetime.datetime.utcfromtimestamp(float(d)), + help='Expiration time expessed as seconds since the epoch.') + + args = parser.parse_args() + + if args.command == 'sign-url': + sign_url( + args.url, args.key_name, args.base64_key, args.expiration_time) diff --git a/cdn/snippets_test.py b/cdn/snippets_test.py new file mode 100644 index 00000000000..245daea13cd --- /dev/null +++ b/cdn/snippets_test.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +# +# Copyright 2017 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for snippets.""" + +import datetime + +import snippets + + +def test_sign_url(capsys): + snippets.sign_url( + 'http://35.186.234.33/index.html', + 'my-key', + 'nZtRohdNF9m3cKM24IcK4w==', + datetime.datetime.utcfromtimestamp(1549751401)) + snippets.sign_url( + 'http://www.example.com/', + 'my-key', + 'nZtRohdNF9m3cKM24IcK4w==', + datetime.datetime.utcfromtimestamp(1549751401)) + snippets.sign_url( + 'http://www.example.com/some/path?some=query&another=param', + 'my-key', + 'nZtRohdNF9m3cKM24IcK4w==', + datetime.datetime.utcfromtimestamp(1549751401)) + + out, _ = capsys.readouterr() + + results = out.splitlines() + assert results[0] == ( + 'http://35.186.234.33/index.html?Expires=1549751401&KeyName=my-key&' + 'Signature=CRFqQnVfFyiUyR63OQf-HRUpIwc=') + assert results[1] == ( + 'http://www.example.com/?Expires=1549751401&KeyName=my-key&' + 'Signature=OqDUFfHpN5Vxga6r80bhsgxKves=') + assert results[2] == ( + 'http://www.example.com/some/path?some=query&another=param&Expires=' + '1549751401&KeyName=my-key&Signature=9Q9TCxSju8-W5nUkk5CuTrun2_o=') diff --git a/cloud-sql/mysql/sqlalchemy/README.md b/cloud-sql/mysql/sqlalchemy/README.md new file mode 100644 index 00000000000..c4dfa654fed --- /dev/null +++ b/cloud-sql/mysql/sqlalchemy/README.md @@ -0,0 +1,70 @@ +# Connecting to Cloud SQL - MySQL + +## Before you begin + +1. If you haven't already, set up a Python Development Environment by following the [python setup guide](https://cloud.google.com/python/setup) and +[create a project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#creating_a_project). + +1. Create a 2nd Gen Cloud SQL Instance by following these +[instructions](https://cloud.google.com/sql/docs/mysql/create-instance). Note the connection string, +database user, and database password that you create. + +1. Create a database for your application by following these +[instructions](https://cloud.google.com/sql/docs/mysql/create-manage-databases). Note the database +name. + +1. Create a service account with the 'Cloud SQL Client' permissions by following these +[instructions](https://cloud.google.com/sql/docs/mysql/connect-external-app#4_if_required_by_your_authentication_method_create_a_service_account). +Download a JSON key to use to authenticate your connection. + +1. Use the information noted in the previous steps: +```bash +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service/account/key.json +export CLOUD_SQL_CONNECTION_NAME='::' +export DB_USER='my-db-user' +export DB_PASS='my-db-pass' +export DB_NAME='my_db' +``` +Note: Saving credentials in environment variables is convenient, but not secure - consider a more +secure solution such as [Cloud KMS](https://cloud.google.com/kms/) to help keep secrets safe. + +## Running locally + +To run this application locally, download and install the `cloud_sql_proxy` by +following the instructions [here](https://cloud.google.com/sql/docs/mysql/sql-proxy#install). + +Once the proxy is ready, use the following command to start the proxy in the +background: +```bash +./cloud_sql_proxy -dir=/cloudsql --instances=$CLOUD_SQL_CONNECTION_NAME --credential_file=$GOOGLE_APPLICATION_CREDENTIALS +``` +Note: Make sure to run the command under a user with write access in the +`/cloudsql` directory. This proxy will use this folder to create a unix socket +the application will use to connect to Cloud SQL. + +Next, setup install the requirements into a virtual enviroment: +```bash +virtualenv --python python3 env +source env/bin/activate +pip install -r requirements.txt +``` + +Finally, start the application: +```bash +python main.py +``` + +Navigate towards `http://127.0.0.1:8080` to verify your application is running correctly. + +## Google App Engine Standard + +To run on GAE-Standard, create an App Engine project by following the setup for these +[instructions](https://cloud.google.com/appengine/docs/standard/python3/quickstart#before-you-begin). + +First, update `app.yaml` with the correct values to pass the environment +variables into the runtime. + +Next, the following command will deploy the application to your Google Cloud project: +```bash +gcloud app deploy +``` diff --git a/cloud-sql/mysql/sqlalchemy/app.yaml b/cloud-sql/mysql/sqlalchemy/app.yaml new file mode 100644 index 00000000000..b6959445df6 --- /dev/null +++ b/cloud-sql/mysql/sqlalchemy/app.yaml @@ -0,0 +1,23 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +runtime: python37 + +# Remember - storing secrets in plaintext is potentially unsafe. Consider using +# something like https://cloud.google.com/kms/ to help keep secrets secret. +env_variables: + CLOUD_SQL_INSTANCE_NAME: :: + DB_USER: my-db-user + DB_PASS: my-db-pass + DB_NAME: my_db diff --git a/cloud-sql/mysql/sqlalchemy/main.py b/cloud-sql/mysql/sqlalchemy/main.py new file mode 100644 index 00000000000..57a5e6fbd4a --- /dev/null +++ b/cloud-sql/mysql/sqlalchemy/main.py @@ -0,0 +1,174 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import logging +import os + +from flask import Flask, render_template, request, Response +import sqlalchemy + + +# Remember - storing secrets in plaintext is potentially unsafe. Consider using +# something like https://cloud.google.com/kms/ to help keep secrets secret. +db_user = os.environ.get("DB_USER") +db_pass = os.environ.get("DB_PASS") +db_name = os.environ.get("DB_NAME") +cloud_sql_connection_name = os.environ.get("CLOUD_SQL_CONNECTION_NAME") + +app = Flask(__name__) + +logger = logging.getLogger() + +# [START cloud_sql_mysql_sqlalchemy_create] +# The SQLAlchemy engine will help manage interactions, including automatically +# managing a pool of connections to your database +db = sqlalchemy.create_engine( + # Equivalent URL: + # mysql+pymysql://:@/?unix_socket=/cloudsql/ + sqlalchemy.engine.url.URL( + drivername='mysql+pymysql', + username=db_user, + password=db_pass, + database=db_name, + query={ + 'unix_socket': '/cloudsql/{}'.format(cloud_sql_connection_name) + } + ), + # ... Specify additional properties here. + # [START_EXCLUDE] + + # [START cloud_sql_mysql_sqlalchemy_limit] + # Pool size is the maximum number of permanent connections to keep. + pool_size=5, + # Temporarily exceeds the set pool_size if no connections are available. + max_overflow=2, + # The total number of concurrent connections for your application will be + # a total of pool_size and max_overflow. + # [END cloud_sql_mysql_sqlalchemy_limit] + + # [START cloud_sql_mysql_sqlalchemy_backoff] + # SQLAlchemy automatically uses delays between failed connection attempts, + # but provides no arguments for configuration. + # [END cloud_sql_mysql_sqlalchemy_backoff] + + # [START cloud_sql_mysql_sqlalchemy_timeout] + # 'pool_timeout' is the maximum number of seconds to wait when retrieving a + # new connection from the pool. After the specified amount of time, an + # exception will be thrown. + pool_timeout=30, # 30 seconds + # [END cloud_sql_mysql_sqlalchemy_timeout] + + # [START cloud_sql_mysql_sqlalchemy_lifetime] + # 'pool_recycle' is the maximum number of seconds a connection can persist. + # Connections that live longer than the specified amount of time will be + # reestablished + pool_recycle=1800, # 30 minutes + # [END cloud_sql_mysql_sqlalchemy_lifetime] + + # [END_EXCLUDE] +) +# [END cloud_sql_mysql_sqlalchemy_create] + + +@app.before_first_request +def create_tables(): + # Create tables (if they don't already exist) + with db.connect() as conn: + conn.execute( + "CREATE TABLE IF NOT EXISTS votes " + "( vote_id SERIAL NOT NULL, time_cast timestamp NOT NULL, " + "candidate CHAR(6) NOT NULL, PRIMARY KEY (vote_id) );" + ) + + +@app.route('/', methods=['GET']) +def index(): + votes = [] + with db.connect() as conn: + # Execute the query and fetch all results + recent_votes = conn.execute( + "SELECT candidate, time_cast FROM votes " + "ORDER BY time_cast DESC LIMIT 5" + ).fetchall() + # Convert the results into a list of dicts representing votes + for row in recent_votes: + votes.append({ + 'candidate': row[0], + 'time_cast': row[1] + }) + + stmt = sqlalchemy.text( + "SELECT COUNT(vote_id) FROM votes WHERE candidate=:candidate") + # Count number of votes for tabs + tab_result = conn.execute(stmt, candidate="TABS").fetchone() + tab_count = tab_result[0] + # Count number of votes for spaces + space_result = conn.execute(stmt, candidate="SPACES").fetchone() + space_count = space_result[0] + + return render_template( + 'index.html', + recent_votes=votes, + tab_count=tab_count, + space_count=space_count + ) + + +@app.route('/', methods=['POST']) +def save_vote(): + # Get the team and time the vote was cast. + team = request.form['team'] + time_cast = datetime.datetime.utcnow() + # Verify that the team is one of the allowed options + if team != "TABS" and team != "SPACES": + logger.warning(team) + return Response( + response="Invalid team specified.", + status=400 + ) + + # [START cloud_sql_mysql_sqlalchemy_connection] + # Preparing a statement before hand can help protect against injections. + stmt = sqlalchemy.text( + "INSERT INTO votes (time_cast, candidate)" + " VALUES (:time_cast, :candidate)" + ) + try: + # Using a with statement ensures that the connection is always released + # back into the pool at the end of statement (even if an error occurs) + with db.connect() as conn: + conn.execute(stmt, time_cast=time_cast, candidate=team) + except Exception as e: + # If something goes wrong, handle the error in this section. This might + # involve retrying or adjusting parameters depending on the situation. + # [START_EXCLUDE] + logger.exception(e) + return Response( + status=500, + response="Unable to successfully cast vote! Please check the " + "application logs for more details." + ) + # [END_EXCLUDE] + # [END cloud_sql_mysql_sqlalchemy_connection] + + return Response( + status=200, + response="Vote successfully cast for '{}' at time {}!".format( + team, time_cast) + ) + + +if __name__ == '__main__': + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/cloud-sql/mysql/sqlalchemy/requirements.txt b/cloud-sql/mysql/sqlalchemy/requirements.txt new file mode 100644 index 00000000000..20a465bc898 --- /dev/null +++ b/cloud-sql/mysql/sqlalchemy/requirements.txt @@ -0,0 +1,3 @@ +Flask==1.0.2 +SQLAlchemy==1.2.17 +PyMySQL==0.9.3 diff --git a/cloud-sql/mysql/sqlalchemy/templates/index.html b/cloud-sql/mysql/sqlalchemy/templates/index.html new file mode 100644 index 00000000000..e390afaa5e2 --- /dev/null +++ b/cloud-sql/mysql/sqlalchemy/templates/index.html @@ -0,0 +1,100 @@ + + + + Tabs VS Spaces + + + + + + +
          +
          +

          + {% if tab_count == space_count %} + TABS and SPACES are evenly matched! + {% elif tab_count > space_count %} + TABS are winning by {{tab_count - space_count}} + {{'votes' if tab_count - space_count > 1 else 'vote'}}! + {% elif space_count > tab_count %} + SPACES are winning by {{space_count - tab_count}} + {{'votes' if space_count - tab_count > 1 else 'vote'}}! + {% endif %} +

          +
          +
          +
          +
          + keyboard_tab +

          {{tab_count}} votes

          + +
          +
          +
          +
          + space_bar +

          {{space_count}} votes

          + +
          +
          +
          +

          Recent Votes

          +
            + {% for vote in recent_votes %} +
          • + {% if vote.candidate == "TABS" %} + keyboard_tab + {% elif vote.candidate == "SPACES" %} + space_bar + {% endif %} + + A vote for {{vote.candidate}} + +

            was cast at {{vote.time_cast}}

            +
            • + {% endfor %} +
          +
          + + + diff --git a/cloud-sql/postgres/sqlalchemy/README.md b/cloud-sql/postgres/sqlalchemy/README.md new file mode 100644 index 00000000000..46f1afabfee --- /dev/null +++ b/cloud-sql/postgres/sqlalchemy/README.md @@ -0,0 +1,70 @@ +# Connecting to Cloud SQL - Postgres + +## Before you begin + +1. If you haven't already, set up a Python Development Environment by following the [python setup guide](https://cloud.google.com/python/setup) and +[create a project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#creating_a_project). + +1. Create a 2nd Gen Cloud SQL Instance by following these +[instructions](https://cloud.google.com/sql/docs/postgres/create-instance). Note the connection +string, database user, and database password that you create. + +1. Create a database for your application by following these +[instructions](https://cloud.google.com/sql/docs/postgres/create-manage-databases). Note the database +name. + +1. Create a service account with the 'Cloud SQL Client' permissions by following these +[instructions](https://cloud.google.com/sql/docs/postgres/connect-external-app#4_if_required_by_your_authentication_method_create_a_service_account). +Download a JSON key to use to authenticate your connection. + +1. Use the information noted in the previous steps: +```bash +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service/account/key.json +export CLOUD_SQL_CONNECTION_NAME='::' +export DB_USER='my-db-user' +export DB_PASS='my-db-pass' +export DB_NAME='my_db' +``` +Note: Saving credentials in environment variables is convenient, but not secure - consider a more +secure solution such as [Cloud KMS](https://cloud.google.com/kms/) to help keep secrets safe. + +## Running locally + +To run this application locally, download and install the `cloud_sql_proxy` by +following the instructions [here](https://cloud.google.com/sql/docs/mysql/sql-proxy#install). + +Once the proxy is ready, use the following command to start the proxy in the +background: +```bash +./cloud_sql_proxy -dir=/cloudsql --instances=$CLOUD_SQL_CONNECTION_NAME --credential_file=$GOOGLE_APPLICATION_CREDENTIALS +``` +Note: Make sure to run the command under a user with write access in the +`/cloudsql` directory. This proxy will use this folder to create a unix socket +the application will use to connect to Cloud SQL. + +Next, setup install the requirements into a virtual enviroment: +```bash +virtualenv --python python3 env +source env/bin/activate +pip install -r requirements.txt +``` + +Finally, start the application: +```bash +python main.py +``` + +Navigate towards `http://127.0.0.1:8080` to verify your application is running correctly. + +## Google App Engine Standard + +To run on GAE-Standard, create an App Engine project by following the setup for these +[instructions](https://cloud.google.com/appengine/docs/standard/python3/quickstart#before-you-begin). + +First, update `app.yaml` with the correct values to pass the environment +variables into the runtime. + +Next, the following command will deploy the application to your Google Cloud project: +```bash +gcloud app deploy +``` diff --git a/cloud-sql/postgres/sqlalchemy/app.yaml b/cloud-sql/postgres/sqlalchemy/app.yaml new file mode 100644 index 00000000000..b6959445df6 --- /dev/null +++ b/cloud-sql/postgres/sqlalchemy/app.yaml @@ -0,0 +1,23 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +runtime: python37 + +# Remember - storing secrets in plaintext is potentially unsafe. Consider using +# something like https://cloud.google.com/kms/ to help keep secrets secret. +env_variables: + CLOUD_SQL_INSTANCE_NAME: :: + DB_USER: my-db-user + DB_PASS: my-db-pass + DB_NAME: my_db diff --git a/cloud-sql/postgres/sqlalchemy/main.py b/cloud-sql/postgres/sqlalchemy/main.py new file mode 100644 index 00000000000..0f262476d3f --- /dev/null +++ b/cloud-sql/postgres/sqlalchemy/main.py @@ -0,0 +1,174 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import logging +import os + +from flask import Flask, render_template, request, Response +import sqlalchemy + + +# Remember - storing secrets in plaintext is potentially unsafe. Consider using +# something like https://cloud.google.com/kms/ to help keep secrets secret. +db_user = os.environ.get("DB_USER") +db_pass = os.environ.get("DB_PASS") +db_name = os.environ.get("DB_NAME") +cloud_sql_connection_name = os.environ.get("CLOUD_SQL_CONNECTION_NAME") + +app = Flask(__name__) + +logger = logging.getLogger() + +# [START cloud_sql_postgres_sqlalchemy_create] +# The SQLAlchemy engine will help manage interactions, including automatically +# managing a pool of connections to your database +db = sqlalchemy.create_engine( + # Equivalent URL: + # postgres+pg8000://:@/?unix_socket=/cloudsql/ + sqlalchemy.engine.url.URL( + drivername='postgres+pg8000', + username=db_user, + password=db_pass, + database=db_name, + query={ + 'unix_sock': '/cloudsql/{}'.format(cloud_sql_connection_name) + } + ), + # ... Specify additional properties here. + # [START_EXCLUDE] + + # [START cloud_sql_postgres_sqlalchemy_limit] + # Pool size is the maximum number of permanent connections to keep. + pool_size=5, + # Temporarily exceeds the set pool_size if no connections are available. + max_overflow=2, + # The total number of concurrent connections for your application will be + # a total of pool_size and max_overflow. + # [END cloud_sql_postgres_sqlalchemy_limit] + + # [START cloud_sql_postgres_sqlalchemy_backoff] + # SQLAlchemy automatically uses delays between failed connection attempts, + # but provides no arguments for configuration. + # [END cloud_sql_postgres_sqlalchemy_backoff] + + # [START cloud_sql_postgres_sqlalchemy_timeout] + # 'pool_timeout' is the maximum number of seconds to wait when retrieving a + # new connection from the pool. After the specified amount of time, an + # exception will be thrown. + pool_timeout=30, # 30 seconds + # [END cloud_sql_postgres_sqlalchemy_timeout] + + # [START cloud_sql_postgres_sqlalchemy_lifetime] + # 'pool_recycle' is the maximum number of seconds a connection can persist. + # Connections that live longer than the specified amount of time will be + # reestablished + pool_recycle=1800, # 30 minutes + # [END cloud_sql_postgres_sqlalchemy_lifetime] + + # [END_EXCLUDE] +) +# [END cloud_sql_postgres_sqlalchemy_create] + + +@app.before_first_request +def create_tables(): + # Create tables (if they don't already exist) + with db.connect() as conn: + conn.execute( + "CREATE TABLE IF NOT EXISTS votes " + "( vote_id SERIAL NOT NULL, time_cast timestamp NOT NULL, " + "candidate VARCHAR(6) NOT NULL, PRIMARY KEY (vote_id) );" + ) + + +@app.route('/', methods=['GET']) +def index(): + votes = [] + with db.connect() as conn: + # Execute the query and fetch all results + recent_votes = conn.execute( + "SELECT candidate, time_cast FROM votes " + "ORDER BY time_cast DESC LIMIT 5" + ).fetchall() + # Convert the results into a list of dicts representing votes + for row in recent_votes: + votes.append({ + 'candidate': row[0], + 'time_cast': row[1] + }) + + stmt = sqlalchemy.text( + "SELECT COUNT(vote_id) FROM votes WHERE candidate=:candidate") + # Count number of votes for tabs + tab_result = conn.execute(stmt, candidate="TABS").fetchone() + tab_count = tab_result[0] + # Count number of votes for spaces + space_result = conn.execute(stmt, candidate="SPACES").fetchone() + space_count = space_result[0] + + return render_template( + 'index.html', + recent_votes=votes, + tab_count=tab_count, + space_count=space_count + ) + + +@app.route('/', methods=['POST']) +def save_vote(): + # Get the team and time the vote was cast. + team = request.form['team'] + time_cast = datetime.datetime.utcnow() + # Verify that the team is one of the allowed options + if team != "TABS" and team != "SPACES": + logger.warning(team) + return Response( + response="Invalid team specified.", + status=400 + ) + + # [START cloud_sql_postgres_sqlalchemy_connection] + # Preparing a statement before hand can help protect against injections. + stmt = sqlalchemy.text( + "INSERT INTO votes (time_cast, candidate)" + " VALUES (:time_cast, :candidate)" + ) + try: + # Using a with statement ensures that the connection is always released + # back into the pool at the end of statement (even if an error occurs) + with db.connect() as conn: + conn.execute(stmt, time_cast=time_cast, candidate=team) + except Exception as e: + # If something goes wrong, handle the error in this section. This might + # involve retrying or adjusting parameters depending on the situation. + # [START_EXCLUDE] + logger.exception(e) + return Response( + status=500, + response="Unable to successfully cast vote! Please check the " + "application logs for more details." + ) + # [END_EXCLUDE] + # [END cloud_sql_postgres_sqlalchemy_connection] + + return Response( + status=200, + response="Vote successfully cast for '{}' at time {}!".format( + team, time_cast) + ) + + +if __name__ == '__main__': + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/cloud-sql/postgres/sqlalchemy/requirements.txt b/cloud-sql/postgres/sqlalchemy/requirements.txt new file mode 100644 index 00000000000..cb393e0dfa1 --- /dev/null +++ b/cloud-sql/postgres/sqlalchemy/requirements.txt @@ -0,0 +1,3 @@ +Flask==1.0.2 +SQLAlchemy==1.2.17 +pg8000==1.13.1 diff --git a/cloud-sql/postgres/sqlalchemy/templates/index.html b/cloud-sql/postgres/sqlalchemy/templates/index.html new file mode 100644 index 00000000000..e390afaa5e2 --- /dev/null +++ b/cloud-sql/postgres/sqlalchemy/templates/index.html @@ -0,0 +1,100 @@ + + + + Tabs VS Spaces + + + + + + +
          +
          +

          + {% if tab_count == space_count %} + TABS and SPACES are evenly matched! + {% elif tab_count > space_count %} + TABS are winning by {{tab_count - space_count}} + {{'votes' if tab_count - space_count > 1 else 'vote'}}! + {% elif space_count > tab_count %} + SPACES are winning by {{space_count - tab_count}} + {{'votes' if space_count - tab_count > 1 else 'vote'}}! + {% endif %} +

          +
          +
          +
          +
          + keyboard_tab +

          {{tab_count}} votes

          + +
          +
          +
          +
          + space_bar +

          {{space_count}} votes

          + +
          +
          +
          +

          Recent Votes

          +
            + {% for vote in recent_votes %} +
          • + {% if vote.candidate == "TABS" %} + keyboard_tab + {% elif vote.candidate == "SPACES" %} + space_bar + {% endif %} + + A vote for {{vote.candidate}} + +

            was cast at {{vote.time_cast}}

            +
            • + {% endfor %} +
          +
          + + + diff --git a/cloud_logging/README.md b/cloud_logging/README.md deleted file mode 100644 index f023b77ed2c..00000000000 --- a/cloud_logging/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# Google Cloud Logging Samples - -This section contains samples for [Google Cloud Logging](https://cloud.google.com/logging). - -## Running the samples - -1. Your environment must be setup with [authentication -information](https://developers.google.com/identity/protocols/application-default-credentials#howtheywork). If you're running in your local development environment and you have the [Google Cloud SDK](https://cloud.google.com/sdk/) installed, you can do this easily by running: - - $ gcloud init - -2. Install dependencies from `requirements.txt`: - - $ pip install -r requirements.txt - -3. Depending on the sample, you may also need to create resources on the [Google Developers Console](https://console.developers.google.com). Refer to the sample description and associated documentation page. - -## Additional resources - -For more information on Cloud Logging you can visit: - -> https://developers.google.com/logging - -For more information on the Clloud Logging API Python library surface you -can visit: - -> https://developers.google.com/resources/api-libraries/documentation/logging/v1beta3/python/latest/ - -For information on the Python Client Library visit: - -> https://developers.google.com/api-client-library/python diff --git a/cloud_logging/api/list_logs.py b/cloud_logging/api/list_logs.py deleted file mode 100644 index 0e0f180ce05..00000000000 --- a/cloud_logging/api/list_logs.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Command-line program to list the logs in a Google Cloud Platform project. - -This sample is used in this section of the documentation: - - https://cloud.google.com/logging/docs - -For more information, see the README.md under /cloud_logging. -""" - -# [START all] -import argparse - -from googleapiclient import discovery -from oauth2client.client import GoogleCredentials - - -# [START list_logs] -def list_logs(project_id, logging_service): - request = logging_service.projects().logs().list(projectsId=project_id) - - while request: - response = request.execute() - if not response: - print("No logs found in {0} project").format(project_id) - return False - for log in response['logs']: - print(log['name']) - - request = logging_service.projects().logs().list_next( - request, response) -# [END list_logs] - - -def main(project_id): - # [START build_service] - credentials = GoogleCredentials.get_application_default() - logging_service = discovery.build( - 'logging', 'v1beta3', credentials=credentials) - # [END build_service] - - list_logs(project_id, logging_service) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('project_id', help='Your Google Cloud project ID.') - - args = parser.parse_args() - - main(args.project_id) -# [END all] diff --git a/cloud_logging/api/list_logs_test.py b/cloud_logging/api/list_logs_test.py deleted file mode 100644 index f7f94cf84b1..00000000000 --- a/cloud_logging/api/list_logs_test.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -import list_logs - - -def test_main(cloud_config, capsys): - list_logs.main(cloud_config.project) - out, _ = capsys.readouterr() - assert re.search(re.compile(r'.*', re.S), out) diff --git a/cloud_logging/api/requirements.txt b/cloud_logging/api/requirements.txt deleted file mode 100644 index c3b2784ce87..00000000000 --- a/cloud_logging/api/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -google-api-python-client==1.5.0 diff --git a/codelabs/flex_and_vision/README.md b/codelabs/flex_and_vision/README.md new file mode 100644 index 00000000000..bbfc45e5d51 --- /dev/null +++ b/codelabs/flex_and_vision/README.md @@ -0,0 +1,111 @@ +# Python Google Cloud Vision sample for Google App Engine Flexible Environment + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=codelabs/flex_and_vision/README.md + +This sample demonstrates how to use the [Google Cloud Vision API](https://cloud.google.com/vision/), [Google Cloud Storage](https://cloud.google.com/storage/), and [Google Cloud Datastore](https://cloud.google.com/datastore/) on [Google App Engine Flexible Environment](https://cloud.google.com/appengine). + +## Setup + +Create a new project with the [Google Cloud Platform console](https://console.cloud.google.com/). +Make a note of your project ID, which may be different than your project name. + +Make sure to [Enable Billing](https://pantheon.corp.google.com/billing?debugUI=DEVELOPERS) +for your project. + +Download the [Google Cloud SDK](https://cloud.google.com/sdk/docs/) to your +local machine. Alternatively, you could use the [Cloud Shell](https://cloud.google.com/shell/docs/quickstart), which comes with the Google Cloud SDK pre-installed. + +Initialize the Google Cloud SDK (skip if using Cloud Shell): + + gcloud init + +Create your App Engine application: + + gcloud app create + +Set an environment variable for your project ID, replacing `[YOUR_PROJECT_ID]` +with your project ID: + + export PROJECT_ID=[YOUR_PROJECT_ID] + +## Getting the sample code + +Run the following command to clone the Github repository: + + git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +Change directory to the sample code location: + + cd python-docs-samples/codelabs/flex_and_vision + +## Authentication + +Enable the APIs: + + gcloud services enable vision.googleapis.com + gcloud services enable storage-component.googleapis.com + gcloud services enable datastore.googleapis.com + +Create a Service Account to access the Google Cloud APIs when testing locally: + + gcloud iam service-accounts create hackathon \ + --display-name "My Hackathon Service Account" + +Give your newly created Service Account appropriate permissions: + + gcloud projects add-iam-policy-binding ${PROJECT_ID} \ + --member serviceAccount:hackathon@${PROJECT_ID}.iam.gserviceaccount.com \ + --role roles/owner + +After creating your Service Account, create a Service Account key: + + gcloud iam service-accounts keys create ~/key.json --iam-account \ + hackathon@${PROJECT_ID}.iam.gserviceaccount.com + +Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to point to where +you just put your Service Account key: + + export GOOGLE_APPLICATION_CREDENTIALS="/home/${USER}/key.json" + +## Running locally + +Create a virtual environment and install dependencies: + + virtualenv -p python3 env + source env/bin/activate + pip install -r requirements.txt + +Create a Cloud Storage bucket. It is recommended that you name it the same as +your project ID: + + gsutil mb gs://${PROJECT_ID} + +Set the environment variable `CLOUD_STORAGE_BUCKET`: + + export CLOUD_STORAGE_BUCKET=${PROJECT_ID} + +Start your application locally: + + python main.py + +Visit `localhost:8080` to view your application running locally. Press `Control-C` +on your command line when you are finished. + +When you are ready to leave your virtual environment: + + deactivate + +## Deploying to App Engine + +Open `app.yaml` and replace with the name of your +Cloud Storage bucket. + +Deploy your application to App Engine using `gcloud`. Please note that this may +take several minutes. + + gcloud app deploy + +Visit `https://[YOUR_PROJECT_ID].appspot.com` to view your deployed application. diff --git a/codelabs/flex_and_vision/app.yaml b/codelabs/flex_and_vision/app.yaml new file mode 100644 index 00000000000..5a2efe3ed5d --- /dev/null +++ b/codelabs/flex_and_vision/app.yaml @@ -0,0 +1,9 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 + +env_variables: + CLOUD_STORAGE_BUCKET: diff --git a/codelabs/flex_and_vision/main.py b/codelabs/flex_and_vision/main.py new file mode 100644 index 00000000000..54242480c76 --- /dev/null +++ b/codelabs/flex_and_vision/main.py @@ -0,0 +1,128 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime +import logging +import os + +from flask import Flask, redirect, render_template, request + +from google.cloud import datastore +from google.cloud import storage +from google.cloud import vision + + +CLOUD_STORAGE_BUCKET = os.environ.get('CLOUD_STORAGE_BUCKET') + + +app = Flask(__name__) + + +@app.route('/') +def homepage(): + # Create a Cloud Datastore client. + datastore_client = datastore.Client() + + # Use the Cloud Datastore client to fetch information from Datastore about + # each photo. + query = datastore_client.query(kind='Faces') + image_entities = list(query.fetch()) + + # Return a Jinja2 HTML template and pass in image_entities as a parameter. + return render_template('homepage.html', image_entities=image_entities) + + +@app.route('/upload_photo', methods=['GET', 'POST']) +def upload_photo(): + photo = request.files['file'] + + # Create a Cloud Storage client. + storage_client = storage.Client() + + # Get the bucket that the file will be uploaded to. + bucket = storage_client.get_bucket(CLOUD_STORAGE_BUCKET) + + # Create a new blob and upload the file's content. + blob = bucket.blob(photo.filename) + blob.upload_from_string( + photo.read(), content_type=photo.content_type) + + # Make the blob publicly viewable. + blob.make_public() + + # Create a Cloud Vision client. + vision_client = vision.ImageAnnotatorClient() + + # Use the Cloud Vision client to detect a face for our image. + source_uri = 'gs://{}/{}'.format(CLOUD_STORAGE_BUCKET, blob.name) + image = vision.types.Image( + source=vision.types.ImageSource(gcs_image_uri=source_uri)) + faces = vision_client.face_detection(image).face_annotations + + # If a face is detected, save to Datastore the likelihood that the face + # displays 'joy,' as determined by Google's Machine Learning algorithm. + if len(faces) > 0: + face = faces[0] + + # Convert the likelihood string. + likelihoods = [ + 'Unknown', 'Very Unlikely', 'Unlikely', 'Possible', 'Likely', + 'Very Likely'] + face_joy = likelihoods[face.joy_likelihood] + else: + face_joy = 'Unknown' + + # Create a Cloud Datastore client. + datastore_client = datastore.Client() + + # Fetch the current date / time. + current_datetime = datetime.now() + + # The kind for the new entity. + kind = 'Faces' + + # The name/ID for the new entity. + name = blob.name + + # Create the Cloud Datastore key for the new entity. + key = datastore_client.key(kind, name) + + # Construct the new entity using the key. Set dictionary values for entity + # keys blob_name, storage_public_url, timestamp, and joy. + entity = datastore.Entity(key) + entity['blob_name'] = blob.name + entity['image_public_url'] = blob.public_url + entity['timestamp'] = current_datetime + entity['joy'] = face_joy + + # Save the new entity to Datastore. + datastore_client.put(entity) + + # Redirect to the home page. + return redirect('/') + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
          {}
          + See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/codelabs/flex_and_vision/main_test.py b/codelabs/flex_and_vision/main_test.py new file mode 100644 index 00000000000..c694656a6c3 --- /dev/null +++ b/codelabs/flex_and_vision/main_test.py @@ -0,0 +1,48 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import requests +import six + +import main + +TEST_PHOTO_URL = ( + 'https://upload.wikimedia.org/wikipedia/commons/5/5e/' + 'John_F._Kennedy%2C_White_House_photo_portrait%2C_looking_up.jpg') + + +@pytest.fixture +def app(): + main.app.testing = True + client = main.app.test_client() + return client + + +def test_index(app): + r = app.get('/') + assert r.status_code == 200 + + +def test_upload_photo(app): + test_photo_data = requests.get(TEST_PHOTO_URL).content + + r = app.post( + '/upload_photo', + data={ + 'file': (six.BytesIO(test_photo_data), 'flex_and_vision.jpg') + } + ) + + assert r.status_code == 302 diff --git a/codelabs/flex_and_vision/requirements.txt b/codelabs/flex_and_vision/requirements.txt new file mode 100644 index 00000000000..e3772e9b5d3 --- /dev/null +++ b/codelabs/flex_and_vision/requirements.txt @@ -0,0 +1,5 @@ +Flask==1.0.2 +gunicorn==19.9.0 +google-cloud-vision==0.35.2 +google-cloud-storage==1.13.2 +google-cloud-datastore==1.7.3 diff --git a/codelabs/flex_and_vision/templates/homepage.html b/codelabs/flex_and_vision/templates/homepage.html new file mode 100644 index 00000000000..5e36e051ef7 --- /dev/null +++ b/codelabs/flex_and_vision/templates/homepage.html @@ -0,0 +1,20 @@ +

          Google Cloud Platform - Face Detection Sample

          + +

          This Python Flask application demonstrates App Engine Flexible, Google Cloud +Storage, Datastore, and the Cloud Vision API.

          + +
          + + + +
          + Upload File:
          + +
          + {% for image_entity in image_entities %} + +

          {{image_entity['blob_name']}} was uploaded {{image_entity['timestamp']}}.

          +

          Joy Likelihood for Face: {{image_entity['joy']}}

          + {% endfor %} + + diff --git a/composer/data/python2_script.py b/composer/data/python2_script.py new file mode 100644 index 00000000000..aec80592f75 --- /dev/null +++ b/composer/data/python2_script.py @@ -0,0 +1,15 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +print 'Output from Python 2.' # noqa diff --git a/composer/rest/README.rst b/composer/rest/README.rst new file mode 100644 index 00000000000..993283e5e05 --- /dev/null +++ b/composer/rest/README.rst @@ -0,0 +1,98 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Composer Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=composer/rest/README.rst + + +This directory contains samples for Google Cloud Composer. `Google Cloud Composer`_ is a managed Apache Airflow service that helps you create, schedule, monitor and manage workflows. Cloud Composer automation helps you create Airflow environments quickly and use Airflow-native tools, such as the powerful Airflow web interface and command line tools, so you can focus on your workflows and not your infrastructure. + + + + +.. _Google Cloud Composer: https://cloud.google.com/composer/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Determine Cloud Storage path for DAGs ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=composer/rest/get_dag_prefix.py,composer/rest/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python get_dag_prefix.py + + usage: get_dag_prefix.py [-h] project_id location composer_environment + + Get a Cloud Composer environment via the REST API. + + This code sample gets a Cloud Composer environment resource and prints the + Cloud Storage path used to store Apache Airflow DAGs. + + positional arguments: + project_id Your Project ID. + location Region of the Cloud Composer environent. + composer_environment Name of the Cloud Composer environent. + + optional arguments: + -h, --help show this help message and exit + + + + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/composer/rest/README.rst.in b/composer/rest/README.rst.in new file mode 100644 index 00000000000..bddbf3314b0 --- /dev/null +++ b/composer/rest/README.rst.in @@ -0,0 +1,26 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Composer + short_name: Cloud Composer + url: https://cloud.google.com/composer/docs + description: > + `Google Cloud Composer`_ is a managed Apache Airflow service that helps + you create, schedule, monitor and manage workflows. Cloud Composer + automation helps you create Airflow environments quickly and use + Airflow-native tools, such as the powerful Airflow web interface and + command line tools, so you can focus on your workflows and not your + infrastructure. + +setup: +- auth +- install_deps + +samples: +- name: Determine Cloud Storage path for DAGs + file: get_dag_prefix.py + show_help: True + +cloud_client_library: false + +folder: composer/rest \ No newline at end of file diff --git a/composer/rest/__init__.py b/composer/rest/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/composer/rest/get_client_id.py b/composer/rest/get_client_id.py new file mode 100644 index 00000000000..789a1a9c093 --- /dev/null +++ b/composer/rest/get_client_id.py @@ -0,0 +1,70 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Get the client ID associated with a Cloud Composer environment.""" + +import argparse + + +def get_client_id(project_id, location, composer_environment): + # [START composer_get_environment_client_id] + import google.auth + import google.auth.transport.requests + import requests + import six.moves.urllib.parse + + # Authenticate with Google Cloud. + # See: https://cloud.google.com/docs/authentication/getting-started + credentials, _ = google.auth.default( + scopes=['https://www.googleapis.com/auth/cloud-platform']) + authed_session = google.auth.transport.requests.AuthorizedSession( + credentials) + + # project_id = 'YOUR_PROJECT_ID' + # location = 'us-central1' + # composer_environment = 'YOUR_COMPOSER_ENVIRONMENT_NAME' + + environment_url = ( + 'https://composer.googleapis.com/v1beta1/projects/{}/locations/{}' + '/environments/{}').format(project_id, location, composer_environment) + composer_response = authed_session.request('GET', environment_url) + environment_data = composer_response.json() + airflow_uri = environment_data['config']['airflowUri'] + + # The Composer environment response does not include the IAP client ID. + # Make a second, unauthenticated HTTP request to the web server to get the + # redirect URI. + redirect_response = requests.get(airflow_uri, allow_redirects=False) + redirect_location = redirect_response.headers['location'] + + # Extract the client_id query parameter from the redirect. + parsed = six.moves.urllib.parse.urlparse(redirect_location) + query_string = six.moves.urllib.parse.parse_qs(parsed.query) + print(query_string['client_id'][0]) + # [END composer_get_environment_client_id] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('project_id', help='Your Project ID.') + parser.add_argument( + 'location', help='Region of the Cloud Composer environent.') + parser.add_argument( + 'composer_environment', help='Name of the Cloud Composer environent.') + + args = parser.parse_args() + get_client_id( + args.project_id, args.location, args.composer_environment) diff --git a/composer/rest/get_client_id_test.py b/composer/rest/get_client_id_test.py new file mode 100644 index 00000000000..52ac8c5f380 --- /dev/null +++ b/composer/rest/get_client_id_test.py @@ -0,0 +1,28 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from .get_client_id import get_client_id + + +PROJECT = os.environ['GOOGLE_CLOUD_PROJECT'] +COMPOSER_LOCATION = os.environ['COMPOSER_LOCATION'] +COMPOSER_ENVIRONMENT = os.environ['COMPOSER_ENVIRONMENT'] + + +def test_get_client_id(capsys): + get_client_id(PROJECT, COMPOSER_LOCATION, COMPOSER_ENVIRONMENT) + out, _ = capsys.readouterr() + assert '.apps.googleusercontent.com' in out diff --git a/composer/rest/get_dag_prefix.py b/composer/rest/get_dag_prefix.py new file mode 100644 index 00000000000..fc44b719b6b --- /dev/null +++ b/composer/rest/get_dag_prefix.py @@ -0,0 +1,63 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Get a Cloud Composer environment via the REST API. + +This code sample gets a Cloud Composer environment resource and prints the +Cloud Storage path used to store Apache Airflow DAGs. +""" + +import argparse + + +def get_dag_prefix(project_id, location, composer_environment): + # [START composer_get_environment_dag_prefix] + import google.auth + import google.auth.transport.requests + + # Authenticate with Google Cloud. + # See: https://cloud.google.com/docs/authentication/getting-started + credentials, _ = google.auth.default( + scopes=['https://www.googleapis.com/auth/cloud-platform']) + authed_session = google.auth.transport.requests.AuthorizedSession( + credentials) + + # project_id = 'YOUR_PROJECT_ID' + # location = 'us-central1' + # composer_environment = 'YOUR_COMPOSER_ENVIRONMENT_NAME' + + environment_url = ( + 'https://composer.googleapis.com/v1beta1/projects/{}/locations/{}' + '/environments/{}').format(project_id, location, composer_environment) + response = authed_session.request('GET', environment_url) + environment_data = response.json() + + # Print the bucket name from the response body. + print(environment_data['config']['dagGcsPrefix']) + # [END composer_get_environment_dag_prefix] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('project_id', help='Your Project ID.') + parser.add_argument( + 'location', help='Region of the Cloud Composer environent.') + parser.add_argument( + 'composer_environment', help='Name of the Cloud Composer environent.') + + args = parser.parse_args() + get_dag_prefix( + args.project_id, args.location, args.composer_environment) diff --git a/composer/rest/get_dag_prefix_test.py b/composer/rest/get_dag_prefix_test.py new file mode 100644 index 00000000000..971a64b5a53 --- /dev/null +++ b/composer/rest/get_dag_prefix_test.py @@ -0,0 +1,28 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from .get_dag_prefix import get_dag_prefix + + +PROJECT = os.environ['GOOGLE_CLOUD_PROJECT'] +COMPOSER_LOCATION = os.environ['COMPOSER_LOCATION'] +COMPOSER_ENVIRONMENT = os.environ['COMPOSER_ENVIRONMENT'] + + +def test_get_dag_prefix(capsys): + get_dag_prefix(PROJECT, COMPOSER_LOCATION, COMPOSER_ENVIRONMENT) + out, _ = capsys.readouterr() + assert 'gs://' in out diff --git a/composer/rest/requirements.txt b/composer/rest/requirements.txt new file mode 100644 index 00000000000..56eda507122 --- /dev/null +++ b/composer/rest/requirements.txt @@ -0,0 +1,3 @@ +google-auth==1.6.2 +requests==2.21.0 +six==1.12.0 diff --git a/composer/tools/README.rst b/composer/tools/README.rst new file mode 100644 index 00000000000..ebf51a76b58 --- /dev/null +++ b/composer/tools/README.rst @@ -0,0 +1,120 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Composer Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=composer/tools/README.rst + + +This directory contains samples for Google Cloud Composer. `Google Cloud Composer`_ is a managed Apache Airflow service that helps you create, schedule, monitor and manage workflows. Cloud Composer automation helps you create Airflow environments quickly and use Airflow-native tools, such as the powerful Airflow web interface and command line tools, so you can focus on your workflows and not your infrastructure. + + + + +.. _Google Cloud Composer: https://cloud.google.com/composer/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Create a new Composer environment based on an existing environment ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=composer/tools/copy_environment.py,composer/tools/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python copy_environment.py + + usage: copy_environment.py [-h] [--running_as_service_account] + [--override_machine_type OVERRIDE_MACHINE_TYPE] + [--override_disk_size_gb OVERRIDE_DISK_SIZE_GB] + [--override_network OVERRIDE_NETWORK] + [--override_subnetwork OVERRIDE_SUBNETWORK] + project location existing_env_name new_env_name + + Clone a composer environment. + + positional arguments: + project Google Cloud Project containing existing Composer + Environment. + location Google Cloud region containing existing Composer + Environment. For example `us-central1`. + existing_env_name The name of the existing Composer Environment. + new_env_name The name to use for the new Composer Environment. + + optional arguments: + -h, --help show this help message and exit + --running_as_service_account + Set this flag if the script is running on a VM with + same service account as used in the Composer + Environment. This avoids creating extra credentials. + --override_machine_type OVERRIDE_MACHINE_TYPE + Optional. Overrides machine type used for Cloud + Composer Environment. Must be a fully specified + machine type URI. + --override_disk_size_gb OVERRIDE_DISK_SIZE_GB + Optional. Overrides the disk size in GB used for Cloud + Composer Environment. + --override_network OVERRIDE_NETWORK + Optional. Overrides the network used for Cloud + Composer Environment. + --override_subnetwork OVERRIDE_SUBNETWORK + Optional. Overrides the subnetwork used for Cloud + Composer Environment. + + + + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/composer/tools/README.rst.in b/composer/tools/README.rst.in new file mode 100644 index 00000000000..fba2f63e42a --- /dev/null +++ b/composer/tools/README.rst.in @@ -0,0 +1,26 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Composer + short_name: Cloud Composer + url: https://cloud.google.com/composer/docs + description: > + `Google Cloud Composer`_ is a managed Apache Airflow service that helps + you create, schedule, monitor and manage workflows. Cloud Composer + automation helps you create Airflow environments quickly and use + Airflow-native tools, such as the powerful Airflow web interface and + command line tools, so you can focus on your workflows and not your + infrastructure. + +setup: +- auth +- install_deps + +samples: +- name: Create a new Composer environment based on an existing environment + file: copy_environment.py + show_help: True + +cloud_client_library: false + +folder: composer/tools \ No newline at end of file diff --git a/composer/tools/__init__.py b/composer/tools/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/composer/tools/copy_environment.py b/composer/tools/copy_environment.py new file mode 100644 index 00000000000..c9d955e641b --- /dev/null +++ b/composer/tools/copy_environment.py @@ -0,0 +1,748 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START composer_copy_environment] +"""Script to create a copy of an existing Cloud Composer Environment. + +Creates a clone of a Composer Environment, copying Environment Configurations, +DAGs/data/plugins/logs, and DAG run history. This script can be useful when +migrating to new Cloud Composer releases. + +To use: +* Upload to cloudshell +* Run + `python copy_environment.py PROJECT LOCATION EXISTING_ENV_NAME NEW_ENV_NAME + [--running_as_service_account]` +""" +from __future__ import print_function + +import argparse +import base64 +import contextlib +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +import uuid +from distutils.spawn import find_executable + +from cryptography import fernet +import google.auth +from google.cloud import storage +from google.oauth2 import service_account +from googleapiclient import discovery, errors +from kubernetes import client, config +from mysql import connector +import six +from six.moves import configparser + +DEFAULT_SCOPES = ["https://www.googleapis.com/auth/cloud-platform"] +EXECUTABLES = ['gcsfuse', 'cloud_sql_proxy', 'mysql', 'gcloud', 'gsutil'] + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Clone a composer environment." + ) + parser.add_argument( + "project", + help="Google Cloud Project containing existing Composer Environment.", + ) + parser.add_argument( + "location", + help="Google Cloud region containing existing Composer Environment. " + "For example `us-central1`.", + ) + parser.add_argument( + "existing_env_name", + help="The name of the existing Composer Environment.", + ) + parser.add_argument( + "new_env_name", + help="The name to use for the new Composer Environment.", + ) + parser.add_argument( + "--running_as_service_account", + action="store_true", + help="Set this flag if the script is running on a VM with same " + "service account as used in the Composer Environment. This avoids " + "creating extra credentials.", + ) + parser.add_argument( + "--override_machine_type", + default=None, + help="Optional. Overrides machine type used for Cloud Composer " + "Environment. Must be a fully specified machine type URI.", + ) + parser.add_argument( + "--override_disk_size_gb", + type=int, + default=0, + help="Optional. Overrides the disk size in GB used for Cloud Composer " + "Environment.", + ) + parser.add_argument( + "--override_network", + default=None, + help="Optional. Overrides the network used for Cloud Composer " + "Environment.", + ) + parser.add_argument( + "--override_subnetwork", + default=None, + help="Optional. Overrides the subnetwork used for Cloud Composer " + "Environment.", + ) + return parser.parse_args() + + +def get_composer_env(composer_client, project_id, location, name): + request = ( + composer_client.projects() + .locations() + .environments() + .get( + name="projects/{}/locations/{}/environments/{}".format( + project_id, location, name + ) + ) + ) + return request.execute() + + +def wait_composer_operation( + composer_client, operation_name, exit_on_error=True +): + while True: + request = ( + composer_client.projects() + .locations() + .operations() + .get(name=operation_name) + ) + operation = request.execute() + if operation.get("done"): + if operation.get("error"): + print("Composer Operation Failed: {}".format(str(operation))) + if exit_on_error: + sys.exit(1) + else: + print("Composer operation successful.") + return operation + time.sleep(10) + + +def wait_sql_operation( + sql_client, sql_project, operation_name, exit_on_error=True +): + while True: + request = sql_client.operations().get( + operation=operation_name, project=sql_project + ) + operation = request.execute() + if operation.get("status", "") == "DONE": + if operation.get("error"): + print("SQL Operation Failed: {}".format(str(operation))) + if exit_on_error: + sys.exit(1) + else: + print("SQL operation successful.") + return operation + time.sleep(5) + + +def create_composer_env_if_not_exist( + composer_client, existing_env, project, location, new_env_name, overrides +): + existing_config = existing_env.get("config", {}) + existing_node_config = existing_config.get("nodeConfig", {}) + existing_software_config = existing_config.get("softwareConfig", {}) + + expected_env = { + "name": "projects/{}/locations/{}/environments/{}".format( + project, location, new_env_name + ), + "config": { + "nodeCount": existing_config.get("nodeCount", 0), + "nodeConfig": { + "location": existing_node_config.get("location", ""), + "machineType": overrides["machineType"] + or existing_node_config.get("machineType", ""), + "network": overrides["network"] + or existing_node_config.get("network", ""), + "subnetwork": overrides["subnetwork"] + or existing_node_config.get("subnetwork", ""), + "diskSizeGb": overrides["diskSizeGb"] + or existing_node_config.get("diskSizeGb", 0), + "oauthScopes": existing_node_config.get("oauthScopes", []), + "serviceAccount": existing_node_config.get( + "serviceAccount", "" + ), + "tags": existing_node_config.get("tags", []), + }, + "softwareConfig": { + "airflowConfigOverrides": existing_software_config.get( + "airflowConfigOverrides", {} + ), + "envVariables": existing_software_config.get( + "envVariables", {} + ), + }, + }, + "labels": existing_env.get("labels", {}), + } + + try: + new_env = get_composer_env( + composer_client, project, location, new_env_name + ) + print( + "Attempting to use existing Composer Environment `{}`. If " + "Environment was not created using this script, configs may " + "differ.".format(new_env_name) + ) + if new_env.get("state") != "RUNNING": + print("Error: Composer Environment {} is not in a RUNNING state.") + sys.exit(1) + except errors.HttpError as error: + if error.resp.status == 404: + print( + "Starting Composer Environment Create Operation. This " + "takes 20 - 60 mins." + ) + request = ( + composer_client.projects() + .locations() + .environments() + .create( + parent="projects/{}/locations/{}".format( + project, location + ), + body=expected_env, + ) + ) + operation = request.execute() + wait_composer_operation(composer_client, operation.get("name")) + else: + raise error + + +def update_pypi_packages(composer_client, existing_env, new_env): + existing_config = existing_env.get("config", {}) + existing_software_config = existing_config.get("softwareConfig", {}) + if existing_software_config.get( + "pypiPackages" + ) and existing_software_config.get("pypiPackages") != new_env.get( + "config", {} + ).get( + "softwareConfig", {} + ).get( + "pypiPackages" + ): + body = { + "name": new_env.get("name"), + "config": { + "softwareConfig": { + "pypiPackages": existing_software_config.get( + "pypiPackages", {} + ) + } + }, + } + print( + "Starting Composer Update PyPI Packages Operation. This takes " + "20 - 30 mins" + ) + request = ( + composer_client.projects() + .locations() + .environments() + .patch( + name=new_env.get("name"), + body=body, + updateMask="config.softwareConfig.pypiPackages", + ) + ) + operation = request.execute() + wait_composer_operation(composer_client, operation.get("name")) + + +def create_service_account_key(iam_client, project, service_account_name): + service_account_key = ( + iam_client.projects() + .serviceAccounts() + .keys() + .create( + name="projects/{}/serviceAccounts/{}".format( + project, service_account_name + ), + body={}, + ) + .execute() + ) + service_account_key_decoded = json.loads( + base64.b64decode(service_account_key.get("privateKeyData", "")) + .decode("utf-8") + ) + time.sleep(5) + return service_account_key_decoded + + +def create_temp_bucket(storage_client, project): + # Bucket names need to start with lowercase letter, end with lowercase + # letter, and contain only lowercase letters, numbers, and dashes + temp_bucket_name = "temp" + str(uuid.uuid4()) + "a" + return storage_client.create_bucket(temp_bucket_name, project=project) + + +def get_sql_project_and_instance(env): + gke_cluster = env.get("config", {}).get("gkeCluster") + airflow_uri = env.get("config", {}).get("airflowUri") + sql_project = re.match( + "https://([^.]*).appspot.com", airflow_uri + ).groups()[0] + sql_instance = ( + re.match( + "projects/[^/]*/zones/[^/]*/clusters/([^/]*)", gke_cluster + ).groups()[0][:-3] + + "sql" + ) + return sql_project, sql_instance + + +def get_sql_instance_service_account(sql_client, project, instance): + return ( + sql_client.instances() + .get(project=project, instance=instance) + .execute() + .get("serviceAccountEmailAddress") + ) + + +def grant_rw_permissions(gcs_bucket, service_account): + try: + gcs_bucket.acl.user(service_account).grant_owner() + gcs_bucket.acl.save() + except Exception: + print( + "Failed to set acls for service account {} on bucket {}.".format( + service_account, gcs_bucket.name + ) + ) + sys.exit(1) + time.sleep(5) + + +def export_data(sql_client, project, instance, gcs_bucket_name, filename): + operation = ( + sql_client.instances() + .export( + project=project, + instance=instance, + body={ + "exportContext": { + "kind": "sql#exportContext", + "fileType": "SQL", + "uri": "gs://" + gcs_bucket_name + "/" + filename, + } + }, + ) + .execute() + ) + print( + "Starting to export Cloud SQL database from old Environment. This " + "takes about 2 mins." + ) + wait_sql_operation(sql_client, project, operation.get("name")) + + +def get_fernet_key(composer_env): + print("Retrieving fernet key for Composer Environment {}.".format( + composer_env.get('name', ''))) + gke_cluster_resource = composer_env.get("config", {}).get("gkeCluster") + project_zone_cluster = re.match( + "projects/([^/]*)/zones/([^/]*)/clusters/([^/]*)", gke_cluster_resource + ).groups() + tmp_dir_name = None + try: + print("Getting cluster credentials {} to retrieve fernet key.".format( + gke_cluster_resource)) + tmp_dir_name = tempfile.mkdtemp() + kubeconfig_file = tmp_dir_name + "/config" + os.environ["KUBECONFIG"] = kubeconfig_file + if subprocess.call( + [ + "gcloud", + "container", + "clusters", + "get-credentials", + project_zone_cluster[2], + "--zone", + project_zone_cluster[1], + "--project", + project_zone_cluster[0] + ] + ): + print("Failed to retrieve cluster credentials: {}.".format( + gke_cluster_resource)) + sys.exit(1) + + kubernetes_client = client.CoreV1Api( + api_client=config.new_client_from_config( + config_file=kubeconfig_file)) + airflow_configmap = kubernetes_client.read_namespaced_config_map( + "airflow-configmap", "default") + config_str = airflow_configmap.data['airflow.cfg'] + with contextlib.closing(six.StringIO(config_str)) as config_buffer: + config_parser = configparser.ConfigParser() + config_parser.readfp(config_buffer) + return config_parser.get("core", "fernet_key") + except Exception as exc: + print("Failed to get fernet key for cluster: {}.".format(str(exc))) + sys.exit(1) + finally: + if tmp_dir_name: + shutil.rmtree(tmp_dir_name) + + +def reencrypt_variables_connections(old_fernet_key_str, new_fernet_key_str): + old_fernet_key = fernet.Fernet(old_fernet_key_str.encode("utf-8")) + new_fernet_key = fernet.Fernet(new_fernet_key_str.encode("utf-8")) + db = connector.connect( + host="127.0.0.1", + user="root", + database="airflow-db", + ) + variable_cursor = db.cursor() + variable_cursor.execute("SELECT id, val, is_encrypted FROM variable") + rows = variable_cursor.fetchall() + for row in rows: + id = row[0] + val = row[1] + is_encrypted = row[2] + if is_encrypted: + updated_val = new_fernet_key.encrypt( + old_fernet_key.decrypt(bytes(val))).decode() + variable_cursor.execute( + "UPDATE variable SET val=%s WHERE id=%s", (updated_val, id)) + db.commit() + + conn_cursor = db.cursor() + conn_cursor.execute( + "SELECT id, password, extra, is_encrypted, is_extra_encrypted FROM " + "connection") + rows = conn_cursor.fetchall() + for row in rows: + id = row[0] + password = row[1] + extra = row[2] + is_encrypted = row[3] + is_extra_encrypted = row[4] + if is_encrypted: + updated_password = new_fernet_key.encrypt( + old_fernet_key.decrypt(bytes(password))).decode() + conn_cursor.execute( + "UPDATE connection SET password=%s WHERE id=%s", + (updated_password, id)) + if is_extra_encrypted: + updated_extra = new_fernet_key.encrypt( + old_fernet_key.decrypt(bytes(extra))).decode() + conn_cursor.execute( + "UPDATE connection SET extra=%s WHERE id=%s", + (updated_extra, id)) + db.commit() + + +def import_data( + sql_client, + service_account_key, + project, + instance, + gcs_bucket, + filename, + old_fernet_key, + new_fernet_key +): + tmp_dir_name = None + fuse_dir = None + proxy_subprocess = None + try: + print("Locally fusing Cloud Storage bucket to access database dump.") + tmp_dir_name = tempfile.mkdtemp() + fuse_dir = tmp_dir_name + "/fuse" + if subprocess.call(["mkdir", fuse_dir]): + print("Failed to make temporary subdir {}.".format(fuse_dir)) + sys.exit(1) + if subprocess.call(["gcsfuse", gcs_bucket, fuse_dir]): + print( + "Failed to fuse bucket {} with temp local directory {}".format( + gcs_bucket, fuse_dir + ) + ) + sys.exit(1) + instance_connection = ( + sql_client.instances() + .get(project=project, instance=instance) + .execute() + .get("connectionName") + ) + proxy_cmd = [ + "cloud_sql_proxy", + "-instances=" + instance_connection + "=tcp:3306", + ] + + if service_account_key: + key_file = tmp_dir_name + "/key.json" + fh = open(key_file, "w") + fh.write(json.dumps(service_account_key)) + fh.close() + proxy_cmd.append("-credential_file=" + key_file) + + print("Starting proxy to new database.") + proxy_subprocess = subprocess.Popen(proxy_cmd, close_fds=True) + time.sleep(2) + if proxy_subprocess.poll() is not None: + print( + "Proxy subprocess failed to start or terminated prematurely." + ) + sys.exit(1) + print("Importing database.") + if subprocess.call( + ["mysql", "-u", "root", "--host", "127.0.0.1"], + stdin=open(fuse_dir + "/" + filename), + ): + print("Failed to import database.") + sys.exit(1) + print("Reencrypting variables and connections.") + reencrypt_variables_connections(old_fernet_key, new_fernet_key) + print("Database import succeeded.") + except Exception as exc: + print("Failed to copy database: {}".format(str(exc))) + sys.exit(1) + finally: + if proxy_subprocess: + proxy_subprocess.kill() + if fuse_dir: + try: + subprocess.call(["fusermount", "-u", fuse_dir]) + except OSError: + subprocess.call(["umount", fuse_dir]) + if tmp_dir_name: + shutil.rmtree(tmp_dir_name) + + +def delete_service_account_key( + iam_client, project, service_account_name, service_account_key +): + iam_client.projects().serviceAccounts().keys().delete( + name="projects/{}/serviceAccounts/{}/keys/{}".format( + project, + service_account_name, + service_account_key["private_key_id"], + ) + ).execute() + + +def delete_bucket(gcs_bucket): + gcs_bucket.delete(force=True) + + +def copy_database(project, existing_env, new_env, running_as_service_account): + print("Starting database transfer.") + gke_service_account_name = None + gke_service_account_key = None + gcs_db_dump_bucket = None + try: + # create default creds clients + default_credentials, _ = google.auth.default(scopes=DEFAULT_SCOPES) + storage_client = storage.Client( + project=project, credentials=default_credentials + ) + iam_client = discovery.build( + "iam", "v1", credentials=default_credentials + ) + + # create service account creds sql client + gke_service_account_name = ( + new_env.get("config", {}) + .get("nodeConfig", {}) + .get("serviceAccount") + ) + gke_service_account_credentials = None + # Only the service account used for Composer Environment has access to + # hidden SQL database. If running in a VM as the service account, use + # default credentials, otherwise create a key and authenticate as the + # service account. + if running_as_service_account: + gke_service_account_credentials = default_credentials + else: + print( + "Obtaining service account `{}` credentials to access SQL " + "database.".format(gke_service_account_name) + ) + gke_service_account_key = create_service_account_key( + iam_client, project, gke_service_account_name + ) + gke_service_account_credentials = ( + service_account.Credentials.from_service_account_info( + gke_service_account_key + ) + ).with_scopes(DEFAULT_SCOPES) + sql_client = discovery.build( + "sqladmin", "v1beta4", credentials=gke_service_account_credentials + ) + + # create a bucket, export data from existing env to bucket, import data + # to new env + print("Creating temporary Cloud Storage bucket for database dump.") + gcs_db_dump_bucket = create_temp_bucket(storage_client, project) + prev_sql_project, prev_sql_instance = get_sql_project_and_instance( + existing_env + ) + new_sql_project, new_sql_instance = get_sql_project_and_instance( + new_env + ) + + print("Granting permissions on bucket to enable database dump.") + grant_rw_permissions( + gcs_db_dump_bucket, + get_sql_instance_service_account( + sql_client, prev_sql_project, prev_sql_instance + ), + ) + print("Exporting database from old Environment.") + export_data( + sql_client, + prev_sql_project, + prev_sql_instance, + gcs_db_dump_bucket.name, + "db_dump.sql", + ) + print("Obtaining fernet keys for Composer Environments.") + old_fernet_key = get_fernet_key(existing_env) + new_fernet_key = get_fernet_key(new_env) + print("Preparing database import to new Environment.") + import_data( + sql_client, + gke_service_account_key, + new_sql_project, + new_sql_instance, + gcs_db_dump_bucket.name, + "db_dump.sql", + old_fernet_key, + new_fernet_key, + ) + finally: + if gke_service_account_key: + print("Deleting temporary service account key.") + delete_service_account_key( + iam_client, + project, + gke_service_account_name, + gke_service_account_key, + ) + if gcs_db_dump_bucket: + print("Deleting temporary Cloud Storage bucket.") + delete_bucket(gcs_db_dump_bucket) + + +def copy_gcs_bucket(existing_env, new_env): + print("Starting to transfer Cloud Storage artifacts.") + existing_bucket = existing_env["config"]["dagGcsPrefix"][:-4] + new_bucket = new_env["config"]["dagGcsPrefix"][:-4] + for subdir in ["dags", "plugins", "data", "logs"]: + subprocess.call( + [ + "gsutil", + "-m", + "cp", + "-r", + existing_bucket + subdir + "/*", + new_bucket + subdir, + ] + ) + + +def clone_environment( + project, + location, + existing_env_name, + new_env_name, + running_as_service_account, + overrides, +): + default_credentials, _ = google.auth.default(scopes=DEFAULT_SCOPES) + composer_client = discovery.build( + "composer", "v1", credentials=default_credentials + ) + + existing_env = get_composer_env( + composer_client, project, location, existing_env_name + ) + create_composer_env_if_not_exist( + composer_client, + existing_env, + project, + location, + new_env_name, + overrides, + ) + new_env = get_composer_env( + composer_client, project, location, new_env_name + ) + update_pypi_packages(composer_client, existing_env, new_env) + new_env = get_composer_env( + composer_client, project, location, new_env_name + ) + copy_database(project, existing_env, new_env, running_as_service_account) + copy_gcs_bucket(existing_env, new_env) + print( + "Composer Environment copy completed. Please check new environment " + "correctness and delete old Environment to avoid incurring " + "additional costs." + ) + + +def check_executables(): + not_found = [ + executable for executable in EXECUTABLES + if not find_executable(executable) + ] + if not_found: + print('Required executables not found: {}'.format(' '.join(not_found))) + sys.exit(1) + + +if __name__ == "__main__": + args = parse_args() + check_executables() + clone_environment( + args.project, + args.location, + args.existing_env_name, + args.new_env_name, + args.running_as_service_account, + { + "machineType": args.override_machine_type, + "network": args.override_network, + "subnetwork": args.override_subnetwork, + "diskSizeGb": args.override_disk_size_gb, + }, + ) + +# [END composer_copy_environment] diff --git a/composer/tools/copy_environment_test.py b/composer/tools/copy_environment_test.py new file mode 100644 index 00000000000..319b5f31589 --- /dev/null +++ b/composer/tools/copy_environment_test.py @@ -0,0 +1,36 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess +import time + +from google.cloud import storage +import mock +import pytest + + +def test_grant_rw_permissions_fails_gracefully(monkeypatch, capsys): + mock_call = mock.Mock() + mock_call.side_effect = RuntimeError() + monkeypatch.setattr(subprocess, 'call', mock_call) + monkeypatch.setattr(time, 'sleep', lambda sec: None) + from . import copy_environment + + with pytest.raises(SystemExit): + copy_environment.grant_rw_permissions( + storage.Bucket(None, name='example-bucket'), + 'serviceaccount@example.com') + + out, _ = capsys.readouterr() + assert 'Failed to set acls for service account' in out diff --git a/composer/tools/requirements.txt b/composer/tools/requirements.txt new file mode 100644 index 00000000000..5f187cbbfd2 --- /dev/null +++ b/composer/tools/requirements.txt @@ -0,0 +1,6 @@ +cryptography==2.5 +google-api-python-client==1.7.8 +google-auth==1.6.2 +google-cloud-storage==1.13.2 +kubernetes==8.0.1 +mysql-connector-python==8.0.15 diff --git a/composer/workflows/README.rst b/composer/workflows/README.rst new file mode 100644 index 00000000000..54bbeb05c0c --- /dev/null +++ b/composer/workflows/README.rst @@ -0,0 +1,63 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Composer Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=composer/workflows/README.rst + + +This directory contains samples for Google Cloud Composer. `Google Cloud Composer`_ is a fully managed workflow orchestration service built on Apache Airflow. + + + + +.. _Google Cloud Composer: https://cloud.google.com/composer/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/composer/workflows/README.rst.in b/composer/workflows/README.rst.in new file mode 100644 index 00000000000..30e1c099745 --- /dev/null +++ b/composer/workflows/README.rst.in @@ -0,0 +1,15 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Composer + short_name: Cloud Composer + url: https://cloud.google.com/composer/docs + description: > + `Google Cloud Composer`_ is a fully managed workflow orchestration + service built on Apache Airflow. + +setup: +- auth +- install_deps + +folder: composer/workflows \ No newline at end of file diff --git a/composer/workflows/__init__.py b/composer/workflows/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/composer/workflows/bashoperator_python2.py b/composer/workflows/bashoperator_python2.py new file mode 100644 index 00000000000..29bcdb7af79 --- /dev/null +++ b/composer/workflows/bashoperator_python2.py @@ -0,0 +1,44 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START composer_bashoperator_python2] +import datetime + +from airflow import models +from airflow.operators import bash_operator + + +yesterday = datetime.datetime.combine( + datetime.datetime.today() - datetime.timedelta(1), + datetime.datetime.min.time()) + + +default_dag_args = { + # Setting start date as yesterday starts the DAG immediately when it is + # detected in the Cloud Storage bucket. + 'start_date': yesterday, +} + +with models.DAG( + 'composer_sample_bashoperator_python2', + schedule_interval=datetime.timedelta(days=1), + default_args=default_dag_args) as dag: + + run_python2 = bash_operator.BashOperator( + task_id='run_python2', + # This example runs a Python script from the data folder to prevent + # Airflow from attempting to parse the script as a DAG. + bash_command='python2 /home/airflow/gcs/data/python2_script.py', + ) +# [END composer_bashoperator_python2] diff --git a/composer/workflows/bashoperator_python2_test.py b/composer/workflows/bashoperator_python2_test.py new file mode 100644 index 00000000000..7e035607f6b --- /dev/null +++ b/composer/workflows/bashoperator_python2_test.py @@ -0,0 +1,26 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import unit_testing + + +def test_dag_import(): + """Test that the DAG file can be successfully imported. + + This tests that the DAG can be parsed, but does not run it in an Airflow + environment. This is a recommended sanity check by the official Airflow + docs: https://airflow.incubator.apache.org/tutorial.html#testing + """ + from . import bashoperator_python2 as module + unit_testing.assert_has_valid_dag(module) diff --git a/composer/workflows/bq_copy_across_locations.py b/composer/workflows/bq_copy_across_locations.py new file mode 100644 index 00000000000..82581d700f1 --- /dev/null +++ b/composer/workflows/bq_copy_across_locations.py @@ -0,0 +1,180 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Example Airflow DAG that performs an export from BQ tables listed in +config file to GCS, copies GCS objects across locations (e.g., from US to +EU) then imports from GCS to BQ. The DAG imports the gcs_to_gcs operator +from plugins and dynamically builds the tasks based on the list of tables. +Lastly, the DAG defines a specific application logger to generate logs. + +This DAG relies on three Airflow variables +(https://airflow.apache.org/concepts.html#variables): +* table_list_file_path - CSV file listing source and target tables, including +Datasets. +* gcs_source_bucket - Google Cloud Storage bucket to use for exporting +BigQuery tables in source. +* gcs_dest_bucket - Google Cloud Storage bucket to use for importing +BigQuery tables in destination. +See https://cloud.google.com/storage/docs/creating-buckets for creating a +bucket. +""" + +# -------------------------------------------------------------------------------- +# Load The Dependencies +# -------------------------------------------------------------------------------- + +import csv +import datetime +import io +import logging + +from airflow import models +from airflow.contrib.operators import bigquery_to_gcs +from airflow.contrib.operators import gcs_to_bq +from airflow.operators import dummy_operator +# Import operator from plugins +from gcs_plugin.operators import gcs_to_gcs + + +# -------------------------------------------------------------------------------- +# Set default arguments +# -------------------------------------------------------------------------------- + +yesterday = datetime.datetime.now() - datetime.timedelta(days=1) + +default_args = { + 'owner': 'airflow', + 'start_date': yesterday, + 'depends_on_past': False, + 'email': [''], + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': datetime.timedelta(minutes=5), +} + +# -------------------------------------------------------------------------------- +# Set variables +# -------------------------------------------------------------------------------- + +# 'table_list_file_path': This variable will contain the location of the master +# file. +table_list_file_path = models.Variable.get('table_list_file_path') + +# Source Bucket +source_bucket = models.Variable.get('gcs_source_bucket') + +# Destination Bucket +dest_bucket = models.Variable.get('gcs_dest_bucket') + +# -------------------------------------------------------------------------------- +# Set GCP logging +# -------------------------------------------------------------------------------- + +logger = logging.getLogger('bq_copy_us_to_eu_01') + +# -------------------------------------------------------------------------------- +# Functions +# -------------------------------------------------------------------------------- + + +def read_table_list(table_list_file): + """ + Reads the table list file that will help in creating Airflow tasks in + the DAG dynamically. + :param table_list_file: (String) The file location of the table list file, + e.g. '/home/airflow/framework/table_list.csv' + :return table_list: (List) List of tuples containing the source and + target tables. + """ + table_list = [] + logger.info('Reading table_list_file from : %s' % str(table_list_file)) + try: + with io.open(table_list_file, 'rt', encoding='utf-8') as csv_file: + csv_reader = csv.reader(csv_file) + next(csv_reader) # skip the headers + for row in csv_reader: + logger.info(row) + table_tuple = { + 'table_source': row[0], + 'table_dest': row[1] + } + table_list.append(table_tuple) + return table_list + except IOError as e: + logger.error('Error opening table_list_file %s: ' % str( + table_list_file), e) + + +# -------------------------------------------------------------------------------- +# Main DAG +# -------------------------------------------------------------------------------- + +# Define a DAG (directed acyclic graph) of tasks. +# Any task you create within the context manager is automatically added to the +# DAG object. +with models.DAG( + 'composer_sample_bq_copy_across_locations', + default_args=default_args, + schedule_interval=None) as dag: + start = dummy_operator.DummyOperator( + task_id='start', + trigger_rule='all_success' + ) + + end = dummy_operator.DummyOperator( + task_id='end', + trigger_rule='all_success' + ) + + # Get the table list from master file + all_records = read_table_list(table_list_file_path) + + # Loop over each record in the 'all_records' python list to build up + # Airflow tasks + for record in all_records: + logger.info('Generating tasks to transfer table: {}'.format(record)) + + table_source = record['table_source'] + table_dest = record['table_dest'] + + BQ_to_GCS = bigquery_to_gcs.BigQueryToCloudStorageOperator( + # Replace ":" with valid character for Airflow task + task_id='{}_BQ_to_GCS'.format(table_source.replace(":", "_")), + source_project_dataset_table=table_source, + destination_cloud_storage_uris=['{}-*.avro'.format( + 'gs://' + source_bucket + '/' + table_source)], + export_format='AVRO' + ) + + GCS_to_GCS = gcs_to_gcs.GoogleCloudStorageToGoogleCloudStorageOperator( + # Replace ":" with valid character for Airflow task + task_id='{}_GCS_to_GCS'.format(table_source.replace(":", "_")), + source_bucket=source_bucket, + source_object='{}-*.avro'.format(table_source), + destination_bucket=dest_bucket, + # destination_object='{}-*.avro'.format(table_dest) + ) + + GCS_to_BQ = gcs_to_bq.GoogleCloudStorageToBigQueryOperator( + # Replace ":" with valid character for Airflow task + task_id='{}_GCS_to_BQ'.format(table_dest.replace(":", "_")), + bucket=dest_bucket, + source_objects=['{}-*.avro'.format(table_source)], + destination_project_dataset_table=table_dest, + source_format='AVRO', + write_disposition='WRITE_TRUNCATE' + ) + + start >> BQ_to_GCS >> GCS_to_GCS >> GCS_to_BQ >> end diff --git a/composer/workflows/bq_copy_across_locations_test.py b/composer/workflows/bq_copy_across_locations_test.py new file mode 100644 index 00000000000..af88bcb8f8d --- /dev/null +++ b/composer/workflows/bq_copy_across_locations_test.py @@ -0,0 +1,54 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import os.path +import sys + +from airflow import models +import pytest + +from . import unit_testing + + +@pytest.fixture(scope='module', autouse=True) +def gcs_plugin(): + plugins_dir = os.path.abspath(os.path.join( + os.path.dirname(__file__), + '..', + '..', + 'third_party', + 'apache-airflow', + 'plugins', + )) + sys.path.append(plugins_dir) + yield + sys.path.remove(plugins_dir) + + +def test_dag(): + """Test that the DAG file can be successfully imported. + + This tests that the DAG can be parsed, but does not run it in an Airflow + environment. This is a recommended sanity check by the official Airflow + docs: https://airflow.incubator.apache.org/tutorial.html#testing + """ + example_file_path = os.path.join( + os.path.abspath(os.path.dirname(__file__)), + 'bq_copy_eu_to_us_sample.csv') + models.Variable.set('table_list_file_path', example_file_path) + models.Variable.set('gcs_source_bucket', 'example-project') + models.Variable.set('gcs_dest_bucket', 'us-central1-f') + from . import bq_copy_across_locations as module + unit_testing.assert_has_valid_dag(module) diff --git a/composer/workflows/bq_copy_eu_to_us_sample.csv b/composer/workflows/bq_copy_eu_to_us_sample.csv new file mode 100644 index 00000000000..fd529d9cde8 --- /dev/null +++ b/composer/workflows/bq_copy_eu_to_us_sample.csv @@ -0,0 +1,3 @@ +Source, Target +nyc-tlc:green.trips_2014,nyc_tlc_EU.trips_2014 +nyc-tlc:green.trips_2015,nyc_tlc_EU.trips_2015 \ No newline at end of file diff --git a/composer/workflows/bq_notify.py b/composer/workflows/bq_notify.py new file mode 100644 index 00000000000..ed76abbd9c6 --- /dev/null +++ b/composer/workflows/bq_notify.py @@ -0,0 +1,189 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Example Airflow DAG that creates a BigQuery dataset, populates the dataset +by performing a queries for recent popular StackOverflow questions against the +public dataset `bigquery-public-data.stackoverflow.posts_questions`. The DAG +exports the results of the query as a CSV to Cloud Storage, and sends an email +with path to the CSV file and the title and view count of the most popular +question. Lastly, the DAG cleans up the BigQuery dataset. + +This DAG relies on three Airflow variables +https://airflow.apache.org/concepts.html#variables +* gcp_project - Google Cloud Project to use for BigQuery. +* gcs_bucket - Google Cloud Storage bucket to use for result CSV file. + See https://cloud.google.com/storage/docs/creating-buckets for creating a + bucket. +* email - The email used to receive DAG updates. +""" + +import datetime + +# [START composer_notify_failure] +from airflow import models +# [END composer_notify_failure] +from airflow.contrib.operators import bigquery_get_data +# [START composer_bigquery] +from airflow.contrib.operators import bigquery_operator +# [END composer_bigquery] +from airflow.contrib.operators import bigquery_to_gcs +# [START composer_bash_bq] +from airflow.operators import bash_operator +# [END composer_bash_bq] +# [START composer_email] +from airflow.operators import email_operator +# [END composer_email] +from airflow.utils import trigger_rule + + +bq_dataset_name = 'airflow_bq_notify_dataset_{{ ds_nodash }}' +bq_recent_questions_table_id = bq_dataset_name + '.recent_questions' +BQ_MOST_POPULAR_TABLE_NAME = 'most_popular' +bq_most_popular_table_id = bq_dataset_name + '.' + BQ_MOST_POPULAR_TABLE_NAME +output_file = 'gs://{gcs_bucket}/recent_questionsS.csv'.format( + gcs_bucket=models.Variable.get('gcs_bucket')) + +# Data from the month of January 2018 +# You may change the query dates to get data from a different time range. You +# may also dynamically pick a date range based on DAG schedule date. Airflow +# macros can be useful for this. For example, {{ macros.ds_add(ds, -7) }} +# corresponds to a date one week (7 days) before the DAG was run. +# https://airflow.apache.org/code.html?highlight=execution_date#airflow.macros.ds_add +max_query_date = '2018-02-01' +min_query_date = '2018-01-01' + +yesterday = datetime.datetime.combine( + datetime.datetime.today() - datetime.timedelta(1), + datetime.datetime.min.time()) + +# [START composer_notify_failure] +default_dag_args = { + 'start_date': yesterday, + # Email whenever an Operator in the DAG fails. + 'email': models.Variable.get('email'), + 'email_on_failure': True, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': datetime.timedelta(minutes=5), + 'project_id': models.Variable.get('gcp_project') +} + +with models.DAG( + 'composer_sample_bq_notify', + schedule_interval=datetime.timedelta(weeks=4), + default_args=default_dag_args) as dag: + # [END composer_notify_failure] + + # [START composer_bash_bq] + # Create BigQuery output dataset. + make_bq_dataset = bash_operator.BashOperator( + task_id='make_bq_dataset', + # Executing 'bq' command requires Google Cloud SDK which comes + # preinstalled in Cloud Composer. + bash_command='bq ls {} || bq mk {}'.format( + bq_dataset_name, bq_dataset_name)) + # [END composer_bash_bq] + + # [START composer_bigquery] + # Query recent StackOverflow questions. + bq_recent_questions_query = bigquery_operator.BigQueryOperator( + task_id='bq_recent_questions_query', + bql=""" + SELECT owner_display_name, title, view_count + FROM `bigquery-public-data.stackoverflow.posts_questions` + WHERE creation_date < CAST('{max_date}' AS TIMESTAMP) + AND creation_date >= CAST('{min_date}' AS TIMESTAMP) + ORDER BY view_count DESC + LIMIT 100 + """.format(max_date=max_query_date, min_date=min_query_date), + use_legacy_sql=False, + destination_dataset_table=bq_recent_questions_table_id) + # [END composer_bigquery] + + # Export query result to Cloud Storage. + export_questions_to_gcs = bigquery_to_gcs.BigQueryToCloudStorageOperator( + task_id='export_recent_questions_to_gcs', + source_project_dataset_table=bq_recent_questions_table_id, + destination_cloud_storage_uris=[output_file], + export_format='CSV') + + # Perform most popular question query. + bq_most_popular_query = bigquery_operator.BigQueryOperator( + task_id='bq_most_popular_question_query', + bql=""" + SELECT title, view_count + FROM `{table}` + ORDER BY view_count DESC + LIMIT 1 + """.format(table=bq_recent_questions_table_id), + use_legacy_sql=False, + destination_dataset_table=bq_most_popular_table_id) + + # Read most popular question from BigQuery to XCom output. + # XCom is the best way to communicate between operators, but can only + # transfer small amounts of data. For passing large amounts of data, store + # the data in Cloud Storage and pass the path to the data if necessary. + # https://airflow.apache.org/concepts.html#xcoms + bq_read_most_popular = bigquery_get_data.BigQueryGetDataOperator( + task_id='bq_read_most_popular', + dataset_id=bq_dataset_name, + table_id=BQ_MOST_POPULAR_TABLE_NAME) + + # [START composer_email] + # Send email confirmation + email_summary = email_operator.EmailOperator( + task_id='email_summary', + to=models.Variable.get('email'), + subject='Sample BigQuery notify data ready', + html_content=""" + Analyzed Stack Overflow posts data from {min_date} 12AM to {max_date} + 12AM. The most popular question was '{question_title}' with + {view_count} views. Top 100 questions asked are now available at: + {export_location}. + """.format( + min_date=min_query_date, + max_date=max_query_date, + question_title=( + '{{ ti.xcom_pull(task_ids=\'bq_read_most_popular\', ' + 'key=\'return_value\')[0][0] }}' + ), + view_count=( + '{{ ti.xcom_pull(task_ids=\'bq_read_most_popular\', ' + 'key=\'return_value\')[0][1] }}' + ), + export_location=output_file)) + # [END composer_email] + + # Delete BigQuery dataset + # Delete the bq table + delete_bq_dataset = bash_operator.BashOperator( + task_id='delete_bq_dataset', + bash_command='bq rm -r -f %s' % bq_dataset_name, + trigger_rule=trigger_rule.TriggerRule.ALL_DONE) + + # Define DAG dependencies. + ( + make_bq_dataset + >> bq_recent_questions_query + >> export_questions_to_gcs + >> delete_bq_dataset + ) + ( + bq_recent_questions_query + >> bq_most_popular_query + >> bq_read_most_popular + >> delete_bq_dataset + ) + export_questions_to_gcs >> email_summary + bq_read_most_popular >> email_summary diff --git a/composer/workflows/bq_notify_test.py b/composer/workflows/bq_notify_test.py new file mode 100644 index 00000000000..3e6f1a5ee56 --- /dev/null +++ b/composer/workflows/bq_notify_test.py @@ -0,0 +1,32 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from airflow import models + +from . import unit_testing + + +def test_dag_import(): + """Test that the DAG file can be successfully imported. + + This tests that the DAG can be parsed, but does not run it in an Airflow + environment. This is a recommended sanity check by the official Airflow + docs: https://airflow.incubator.apache.org/tutorial.html#testing + """ + models.Variable.set('gcs_bucket', 'example_bucket') + models.Variable.set('gcp_project', 'example-project') + models.Variable.set('gce_zone', 'us-central1-f') + models.Variable.set('email', 'notify@example.com') + from . import bq_notify as module + unit_testing.assert_has_valid_dag(module) diff --git a/composer/workflows/connections.py b/composer/workflows/connections.py new file mode 100644 index 00000000000..c4a6b5582ce --- /dev/null +++ b/composer/workflows/connections.py @@ -0,0 +1,58 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Demonstrates how to use connections in an Airflow DAG.""" + +import datetime + +from airflow import models +from airflow.contrib.operators import bigquery_operator + + +yesterday = datetime.datetime.combine( + datetime.datetime.today() - datetime.timedelta(1), + datetime.datetime.min.time()) + +default_dag_args = { + # Setting start date as yesterday starts the DAG immediately when it is + # detected in the Cloud Storage bucket. + 'start_date': yesterday, +} + +# Define a DAG (directed acyclic graph) of tasks. +# Any task you create within the context manager is automatically added to the +# DAG object. +with models.DAG( + 'composer_sample_connections', + schedule_interval=datetime.timedelta(days=1), + default_args=default_dag_args) as dag: + # [START composer_connections_default] + task_default = bigquery_operator.BigQueryOperator( + task_id='task_default_connection', + bql='SELECT 1', use_legacy_sql=False) + # [END composer_connections_default] + # [START composer_connections_explicit] + task_explicit = bigquery_operator.BigQueryOperator( + task_id='task_explicit_connection', + bql='SELECT 1', use_legacy_sql=False, + # Composer creates a 'google_cloud_default' connection by default. + bigquery_conn_id='google_cloud_default') + # [END composer_connections_explicit] + # [START composer_connections_custom] + task_custom = bigquery_operator.BigQueryOperator( + task_id='task_custom_connection', + bql='SELECT 1', use_legacy_sql=False, + # Set a connection ID to use a connection that you have created. + bigquery_conn_id='my_gcp_connection') + # [END composer_connections_custom] diff --git a/composer/workflows/connections_test.py b/composer/workflows/connections_test.py new file mode 100644 index 00000000000..808512936a3 --- /dev/null +++ b/composer/workflows/connections_test.py @@ -0,0 +1,26 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import unit_testing + + +def test_dag_import(): + """Test that the DAG file can be successfully imported. + + This tests that the DAG can be parsed, but does not run it in an Airflow + environment. This is a recommended sanity check by the official Airflow + docs: https://airflow.incubator.apache.org/tutorial.html#testing + """ + from . import connections as module + unit_testing.assert_has_valid_dag(module) diff --git a/composer/workflows/dependencies/__init__.py b/composer/workflows/dependencies/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/composer/workflows/dependencies/coin_module.py b/composer/workflows/dependencies/coin_module.py new file mode 100644 index 00000000000..ee8fb736cf9 --- /dev/null +++ b/composer/workflows/dependencies/coin_module.py @@ -0,0 +1,32 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A custom Python package example. + +This package requires that your environment has the scipy PyPI package +installed. """ + +import numpy as np # numpy is installed by default in Composer. +from scipy import special # scipy is not. + + +def flip_coin(): + """Return "Heads" or "Tails" depending on a calculation.""" + # Returns a 2x2 randomly sampled array of values in the range [-5, 5] + rand_array = 10 * np.random.random((2, 2)) - 5 + # Computes the average of this + avg = rand_array.mean() + # Returns the Gaussian CDF of this average + ndtr = special.ndtr(avg) + return "Heads" if ndtr > .5 else "Tails" diff --git a/composer/workflows/kubernetes_pod_operator.py b/composer/workflows/kubernetes_pod_operator.py new file mode 100644 index 00000000000..6b282bd89e9 --- /dev/null +++ b/composer/workflows/kubernetes_pod_operator.py @@ -0,0 +1,223 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""An example DAG demonstrating Kubernetes Pod Operator.""" + +# [START composer_kubernetespodoperator] +import datetime + +from airflow import models +# [END composer_kubernetespodoperator] +from airflow.contrib.kubernetes import pod +from airflow.contrib.kubernetes import secret +# [START composer_kubernetespodoperator] +from airflow.contrib.operators import kubernetes_pod_operator + +# [END composer_kubernetespodoperator] + +# A Secret is an object that contains a small amount of sensitive data such as +# a password, a token, or a key. Such information might otherwise be put in a +# Pod specification or in an image; putting it in a Secret object allows for +# more control over how it is used, and reduces the risk of accidental +# exposure. +secret_file = secret.Secret( + # Mounts the secret as a file in RAM-backed tmpfs. + deploy_type='volume', + # File path of where to deploy the target, since deploy_type is 'volume' + # rather than 'env'. + deploy_target='/etc/sql_conn', + # Name of secret in Kubernetes, if the secret is not already defined in + # Kubernetes using kubectl the Pod will fail to find the secret, and in + # turn, fail to launch. + secret='airflow-secrets', + # Key of the secret within Kubernetes. + key='sql_alchemy_conn') + +secret_env = secret.Secret( + # Expose the secret as environment variable. + deploy_type='env', + # The name of the environment variable, since deploy_type is `env` rather + # than `volume`. + deploy_target='SQL_CONN', + secret='airflow-secrets', + key='sql_alchemy_conn') + +# [START composer_kubernetespodoperator] +YESTERDAY = datetime.datetime.now() - datetime.timedelta(days=1) + +# If a Pod fails to launch, or has an error occur in the container, Airflow +# will show the task as failed, as well as contain all of the task logs +# required to debug. +with models.DAG( + dag_id='composer_sample_kubernetes_pod', + schedule_interval=datetime.timedelta(days=1), + start_date=YESTERDAY) as dag: + # Only name, namespace, image, and task_id are required to create a + # KubernetesPodOperator. In Cloud Composer, currently the operator defaults + # to using the config file found at `/home/airflow/composer_kube_config if + # no `config_file` parameter is specified. By default it will contain the + # credentials for Cloud Composer's Google Kubernetes Engine cluster that is + # created upon environment creation. + kubernetes_min_pod = kubernetes_pod_operator.KubernetesPodOperator( + # The ID specified for the task. + task_id='pod-ex-minimum', + # Name of task you want to run, used to generate Pod ID. + name='pod-ex-minimum', + # Entrypoint of the container, if not specified the Docker container's + # entrypoint is used. The cmds parameter is templated. + cmds=['echo'], + # The namespace to run within Kubernetes, default namespace is + # `default`. There is the potential for the resource starvation of + # Airflow workers and scheduler within the Cloud Composer environment, + # the recommended solution is to increase the amount of nodes in order + # to satisfy the computing requirements. Alternatively, launching pods + # into a custom namespace will stop fighting over resources. + namespace='default', + # Docker image specified. Defaults to hub.docker.com, but any fully + # qualified URLs will point to a custom repository. Supports private + # gcr.io images if the Composer Environment is under the same + # project-id as the gcr.io images. + image='gcr.io/gcp-runtimes/ubuntu_16_0_4') + # [END composer_kubernetespodoperator] + + kubenetes_template_ex = kubernetes_pod_operator.KubernetesPodOperator( + task_id='ex-kube-templates', + name='ex-kube-templates', + namespace='default', + image='bash', + # All parameters below are able to be templated with jinja -- cmds, + # arguments, env_vars, and config_file. For more information visit: + # https://airflow.apache.org/code.html#default-variables + + # Entrypoint of the container, if not specified the Docker container's + # entrypoint is used. The cmds parameter is templated. + cmds=['echo'], + # DS in jinja is the execution date as YYYY-MM-DD, this docker image + # will echo the execution date. Arguments to the entrypoint. The docker + # image's CMD is used if this is not provided. The arguments parameter + # is templated. + arguments=['{{ ds }}'], + # The var template variable allows you to access variables defined in + # Airflow UI. In this case we are getting the value of my_value and + # setting the environment variable `MY_VALUE`. The pod will fail if + # `my_value` is not set in the Airflow UI. + env_vars={'MY_VALUE': '{{ var.value.my_value }}'}, + # Sets the config file to the specified airflow.cfg airflow home. If + # the configuration file does not exist or does not provide valid + # credentials the pod will fail to launch. + config_file="{{ conf.get('core', 'airflow_home') }}/config") + + kubernetes_secret_vars_ex = kubernetes_pod_operator.KubernetesPodOperator( + task_id='ex-kube-secrets', + name='ex-kube-secrets', + namespace='default', + image='ubuntu', + # The secrets to pass to Pod, the Pod will fail to create if the + # secrets you specify in a Secret object do not exist in Kubernetes. + secrets=[secret_env, secret_file], + # env_vars allows you to specify environment variables for your + # container to use. env_vars is templated. + env_vars={'EXAMPLE_VAR': '/example/value'}) + + # [START composer_kubernetespodaffinity] + kubernetes_affinity_ex = kubernetes_pod_operator.KubernetesPodOperator( + task_id='ex-pod-affinity', + name='ex-pod-affinity', + namespace='default', + image='perl', + cmds=['perl'], + arguments=['-Mbignum=bpi', '-wle', 'print bpi(2000)'], + # affinity allows you to constrain which nodes your pod is eligible to + # be scheduled on, based on labels on the node. In this case, if the + # label 'cloud.google.com/gke-nodepool' with value + # 'nodepool-label-value' or 'nodepool-label-value2' is not found on any + # nodes, it will fail to schedule. + affinity={ + 'nodeAffinity': { + # requiredDuringSchedulingIgnoredDuringExecution means in order + # for a pod to be scheduled on a node, the node must have the + # specified labels. However, if labels on a node change at + # runtime such that the affinity rules on a pod are no longer + # met, the pod will still continue to run on the node. + 'requiredDuringSchedulingIgnoredDuringExecution': { + 'nodeSelectorTerms': [{ + 'matchExpressions': [{ + # When nodepools are created in Google Kubernetes + # Engine, the nodes inside of that nodepool are + # automatically assigned the label + # 'cloud.google.com/gke-nodepool' with the value of + # the nodepool's name. + 'key': 'cloud.google.com/gke-nodepool', + 'operator': 'In', + # The label key's value that pods can be scheduled + # on. + 'values': [ + 'node-pool-name-1', + 'node-pool-name-2', + ] + }] + }] + } + } + }) + # [END composer_kubernetespodaffinity] + + kubernetes_full_pod = kubernetes_pod_operator.KubernetesPodOperator( + task_id='ex-all-configs', + name='pi', + namespace='default', + image='perl', + # Entrypoint of the container, if not specified the Docker container's + # entrypoint is used. The cmds parameter is templated. + cmds=['perl'], + # Arguments to the entrypoint. The docker image's CMD is used if this + # is not provided. The arguments parameter is templated. + arguments=['-Mbignum=bpi', '-wle', 'print bpi(2000)'], + # The secrets to pass to Pod, the Pod will fail to create if the + # secrets you specify in a Secret object do not exist in Kubernetes. + secrets=[], + # Labels to apply to the Pod. + labels={'pod-label': 'label-name'}, + # Timeout to start up the Pod, default is 120. + startup_timeout_seconds=120, + # The environment variables to be initialized in the container + # env_vars are templated. + env_vars={'EXAMPLE_VAR': '/example/value'}, + # If true, logs stdout output of container. Defaults to True. + get_logs=True, + # Determines when to pull a fresh image, if 'IfNotPresent' will cause + # the Kubelet to skip pulling an image if it already exists. If you + # want to always pull a new image, set it to 'Always'. + image_pull_policy='Always', + # Annotations are non-identifying metadata you can attach to the Pod. + # Can be a large range of data, and can include characters that are not + # permitted by labels. + annotations={'key1': 'value1'}, + # Resource specifications for Pod, this will allow you to set both cpu + # and memory limits and requirements. + resources=pod.Resources(), + # Specifies path to kubernetes config. If no config is specified will + # default to '~/.kube/config'. The config_file is templated. + config_file='/home/airflow/composer_kube_config', + # If true, the content of /airflow/xcom/return.json from container will + # also be pushed to an XCom when the container ends. + xcom_push=False, + # List of Volume objects to pass to the Pod. + volumes=[], + # List of VolumeMount objects to pass to the Pod. + volume_mounts=[], + # Affinity determines which nodes the Pod can run on based on the + # config. For more information see: + # https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + affinity={}) diff --git a/composer/workflows/kubernetes_pod_operator_test.py b/composer/workflows/kubernetes_pod_operator_test.py new file mode 100644 index 00000000000..0cd9b8f7e38 --- /dev/null +++ b/composer/workflows/kubernetes_pod_operator_test.py @@ -0,0 +1,26 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import unit_testing + + +def test_dag_import(): + """Test that the DAG file can be successfully imported. + + This tests that the DAG can be parsed, but does not run it in an Airflow + environment. This is a recommended sanity check by the official Airflow + docs: https://airflow.incubator.apache.org/tutorial.html#testing + """ + from . import kubernetes_pod_operator as module + unit_testing.assert_has_valid_dag(module) diff --git a/composer/workflows/pythonvirtualenvoperator_python2.py b/composer/workflows/pythonvirtualenvoperator_python2.py new file mode 100644 index 00000000000..af8c983a9dd --- /dev/null +++ b/composer/workflows/pythonvirtualenvoperator_python2.py @@ -0,0 +1,63 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START composer_pythonvirtualenvoperator_python2] +import datetime + +from airflow import models +from airflow.operators import python_operator + + +def python2_function(): + """A function which has not been converted to Python 3.""" + # Use the global variable virtualenv_string_args to pass in values when the + # Python version differs from that used by the Airflow process. + global virtualenv_string_args + + # Imports must happen within the function when run with the + # PythonVirtualenvOperator. + import cStringIO + import logging + + arg0 = virtualenv_string_args[0] + buffer = cStringIO.StringIO() + buffer.write('Wrote an ASCII string to buffer:\n') + buffer.write(arg0) + logging.info(buffer.getvalue()) + + +yesterday = datetime.datetime.combine( + datetime.datetime.today() - datetime.timedelta(1), + datetime.datetime.min.time()) + + +default_dag_args = { + # Setting start date as yesterday starts the DAG immediately when it is + # detected in the Cloud Storage bucket. + 'start_date': yesterday, +} + +with models.DAG( + 'composer_sample_pythonvirtualenvoperator_python2', + schedule_interval=datetime.timedelta(days=1), + default_args=default_dag_args) as dag: + + # Use the PythonVirtualenvOperator to select an explicit python_version. + run_python2 = python_operator.PythonVirtualenvOperator( + task_id='run_python2', + python_callable=python2_function, + python_version='2', + string_args=['An example input string'], + ) +# [END composer_pythonvirtualenvoperator_python2] diff --git a/composer/workflows/pythonvirtualenvoperator_python2_test.py b/composer/workflows/pythonvirtualenvoperator_python2_test.py new file mode 100644 index 00000000000..5ec7c28082e --- /dev/null +++ b/composer/workflows/pythonvirtualenvoperator_python2_test.py @@ -0,0 +1,26 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import unit_testing + + +def test_dag_import(): + """Test that the DAG file can be successfully imported. + + This tests that the DAG can be parsed, but does not run it in an Airflow + environment. This is a recommended sanity check by the official Airflow + docs: https://airflow.incubator.apache.org/tutorial.html#testing + """ + from . import pythonvirtualenvoperator_python2 as module + unit_testing.assert_has_valid_dag(module) diff --git a/composer/workflows/quickstart.py b/composer/workflows/quickstart.py new file mode 100644 index 00000000000..f9ab340b1bc --- /dev/null +++ b/composer/workflows/quickstart.py @@ -0,0 +1,105 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START composer_quickstart] +"""Example Airflow DAG that creates a Cloud Dataproc cluster, runs the Hadoop +wordcount example, and deletes the cluster. + +This DAG relies on three Airflow variables +https://airflow.apache.org/concepts.html#variables +* gcp_project - Google Cloud Project to use for the Cloud Dataproc cluster. +* gce_zone - Google Compute Engine zone where Cloud Dataproc cluster should be + created. +* gcs_bucket - Google Cloud Storage bucket to use for result of Hadoop job. + See https://cloud.google.com/storage/docs/creating-buckets for creating a + bucket. +""" + +import datetime +import os + +from airflow import models +from airflow.contrib.operators import dataproc_operator +from airflow.utils import trigger_rule + +# Output file for Cloud Dataproc job. +output_file = os.path.join( + models.Variable.get('gcs_bucket'), 'wordcount', + datetime.datetime.now().strftime('%Y%m%d-%H%M%S')) + os.sep +# Path to Hadoop wordcount example available on every Dataproc cluster. +WORDCOUNT_JAR = ( + 'file:///usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar' +) +# Arguments to pass to Cloud Dataproc job. +wordcount_args = ['wordcount', 'gs://pub/shakespeare/rose.txt', output_file] + +yesterday = datetime.datetime.combine( + datetime.datetime.today() - datetime.timedelta(1), + datetime.datetime.min.time()) + +default_dag_args = { + # Setting start date as yesterday starts the DAG immediately when it is + # detected in the Cloud Storage bucket. + 'start_date': yesterday, + # To email on failure or retry set 'email' arg to your email and enable + # emailing here. + 'email_on_failure': False, + 'email_on_retry': False, + # If a task fails, retry it once after waiting at least 5 minutes + 'retries': 1, + 'retry_delay': datetime.timedelta(minutes=5), + 'project_id': models.Variable.get('gcp_project') +} + +# [START composer_quickstart_schedule] +with models.DAG( + 'composer_sample_quickstart', + # Continue to run DAG once per day + schedule_interval=datetime.timedelta(days=1), + default_args=default_dag_args) as dag: + # [END composer_quickstart_schedule] + + # Create a Cloud Dataproc cluster. + create_dataproc_cluster = dataproc_operator.DataprocClusterCreateOperator( + task_id='create_dataproc_cluster', + # Give the cluster a unique name by appending the date scheduled. + # See https://airflow.apache.org/code.html#default-variables + cluster_name='quickstart-cluster-{{ ds_nodash }}', + num_workers=2, + zone=models.Variable.get('gce_zone'), + master_machine_type='n1-standard-1', + worker_machine_type='n1-standard-1') + + # Run the Hadoop wordcount example installed on the Cloud Dataproc cluster + # master node. + run_dataproc_hadoop = dataproc_operator.DataProcHadoopOperator( + task_id='run_dataproc_hadoop', + main_jar=WORDCOUNT_JAR, + cluster_name='quickstart-cluster-{{ ds_nodash }}', + arguments=wordcount_args) + + # Delete Cloud Dataproc cluster. + delete_dataproc_cluster = dataproc_operator.DataprocClusterDeleteOperator( + task_id='delete_dataproc_cluster', + cluster_name='quickstart-cluster-{{ ds_nodash }}', + # Setting trigger_rule to ALL_DONE causes the cluster to be deleted + # even if the Dataproc job fails. + trigger_rule=trigger_rule.TriggerRule.ALL_DONE) + + # [START composer_quickstart_steps] + # Define DAG dependencies. + create_dataproc_cluster >> run_dataproc_hadoop >> delete_dataproc_cluster + # [END composer_quickstart_steps] + +# [END composer_quickstart] diff --git a/composer/workflows/quickstart_test.py b/composer/workflows/quickstart_test.py new file mode 100644 index 00000000000..f7bb96070c3 --- /dev/null +++ b/composer/workflows/quickstart_test.py @@ -0,0 +1,31 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from airflow import models + +from . import unit_testing + + +def test_dag_import(): + """Test that the DAG file can be successfully imported. + + This tests that the DAG can be parsed, but does not run it in an Airflow + environment. This is a recommended sanity check by the official Airflow + docs: https://airflow.incubator.apache.org/tutorial.html#testing + """ + models.Variable.set('gcs_bucket', 'example_bucket') + models.Variable.set('gcp_project', 'example-project') + models.Variable.set('gce_zone', 'us-central1-f') + from . import quickstart as module + unit_testing.assert_has_valid_dag(module) diff --git a/composer/workflows/requirements.txt b/composer/workflows/requirements.txt new file mode 100644 index 00000000000..361b8e08782 --- /dev/null +++ b/composer/workflows/requirements.txt @@ -0,0 +1,4 @@ +apache-airflow[gcp_api]==1.10.2 +kubernetes==8.0.1 +scipy==1.2.0 +numpy==1.16.1 diff --git a/composer/workflows/simple.py b/composer/workflows/simple.py new file mode 100644 index 00000000000..1722eed9ec2 --- /dev/null +++ b/composer/workflows/simple.py @@ -0,0 +1,70 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""An example DAG demonstrating simple Apache Airflow operators.""" + +# [START composer_simple] +from __future__ import print_function + +# [START composer_simple_define_dag] +import datetime + +from airflow import models +# [END composer_simple_define_dag] +# [START composer_simple_operators] +from airflow.operators import bash_operator +from airflow.operators import python_operator +# [END composer_simple_operators] + + +# [START composer_simple_define_dag] +default_dag_args = { + # The start_date describes when a DAG is valid / can be run. Set this to a + # fixed point in time rather than dynamically, since it is evaluated every + # time a DAG is parsed. See: + # https://airflow.apache.org/faq.html#what-s-the-deal-with-start-date + 'start_date': datetime.datetime(2018, 1, 1), +} + +# Define a DAG (directed acyclic graph) of tasks. +# Any task you create within the context manager is automatically added to the +# DAG object. +with models.DAG( + 'composer_sample_simple_greeting', + schedule_interval=datetime.timedelta(days=1), + default_args=default_dag_args) as dag: + # [END composer_simple_define_dag] + # [START composer_simple_operators] + def greeting(): + import logging + logging.info('Hello World!') + + # An instance of an operator is called a task. In this case, the + # hello_python task calls the "greeting" Python function. + hello_python = python_operator.PythonOperator( + task_id='hello', + python_callable=greeting) + + # Likewise, the goodbye_bash task calls a Bash script. + goodbye_bash = bash_operator.BashOperator( + task_id='bye', + bash_command='echo Goodbye.') + # [END composer_simple_operators] + + # [START composer_simple_relationships] + # Define the order in which the tasks complete by using the >> and << + # operators. In this example, hello_python executes before goodbye_bash. + hello_python >> goodbye_bash + # [END composer_simple_relationships] +# [END composer_simple] diff --git a/composer/workflows/simple_test.py b/composer/workflows/simple_test.py new file mode 100644 index 00000000000..b7feec2d27e --- /dev/null +++ b/composer/workflows/simple_test.py @@ -0,0 +1,26 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import unit_testing + + +def test_dag_import(): + """Test that the DAG file can be successfully imported. + + This tests that the DAG can be parsed, but does not run it in an Airflow + environment. This is a recommended sanity check by the official Airflow + docs: https://airflow.incubator.apache.org/tutorial.html#testing + """ + from . import simple as module + unit_testing.assert_has_valid_dag(module) diff --git a/composer/workflows/trigger_response_dag.py b/composer/workflows/trigger_response_dag.py new file mode 100644 index 00000000000..dd933c54404 --- /dev/null +++ b/composer/workflows/trigger_response_dag.py @@ -0,0 +1,45 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DAG running in response to a Cloud Storage bucket change.""" + +# [START composer_trigger_response_dag] +import datetime + +import airflow +from airflow.operators import bash_operator + + +default_args = { + 'owner': 'Composer Example', + 'depends_on_past': False, + 'email': [''], + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': datetime.timedelta(minutes=5), + 'start_date': datetime.datetime(2017, 1, 1), +} + +with airflow.DAG( + 'composer_sample_trigger_response_dag', + default_args=default_args, + # Not scheduled, trigger only + schedule_interval=None) as dag: + + # Print the dag_run's configuration, which includes information about the + # Cloud Storage object change. + print_gcs_info = bash_operator.BashOperator( + task_id='print_gcs_info', bash_command='echo {{ dag_run.conf }}') +# [END composer_trigger_response_dag] diff --git a/composer/workflows/trigger_response_dag_test.py b/composer/workflows/trigger_response_dag_test.py new file mode 100644 index 00000000000..c785408375d --- /dev/null +++ b/composer/workflows/trigger_response_dag_test.py @@ -0,0 +1,26 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import unit_testing + + +def test_dag_import(): + """Test that the DAG file can be successfully imported. + + This tests that the DAG can be parsed, but does not run it in an Airflow + environment. This is a recommended sanity check by the official Airflow + docs: https://airflow.incubator.apache.org/tutorial.html#testing + """ + from . import trigger_response_dag as module + unit_testing.assert_has_valid_dag(module) diff --git a/composer/workflows/unit_testing.py b/composer/workflows/unit_testing.py new file mode 100644 index 00000000000..6c8010721ba --- /dev/null +++ b/composer/workflows/unit_testing.py @@ -0,0 +1,33 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utilities for unit testing DAGs.""" + +# [START composer_dag_unit_testing] +from airflow import models + + +def assert_has_valid_dag(module): + """Assert that a module contains a valid DAG.""" + + no_dag_found = True + + for dag in vars(module).values(): + if isinstance(dag, models.DAG): + no_dag_found = False + dag.test_cycle() # Throws if a task cycle is found. + + if no_dag_found: + raise AssertionError('module does not contain a valid DAG') +# [END composer_dag_unit_testing] diff --git a/composer/workflows/unit_testing_cycle.py b/composer/workflows/unit_testing_cycle.py new file mode 100644 index 00000000000..ae30f015026 --- /dev/null +++ b/composer/workflows/unit_testing_cycle.py @@ -0,0 +1,35 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""An example DAG demonstrating a cyle in the task IDs.""" + +import datetime + +from airflow import models +from airflow.operators import dummy_operator + + +yesterday = datetime.datetime.now() - datetime.timedelta(days=1) + +default_dag_args = { + 'start_date': yesterday, +} + +with models.DAG( + 'composer_sample_cycle', + schedule_interval=datetime.timedelta(days=1), + default_args=default_dag_args) as dag: + start = dummy_operator.DummyOperator(task_id='oops_a_cycle') + end = dummy_operator.DummyOperator(task_id='oops_a_cycle') + start >> end diff --git a/composer/workflows/unit_testing_test.py b/composer/workflows/unit_testing_test.py new file mode 100644 index 00000000000..e2dc188edc7 --- /dev/null +++ b/composer/workflows/unit_testing_test.py @@ -0,0 +1,46 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from airflow import exceptions +import pytest + +from . import unit_testing + + +def test_dag_no_dag(): + from . import unit_testing as module # Does not contain a DAG. + with pytest.raises(AssertionError): + unit_testing.assert_has_valid_dag(module) + + +def test_dag_has_cycle(): + from . import unit_testing_cycle as module + with pytest.raises(exceptions.AirflowDagCycleException): + unit_testing.assert_has_valid_dag(module) + + +# [START composer_dag_unit_testing] +def test_dag_with_variables(): + from airflow import models + + # Set any Airflow variables before importing the DAG module. + models.Variable.set('gcp_project', 'example-project') + + # Importing the module verifies that there are no syntax errors. + from . import unit_testing_variables as module + + # The assert_has_valid_dag verifies that the module contains an Airflow DAG + # and that the DAG contains no cycles. + unit_testing.assert_has_valid_dag(module) +# [END composer_dag_unit_testing] diff --git a/composer/workflows/unit_testing_variables.py b/composer/workflows/unit_testing_variables.py new file mode 100644 index 00000000000..c1c6e06a48b --- /dev/null +++ b/composer/workflows/unit_testing_variables.py @@ -0,0 +1,39 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""An example DAG demonstrating use of variables and how to test it.""" + +import datetime + +from airflow import models +from airflow.operators import bash_operator +from airflow.operators import dummy_operator + + +yesterday = datetime.datetime.now() - datetime.timedelta(days=1) + +default_dag_args = { + 'start_date': yesterday, +} + +with models.DAG( + 'composer_sample_cycle', + schedule_interval=datetime.timedelta(days=1), + default_args=default_dag_args) as dag: + start = dummy_operator.DummyOperator(task_id='start') + end = dummy_operator.DummyOperator(task_id='end') + variable_example = bash_operator.BashOperator( + task_id='variable_example', + bash_command='echo project_id=' + models.Variable.get('gcp_project')) + start >> variable_example >> end diff --git a/composer/workflows/use_local_deps.py b/composer/workflows/use_local_deps.py new file mode 100644 index 00000000000..916ca3ed87e --- /dev/null +++ b/composer/workflows/use_local_deps.py @@ -0,0 +1,39 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A DAG consisting of a BashOperator that prints the result of a coin flip.""" + +import datetime + +import airflow +from airflow.operators import bash_operator + +# [START composer_dag_local_deps] +from dependencies import coin_module +# [END composer_dag_local_deps] + +default_args = { + 'start_date': + datetime.datetime.combine( + datetime.datetime.today() - datetime.timedelta(days=1), + datetime.datetime.min.time()), +} + +with airflow.DAG( + 'composer_sample_dependencies_dag', + default_args=default_args) as dag: + t1 = bash_operator.BashOperator( + task_id='print_coin_result', + bash_command='echo "{0}"'.format(coin_module.flip_coin()), + dag=dag) diff --git a/composer/workflows/use_local_deps_test.py b/composer/workflows/use_local_deps_test.py new file mode 100644 index 00000000000..071bdec6aa3 --- /dev/null +++ b/composer/workflows/use_local_deps_test.py @@ -0,0 +1,43 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os.path +import sys + +import pytest + +from . import unit_testing + + +@pytest.fixture(scope='module', autouse=True) +def local_deps(): + """Add local directory to the PYTHONPATH to allow absolute imports. + + Relative imports do not work in Airflow workflow definitions. + """ + workflows_dir = os.path.abspath(os.path.dirname(__file__)) + sys.path.append(workflows_dir) + yield + sys.path.remove(workflows_dir) + + +def test_dag_import(): + """Test that the DAG file can be successfully imported. + + This tests that the DAG can be parsed, but does not run it in an Airflow + environment. This is a recommended sanity check by the official Airflow + docs: https://airflow.incubator.apache.org/tutorial.html#testing + """ + from . import use_local_deps as module + unit_testing.assert_has_valid_dag(module) diff --git a/compute/README.md b/compute/README.md index f0bc60f8b0a..3eb22ea4512 100644 --- a/compute/README.md +++ b/compute/README.md @@ -1,15 +1,16 @@ # Google Compute Engine Samples +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=compute/README.md + This section contains samples for [Google Compute Engine](https://cloud.google.com/compute). ## Running the samples 1. Your environment must be setup with [authentication -information](https://developers.google.com/identity/protocols/application-default-credentials#howtheywork). *Note* that Cloud Monitoring does not currently work -with `gcloud auth`. You will need to use a *service account* when running -locally and set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable. - - $ export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service_account.json +information](https://developers.google.com/identity/protocols/application-default-credentials#howtheywork). If you're running on Compute Engine, this is already setup. 2. Install dependencies from `requirements.txt` @@ -23,11 +24,6 @@ For more information on Compute Engine you can visit: > https://cloud.google.com/compute -For more information on the Cloud Monitoring API Python library surface you -can visit: - -> https://developers.google.com/resources/api-libraries/documentation/compute/v1/python/latest/ - For information on the Python Client Library visit: > https://developers.google.com/api-client-library/python diff --git a/compute/api/README.md b/compute/api/README.md index 3f95f09a9e0..757a4a2a79a 100644 --- a/compute/api/README.md +++ b/compute/api/README.md @@ -1,9 +1,17 @@ # Compute Engine API Samples +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=compute/api/README.md + -These samples are used on the following documentation page: +These samples are used on the following documentation pages: -> https://cloud.google.com/compute/docs/tutorials/python-guide +> +* https://cloud.google.com/compute/docs/tutorials/python-guide +* https://cloud.google.com/compute/docs/instances/create-start-instance +* https://cloud.google.com/compute/docs/api/how-tos/api-requests-responses diff --git a/compute/api/create_instance.py b/compute/api/create_instance.py index 681b76e1a09..46418d6f6fb 100644 --- a/compute/api/create_instance.py +++ b/compute/api/create_instance.py @@ -28,22 +28,25 @@ import os import time -from googleapiclient import discovery -from oauth2client.client import GoogleCredentials +import googleapiclient.discovery from six.moves import input # [START list_instances] def list_instances(compute, project, zone): result = compute.instances().list(project=project, zone=zone).execute() - return result['items'] + return result['items'] if 'items' in result else None # [END list_instances] # [START create_instance] def create_instance(compute, project, zone, name, bucket): - source_disk_image = \ - "projects/debian-cloud/global/images/debian-7-wheezy-v20150320" + # Get the latest Debian Jessie image. + image_response = compute.images().getFromFamily( + project='debian-cloud', family='debian-9').execute() + source_disk_image = image_response['selfLink'] + + # Configure the machine machine_type = "zones/%s/machineTypes/n1-standard-1" % zone startup_script = open( os.path.join( @@ -142,8 +145,7 @@ def wait_for_operation(compute, project, zone, operation): # [START run] def main(project, bucket, zone, instance_name, wait=True): - credentials = GoogleCredentials.get_application_default() - compute = discovery.build('compute', 'v1', credentials=credentials) + compute = googleapiclient.discovery.build('compute', 'v1') print('Creating instance.') diff --git a/compute/api/create_instance_test.py b/compute/api/create_instance_test.py index f85dcf09d0e..95244e95d6b 100644 --- a/compute/api/create_instance_test.py +++ b/compute/api/create_instance_test.py @@ -11,17 +11,22 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import re +from gcp_devrel.testing.flaky import flaky + from create_instance import main -from gcp.testing.flaky import flaky + +PROJECT = os.environ['GCLOUD_PROJECT'] +BUCKET = os.environ['CLOUD_STORAGE_BUCKET'] @flaky -def test_main(cloud_config, capsys): +def test_main(capsys): main( - cloud_config.project, - cloud_config.storage_bucket, + PROJECT, + BUCKET, 'us-central1-f', 'test-instance', wait=False) diff --git a/compute/api/requirements.txt b/compute/api/requirements.txt index c3b2784ce87..7e4359ce08d 100644 --- a/compute/api/requirements.txt +++ b/compute/api/requirements.txt @@ -1 +1,3 @@ -google-api-python-client==1.5.0 +google-api-python-client==1.7.8 +google-auth==1.6.2 +google-auth-httplib2==0.0.3 diff --git a/compute/auth/access_token.py b/compute/auth/access_token.py new file mode 100644 index 00000000000..3fc7c553a5e --- /dev/null +++ b/compute/auth/access_token.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Example of authenticating using access tokens directly on Compute Engine. + +For more information, see the README.md under /compute. +""" + +# [START all] + +import argparse + +import requests + + +METADATA_URL = 'http://metadata.google.internal/computeMetadata/v1/' +METADATA_HEADERS = {'Metadata-Flavor': 'Google'} +SERVICE_ACCOUNT = 'default' + + +def get_access_token(): + url = '{}instance/service-accounts/{}/token'.format( + METADATA_URL, SERVICE_ACCOUNT) + + # Request an access token from the metadata server. + r = requests.get(url, headers=METADATA_HEADERS) + r.raise_for_status() + + # Extract the access token from the response. + access_token = r.json()['access_token'] + + return access_token + + +def list_buckets(project_id, access_token): + url = 'https://www.googleapis.com/storage/v1/b' + params = { + 'project': project_id + } + headers = { + 'Authorization': 'Bearer {}'.format(access_token) + } + + r = requests.get(url, params=params, headers=headers) + r.raise_for_status() + + return r.json() + + +def main(project_id): + access_token = get_access_token() + buckets = list_buckets(project_id, access_token) + print(buckets) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('project_id', help='Your Google Cloud project ID.') + + args = parser.parse_args() + + main(args.project_id) +# [END all] diff --git a/compute/auth/access_token_test.py b/compute/auth/access_token_test.py new file mode 100644 index 00000000000..4d65b020da8 --- /dev/null +++ b/compute/auth/access_token_test.py @@ -0,0 +1,39 @@ +# Copyright 2016, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import mock + +import access_token + +PROJECT = os.environ['GCLOUD_PROJECT'] + + +@mock.patch('access_token.requests') +def test_main(requests_mock): + metadata_response = mock.Mock() + metadata_response.status_code = 200 + metadata_response.json.return_value = { + 'access_token': '123' + } + bucket_response = mock.Mock() + bucket_response.status_code = 200 + bucket_response.json.return_value = [{'bucket': 'name'}] + + requests_mock.get.side_effect = [ + metadata_response, bucket_response] + + access_token.main(PROJECT) + + assert requests_mock.get.call_count == 2 diff --git a/compute/auth/application_default.py b/compute/auth/application_default.py new file mode 100644 index 00000000000..f2ca6b085e6 --- /dev/null +++ b/compute/auth/application_default.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Example of authenticating using Application Default Credentials on +Compute Engine. + +For more information, see the README.md under /compute. +""" + +# [START all] + +import argparse + +import googleapiclient.discovery + + +def create_service(): + # Construct the service object for interacting with the Cloud Storage API - + # the 'storage' service, at version 'v1'. + # Authentication is provided by application default credentials. + # When running locally, these are available after running + # `gcloud auth application-default login`. When running on Compute + # Engine, these are available from the environment. + return googleapiclient.discovery.build('storage', 'v1') + + +def list_buckets(service, project_id): + buckets = service.buckets().list(project=project_id).execute() + return buckets + + +def main(project_id): + service = create_service() + buckets = list_buckets(service, project_id) + print(buckets) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('project_id', help='Your Google Cloud Project ID.') + + args = parser.parse_args() + + main(args.project_id) +# [END all] diff --git a/compute/auth/application_default_test.py b/compute/auth/application_default_test.py new file mode 100644 index 00000000000..6d23b71639c --- /dev/null +++ b/compute/auth/application_default_test.py @@ -0,0 +1,22 @@ +# Copyright 2016, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from application_default import main + +PROJECT = os.environ['GCLOUD_PROJECT'] + + +def test_main(): + main(PROJECT) diff --git a/compute/auth/requirements.txt b/compute/auth/requirements.txt new file mode 100644 index 00000000000..707b08bdc29 --- /dev/null +++ b/compute/auth/requirements.txt @@ -0,0 +1,4 @@ +requests==2.21.0 +google-api-python-client==1.7.8 +google-auth==1.6.2 +google-auth-httplib2==0.0.3 diff --git a/compute/autoscaler/demo/frontend.py b/compute/autoscaler/demo/frontend.py index 294cb2f45d0..188c48b87ad 100644 --- a/compute/autoscaler/demo/frontend.py +++ b/compute/autoscaler/demo/frontend.py @@ -25,7 +25,7 @@ try: import BaseHTTPServer import SocketServer -except: +except ImportError: import http.server as BaseHTTPServer import socketserver as SocketServer diff --git a/compute/autoscaler/demo/frontend_test.py b/compute/autoscaler/demo/frontend_test.py index 09bae5d8e77..41aa9938c98 100644 --- a/compute/autoscaler/demo/frontend_test.py +++ b/compute/autoscaler/demo/frontend_test.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import frontend import pytest +import frontend + class FakeTime(object): """Fake implementations of GetUserCpuTime, GetUserCpuTime and BusyWait. diff --git a/compute/encryption/generate_wrapped_rsa_key.py b/compute/encryption/generate_wrapped_rsa_key.py new file mode 100644 index 00000000000..8df4a4e47d4 --- /dev/null +++ b/compute/encryption/generate_wrapped_rsa_key.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Example of authenticating using access tokens directly on Compute Engine. + +For more information, see the README.md under /compute. +""" + +# [START all] + +import argparse +import base64 +import os + +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import padding +import requests + + +GOOGLE_PUBLIC_CERT_URL = ( + 'https://cloud-certs.storage.googleapis.com/google-cloud-csek-ingress.pem') + + +def get_google_public_cert_key(): + r = requests.get(GOOGLE_PUBLIC_CERT_URL) + r.raise_for_status() + + # Load the certificate. + certificate = x509.load_pem_x509_certificate( + r.text.encode('utf-8'), default_backend()) + + # Get the certicate's public key. + public_key = certificate.public_key() + + return public_key + + +def wrap_rsa_key(public_key, private_key_bytes): + # Use the Google public key to encrypt the customer private key. + # This means that only the Google private key is capable of decrypting + # the customer private key. + wrapped_key = public_key.encrypt( + private_key_bytes, + padding.OAEP( + mgf=padding.MGF1(algorithm=hashes.SHA1()), + algorithm=hashes.SHA1(), + label=None)) + encoded_wrapped_key = base64.b64encode(wrapped_key) + return encoded_wrapped_key + + +def main(key_file): + # Generate a new 256-bit private key if no key is specified. + if not key_file: + customer_key_bytes = os.urandom(32) + else: + with open(key_file, 'rb') as f: + customer_key_bytes = f.read() + + google_public_key = get_google_public_cert_key() + wrapped_rsa_key = wrap_rsa_key(google_public_key, customer_key_bytes) + + print('Base-64 encoded private key: {}'.format( + base64.b64encode(customer_key_bytes).decode('utf-8'))) + print('Wrapped RSA key: {}'.format(wrapped_rsa_key.decode('utf-8'))) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--key_file', help='File containing your binary private key.') + + args = parser.parse_args() + + main(args.key_file) +# [END all] diff --git a/compute/encryption/generate_wrapped_rsa_key_test.py b/compute/encryption/generate_wrapped_rsa_key_test.py new file mode 100644 index 00000000000..ea5ed6ea524 --- /dev/null +++ b/compute/encryption/generate_wrapped_rsa_key_test.py @@ -0,0 +1,51 @@ +# Copyright 2016, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import googleapiclient.discovery + +import generate_wrapped_rsa_key + +PROJECT = os.environ['GCLOUD_PROJECT'] + + +def test_main(): + generate_wrapped_rsa_key.main(None) + + +def test_create_disk(): + compute = googleapiclient.discovery.build('compute', 'beta') + + # Generate the key. + key_bytes = os.urandom(32) + google_public_key = generate_wrapped_rsa_key.get_google_public_cert_key() + wrapped_rsa_key = generate_wrapped_rsa_key.wrap_rsa_key( + google_public_key, key_bytes) + + # Create the disk, if the encryption key is invalid, this will raise. + compute.disks().insert( + project=PROJECT, + zone='us-central1-f', + body={ + 'name': 'new-encrypted-disk', + 'diskEncryptionKey': { + 'rsaEncryptedKey': wrapped_rsa_key.decode('utf-8') + } + }).execute() + + # Delete the disk. + compute.disks().delete( + project=PROJECT, + zone='us-central1-f', + disk='new-encrypted-disk').execute() diff --git a/compute/encryption/requirements.txt b/compute/encryption/requirements.txt new file mode 100644 index 00000000000..5078ceb8c10 --- /dev/null +++ b/compute/encryption/requirements.txt @@ -0,0 +1,5 @@ +cryptography==2.5 +requests==2.21.0 +google-api-python-client==1.7.8 +google-auth==1.6.2 +google-auth-httplib2==0.0.3 diff --git a/compute/metadata/README.md b/compute/metadata/README.md new file mode 100644 index 00000000000..fdb92240af2 --- /dev/null +++ b/compute/metadata/README.md @@ -0,0 +1,15 @@ +# Compute Engine Metadata Samples + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=compute/metadata/README.md + +These samples demonstrate interacting with the Compute Engine metadata service. These samples must be run on Compute Engine. + + +These samples are used on the following documentation page: + +> https://cloud.google.com/compute/docs/metadata + + diff --git a/compute/metadata/main.py b/compute/metadata/main.py new file mode 100644 index 00000000000..428a2577f9f --- /dev/null +++ b/compute/metadata/main.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Example of using the Compute Engine API to watch for maintenance notices. + +For more information, see the README.md under /compute. +""" + +# [START all] + +import time + +import requests + + +METADATA_URL = 'http://metadata.google.internal/computeMetadata/v1/' +METADATA_HEADERS = {'Metadata-Flavor': 'Google'} + + +def wait_for_maintenance(callback): + url = METADATA_URL + 'instance/maintenance-event' + last_maintenance_event = None + # [START hanging_get] + last_etag = '0' + + while True: + r = requests.get( + url, + params={'last_etag': last_etag, 'wait_for_change': True}, + headers=METADATA_HEADERS) + + # During maintenance the service can return a 503, so these should + # be retried. + if r.status_code == 503: + time.sleep(1) + continue + r.raise_for_status() + + last_etag = r.headers['etag'] + # [END hanging_get] + + if r.text == 'NONE': + maintenance_event = None + else: + maintenance_event = r.text + + if maintenance_event != last_maintenance_event: + last_maintenance_event = maintenance_event + callback(maintenance_event) + + +def maintenance_callback(event): + if event: + print('Undergoing host maintenance: {}'.format(event)) + else: + print('Finished host maintenance') + + +def main(): + wait_for_maintenance(maintenance_callback) + + +if __name__ == '__main__': + main() +# [END all] diff --git a/compute/metadata/main_test.py b/compute/metadata/main_test.py new file mode 100644 index 00000000000..51ef9bda42e --- /dev/null +++ b/compute/metadata/main_test.py @@ -0,0 +1,51 @@ +# Copyright 2016, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import requests + +import main + + +@mock.patch('main.requests') +def test_wait_for_maintenance(requests_mock): + # Response 1 is a host maintenance event. + response1_mock = mock.Mock() + response1_mock.status_code = 200 + response1_mock.text = 'MIGRATE_ON_HOST_MAINTENANCE' + response1_mock.headers = {'etag': 1} + # Response 2 is the end of the event. + response2_mock = mock.Mock() + response2_mock.status_code = 200 + response2_mock.text = 'NONE' + response2_mock.headers = {'etag': 2} + # Response 3 is a 503 + response3_mock = mock.Mock() + response3_mock.status_code = 503 + + requests_mock.codes.ok = requests.codes.ok + requests_mock.get.side_effect = [ + response1_mock, response2_mock, response3_mock, response2_mock, + StopIteration()] + + callback_mock = mock.Mock() + + try: + main.wait_for_maintenance(callback_mock) + except StopIteration: + pass + + assert callback_mock.call_count == 2 + assert callback_mock.call_args_list[0][0] == ( + 'MIGRATE_ON_HOST_MAINTENANCE',) + assert callback_mock.call_args_list[1][0] == (None,) diff --git a/compute/metadata/requirements.txt b/compute/metadata/requirements.txt new file mode 100644 index 00000000000..da3fa4c34e3 --- /dev/null +++ b/compute/metadata/requirements.txt @@ -0,0 +1 @@ +requests==2.21.0 diff --git a/compute/oslogin/requirements.txt b/compute/oslogin/requirements.txt new file mode 100644 index 00000000000..a1a63f75b4b --- /dev/null +++ b/compute/oslogin/requirements.txt @@ -0,0 +1,3 @@ +google-api-python-client==1.7.4 +google-auth==1.6.1 +google-auth-httplib2==0.0.3 diff --git a/compute/oslogin/service_account_ssh.py b/compute/oslogin/service_account_ssh.py new file mode 100644 index 00000000000..ce854e63829 --- /dev/null +++ b/compute/oslogin/service_account_ssh.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python + +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Example of using the OS Login API to apply public SSH keys for a service +account, and use that service account to execute commands on a remote +instance over SSH. This example uses zonal DNS names to address instances +on the same internal VPC network. +""" + +# [START imports_and_variables] +import time +import subprocess +import uuid +import logging +import requests +import argparse +import googleapiclient.discovery + +# Global variables +SERVICE_ACCOUNT_METADATA_URL = ( + 'http://metadata.google.internal/computeMetadata/v1/instance/' + 'service-accounts/default/email') +HEADERS = {'Metadata-Flavor': 'Google'} + +# [END imports_and_variables] + + +# [START run_command_local] +def execute(cmd, cwd=None, capture_output=False, env=None, raise_errors=True): + """Execute an external command (wrapper for Python subprocess).""" + logging.info('Executing command: {cmd}'.format(cmd=str(cmd))) + stdout = subprocess.PIPE if capture_output else None + process = subprocess.Popen(cmd, cwd=cwd, env=env, stdout=stdout) + output = process.communicate()[0] + returncode = process.returncode + if returncode: + # Error + if raise_errors: + raise subprocess.CalledProcessError(returncode, cmd) + else: + logging.info('Command returned error status %s', returncode) + if output: + logging.info(output) + return returncode, output +# [END run_command_local] + + +# [START create_key] +def create_ssh_key(oslogin, account, private_key_file=None, expire_time=300): + """Generate an SSH key pair and apply it to the specified account.""" + private_key_file = private_key_file or '/tmp/key-' + str(uuid.uuid4()) + execute(['ssh-keygen', '-t', 'rsa', '-N', '', '-f', private_key_file]) + + with open(private_key_file + '.pub', 'r') as original: + public_key = original.read().strip() + + # Expiration time is in microseconds. + expiration = int((time.time() + expire_time) * 1000000) + + body = { + 'key': public_key, + 'expirationTimeUsec': expiration, + } + oslogin.users().importSshPublicKey(parent=account, body=body).execute() + return private_key_file +# [END create_key] + + +# [START run_command_remote] +def run_ssh(cmd, private_key_file, username, hostname): + """Run a command on a remote system.""" + ssh_command = [ + 'ssh', '-i', private_key_file, '-o', 'StrictHostKeyChecking=no', + '{username}@{hostname}'.format(username=username, hostname=hostname), + cmd, + ] + ssh = subprocess.Popen( + ssh_command, shell=False, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + result = ssh.stdout.readlines() + return result if result else ssh.stderr.readlines() + +# [END run_command_remote] + + +# [START main] +def main(cmd, project, instance=None, zone=None, + oslogin=None, account=None, hostname=None): + """Run a command on a remote system.""" + + # Create the OS Login API object. + oslogin = oslogin or googleapiclient.discovery.build('oslogin', 'v1') + + # Identify the service account ID if it is not already provided. + account = account or requests.get( + SERVICE_ACCOUNT_METADATA_URL, headers=HEADERS).text + if not account.startswith('users/'): + account = 'users/' + account + + # Create a new SSH key pair and associate it with the service account. + private_key_file = create_ssh_key(oslogin, account) + + # Using the OS Login API, get the POSIX user name from the login profile + # for the service account. + profile = oslogin.users().getLoginProfile(name=account).execute() + username = profile.get('posixAccounts')[0].get('username') + + # Create the hostname of the target instance using the instance name, + # the zone where the instance is located, and the project that owns the + # instance. + hostname = hostname or '{instance}.{zone}.c.{project}.internal'.format( + instance=instance, zone=zone, project=project) + + # Run a command on the remote instance over SSH. + result = run_ssh(cmd, private_key_file, username, hostname) + + # Print the command line output from the remote instance. + # Use .rstrip() rather than end='' for Python 2 compatability. + for line in result: + print(line.decode('utf-8').rstrip('\n\r')) + + # Shred the private key and delete the pair. + execute(['shred', private_key_file]) + execute(['rm', private_key_file]) + execute(['rm', private_key_file + '.pub']) + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--cmd', default='uname -a', + help='The command to run on the remote instance.') + parser.add_argument( + '--project', + help='Your Google Cloud project ID.') + parser.add_argument( + '--zone', + help='The zone where the target instance is locted.') + parser.add_argument( + '--instance', + help='The target instance for the ssh command.') + parser.add_argument( + '--account', + help='The service account email.') + parser.add_argument( + '--hostname', + help='The external IP address or hostname for the target instance.') + args = parser.parse_args() + + main(args.cmd, args.project, instance=args.instance, zone=args.zone, + account=args.account, hostname=args.hostname) + +# [END main] diff --git a/compute/oslogin/service_account_ssh_test.py b/compute/oslogin/service_account_ssh_test.py new file mode 100644 index 00000000000..b49e43fa2f2 --- /dev/null +++ b/compute/oslogin/service_account_ssh_test.py @@ -0,0 +1,255 @@ +# Copyright 2019, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +import random +import base64 +import json +import googleapiclient.discovery +from google.oauth2 import service_account +from service_account_ssh import main + +''' +The service account that runs this test must have the following roles: +- roles/compute.instanceAdmin.v1 +- roles/compute.securityAdmin +- roles/iam.serviceAccountAdmin +- roles/iam.serviceAccountKeyAdmin +- roles/iam.serviceAccountUser +The Project Editor legacy role is not sufficient because it does not grant +several necessary permissions. +''' + + +def test_main(capsys): + + # Initialize variables. + cmd = 'uname -a' + project = os.environ['GCLOUD_PROJECT'] + test_id = 'oslogin-test-{id}'.format(id=str(random.randint(0, 1000000))) + zone = 'us-east1-d' + image_family = 'projects/debian-cloud/global/images/family/debian-9' + machine_type = 'zones/{zone}/machineTypes/f1-micro'.format(zone=zone) + account_email = '{test_id}@{project}.iam.gserviceaccount.com'.format( + test_id=test_id, project=project) + + # Initialize the necessary APIs. + iam = googleapiclient.discovery.build( + 'iam', 'v1', cache_discovery=False) + compute = googleapiclient.discovery.build( + 'compute', 'v1', cache_discovery=False) + + # Create the necessary test resources and retrieve the service account + # email and account key. + try: + print('Creating test resources.') + service_account_key = setup_resources( + compute, iam, project, test_id, zone, image_family, + machine_type, account_email) + except Exception: + print('Cleaning up partially created test resources.') + cleanup_resources(compute, iam, project, test_id, zone, account_email) + raise Exception('Could not set up the necessary test resources.') + + # Get the target host name for the instance + hostname = compute.instances().get( + project=project, + zone=zone, + instance=test_id, + fields='networkInterfaces/accessConfigs/natIP' + ).execute()['networkInterfaces'][0]['accessConfigs'][0]['natIP'] + + # Create a credentials object and use it to initialize the OS Login API. + credentials = service_account.Credentials.from_service_account_info( + json.loads(base64.b64decode( + service_account_key['privateKeyData']).decode('utf-8'))) + + oslogin = googleapiclient.discovery.build( + 'oslogin', 'v1', cache_discovery=False, credentials=credentials) + account = 'users/' + account_email + + # Give OS Login some time to catch up. + time.sleep(120) + + # Test SSH to the instance. + try: + main(cmd, project, test_id, zone, oslogin, account, hostname) + out, _ = capsys.readouterr() + assert_value = 'Linux {test_id}'.format(test_id=test_id) + assert assert_value in out + except Exception: + raise Exception('SSH to the test instance failed.') + + finally: + cleanup_resources(compute, iam, project, test_id, zone, account_email) + + +def setup_resources( + compute, iam, project, test_id, zone, + image_family, machine_type, account_email): + + # Create a temporary service account. + iam.projects().serviceAccounts().create( + name='projects/' + project, + body={ + 'accountId': test_id + }).execute() + + # Grant the service account access to itself. + iam.projects().serviceAccounts().setIamPolicy( + resource='projects/' + project + '/serviceAccounts/' + account_email, + body={ + 'policy': { + 'bindings': [ + { + 'members': [ + 'serviceAccount:' + account_email + ], + 'role': 'roles/iam.serviceAccountUser' + } + ] + } + }).execute() + + # Create a service account key. + service_account_key = iam.projects().serviceAccounts().keys().create( + name='projects/' + project + '/serviceAccounts/' + account_email, + body={} + ).execute() + + # Create a temporary firewall on the default network to allow SSH tests + # only for instances with the temporary service account. + firewall_config = { + 'name': test_id, + 'network': '/global/networks/default', + 'targetServiceAccounts': [ + account_email + ], + 'sourceRanges': [ + '0.0.0.0/0' + ], + 'allowed': [{ + 'IPProtocol': 'tcp', + 'ports': [ + '22' + ], + }] + } + + compute.firewalls().insert( + project=project, + body=firewall_config).execute() + + # Create a new test instance. + instance_config = { + 'name': test_id, + 'machineType': machine_type, + 'disks': [ + { + 'boot': True, + 'autoDelete': True, + 'initializeParams': { + 'sourceImage': image_family, + } + } + ], + 'networkInterfaces': [{ + 'network': 'global/networks/default', + 'accessConfigs': [ + {'type': 'ONE_TO_ONE_NAT', 'name': 'External NAT'} + ] + }], + 'serviceAccounts': [{ + 'email': account_email, + 'scopes': [ + 'https://www.googleapis.com/auth/cloud-platform' + ] + }], + 'metadata': { + 'items': [{ + 'key': 'enable-oslogin', + 'value': 'TRUE' + }] + } + } + + operation = compute.instances().insert( + project=project, + zone=zone, + body=instance_config).execute() + + # Wait for the instance to start. + while compute.zoneOperations().get( + project=project, + zone=zone, + operation=operation['name']).execute()['status'] != 'DONE': + time.sleep(5) + + # Grant the service account osLogin access on the test instance. + compute.instances().setIamPolicy( + project=project, + zone=zone, + resource=test_id, + body={ + 'bindings': [ + { + 'members': [ + 'serviceAccount:' + account_email + ], + 'role': 'roles/compute.osLogin' + } + ] + }).execute() + + # Wait for the IAM policy to take effect. + while compute.instances().getIamPolicy( + project=project, + zone=zone, + resource=test_id, + fields='bindings/role' + ).execute()['bindings'][0]['role'] != 'roles/compute.osLogin': + time.sleep(5) + + return service_account_key + + +def cleanup_resources(compute, iam, project, test_id, zone, account_email): + + # Delete the temporary firewall. + try: + compute.firewalls().delete( + project=project, + firewall=test_id).execute() + except Exception: + pass + + # Delete the test instance. + try: + delete = compute.instances().delete( + project=project, zone=zone, instance=test_id).execute() + + while compute.zoneOperations().get( + project=project, zone=zone, operation=delete['name'] + ).execute()['status'] != 'DONE': + time.sleep(5) + except Exception: + pass + + # Delete the temporary service account and its associated keys. + try: + iam.projects().serviceAccounts().delete( + name='projects/' + project + '/serviceAccounts/' + account_email + ).execute() + except Exception: + pass diff --git a/compute/xmpp_wikibot/README.md b/compute/xmpp_wikibot/README.md new file mode 100644 index 00000000000..e1dd45ecd56 --- /dev/null +++ b/compute/xmpp_wikibot/README.md @@ -0,0 +1,80 @@ +# Wikibot example that can be run on Google Compute Engine + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=compute/xmpp_wikibot/README.md + +This sample shows how to use the [SleekXMPP](http://sleekxmpp.com/index.html) +client and [Flask](http://flask.pocoo.org/) to build a simple chatbot that can +be run on [Google Compute Engine](https://cloud.google.com/compute/). The +chatbot does two things: + +1. Sends messages to XMPP users via http get: + * The server is running on port 5000 + * if running on virtual machine use: + `http://:5000/send_message?recipient=&message=` + * If running locally use: + `http://localhost:5000/send_message?recipient=&message=` + +2. Responds to incoming messages with a Wikipedia page on the topic: + * Send a message with a topic (e.g., 'Hawaii') to the XMPP account the + server is using + * It should respond with a Wikipedia page (when one exists) + +## Setup + +Follow the instructions at the +[Compute Engine Quickstart Guide](https://cloud.google.com/compute/docs/quickstart-linux) +on how to create a project, create a virtual machine, and connect to your +instance via SSH. Once you have done this, you may jump to +[Installing files and dependencies](#installing-files-and-dependencies). + +You should also download the [Google Cloud SDK](https://cloud.google.com/sdk/). +It will allow you to access many of the features of Google Compute Engine via +your local machine. + +**IMPORTANT** You must enable tcp traffic on port 5000 to send messages to the +XMPP server. This can be done by running the following SDK commands: + + gcloud config set project + + gcloud compute firewall-rules create wikibot-server-rule --allow tcp:5000 --source-ranges=0.0.0.0/0 + +Or you can create a new firewall rule via the UI in the +[Networks](https://console.cloud.google.com/networking/networks/list) section of +the Google Cloud Console. + +### Installing files and dependencies + +First, install the `wikibot.py` and `requirements.txt` files onto your remote +instance. See the guide on +[Transferring Files](https://cloud.google.com/compute/docs/instances/transfer-files) +for more information on how to do this using the Mac file browser, `scp`, or +the Google Cloud SDK. + +Before running or deploying this application, you must install the dependencies +using [pip](http://pip.readthedocs.io/en/stable/): + + pip install -r requirements.txt + + +## Running the sample + +You'll need to have an XMPP account prior to actually running the sample. +If you do not have one, you can easily create an account at one of the many +XMPP servers such as [xmpp.jp](http://xmpp.jp). +Once you have an account, run the following command: + + python wikibot.py -j '' -p '' + +Where the username (e.g., 'bob@xmpp.jp') and password for the account that +you'd like to use for your chatbot are passed in as arguments. + +Enter control-C to stop the server + + +### Running on your local machine + +You may also run the sample locally by simply copying `wikibot.py` to a project +directory and installing all python dependencies there. diff --git a/compute/xmpp_wikibot/requirements.txt b/compute/xmpp_wikibot/requirements.txt new file mode 100644 index 00000000000..a361532b387 --- /dev/null +++ b/compute/xmpp_wikibot/requirements.txt @@ -0,0 +1,4 @@ +Flask==1.0.2 +requests==2.21.0 +sleekxmpp==1.3.3 +six==1.12.0 diff --git a/compute/xmpp_wikibot/wikibot.py b/compute/xmpp_wikibot/wikibot.py new file mode 100644 index 00000000000..a65c67c235c --- /dev/null +++ b/compute/xmpp_wikibot/wikibot.py @@ -0,0 +1,153 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Wikibot server example using SleekXMPP client library""" + +import argparse +import getpass +import json +import re +import sys +import urllib + +from flask import Flask, request +import requests +from six.moves import html_parser +import sleekxmpp + +app = Flask(__name__) + + +@app.route('/send_message', methods=['GET']) +def send_message(): + recipient = request.args.get('recipient') + message = request.args.get('message') + + if chat_client and recipient and message: + chat_client.send_message(mto=recipient, mbody=message) + return 'message sent to {} with body: {}'.format(recipient, message) + else: + return 'message failed to send', 400 + + +class WikiBot(sleekxmpp.ClientXMPP): + """A simple SleekXMPP bot that will take messages, look up their content on + wikipedia and provide a link to the page if it exists. + """ + + def __init__(self, jid, password): + sleekxmpp.ClientXMPP.__init__(self, jid, password) + + # The session_start event will be triggered when + # the bot establishes its connection with the server + # and the XML streams are ready for use. We want to + # listen for this event so that we we can initialize + # our roster. + self.add_event_handler('session_start', self.start) + + # The message event is triggered whenever a message + # stanza is received. Be aware that that includes + # MUC messages and error messages. + self.add_event_handler('message', self.message) + + # Register plugins. Note that while plugins may have + # interdependencies, the order you register them in doesn't matter. + self.register_plugin('xep_0030') # Service Discovery + self.register_plugin('xep_0004') # Data Forms + self.register_plugin('xep_0060') # PubSub + self.register_plugin('xep_0199') # XMPP Ping + + def start(self, event): + """Process the session_start event. + + Typical actions for the session_start event are requesting the roster + and broadcasting an initial presence stanza. + + Arguments: + event -- An empty dictionary. The session_start event does not + provide any additional data. + """ + self.send_presence() + self.get_roster() + + def message(self, msg): + """Process incoming message stanzas. + + Be aware that this also includes MUC messages and error messages. It is + usually a good idea to check the messages's type before processing or + sending replies. If the message is the appropriate type, then the bot + checks wikipedia to see if the message string exists as a page on the + site. If so, it sends this link back to the sender in the reply. + + Arguments: + msg -- The received message stanza. See the SleekXMPP documentation + for stanza objects and the Message stanza to see how it may be + used. + """ + if msg['type'] in ('chat', 'normal'): + msg_body = msg['body'] + encoded_body = urllib.quote_plus(msg_body) + response = requests.get( + 'https://en.wikipedia.org/w/api.php?' + 'action=query&list=search&format=json&srprop=snippet&' + 'srsearch={}'.format(encoded_body)) + doc = json.loads(response.content) + + results = doc.get('query', {}).get('search') + if not results: + msg.reply('I wasn\'t able to locate info on "{}" Sorry'.format( + msg_body)).send() + return + + snippet = results[0]['snippet'] + title = urllib.quote_plus(results[0]['title']) + + # Strip out html + snippet = html_parser.HTMLParser().unescape( + re.sub(r'<[^>]*>', '', snippet)) + msg.reply(u'{}...\n(http://en.wikipedia.org/w/?title={})'.format( + snippet, title)).send() + + +if __name__ == '__main__': + # Setup the command line arguments. + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + + # JID and password options. + parser.add_argument('-j', '--jid', help='JID to use', required=True) + parser.add_argument('-p', '--password', help='password to use') + + args = parser.parse_args() + + if args.password is None: + args.password = getpass.getpass('Password: ') + + xmpp = WikiBot(args.jid, args.password) + + chat_client = xmpp # set the global variable + + try: + # Connect to the XMPP server and start processing XMPP stanzas. + if not xmpp.connect(): + print('Unable to connect.') + sys.exit(1) + + xmpp.process(block=False) + + app.run(threaded=True, use_reloader=False, host='0.0.0.0', debug=False) + print('Done') + finally: + xmpp.disconnect() diff --git a/conftest.py b/conftest.py index 8bf0e8d2dcf..104baa32204 100644 --- a/conftest.py +++ b/conftest.py @@ -14,36 +14,25 @@ import os +import mock import pytest +PROJECT = os.environ['GCLOUD_PROJECT'] -class Namespace: - def __init__(self, **kwargs): - self.__dict__.update(kwargs) +@pytest.fixture +def api_client_inject_project_id(): + """Patches all googleapiclient requests to replace 'YOUR_PROJECT_ID' with + the project ID.""" + import googleapiclient.http -@pytest.fixture(scope='session') -def cloud_config(): - """Provides a configuration object as a proxy to environment variables.""" - return Namespace( - project=os.environ.get('GCLOUD_PROJECT'), - storage_bucket=os.environ.get('CLOUD_STORAGE_BUCKET'), - client_secrets=os.environ.get('GOOGLE_CLIENT_SECRETS')) + old_execute = googleapiclient.http.HttpRequest.execute + def new_execute(self, http=None, num_retries=0): + self.uri = self.uri.replace('YOUR_PROJECT_ID', PROJECT) + return old_execute(self, http=http, num_retries=num_retries) -def get_resource_path(resource, local_path): - local_resource_path = os.path.join(local_path, 'resources', *resource) - - if os.path.exists(local_resource_path): - return local_resource_path - else: - raise EnvironmentError('Resource {} not found.'.format( - os.path.join(*resource))) - - -@pytest.fixture(scope='module') -def resource(request): - """Provides a function that returns the full path to a local or global - testing resource""" - local_path = os.path.dirname(request.module.__file__) - return lambda *args: get_resource_path(args, local_path) + with mock.patch( + 'googleapiclient.http.HttpRequest.execute', + new=new_execute): + yield diff --git a/container_engine/api-client/README.rst b/container_engine/api-client/README.rst new file mode 100644 index 00000000000..99c2ffb4cea --- /dev/null +++ b/container_engine/api-client/README.rst @@ -0,0 +1,93 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Container Engine Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=container_engine/api-client/README.rst + + +This directory contains samples for Google Container Engine. `Google Container Engine`_ runs Docker containers on Google Cloud Platform, powered by Kubernetes. + + + + +.. _Google Container Engine: https://cloud.google.com/container-engine/docs/ + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Snippets ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=container_engine/api-client/snippets.py,container_engine/api-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python snippets.py + + usage: snippets.py [-h] {list_clusters_and_nodepools} ... + + positional arguments: + {list_clusters_and_nodepools} + list_clusters_and_nodepools + Lists all clusters and associated node pools. + + optional arguments: + -h, --help show this help message and exit + + + + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/container_engine/api-client/README.rst.in b/container_engine/api-client/README.rst.in new file mode 100644 index 00000000000..c729b78e815 --- /dev/null +++ b/container_engine/api-client/README.rst.in @@ -0,0 +1,20 @@ +# This file is used to generate README.rst + +product: + name: Google Container Engine + short_name: Container Engine + url: https://cloud.google.com/container-engine/docs/ + description: > + `Google Container Engine`_ runs Docker containers on Google Cloud Platform, + powered by Kubernetes. + +setup: +- auth +- install_deps + +samples: +- name: Snippets + file: snippets.py + show_help: true + +folder: container_engine/api-client \ No newline at end of file diff --git a/container_engine/api-client/requirements.txt b/container_engine/api-client/requirements.txt new file mode 100644 index 00000000000..7e4359ce08d --- /dev/null +++ b/container_engine/api-client/requirements.txt @@ -0,0 +1,3 @@ +google-api-python-client==1.7.8 +google-auth==1.6.2 +google-auth-httplib2==0.0.3 diff --git a/container_engine/api-client/snippets.py b/container_engine/api-client/snippets.py new file mode 100644 index 00000000000..044a9a9c882 --- /dev/null +++ b/container_engine/api-client/snippets.py @@ -0,0 +1,62 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +import googleapiclient.discovery + + +def list_clusters_and_nodepools(project_id, zone): + """Lists all clusters and associated node pools.""" + service = googleapiclient.discovery.build('container', 'v1') + clusters_resource = service.projects().zones().clusters() + + clusters_response = clusters_resource.list( + projectId=project_id, zone=zone).execute() + + for cluster in clusters_response.get('clusters', []): + print('Cluster: {}, Status: {}, Current Master Version: {}'.format( + cluster['name'], cluster['status'], + cluster['currentMasterVersion'])) + + nodepools_response = clusters_resource.nodePools().list( + projectId=project_id, zone=zone, + clusterId=cluster['name']).execute() + + for nodepool in nodepools_response['nodePools']: + print( + ' -> Pool: {}, Status: {}, Machine Type: {}, ' + 'Autoscaling: {}'.format( + nodepool['name'], nodepool['status'], + nodepool['config']['machineType'], + nodepool.get('autoscaling', {}).get('enabled', False))) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + subparsers = parser.add_subparsers(dest='command') + list_clusters_and_nodepools_parser = subparsers.add_parser( + 'list_clusters_and_nodepools', + help=list_clusters_and_nodepools.__doc__) + list_clusters_and_nodepools_parser.add_argument('project_id') + list_clusters_and_nodepools_parser.add_argument('zone') + + args = parser.parse_args() + + if args.command == 'list_clusters_and_nodepools': + list_clusters_and_nodepools(args.project_id, args.zone) + else: + parser.print_help() diff --git a/container_engine/api-client/snippets_test.py b/container_engine/api-client/snippets_test.py new file mode 100644 index 00000000000..dc741d6e801 --- /dev/null +++ b/container_engine/api-client/snippets_test.py @@ -0,0 +1,22 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import snippets + + +def test_list_clusters_and_nodepools(): + project_id = os.environ['GCLOUD_PROJECT'] + snippets.list_clusters_and_nodepools(project_id, 'us-central1-f') diff --git a/container_engine/django_tutorial/Dockerfile b/container_engine/django_tutorial/Dockerfile index 220f232cb71..ea0d23a3dc2 100644 --- a/container_engine/django_tutorial/Dockerfile +++ b/container_engine/django_tutorial/Dockerfile @@ -20,12 +20,12 @@ FROM gcr.io/google_appengine/python # Create a virtualenv for the application dependencies. -# # If you want to use Python 3, add the -p python3.4 flag. -RUN virtualenv /env +# # If you want to use Python 2, use the -p python2.7 flag. +RUN virtualenv -p python3 /env ENV PATH /env/bin:$PATH ADD requirements.txt /app/requirements.txt -RUN /env/bin/pip install -r /app/requirements.txt +RUN /env/bin/pip install --upgrade pip && /env/bin/pip install -r /app/requirements.txt ADD . /app CMD gunicorn -b :$PORT mysite.wsgi diff --git a/container_engine/django_tutorial/Makefile b/container_engine/django_tutorial/Makefile index 8941baa4d3b..a06da05cc66 100644 --- a/container_engine/django_tutorial/Makefile +++ b/container_engine/django_tutorial/Makefile @@ -6,12 +6,12 @@ all: deploy .PHONY: create-cluster create-cluster: gcloud container clusters create polls \ - --scope "https://www.googleapis.com/auth/userinfo.email","cloud-platform" + --scopes "https://www.googleapis.com/auth/userinfo.email","cloud-platform" .PHONY: create-bucket create-bucket: gsutil mb gs://$(GCLOUD_PROJECT) - gsutil defacl set public-read gs://$(GCLOUD_PROJECT) + gsutil defacl set public-read gs://$(GCLOUD_PROJECT) .PHONY: build build: @@ -19,7 +19,7 @@ build: .PHONY: push push: build - gcloud docker push gcr.io/$(GCLOUD_PROJECT)/polls + gcloud docker push -- gcr.io/$(GCLOUD_PROJECT)/polls .PHONY: template template: @@ -31,7 +31,7 @@ deploy: push template .PHONY: update update: - kubectl rolling-update polls --image=gcr.io/${GCLOUD_PROJECT}/polls + kubectl patch deployment polls -p "{\"spec\":{\"template\":{\"metadata\":{\"labels\":{\"date\":\"`date +'%s'`\"}}}}}" .PHONY: delete delete: diff --git a/container_engine/django_tutorial/README.md b/container_engine/django_tutorial/README.md index ccf15134928..1d45703e2b8 100644 --- a/container_engine/django_tutorial/README.md +++ b/container_engine/django_tutorial/README.md @@ -1,197 +1,26 @@ # Getting started with Django on Google Container Engine -This repository is an example of how to run a [Django](https://www.djangoproject.com/) -app on Google Container Engine. It uses the [Writing your first Django app](https://docs.djangoproject.com/en/1 -.9/intro/tutorial01/) Polls application as the example app to deploy. From here on out, we refer to this app as -the 'polls' application. - -## Pre-requisites - -1. Create a project in the [Google Cloud Platform Console](https://console.cloud.google.com). - -2. [Enable billing](https://console.cloud.google.com/project/_/settings) for your project. - -3. [Enable APIs](https://console.cloud.google.com/flows/enableapi?apiid=datastore,pubsub,storage_api,logging,plus) for your project. The provided link will enable all necessary APIs, but if you wish to do so manually you will need Datastore, Pub/Sub, Storage, and Logging. - -4. Install the [Google Cloud SDK](https://cloud.google.com/sdk) - - $ curl https://sdk.cloud.google.com | bash - $ gcloud init - -5. Install [Docker](https://www.docker.com/). - -## Makefile - -Several commands listed below are provided in simpler form via the Makefile. Many of them use the GCLOUD_PROJECT -environment variable, which will be picked up from your gcloud config. Make sure you set this to the correct project, - - gcloud config set project - -### Create a cluster - -Create a cluster for the bookshelf application: - - gcloud container clusters create bookshelf \ - --scope "https://www.googleapis.com/auth/userinfo.email","cloud-platform" \ - --num-nodes 2 - gcloud container clusters get-credentials bookshelf - -The scopes specified in the `--scope` argument allows nodes in the cluster to access Google Cloud Platform APIs, such as the Cloud Datastore API. - -### Create a Cloud Storage bucket - -The bookshelf application uses [Google Cloud Storage](https://cloud.google.com/storage) to store image files. Create a bucket for your project: - - gsutil mb gs:// - gsutil defacl set public-read gs:// - - -## Setup the database - -This tutorial assumes you are setting up Django using a SQL database, which is the easiest way to run Django. If you have an existing SQL database running, you can use that, but if not, these are the instructions for creating a managed MySQL instance using CloudSQL. - - -* Create a [CloudSQL instance](https://console.cloud.google.com/project/_/sql/create) - - * In the instances list, click your Cloud SQL instance. - - * Click Access Control. - - * In the IP address subsection, click Request an IPv4 address to enable access to the Cloud SQL instance through an - IPv4 address. It will take a moment to initialize the new IP address. - - * Also under Access Control, in the Authorization subsection, under Allowed Networks, click the add (+) button . - - * In the Networks field, enter 0.0.0.0/0. This value allows access by all IP addresses. - - * Click Save. - -Note: setting allowed networks to 0.0.0.0/0 opens your SQL instance to traffic from any computer. For production databases, it's highly recommended to limit the authorized networks to only IP ranges that need access. - -* Alternatively, the instance can be created with the gcloud command line tool as follows, substituting `your-root-pw - with a strong, unique password. - - `gcloud sql instances create --assign-ip --authorized-networks=0.0.0.0/0 set-root-password=your-root-pw` - -* Create a Database And User - - * Using the root password created in the last step to create a new database, user, and password using your preferred MySQL client. Alternatively, follow these instructions to create the database and user from the console. - * From the CloudSQL instance in the console, click New user. - * Enter a username and password for the application. For example, name the user "pythonapp" and give it a randomly - generated password. - * Click Add. - * Click Databases and then click New database. - * For Name, enter the name of your database (for example, "polls"), and click Add. - -Once you have a SQL host, configuring mysite/settings.py to point to your database. Change `your-database-name`, -`your-database-user`, `your-database-host` , and `your-database-password` to match the settings created above. Note the -instance name is not used in this configuration, and the host name is the IP address you created. - - -## Running locally +[![Open in Cloud Shell][shell_img]][shell_link] -First make sure you have Django installed. It's recommended you do so in a -[virtualenv](https://virtualenv.pypa.io/en/latest/). The requirements.txt -contains just the Django dependency. +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=container_engine/django_tutorial/README.md - pip install -r requirements.txt - -Once the database is setup, run the migrations. - - python manage.py migrate - -If you'd like to use the admin console, create a superuser. - - python manage.py createsuperuser - -The app can be run locally the same way as any other Django app. - - python manage.py runserver - -Now you can view the admin panel of your local site at - - http://localhost:8080/admin - -# Deploying To Google Container Engine (Kubernetes) - -## Build the polls container - -Before the application can be deployed to Container Engine, you will need build and push the image to [Google Container Registry](https://cloud.google.com/container-registry/). - - docker build -t gcr.io/your-project-id/polls . - gcloud docker push gcr.io/your-project-id/polls - -Alternatively, this can be done using - - make push - - -## Deploy to the application - -### Serve the static content - -Collect all the static assets into the static/ folder. - - python manage.py collectstatic - -When DEBUG is enabled, Django can serve the files directly from that folder. For production purposes, you should -serve static assets from a CDN. Here are instructions for how to do this using Google Cloud Storage. - -Upload it to CloudStorage using the `gsutil rsync` command - - gsutil rsync -R static/ gs:///static - -Now your static content can be served from the following URL: - - http://storage.googleapis.com/` - -### Create the Kubernetes resources - -This application is represented in a single Kubernetes config, called `polls`. First, replace the -GCLOUD_PROJECT in `polls.yaml` with your project ID. Alternatively, run `make template` with your -GCLOUD_PROJECT environment variable set. - - kubectl create -f polls.yaml - -Alternatively this create set can be done using the Makefile - - make deploy - -Once the resources are created, there should be 3 `polls` pods on the cluster. To see the pods and ensure that -they are running: - - kubectl get pods - -If the pods are not ready or if you see restarts, you can get the logs for a particular pod to figure out the issue: - - kubectl logs pod-id - -Once the pods are ready, you can get the public IP address of the load balancer: - - kubectl get services polls - -You can then browse to the public IP address in your browser to see the bookshelf application. - -When you are ready to update the replication controller with a new image you built, the following command will do a -rolling update - - kubectl rolling-update polls --image=gcr.io/${GCLOUD_PROJECT}/polls - -which can also be done with the `make update` command. +This repository is an example of how to run a [Django](https://www.djangoproject.com/) +app on Google Container Engine. It uses the +[Writing your first Django app](https://docs.djangoproject.com/en/1.11/intro/tutorial01/) +Polls application (parts 1 and 2) as the example app to deploy. From here on +out, we refer to this app as the 'polls' application. -## Issues +# Tutorial +See our [Django on Container Engine](https://cloud.google.com/python/django/container-engine) tutorial for instructions for setting up and deploying this sample application. -Please use the Issue Tracker for any issues or questions. ## Contributing changes -* See [CONTRIBUTING.md](CONTRIBUTING.md) +* See [CONTRIBUTING.md](https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/CONTRIBUTING.md) ## Licensing -* See [LICENSE](LICENSE) +* See [LICENSE](https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/LICENSE) diff --git a/container_engine/django_tutorial/mysite/settings.py b/container_engine/django_tutorial/mysite/settings.py index b91a24dd64f..bdcb39913e9 100644 --- a/container_engine/django_tutorial/mysite/settings.py +++ b/container_engine/django_tutorial/mysite/settings.py @@ -26,7 +26,10 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = [] +# SECURITY WARNING: If you deploy a Django app to production, make sure to set +# an appropriate host here. +# See https://docs.djangoproject.com/en/1.10/ref/settings/ +ALLOWED_HOSTS = ['*'] # Application definition @@ -40,15 +43,14 @@ 'polls' ) -MIDDLEWARE_CLASSES = ( +MIDDLEWARE = ( + 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django.middleware.security.SecurityMiddleware', ) ROOT_URLCONF = 'mysite.urls' @@ -77,12 +79,14 @@ # [START dbconfig] DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.mysql', - 'NAME': '', - 'USER': '', - 'PASSWORD': '', - 'HOST': '', - 'PORT': '3306', + # If you are using Cloud SQL for MySQL rather than PostgreSQL, set + # 'ENGINE': 'django.db.backends.mysql' instead of the following. + 'ENGINE': 'django.db.backends.postgresql', + 'NAME': 'polls', + 'USER': os.getenv('DATABASE_USER'), + 'PASSWORD': os.getenv('DATABASE_PASSWORD'), + 'HOST': '127.0.0.1', + 'PORT': '5432', } } # [END dbconfig] @@ -105,7 +109,7 @@ # [START staticurl] STATIC_URL = '/static/' -# STATIC_URL = 'https://storage.googleapis.com//static/' +# STATIC_URL = 'https://storage.googleapis.com//static/' # [END staticurl] STATIC_ROOT = 'static/' diff --git a/container_engine/django_tutorial/mysite/urls.py b/container_engine/django_tutorial/mysite/urls.py index 903256505d7..bbb417c431e 100644 --- a/container_engine/django_tutorial/mysite/urls.py +++ b/container_engine/django_tutorial/mysite/urls.py @@ -13,13 +13,14 @@ # limitations under the License. from django.conf import settings -from django.conf.urls import include, url from django.contrib import admin from django.contrib.staticfiles.urls import staticfiles_urlpatterns +from django.urls import include, path + urlpatterns = [ - url(r'^', include('polls.urls')), - url(r'^admin/', admin.site.urls) + path('admin/', admin.site.urls), + path('', include('polls.urls')), ] # Only serve static files from Django during development diff --git a/container_engine/django_tutorial/polls.yaml b/container_engine/django_tutorial/polls.yaml index 5a19a02112f..6ccc2d337b0 100644 --- a/container_engine/django_tutorial/polls.yaml +++ b/container_engine/django_tutorial/polls.yaml @@ -19,12 +19,12 @@ # instances of the bookshelf app are running on the cluster. # For more info about Pods see: # https://cloud.google.com/container-engine/docs/pods/ -# For more info about Replication Controllers: -# https://cloud.google.com/container-engine/docs/replicationcontrollers/ +# For more info about Deployments: +# https://kubernetes.io/docs/user-guide/deployments/ -# [START replication_controller] -apiVersion: v1 -kind: ReplicationController +# [START kubernetes_deployment] +apiVersion: extensions/v1beta1 +kind: Deployment metadata: name: polls labels: @@ -39,14 +39,54 @@ spec: containers: - name: polls-app # Replace with your project ID or use `make template` - image: gcr.io/$GCLOUD_PROJECT/polls + image: gcr.io//polls # This setting makes nodes pull the docker image every time before # starting the pod. This is useful when debugging, but should be turned # off in production. imagePullPolicy: Always + env: + # [START cloudsql_secrets] + - name: DATABASE_USER + valueFrom: + secretKeyRef: + name: cloudsql + key: username + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: cloudsql + key: password + # [END cloudsql_secrets] ports: - containerPort: 8080 -# [END replication_controller] + + # [START proxy_container] + - image: gcr.io/cloudsql-docker/gce-proxy:1.05 + name: cloudsql-proxy + command: ["/cloud_sql_proxy", "--dir=/cloudsql", + "-instances==tcp:5432", + "-credential_file=/secrets/cloudsql/credentials.json"] + volumeMounts: + - name: cloudsql-oauth-credentials + mountPath: /secrets/cloudsql + readOnly: true + - name: ssl-certs + mountPath: /etc/ssl/certs + - name: cloudsql + mountPath: /cloudsql + # [END proxy_container] + # [START volumes] + volumes: + - name: cloudsql-oauth-credentials + secret: + secretName: cloudsql-oauth-credentials + - name: ssl-certs + hostPath: + path: /etc/ssl/certs + - name: cloudsql + emptyDir: + # [END volumes] +# [END kubernetes_deployment] --- diff --git a/container_engine/django_tutorial/polls/migrations/0001_initial.py b/container_engine/django_tutorial/polls/migrations/0001_initial.py index 2ae7f95ca7f..0609ba72f43 100644 --- a/container_engine/django_tutorial/polls/migrations/0001_initial.py +++ b/container_engine/django_tutorial/polls/migrations/0001_initial.py @@ -17,7 +17,9 @@ class Migration(migrations.Migration): migrations.CreateModel( name='Choice', fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('id', models.AutoField( + auto_created=True, primary_key=True, serialize=False, + verbose_name='ID')), ('choice_text', models.CharField(max_length=200)), ('votes', models.IntegerField(default=0)), ], @@ -25,14 +27,19 @@ class Migration(migrations.Migration): migrations.CreateModel( name='Question', fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('id', models.AutoField( + auto_created=True, primary_key=True, serialize=False, + verbose_name='ID')), ('question_text', models.CharField(max_length=200)), - ('pub_date', models.DateTimeField(verbose_name=b'date published')), + ('pub_date', models.DateTimeField( + verbose_name=b'date published')), ], ), migrations.AddField( model_name='choice', name='question', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='polls.Question'), + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to='polls.Question'), ), ] diff --git a/container_engine/django_tutorial/polls/urls.py b/container_engine/django_tutorial/polls/urls.py index 5927a172ba4..672ee5c879a 100644 --- a/container_engine/django_tutorial/polls/urls.py +++ b/container_engine/django_tutorial/polls/urls.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from django.conf.urls import url +from django.urls import path from . import views urlpatterns = [ - url(r'^$', views.index, name='index') + path('', views.index, name='index') ] diff --git a/container_engine/django_tutorial/requirements.txt b/container_engine/django_tutorial/requirements.txt index 6d724096fce..c19a8e2bed0 100644 --- a/container_engine/django_tutorial/requirements.txt +++ b/container_engine/django_tutorial/requirements.txt @@ -1,4 +1,5 @@ -Django==1.9.3 -mysqlclient==1.3.7 -wheel==0.29.0 -gunicorn==19.4.5 +Django==2.1.5 +mysqlclient==1.4.1 +wheel==0.32.3 +gunicorn==19.9.0 +psycopg2==2.7.7 diff --git a/datalabeling/README.rst b/datalabeling/README.rst new file mode 100644 index 00000000000..bf5949b8cb7 --- /dev/null +++ b/datalabeling/README.rst @@ -0,0 +1,78 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Data Labeling Service Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=datalabeling/README.rst + + +This directory contains samples for Google Cloud Data Labeling Service. `Google Cloud Data Labeling Service`_ allows developers to request having human labelers label a collection of data that you plan to use to train a custom machine learning model. + + + + +.. _Google Cloud Data Labeling Service: https://cloud.google.com/data-labeling/docs/ + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/datalabeling/README.rst.in b/datalabeling/README.rst.in new file mode 100644 index 00000000000..c87a1ff89b4 --- /dev/null +++ b/datalabeling/README.rst.in @@ -0,0 +1,18 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Data Labeling Service + short_name: Cloud Data Labeling + url: https://cloud.google.com/data-labeling/docs/ + description: > + `Google Cloud Data Labeling Service`_ allows developers to request having + human labelers label a collection of data that you plan to use to train a + custom machine learning model. + +setup: +- auth +- install_deps + +cloud_client_library: true + +folder: datalabeling \ No newline at end of file diff --git a/datalabeling/create_annotation_spec_set.py b/datalabeling/create_annotation_spec_set.py new file mode 100644 index 00000000000..29eab029e53 --- /dev/null +++ b/datalabeling/create_annotation_spec_set.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + + +# [START datalabeling_create_annotation_spec_set_beta] +def create_annotation_spec_set(project_id): + """Creates a data labeling annotation spec set for the given + Google Cloud project. + """ + from google.cloud import datalabeling_v1beta1 as datalabeling + client = datalabeling.DataLabelingServiceClient() + + project_path = client.project_path(project_id) + + annotation_spec_1 = datalabeling.types.AnnotationSpec( + display_name='label_1', + description='label_description_1' + ) + + annotation_spec_2 = datalabeling.types.AnnotationSpec( + display_name='label_2', + description='label_description_2' + ) + + annotation_spec_set = datalabeling.types.AnnotationSpecSet( + display_name='YOUR_ANNOTATION_SPEC_SET_DISPLAY_NAME', + description='YOUR_DESCRIPTION', + annotation_specs=[annotation_spec_1, annotation_spec_2] + ) + + response = client.create_annotation_spec_set( + project_path, annotation_spec_set) + + # The format of the resource name: + # project_id/{project_id}/annotationSpecSets/{annotationSpecSets_id} + print('The annotation_spec_set resource name: {}'.format(response.name)) + print('Display name: {}'.format(response.display_name)) + print('Description: {}'.format(response.description)) + print('Annotation specs:') + for annotation_spec in response.annotation_specs: + print('\tDisplay name: {}'.format(annotation_spec.display_name)) + print('\tDescription: {}\n'.format(annotation_spec.description)) + + return response +# [END datalabeling_create_annotation_spec_set_beta] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + + parser.add_argument( + '--project-id', + help='Project ID. Required.', + required=True + ) + + args = parser.parse_args() + + create_annotation_spec_set(args.project_id) diff --git a/datalabeling/create_annotation_spec_set_test.py b/datalabeling/create_annotation_spec_set_test.py new file mode 100644 index 00000000000..0214fa7967c --- /dev/null +++ b/datalabeling/create_annotation_spec_set_test.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import create_annotation_spec_set +from google.cloud import datalabeling_v1beta1 as datalabeling +import pytest + +PROJECT_ID = os.getenv('GCLOUD_PROJECT') + + +@pytest.mark.slow +def test_create_annotation_spec_set(capsys): + response = create_annotation_spec_set.create_annotation_spec_set( + PROJECT_ID) + out, _ = capsys.readouterr() + assert 'The annotation_spec_set resource name:' in out + + # Delete the created annotation spec set. + annotation_spec_set_name = response.name + client = datalabeling.DataLabelingServiceClient() + client.delete_annotation_spec_set(annotation_spec_set_name) diff --git a/datalabeling/create_instruction.py b/datalabeling/create_instruction.py new file mode 100644 index 00000000000..c2d608f402c --- /dev/null +++ b/datalabeling/create_instruction.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + + +# [START datalabeling_create_instruction_beta] +def create_instruction(project_id, data_type, instruction_gcs_uri): + """ Creates a data labeling PDF instruction for the given Google Cloud + project. The PDF file should be uploaded to the project in + Google Cloud Storage. + """ + from google.cloud import datalabeling_v1beta1 as datalabeling + client = datalabeling.DataLabelingServiceClient() + + project_path = client.project_path(project_id) + + pdf_instruction = datalabeling.types.PdfInstruction( + gcs_file_uri=instruction_gcs_uri) + + instruction = datalabeling.types.Instruction( + display_name='YOUR_INSTRUCTION_DISPLAY_NAME', + description='YOUR_DESCRIPTION', + data_type=data_type, + pdf_instruction=pdf_instruction + ) + + operation = client.create_instruction(project_path, instruction) + + result = operation.result() + + # The format of the resource name: + # project_id/{project_id}/instruction/{instruction_id} + print('The instruction resource name: {}\n'.format(result.name)) + print('Display name: {}'.format(result.display_name)) + print('Description: {}'.format(result.description)) + print('Create time:') + print('\tseconds: {}'.format(result.create_time.seconds)) + print('\tnanos: {}'.format(result.create_time.nanos)) + print('Data type: {}'.format( + datalabeling.enums.DataType(result.data_type).name)) + print('Pdf instruction:') + print('\tGcs file uri: {}'.format( + result.pdf_instruction.gcs_file_uri)) + + return result +# [END datalabeling_create_instruction_beta] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + + parser.add_argument( + '--project-id', + help='Project ID. Required.', + required=True + ) + + parser.add_argument( + '--data-type', + help='Data type. Only support IMAGE, VIDEO, TEXT and AUDIO. Required.', + required=True + ) + + parser.add_argument( + '--instruction-gcs-uri', + help='The URI of Google Cloud Storage of the instruction. Required.', + required=True + ) + + args = parser.parse_args() + + create_instruction( + args.project_id, + args.data_type, + args.instruction_gcs_uri + ) diff --git a/datalabeling/create_instruction_test.py b/datalabeling/create_instruction_test.py new file mode 100644 index 00000000000..43cf90e0262 --- /dev/null +++ b/datalabeling/create_instruction_test.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import create_instruction +from google.cloud import datalabeling_v1beta1 as datalabeling +import pytest + +PROJECT_ID = os.getenv('GCLOUD_PROJECT') +INSTRUCTION_GCS_URI = ('gs://cloud-samples-data/datalabeling' + '/instruction/test.pdf') + + +@pytest.mark.slow +def test_create_instruction(capsys): + result = create_instruction.create_instruction( + PROJECT_ID, + 'IMAGE', + INSTRUCTION_GCS_URI + ) + out, _ = capsys.readouterr() + assert 'The instruction resource name: ' in out + + # Delete the created instruction. + instruction_name = result.name + client = datalabeling.DataLabelingServiceClient() + client.delete_instruction(instruction_name) diff --git a/datalabeling/export_data.py b/datalabeling/export_data.py new file mode 100644 index 00000000000..2487124c008 --- /dev/null +++ b/datalabeling/export_data.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + + +# [START datalabeling_export_data_beta] +def export_data(dataset_resource_name, annotated_dataset_resource_name, + export_gcs_uri): + """Exports a dataset from the given Google Cloud project.""" + from google.cloud import datalabeling_v1beta1 as datalabeling + client = datalabeling.DataLabelingServiceClient() + + gcs_destination = datalabeling.types.GcsDestination( + output_uri=export_gcs_uri, mime_type='text/csv') + + output_config = datalabeling.types.OutputConfig( + gcs_destination=gcs_destination) + + response = client.export_data( + dataset_resource_name, + annotated_dataset_resource_name, + output_config + ) + + print('Dataset ID: {}\n'.format(response.result().dataset)) + print('Output config:') + print('\tGcs destination:') + print('\t\tOutput URI: {}\n'.format( + response.result().output_config.gcs_destination.output_uri)) +# [END datalabeling_export_data_beta] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + + parser.add_argument( + '--dataset-resource-name', + help='Dataset resource name. Required.', + required=True + ) + + parser.add_argument( + '--annotated-dataset-resource-name', + help='Annotated Dataset resource name. Required.', + required=True + ) + + parser.add_argument( + '--export-gcs-uri', + help='The export GCS URI. Required.', + required=True + ) + + args = parser.parse_args() + + export_data( + args.dataset_resource_name, + args.annotated_dataset_resource_name, + args.export_gcs_uri + ) diff --git a/datalabeling/import_data.py b/datalabeling/import_data.py new file mode 100644 index 00000000000..a529694128a --- /dev/null +++ b/datalabeling/import_data.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + + +# [START datalabeling_import_data_beta] +def import_data(dataset_resource_name, data_type, input_gcs_uri): + """Imports data to the given Google Cloud project and dataset.""" + from google.cloud import datalabeling_v1beta1 as datalabeling + client = datalabeling.DataLabelingServiceClient() + + gcs_source = datalabeling.types.GcsSource( + input_uri=input_gcs_uri, mime_type='text/csv') + + csv_input_config = datalabeling.types.InputConfig( + data_type=data_type, gcs_source=gcs_source) + + response = client.import_data(dataset_resource_name, csv_input_config) + + result = response.result() + + # The format of resource name: + # project_id/{project_id}/datasets/{dataset_id} + print('Dataset resource name: {}\n'.format(result.dataset)) + + return result +# [END datalabeling_import_data_beta] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + + parser.add_argument( + '--dataset-resource-name', + help='Dataset resource name. Required.', + required=True + ) + + parser.add_argument( + '--data-type', + help='Data type. Only support IMAGE, VIDEO, TEXT and AUDIO. Required.', + required=True + ) + + parser.add_argument( + '--input-gcs-uri', + help='The GCS URI of the input dataset. Required.', + required=True + ) + + args = parser.parse_args() + + import_data(args.dataset_resource_name, args.data_type, args.input_gcs_uri) diff --git a/datalabeling/import_data_test.py b/datalabeling/import_data_test.py new file mode 100644 index 00000000000..6a389e94204 --- /dev/null +++ b/datalabeling/import_data_test.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import import_data +import manage_dataset +import pytest + +PROJECT_ID = os.getenv('GCLOUD_PROJECT') +INPUT_GCS_URI = 'gs://cloud-samples-data/datalabeling/image/image_dataset.csv' + + +@pytest.fixture(scope='function') +def dataset(): + # create a temporary dataset + dataset = manage_dataset.create_dataset(PROJECT_ID) + + yield dataset + + # tear down + manage_dataset.delete_dataset(dataset.name) + + +@pytest.mark.slow +def test_import_data(capsys, dataset): + import_data.import_data(dataset.name, 'IMAGE', INPUT_GCS_URI) + out, _ = capsys.readouterr() + assert 'Dataset resource name: ' in out diff --git a/datalabeling/label_image.py b/datalabeling/label_image.py new file mode 100644 index 00000000000..7984540ff70 --- /dev/null +++ b/datalabeling/label_image.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + + +# [START datalabeling_label_image_beta] +def label_image(dataset_resource_name, instruction_resource_name, + annotation_spec_set_resource_name): + """Labels an image dataset.""" + from google.cloud import datalabeling_v1beta1 as datalabeling + client = datalabeling.DataLabelingServiceClient() + + basic_config = datalabeling.types.HumanAnnotationConfig( + instruction=instruction_resource_name, + annotated_dataset_display_name='YOUR_ANNOTATED_DATASET_DISPLAY_NAME', + label_group='YOUR_LABEL_GROUP', + replica_count=1 + ) + + feature = datalabeling.enums.LabelImageRequest.Feature.CLASSIFICATION + + config = datalabeling.types.ImageClassificationConfig( + annotation_spec_set=annotation_spec_set_resource_name, + allow_multi_label=False, + answer_aggregation_type=datalabeling.enums.StringAggregationType + .MAJORITY_VOTE + ) + + response = client.label_image( + dataset_resource_name, + basic_config, + feature, + image_classification_config=config + ) + + print('Label_image operation name: {}'.format(response.operation.name)) + return response +# [END datalabeling_label_image_beta] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + + parser.add_argument( + '--dataset-resource-name', + help='Dataset resource name. Required.', + required=True + ) + + parser.add_argument( + '--instruction-resource-name', + help='Instruction resource name. Required.', + required=True + ) + + parser.add_argument( + '--annotation-spec-set-resource-name', + help='Annotation spec set resource name. Required.', + required=True + ) + + args = parser.parse_args() + + label_image( + args.dataset_resource_name, + args.instruction_resource_name, + args.annotation_spec_set_resource_name + ) diff --git a/datalabeling/label_image_test.py b/datalabeling/label_image_test.py new file mode 100644 index 00000000000..e6bb3a2814a --- /dev/null +++ b/datalabeling/label_image_test.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import create_annotation_spec_set +import create_instruction +from google.cloud import datalabeling_v1beta1 as datalabeling +import import_data +import label_image +import manage_dataset +import pytest + +PROJECT_ID = os.getenv('GCLOUD_PROJECT') +INPUT_GCS_URI = 'gs://cloud-samples-data/datalabeling/image/image_dataset.csv' + + +@pytest.fixture(scope='function') +def dataset(): + # create a temporary dataset + dataset = manage_dataset.create_dataset(PROJECT_ID) + + # import some data to it + import_data.import_data(dataset.name, 'IMAGE', INPUT_GCS_URI) + + yield dataset + + # tear down + manage_dataset.delete_dataset(dataset.name) + + +@pytest.fixture(scope='function') +def annotation_spec_set(): + # create a temporary annotation_spec_set + response = create_annotation_spec_set.create_annotation_spec_set( + PROJECT_ID) + + yield response + + # tear down + client = datalabeling.DataLabelingServiceClient() + client.delete_annotation_spec_set(response.name) + + +@pytest.fixture(scope='function') +def instruction(): + # create a temporary instruction + instruction = create_instruction.create_instruction( + PROJECT_ID, 'IMAGE', + 'gs://cloud-samples-data/datalabeling/instruction/test.pdf') + + yield instruction + + # tear down + client = datalabeling.DataLabelingServiceClient() + client.delete_instruction(instruction.name) + + +# Passing in dataset as the last argument in test_label_image since it needs +# to be deleted before the annotation_spec_set can be deleted. +@pytest.mark.slow +def test_label_image(capsys, annotation_spec_set, instruction, dataset): + + # Start labeling. + response = label_image.label_image( + dataset.name, + instruction.name, + annotation_spec_set.name + ) + out, _ = capsys.readouterr() + assert 'Label_image operation name: ' in out + operation_name = response.operation.name + + # Cancels the labeling operation. + response.cancel() + assert response.cancelled() is True + + client = datalabeling.DataLabelingServiceClient() + client.transport._operations_client.cancel_operation( + operation_name) diff --git a/datalabeling/label_text.py b/datalabeling/label_text.py new file mode 100644 index 00000000000..107bb8d257d --- /dev/null +++ b/datalabeling/label_text.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + + +# [START datalabeling_label_text_beta] +def label_text(dataset_resource_name, instruction_resource_name, + annotation_spec_set_resource_name): + """Labels a text dataset.""" + from google.cloud import datalabeling_v1beta1 as datalabeling + client = datalabeling.DataLabelingServiceClient() + + basic_config = datalabeling.types.HumanAnnotationConfig( + instruction=instruction_resource_name, + annotated_dataset_display_name='YOUR_ANNOTATED_DATASET_DISPLAY_NAME', + label_group='YOUR_LABEL_GROUP', + replica_count=1 + ) + + feature = (datalabeling.enums.LabelTextRequest. + Feature.TEXT_ENTITY_EXTRACTION) + + config = datalabeling.types.TextEntityExtractionConfig( + annotation_spec_set=annotation_spec_set_resource_name) + + response = client.label_text( + dataset_resource_name, + basic_config, + feature, + text_entity_extraction_config=config + ) + + print('Label_text operation name: {}'.format(response.operation.name)) + return response +# [END datalabeling_label_text_beta] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + + parser.add_argument( + '--dataset-resource-name', + help='Dataset resource name. Required.', + required=True + ) + + parser.add_argument( + '--instruction-resource-name', + help='Instruction resource name. Required.', + required=True + ) + + parser.add_argument( + '--annotation-spec-set-resource-name', + help='Annotation spec set resource name. Required.', + required=True + ) + + args = parser.parse_args() + + label_text( + args.dataset_resource_name, + args.instruction_resource_name, + args.annotation_spec_set_resource_name + ) diff --git a/datalabeling/label_text_test.py b/datalabeling/label_text_test.py new file mode 100644 index 00000000000..0a7b8bb06db --- /dev/null +++ b/datalabeling/label_text_test.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import create_annotation_spec_set +import create_instruction +from google.cloud import datalabeling_v1beta1 as datalabeling +import import_data +import label_text +import manage_dataset +import pytest + +PROJECT_ID = os.getenv('GCLOUD_PROJECT') +INPUT_GCS_URI = 'gs://cloud-samples-data/datalabeling/text/text_dataset.csv' + + +@pytest.fixture(scope='function') +def dataset(): + # create a temporary dataset + dataset = manage_dataset.create_dataset(PROJECT_ID) + + # import some data to it + import_data.import_data(dataset.name, 'TEXT', INPUT_GCS_URI) + + yield dataset + + # tear down + manage_dataset.delete_dataset(dataset.name) + + +@pytest.fixture(scope='function') +def annotation_spec_set(): + # create a temporary annotation_spec_set + response = create_annotation_spec_set.create_annotation_spec_set( + PROJECT_ID) + + yield response + + # tear down + client = datalabeling.DataLabelingServiceClient() + client.delete_annotation_spec_set(response.name) + + +@pytest.fixture(scope='function') +def instruction(): + # create a temporary instruction + instruction = create_instruction.create_instruction( + PROJECT_ID, 'TEXT', + 'gs://cloud-samples-data/datalabeling/instruction/test.pdf') + + yield instruction + + # tear down + client = datalabeling.DataLabelingServiceClient() + client.delete_instruction(instruction.name) + + +# Passing in dataset as the last argument in test_label_image since it needs +# to be deleted before the annotation_spec_set can be deleted. +@pytest.mark.slow +def test_label_text(capsys, annotation_spec_set, instruction, dataset): + + # Start labeling. + response = label_text.label_text( + dataset.name, + instruction.name, + annotation_spec_set.name + ) + out, _ = capsys.readouterr() + assert 'Label_text operation name: ' in out + operation_name = response.operation.name + + # Cancels the labeling operation. + response.cancel() + assert response.cancelled() is True + + client = datalabeling.DataLabelingServiceClient() + client.transport._operations_client.cancel_operation( + operation_name) diff --git a/datalabeling/label_video.py b/datalabeling/label_video.py new file mode 100644 index 00000000000..45edfaf23f6 --- /dev/null +++ b/datalabeling/label_video.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + + +# [START datalabeling_label_video_beta] +def label_video(dataset_resource_name, instruction_resource_name, + annotation_spec_set_resource_name): + """Labels a video dataset.""" + from google.cloud import datalabeling_v1beta1 as datalabeling + client = datalabeling.DataLabelingServiceClient() + + basic_config = datalabeling.types.HumanAnnotationConfig( + instruction=instruction_resource_name, + annotated_dataset_display_name='YOUR_ANNOTATED_DATASET_DISPLAY_NAME', + label_group='YOUR_LABEL_GROUP', + replica_count=1 + ) + + feature = datalabeling.enums.LabelVideoRequest.Feature.OBJECT_TRACKING + + config = datalabeling.types.ObjectTrackingConfig( + annotation_spec_set=annotation_spec_set_resource_name + ) + + response = client.label_video( + dataset_resource_name, + basic_config, + feature, + object_tracking_config=config + ) + + print('Label_video operation name: {}'.format(response.operation.name)) + return response +# [END datalabeling_label_video_beta] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + + parser.add_argument( + '--dataset-resource-name', + help='Dataset resource name. Required.', + required=True + ) + + parser.add_argument( + '--instruction-resource-name', + help='Instruction resource name. Required.', + required=True + ) + + parser.add_argument( + '--annotation-spec-set-resource-name', + help='Annotation spec set resource name. Required.', + required=True + ) + + args = parser.parse_args() + + label_video( + args.dataset_resource_name, + args.instruction_resource_name, + args.annotation_spec_set_resource_name + ) diff --git a/datalabeling/label_video_test.py b/datalabeling/label_video_test.py new file mode 100644 index 00000000000..c3dfca367f8 --- /dev/null +++ b/datalabeling/label_video_test.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import create_annotation_spec_set +import create_instruction +from google.cloud import datalabeling_v1beta1 as datalabeling +import import_data +import label_video +import manage_dataset +import pytest + +PROJECT_ID = os.getenv('GCLOUD_PROJECT') +INPUT_GCS_URI = 'gs://cloud-samples-data/datalabeling/videos/video_dataset.csv' + + +@pytest.fixture(scope='function') +def dataset(): + # create a temporary dataset + dataset = manage_dataset.create_dataset(PROJECT_ID) + + # import some data to it + import_data.import_data(dataset.name, 'VIDEO', INPUT_GCS_URI) + + yield dataset + + # tear down + manage_dataset.delete_dataset(dataset.name) + + +@pytest.fixture(scope='function') +def annotation_spec_set(): + # create a temporary annotation_spec_set + response = create_annotation_spec_set.create_annotation_spec_set( + PROJECT_ID) + + yield response + + # tear down + client = datalabeling.DataLabelingServiceClient() + client.delete_annotation_spec_set(response.name) + + +@pytest.fixture(scope='function') +def instruction(): + # create a temporary instruction + instruction = create_instruction.create_instruction( + PROJECT_ID, 'VIDEO', + 'gs://cloud-samples-data/datalabeling/instruction/test.pdf') + + yield instruction + + # tear down + client = datalabeling.DataLabelingServiceClient() + client.delete_instruction(instruction.name) + + +# Passing in dataset as the last argument in test_label_image since it needs +# to be deleted before the annotation_spec_set can be deleted. +@pytest.mark.slow +def test_label_video(capsys, annotation_spec_set, instruction, dataset): + + # Start labeling. + response = label_video.label_video( + dataset.name, + instruction.name, + annotation_spec_set.name + ) + out, _ = capsys.readouterr() + assert 'Label_video operation name: ' in out + operation_name = response.operation.name + + # Cancels the labeling operation. + response.cancel() + assert response.cancelled() is True + + client = datalabeling.DataLabelingServiceClient() + client.transport._operations_client.cancel_operation( + operation_name) diff --git a/datalabeling/manage_dataset.py b/datalabeling/manage_dataset.py new file mode 100644 index 00000000000..a13f5ad2ca0 --- /dev/null +++ b/datalabeling/manage_dataset.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + + +# [START datalabeling_create_dataset_beta] +def create_dataset(project_id): + """Creates a dataset for the given Google Cloud project.""" + from google.cloud import datalabeling_v1beta1 as datalabeling + client = datalabeling.DataLabelingServiceClient() + + formatted_project_name = client.project_path(project_id) + + dataset = datalabeling.types.Dataset( + display_name='YOUR_ANNOTATION_SPEC_SET_DISPLAY_NAME', + description='YOUR_DESCRIPTION' + ) + + response = client.create_dataset(formatted_project_name, dataset) + + # The format of resource name: + # project_id/{project_id}/datasets/{dataset_id} + print('The dataset resource name: {}\n'.format(response.name)) + print('Display name: {}'.format(response.display_name)) + print('Description: {}'.format(response.description)) + print('Create time:') + print('\tseconds: {}'.format(response.create_time.seconds)) + print('\tnanos: {}'.format(response.create_time.nanos)) + + return response +# [END datalabeling_create_dataset_beta] + + +# [START datalabeling_list_datasets_beta] +def list_datasets(project_id): + """Lists datasets for the given Google Cloud project.""" + from google.cloud import datalabeling_v1beta1 as datalabeling + client = datalabeling.DataLabelingServiceClient() + + formatted_project_name = client.project_path(project_id) + + response = client.list_datasets(formatted_project_name) + for element in response: + # The format of resource name: + # project_id/{project_id}/datasets/{dataset_id} + print('The dataset resource name: {}\n'.format(element.name)) + print('Display name: {}'.format(element.display_name)) + print('Description: {}'.format(element.description)) + print('Create time:') + print('\tseconds: {}'.format(element.create_time.seconds)) + print('\tnanos: {}'.format(element.create_time.nanos)) +# [END datalabeling_list_datasets_beta] + + +# [START datalabeling_get_dataset_beta] +def get_dataset(dataset_resource_name): + """Gets a dataset for the given Google Cloud project.""" + from google.cloud import datalabeling_v1beta1 as datalabeling + client = datalabeling.DataLabelingServiceClient() + + response = client.get_dataset(dataset_resource_name) + + print('The dataset resource name: {}\n'.format(response.name)) + print('Display name: {}'.format(response.display_name)) + print('Description: {}'.format(response.description)) + print('Create time:') + print('\tseconds: {}'.format(response.create_time.seconds)) + print('\tnanos: {}'.format(response.create_time.nanos)) +# [END datalabeling_get_dataset_beta] + + +# [START datalabeling_delete_dataset_beta] +def delete_dataset(dataset_resource_name): + """Deletes a dataset for the given Google Cloud project.""" + from google.cloud import datalabeling_v1beta1 as datalabeling + client = datalabeling.DataLabelingServiceClient() + + response = client.delete_dataset(dataset_resource_name) + + print('Dataset deleted. {}\n'.format(response)) +# [END datalabeling_delete_dataset_beta] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + + subparsers = parser.add_subparsers(dest='command') + + create_parser = subparsers.add_parser( + 'create', help='Create a new dataset.') + create_parser.add_argument( + '--project-id', + help='Project ID. Required.', + required=True + ) + + list_parser = subparsers.add_parser('list', help='List all datasets.') + list_parser.add_argument( + '--project-id', + help='Project ID. Required.', + required=True + ) + + get_parser = subparsers.add_parser( + 'get', help='Get a dataset by the dataset resource name.') + get_parser.add_argument( + '--dataset-resource-name', + help='The dataset resource name. Used in the get or delete operation.', + required=True + ) + + delete_parser = subparsers.add_parser( + 'delete', help='Delete a dataset by the dataset resource name.') + delete_parser.add_argument( + '--dataset-resource-name', + help='The dataset resource name. Used in the get or delete operation.', + required=True + ) + + args = parser.parse_args() + + if args.command == 'create': + create_dataset(args.project_id) + elif args.command == 'list': + list_datasets(args.project_id) + elif args.command == 'get': + get_dataset(args.dataset_resource_name) + elif args.command == 'delete': + delete_dataset(args.dataset_resource_name) diff --git a/datalabeling/manage_dataset_test.py b/datalabeling/manage_dataset_test.py new file mode 100644 index 00000000000..ac7cd83fae5 --- /dev/null +++ b/datalabeling/manage_dataset_test.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import manage_dataset +import pytest + +PROJECT_ID = os.getenv("GCLOUD_PROJECT") + + +@pytest.fixture(scope='function') +def dataset(): + # create a temporary dataset + dataset = manage_dataset.create_dataset(PROJECT_ID) + + yield dataset + + # tear down + manage_dataset.delete_dataset(dataset.name) + + +def test_create_dataset(capsys): + response = manage_dataset.create_dataset(PROJECT_ID) + out, _ = capsys.readouterr() + assert "The dataset resource name:" in out + + # clean up + manage_dataset.delete_dataset(response.name) + + +def test_list_dataset(capsys, dataset): + manage_dataset.list_datasets(PROJECT_ID) + out, _ = capsys.readouterr() + assert dataset.name in out + + +def test_get_dataset(capsys, dataset): + manage_dataset.get_dataset(dataset.name) + out, _ = capsys.readouterr() + assert "The dataset resource name:" in out + + +def test_delete_dataset(capsys): + # Creates a dataset. + response = manage_dataset.create_dataset(PROJECT_ID) + + manage_dataset.delete_dataset(response.name) + out, _ = capsys.readouterr() + assert "Dataset deleted." in out diff --git a/datalabeling/requirements.txt b/datalabeling/requirements.txt new file mode 100644 index 00000000000..6cc7309cddf --- /dev/null +++ b/datalabeling/requirements.txt @@ -0,0 +1 @@ +google-cloud-datalabeling==0.1.1 diff --git a/dataproc/README.md b/dataproc/README.md new file mode 100644 index 00000000000..1d919e4631d --- /dev/null +++ b/dataproc/README.md @@ -0,0 +1,90 @@ +# Cloud Dataproc API Example + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=dataproc/README.md + +Sample command-line programs for interacting with the Cloud Dataproc API. + + +Please see [the tutorial on the using the Dataproc API with the Python client +library](https://cloud.google.com/dataproc/docs/tutorials/python-library-example) +for more information. + +Note that while this sample demonstrates interacting with Dataproc via the API, the functionality +demonstrated here could also be accomplished using the Cloud Console or the gcloud CLI. + +`list_clusters.py` is a simple command-line program to demonstrate connecting to the +Dataproc API and listing the clusters in a region + +`create_cluster_and_submit_job.py` demonstrates how to create a cluster, submit the +`pyspark_sort.py` job, download the output from Google Cloud Storage, and output the result. + +`pyspark_sort.py_gcs` is the asme as `pyspark_sort.py` but demonstrates + reading from a GCS bucket. + +## Prerequisites to run locally: + +* [pip](https://pypi.python.org/pypi/pip) + +Go to the [Google Cloud Console](https://console.cloud.google.com). + +Under API Manager, search for the Google Cloud Dataproc API and enable it. + +## Set Up Your Local Dev Environment + +To install, run the following commands. If you want to use [virtualenv](https://virtualenv.readthedocs.org/en/latest/) +(recommended), run the commands within a virtualenv. + + * pip install -r requirements.txt + +## Authentication + +Please see the [Google cloud authentication guide](https://cloud.google.com/docs/authentication/). +The recommended approach to running these samples is a Service Account with a JSON key. + +## Environment Variables + +Set the following environment variables: + + GOOGLE_CLOUD_PROJECT=your-project-id + REGION=us-central1 # or your region + CLUSTER_NAME=waprin-spark7 + ZONE=us-central1-b + +## Running the samples + +To run list_clusters.py: + + python list_clusters.py $GOOGLE_CLOUD_PROJECT --region=$REGION + +`submit_job_to_cluster.py` can create the Dataproc cluster, or use an existing one. +If you'd like to create a cluster ahead of time, either use the +[Cloud Console](console.cloud.google.com) or run: + + gcloud dataproc clusters create your-cluster-name + +To run submit_job_to_cluster.py, first create a GCS bucket for Dataproc to stage files, from the Cloud Console or with +gsutil: + + gsutil mb gs:// + +Set the environment variable's name: + + BUCKET=your-staging-bucket + CLUSTER=your-cluster-name + +Then, if you want to rely on an existing cluster, run: + + python submit_job_to_cluster.py --project_id=$GOOGLE_CLOUD_PROJECT --zone=us-central1-b --cluster_name=$CLUSTER --gcs_bucket=$BUCKET + +Otherwise, if you want the script to create a new cluster for you: + + python submit_job_to_cluster.py --project_id=$GOOGLE_CLOUD_PROJECT --zone=us-central1-b --cluster_name=$CLUSTER --gcs_bucket=$BUCKET --create_new_cluster + +This will setup a cluster, upload the PySpark file, submit the job, print the result, then +delete the cluster. + +You can optionally specify a `--pyspark_file` argument to change from the default +`pyspark_sort.py` included in this script to a new script. diff --git a/dataproc/dataproc_e2e_test.py b/dataproc/dataproc_e2e_test.py new file mode 100644 index 00000000000..0a45d080122 --- /dev/null +++ b/dataproc/dataproc_e2e_test.py @@ -0,0 +1,32 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Integration tests for Dataproc samples. + +Creates a Dataproc cluster, uploads a pyspark file to Google Cloud Storage, +submits a job to Dataproc that runs the pyspark file, then downloads +the output logs from Cloud Storage and verifies the expected output.""" + +import os + +import submit_job_to_cluster + +PROJECT = os.environ['GCLOUD_PROJECT'] +BUCKET = os.environ['CLOUD_STORAGE_BUCKET'] +CLUSTER_NAME = 'testcluster3' +ZONE = 'us-central1-b' + + +def test_e2e(): + output = submit_job_to_cluster.main( + PROJECT, ZONE, CLUSTER_NAME, BUCKET) + assert b"['Hello,', 'dog', 'elephant', 'panther', 'world!']" in output diff --git a/dataproc/list_clusters.py b/dataproc/list_clusters.py new file mode 100644 index 00000000000..9bbaa3b09c6 --- /dev/null +++ b/dataproc/list_clusters.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Sample command-line program for listing Google Dataproc Clusters +""" + +import argparse + +import googleapiclient.discovery + + +# [START dataproc_list_clusters] +def list_clusters(dataproc, project, region): + result = dataproc.projects().regions().clusters().list( + projectId=project, + region=region).execute() + return result +# [END dataproc_list_clusters] + + +# [START dataproc_get_client] +def get_client(): + """Builds a client to the dataproc API.""" + dataproc = googleapiclient.discovery.build('dataproc', 'v1') + return dataproc +# [END dataproc_get_client] + + +def main(project_id, region): + dataproc = get_client() + result = list_clusters(dataproc, project_id, region) + print(result) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + 'project_id', help='Project ID you want to access.'), + # Sets the region to "global" if it's not provided + # Note: sub-regions (e.g.: us-central1-a/b) are currently not supported + parser.add_argument( + '--region', default='global', help='Region to list clusters') + + args = parser.parse_args() + main(args.project_id, args.region) diff --git a/dataproc/pyspark_sort.py b/dataproc/pyspark_sort.py new file mode 100644 index 00000000000..0ce2350ad02 --- /dev/null +++ b/dataproc/pyspark_sort.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Sample pyspark script to be uploaded to Cloud Storage and run on +Cloud Dataproc. + +Note this file is not intended to be run directly, but run inside a PySpark +environment. +""" + +# [START dataproc_pyspark_sort] +import pyspark + +sc = pyspark.SparkContext() +rdd = sc.parallelize(['Hello,', 'world!', 'dog', 'elephant', 'panther']) +words = sorted(rdd.collect()) +print(words) +# [END dataproc_pyspark_sort] diff --git a/dataproc/pyspark_sort_gcs.py b/dataproc/pyspark_sort_gcs.py new file mode 100644 index 00000000000..f1961c378d3 --- /dev/null +++ b/dataproc/pyspark_sort_gcs.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Sample pyspark script to be uploaded to Cloud Storage and run on +Cloud Dataproc. + +Note this file is not intended to be run directly, but run inside a PySpark +environment. + +This file demonstrates how to read from a GCS bucket. See README.md for more +information. +""" + +# [START dataproc_pyspark_sort_gcs] +import pyspark + +sc = pyspark.SparkContext() +rdd = sc.textFile('gs://path-to-your-GCS-file') +print(sorted(rdd.collect())) +# [END dataproc_pyspark_sort_gcs] diff --git a/dataproc/python-api-walkthrough.md b/dataproc/python-api-walkthrough.md new file mode 100644 index 00000000000..0004e2419cd --- /dev/null +++ b/dataproc/python-api-walkthrough.md @@ -0,0 +1,165 @@ +# Use the Python Client Library to call Cloud Dataproc APIs + +Estimated completion time: + +## Overview + +This [Cloud Shell](https://cloud.google.com/shell/docs/) walkthrough leads you +through the steps to use the +[Google APIs Client Library for Python](http://code.google.com/p/google-api-python-client/ ) +to programmatically interact with [Cloud Dataproc](https://cloud.google.com/dataproc/docs/). + +As you follow this walkthrough, you run Python code that calls +[Cloud Dataproc REST API](https://cloud.google.com//dataproc/docs/reference/rest/) +methods to: + +* create a Cloud Dataproc cluster +* submit a small PySpark word sort job to run on the cluster +* get job status +* tear down the cluster after job completion + +## Using the walkthrough + +The `submit_job_to_cluster.py file` used in this walkthrough is opened in the +Cloud Shell editor when you launch the walkthrough. You can view +the code as your follow the walkthrough steps. + +**For more information**: See [Cloud Dataproc→Use the Python Client Library](https://cloud.google.com/dataproc/docs/tutorials/python-library-example) for +an explanation of how the code works. + +**To reload this walkthrough:** Run the following command from the +`~/python-docs-samples/dataproc` directory in Cloud Shell: + + cloudshell launch-tutorial python-api-walkthrough.md + +**To copy and run commands**: Click the "Paste in Cloud Shell" button + () + on the side of a code box, then press `Enter` to run the command. + +## Prerequisites (1) + +1. Create or select a Google Cloud Platform project to use for this tutorial. + * + +1. Enable the Cloud Dataproc, Compute Engine, and Cloud Storage APIs in your project. + * + +## Prerequisites (2) + +1. This walkthrough uploads a PySpark file (`pyspark_sort.py`) to a + [Cloud Storage bucket](https://cloud.google.com/storage/docs/key-terms#buckets) in + your project. + * You can use the [Cloud Storage browser page](https://console.cloud.google.com/storage/browser) + in Google Cloud Platform Console to view existing buckets in your project. + +     **OR** + + * To create a new bucket, run the following command. Your bucket name must be unique. + ```bash + gsutil mb -p {{project-id}} gs://your-bucket-name + ``` + +1. Set environment variables. + + * Set the name of your bucket. + ```bash + BUCKET=your-bucket-name + ``` + +## Prerequisites (3) + +1. Set up a Python + [virtual environment](https://virtualenv.readthedocs.org/en/latest/) + in Cloud Shell. + + * Create the virtual environment. + ```bash + virtualenv ENV + ``` + * Activate the virtual environment. + ```bash + source ENV/bin/activate + ``` + +1. Install library dependencies in Cloud Shell. + ```bash + pip install -r requirements.txt + ``` + +## Create a cluster and submit a job + +1. Set a name for your new cluster. + ```bash + CLUSTER=new-cluster-name + ``` + +1. Set a [zone](https://cloud.google.com/compute/docs/regions-zones/#available) + where your new cluster will be located. You can change the + "us-central1-a" zone that is pre-set in the following command. + ```bash + ZONE=us-central1-a + ``` + +1. Run `submit_job.py` with the `--create_new_cluster` flag + to create a new cluster and submit the `pyspark_sort.py` job + to the cluster. + + ```bash + python submit_job_to_cluster.py \ + --project_id={{project-id}} \ + --cluster_name=$CLUSTER \ + --zone=$ZONE \ + --gcs_bucket=$BUCKET \ + --create_new_cluster + ``` + +## Job Output + +Job output in Cloud Shell shows cluster creation, job submission, + job completion, and then tear-down of the cluster. + + ... + Creating cluster... + Cluster created. + Uploading pyspark file to GCS + new-cluster-name - RUNNING + Submitted job ID ... + Waiting for job to finish... + Job finished. + Downloading output file + ..... + ['Hello,', 'dog', 'elephant', 'panther', 'world!'] + ... + Tearing down cluster + ``` +## Congratulations on Completing the Walkthrough! + + +--- + +### Next Steps: + +* **View job details from the Console.** View job details by selecting the + PySpark job from the Cloud Dataproc + [Jobs page](https://console.cloud.google.com/dataproc/jobs) + in the Google Cloud Platform Console. + +* **Delete resources used in the walkthrough.** + The `submit_job.py` job deletes the cluster that it created for this + walkthrough. + + If you created a bucket to use for this walkthrough, + you can run the following command to delete the + Cloud Storage bucket (the bucket must be empty). + ```bash + gsutil rb gs://$BUCKET + ``` + You can run the following command to delete the bucket **and all + objects within it. Note: the deleted objects cannot be recovered.** + ```bash + gsutil rm -r gs://$BUCKET + ``` + +* **For more information.** See the [Cloud Dataproc documentation](https://cloud.google.com/dataproc/docs/) + for API reference and product feature information. + diff --git a/dataproc/requirements.txt b/dataproc/requirements.txt new file mode 100644 index 00000000000..bc5d62ef28d --- /dev/null +++ b/dataproc/requirements.txt @@ -0,0 +1,5 @@ +google-api-python-client==1.7.8 +google-auth==1.6.2 +google-auth-httplib2==0.0.3 +google-cloud==0.34.0 +google-cloud-storage==1.13.2 diff --git a/dataproc/submit_job_to_cluster.py b/dataproc/submit_job_to_cluster.py new file mode 100644 index 00000000000..f06d5981c16 --- /dev/null +++ b/dataproc/submit_job_to_cluster.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Sample command-line program for listing Google Dataproc Clusters""" + +import argparse +import os + +from google.cloud import storage +import googleapiclient.discovery + +DEFAULT_FILENAME = 'pyspark_sort.py' + + +def get_default_pyspark_file(): + """Gets the PySpark file from this directory""" + current_dir = os.path.dirname(os.path.abspath(__file__)) + f = open(os.path.join(current_dir, DEFAULT_FILENAME), 'rb') + return f, DEFAULT_FILENAME + + +def get_pyspark_file(filename): + f = open(filename, 'rb') + return f, os.path.basename(filename) + + +def get_region_from_zone(zone): + try: + region_as_list = zone.split('-')[:-1] + return '-'.join(region_as_list) + except (AttributeError, IndexError, ValueError): + raise ValueError('Invalid zone provided, please check your input.') + + +def upload_pyspark_file(project_id, bucket_name, filename, file): + """Uploads the PySpark file in this directory to the configured + input bucket.""" + print('Uploading pyspark file to GCS') + client = storage.Client(project=project_id) + bucket = client.get_bucket(bucket_name) + blob = bucket.blob(filename) + blob.upload_from_file(file) + + +def download_output(project_id, cluster_id, output_bucket, job_id): + """Downloads the output file from Cloud Storage and returns it as a + string.""" + print('Downloading output file') + client = storage.Client(project=project_id) + bucket = client.get_bucket(output_bucket) + output_blob = ( + 'google-cloud-dataproc-metainfo/{}/jobs/{}/driveroutput.000000000' + .format(cluster_id, job_id)) + return bucket.blob(output_blob).download_as_string() + + +# [START dataproc_create_cluster] +def create_cluster(dataproc, project, zone, region, cluster_name): + print('Creating cluster...') + zone_uri = \ + 'https://www.googleapis.com/compute/v1/projects/{}/zones/{}'.format( + project, zone) + cluster_data = { + 'projectId': project, + 'clusterName': cluster_name, + 'config': { + 'gceClusterConfig': { + 'zoneUri': zone_uri + }, + 'masterConfig': { + 'numInstances': 1, + 'machineTypeUri': 'n1-standard-1' + }, + 'workerConfig': { + 'numInstances': 2, + 'machineTypeUri': 'n1-standard-1' + } + } + } + result = dataproc.projects().regions().clusters().create( + projectId=project, + region=region, + body=cluster_data).execute() + return result +# [END dataproc_create_cluster] + + +def wait_for_cluster_creation(dataproc, project_id, region, cluster_name): + print('Waiting for cluster creation...') + + while True: + result = dataproc.projects().regions().clusters().list( + projectId=project_id, + region=region).execute() + cluster_list = result['clusters'] + cluster = [c + for c in cluster_list + if c['clusterName'] == cluster_name][0] + if cluster['status']['state'] == 'ERROR': + raise Exception(result['status']['details']) + if cluster['status']['state'] == 'RUNNING': + print("Cluster created.") + break + + +# [START dataproc_list_clusters_with_detail] +def list_clusters_with_details(dataproc, project, region): + result = dataproc.projects().regions().clusters().list( + projectId=project, + region=region).execute() + cluster_list = result['clusters'] + for cluster in cluster_list: + print("{} - {}" + .format(cluster['clusterName'], cluster['status']['state'])) + return result +# [END dataproc_list_clusters_with_detail] + + +def get_cluster_id_by_name(cluster_list, cluster_name): + """Helper function to retrieve the ID and output bucket of a cluster by + name.""" + cluster = [c for c in cluster_list if c['clusterName'] == cluster_name][0] + return cluster['clusterUuid'], cluster['config']['configBucket'] + + +# [START dataproc_submit_pyspark_job] +def submit_pyspark_job(dataproc, project, region, + cluster_name, bucket_name, filename): + """Submits the Pyspark job to the cluster, assuming `filename` has + already been uploaded to `bucket_name`""" + job_details = { + 'projectId': project, + 'job': { + 'placement': { + 'clusterName': cluster_name + }, + 'pysparkJob': { + 'mainPythonFileUri': 'gs://{}/{}'.format(bucket_name, filename) + } + } + } + result = dataproc.projects().regions().jobs().submit( + projectId=project, + region=region, + body=job_details).execute() + job_id = result['reference']['jobId'] + print('Submitted job ID {}'.format(job_id)) + return job_id +# [END dataproc_submit_pyspark_job] + + +# [START dataproc_delete] +def delete_cluster(dataproc, project, region, cluster): + print('Tearing down cluster') + result = dataproc.projects().regions().clusters().delete( + projectId=project, + region=region, + clusterName=cluster).execute() + return result +# [END dataproc_delete] + + +# [START dataproc_wait] +def wait_for_job(dataproc, project, region, job_id): + print('Waiting for job to finish...') + while True: + result = dataproc.projects().regions().jobs().get( + projectId=project, + region=region, + jobId=job_id).execute() + # Handle exceptions + if result['status']['state'] == 'ERROR': + raise Exception(result['status']['details']) + elif result['status']['state'] == 'DONE': + print('Job finished.') + return result +# [END dataproc_wait] + + +# [START dataproc_get_client] +def get_client(): + """Builds an http client authenticated with the service account + credentials.""" + dataproc = googleapiclient.discovery.build('dataproc', 'v1') + return dataproc +# [END dataproc_get_client] + + +def main(project_id, zone, cluster_name, bucket_name, + pyspark_file=None, create_new_cluster=True): + dataproc = get_client() + region = get_region_from_zone(zone) + try: + if pyspark_file: + spark_file, spark_filename = get_pyspark_file(pyspark_file) + else: + spark_file, spark_filename = get_default_pyspark_file() + + if create_new_cluster: + create_cluster( + dataproc, project_id, zone, region, cluster_name) + wait_for_cluster_creation( + dataproc, project_id, region, cluster_name) + + upload_pyspark_file( + project_id, bucket_name, spark_filename, spark_file) + + cluster_list = list_clusters_with_details( + dataproc, project_id, region)['clusters'] + + (cluster_id, output_bucket) = ( + get_cluster_id_by_name(cluster_list, cluster_name)) + + # [START dataproc_call_submit_pyspark_job] + job_id = submit_pyspark_job( + dataproc, project_id, region, + cluster_name, bucket_name, spark_filename) + # [END dataproc_call_submit_pyspark_job] + wait_for_job(dataproc, project_id, region, job_id) + + output = download_output(project_id, cluster_id, output_bucket, job_id) + print('Received job output {}'.format(output)) + return output + finally: + if create_new_cluster: + delete_cluster(dataproc, project_id, region, cluster_name) + spark_file.close() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + '--project_id', help='Project ID you want to access.', required=True), + parser.add_argument( + '--zone', help='Zone to create clusters in/connect to', required=True) + parser.add_argument( + '--cluster_name', + help='Name of the cluster to create/connect to', required=True) + parser.add_argument( + '--gcs_bucket', help='Bucket to upload Pyspark file to', required=True) + parser.add_argument( + '--pyspark_file', help='Pyspark filename. Defaults to pyspark_sort.py') + parser.add_argument( + '--create_new_cluster', + action='store_true', help='States if the cluster should be created') + + args = parser.parse_args() + main( + args.project_id, args.zone, args.cluster_name, + args.gcs_bucket, args.pyspark_file, args.create_new_cluster) diff --git a/datastore/README.md b/datastore/README.md deleted file mode 100644 index ebea408ecfa..00000000000 --- a/datastore/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# Google Cloud Datastore Samples - -This section contains samples for [Google Cloud Datastore](https://cloud.google.com/datastore). - -## Other Samples - -* [Google App Engine & NDB](../appengine/ndb). -* [Blog Sample: Introduction to Data Models in Cloud Datastore](../blog/introduction_to_data_models_in_cloud_datastore). diff --git a/datastore/api/README.md b/datastore/api/README.md deleted file mode 100644 index a53482eda9d..00000000000 --- a/datastore/api/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# Cloud Datastore API Samples - - - diff --git a/datastore/api/index.yaml b/datastore/api/index.yaml deleted file mode 100644 index 8077e5fa015..00000000000 --- a/datastore/api/index.yaml +++ /dev/null @@ -1,29 +0,0 @@ -indexes: -- kind: Task - properties: - - name: done - - name: priority - direction: desc -- kind: Task - properties: - - name: priority - - name: percent_complete -- kind: Task - properties: - - name: priority - direction: desc - - name: created -- kind: Task - properties: - - name: priority - - name: created -- kind: Task - properties: - - name: type - - name: priority -- kind: Task - properties: - - name: priority - - name: done - - name: created - direction: desc diff --git a/datastore/api/requirements.txt b/datastore/api/requirements.txt deleted file mode 100644 index b40295d1af9..00000000000 --- a/datastore/api/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -gcloud==0.10.1 diff --git a/datastore/api/snippets.py b/datastore/api/snippets.py deleted file mode 100644 index 8bb2c306d6e..00000000000 --- a/datastore/api/snippets.py +++ /dev/null @@ -1,816 +0,0 @@ -# Copyright 2016, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -from collections import defaultdict -import datetime -from pprint import pprint - -import gcloud -from gcloud import datastore - - -def incomplete_key(client): - # [START incomplete_key] - key = client.key('Task') - # [END incomplete_key] - - return key - - -def named_key(client): - # [START named_key] - key = client.key('Task', 'sample_task') - # [END named_key] - - return key - - -def key_with_parent(client): - # [START key_with_parent] - key = client.key('TaskList', 'default', 'Task', 'sample_task') - # Alternatively - parent_key = client.key('TaskList', 'default') - key = client.key('Task', 'sample_task', parent=parent_key) - # [END key_with_parent] - - return key - - -def key_with_multilevel_parent(client): - # [START key_with_multilevel_parent] - key = client.key( - 'User', 'alice', - 'TaskList', 'default', - 'Task', 'sample_task') - # [END key_with_multilevel_parent] - - return key - - -def basic_entity(client): - # [START basic_entity] - task = datastore.Entity(client.key('Task')) - task.update({ - 'type': 'Personal', - 'done': False, - 'priority': 4, - 'description': 'Learn Cloud Datastore' - }) - # [END basic_entity] - - return task - - -def entity_with_parent(client): - # [START entity_with_parent] - key_with_parent = client.key( - 'TaskList', 'default', 'Task', 'sample_task') - - task = datastore.Entity(key=key_with_parent) - - task.update({ - 'type': 'Personal', - 'done': False, - 'priority': 4, - 'description': 'Learn Cloud Datastore' - }) - # [END entity_with_parent] - - return task - - -def properties(client): - key = client.key('Task') - # [START properties] - task = datastore.Entity( - key, - exclude_from_indexes=['description']) - task.update({ - 'type': 'Personal', - 'description': 'Learn Cloud Datastore', - 'created': datetime.datetime.utcnow(), - 'done': False, - 'priority': 4, - 'percent_complete': 10.5, - }) - # [END properties] - - return task - - -def array_value(client): - key = client.key('Task') - # [START array_value] - task = datastore.Entity(key) - task.update({ - 'tags': [ - 'fun', - 'programming' - ], - 'collaborators': [ - 'alice', - 'bob' - ] - }) - # [END array_value] - - return task - - -def upsert(client): - # [START upsert] - complete_key = client.key('Task', 'sample_task') - - task = datastore.Entity(key=complete_key) - - task.update({ - 'type': 'Personal', - 'done': False, - 'priority': 4, - 'description': 'Learn Cloud Datastore' - }) - - client.put(task) - # [END upsert] - - return task - - -def insert(client): - # [START insert] - with client.transaction(): - incomplete_key = client.key('Task') - - task = datastore.Entity(key=incomplete_key) - - task.update({ - 'type': 'Personal', - 'done': False, - 'priority': 4, - 'description': 'Learn Cloud Datastore' - }) - - client.put(task) - # [END insert] - - return task - - -def update(client): - # Create the entity we're going to update. - upsert(client) - - # [START update] - with client.transaction(): - key = client.key('Task', 'sample_task') - task = client.get(key) - - task['done'] = True - - client.put(task) - # [END update] - - return task - - -def lookup(client): - # Create the entity that we're going to look up. - upsert(client) - - # [START lookup] - key = client.key('Task', 'sample_task') - task = client.get(key) - # [END lookup] - - return task - - -def delete(client): - # Create the entity we're going to delete. - upsert(client) - - # [START delete] - key = client.key('Task', 'sample_task') - client.delete(key) - # [END delete] - - return key - - -def batch_upsert(client): - task1 = datastore.Entity(client.key('Task', 1)) - - task1.update({ - 'type': 'Personal', - 'done': False, - 'priority': 4, - 'description': 'Learn Cloud Datastore' - }) - - task2 = datastore.Entity(client.key('Task', 2)) - - task2.update({ - 'type': 'Work', - 'done': False, - 'priority': 8, - 'description': 'Integrate Cloud Datastore' - }) - - # [START batch_upsert] - client.put_multi([task1, task2]) - # [END batch_upsert] - - return task1, task2 - - -def batch_lookup(client): - # Create the entities we will lookup. - batch_upsert(client) - - keys = [ - client.key('Task', 1), - client.key('Task', 2) - ] - - # [START batch_lookup] - tasks = client.get_multi(keys) - # [END batch_lookup] - - return tasks - - -def batch_delete(client): - # Create the entities we will delete. - batch_upsert(client) - - keys = [ - client.key('Task', 1), - client.key('Task', 2) - ] - - # [START batch_delete] - client.delete_multi(keys) - # [END batch_delete] - - return keys - - -def unindexed_property_query(client): - # Create the entity that we're going to query. - upsert(client) - - # [START unindexed_property_query] - query = client.query(kind='Task') - query.add_filter('description', '=', 'Learn Cloud Datastore') - # [END unindexed_property_query] - - return list(query.fetch()) - - -def basic_query(client): - # Create the entity that we're going to query. - upsert(client) - - # [START basic_query] - query = client.query(kind='Task') - query.add_filter('done', '=', False) - query.add_filter('priority', '>=', 4) - query.order = ['-priority'] - # [END basic_query] - - return list(query.fetch()) - - -def projection_query(client): - # Create the entity that we're going to query. - task = datastore.Entity(client.key('Task')) - task.update({ - 'type': 'Personal', - 'done': False, - 'priority': 4, - 'description': 'Learn Cloud Datastore', - 'percent_complete': 0.5 - }) - client.put(task) - - # [START projection_query] - query = client.query(kind='Task') - query.projection = ['priority', 'percent_complete'] - # [END projection_query] - - # [START run_query_projection] - priorities = [] - percent_completes = [] - - for task in query.fetch(): - priorities.append(task['priority']) - percent_completes.append(task['priority']) - # [END run_query_projection] - - return priorities, percent_completes - - -def ancestor_query(client): - task = datastore.Entity( - client.key('TaskList', 'default', 'Task')) - task.update({ - 'type': 'Personal', - 'description': 'Learn Cloud Datastore', - }) - client.put(task) - - # [START ancestor_query] - ancestor = client.key('TaskList', 'default') - query = client.query(kind='Task', ancestor=ancestor) - # [END ancestor_query] - - return list(query.fetch()) - - -def run_query(client): - # [START run_query] - query = client.query() - results = list(query.fetch()) - # [END run_query] - - return results - - -def limit(client): - # [START limit] - query = client.query() - tasks = list(query.fetch(limit=5)) - # [END limit] - - return tasks - - -def cursor_paging(client): - # [START cursor_paging] - - def get_one_page_of_tasks(cursor=None): - query = client.query(kind='Task') - query_iter = query.fetch(start_cursor=cursor, limit=5) - tasks, _, cursor = query_iter.next_page() - - return tasks, cursor - # [END cursor_paging] - - page_one, cursor_one = get_one_page_of_tasks() - page_two, cursor_two = get_one_page_of_tasks(cursor=cursor_one) - return page_one, cursor_one, page_two, cursor_two - - -def property_filter(client): - # Create the entity that we're going to query. - upsert(client) - - # [START property_filter] - query = client.query(kind='Task') - query.add_filter('done', '=', False) - # [END property_filter] - - return list(query.fetch()) - - -def composite_filter(client): - # Create the entity that we're going to query. - upsert(client) - - # [START composite_filter] - query = client.query(kind='Task') - query.add_filter('done', '=', False) - query.add_filter('priority', '=', 4) - # [END composite_filter] - - return list(query.fetch()) - - -def key_filter(client): - # Create the entity that we're going to query. - upsert(client) - - # [START key_filter] - query = client.query(kind='Task') - first_key = client.key('Task', 'first_task') - query.add_filter('__key__', '>', first_key) - # [END key_filter] - - return list(query.fetch()) - - -def ascending_sort(client): - # Create the entity that we're going to query. - task = upsert(client) - task['created'] = datetime.datetime.utcnow() - client.put(task) - - # [START ascending_sort] - query = client.query(kind='Task') - query.order = ['created'] - # [END ascending_sort] - - return list(query.fetch()) - - -def descending_sort(client): - # Create the entity that we're going to query. - task = upsert(client) - task['created'] = datetime.datetime.utcnow() - client.put(task) - - # [START descending_sort] - query = client.query(kind='Task') - query.order = ['-created'] - # [END descending_sort] - - return list(query.fetch()) - - -def multi_sort(client): - # Create the entity that we're going to query. - task = upsert(client) - task['created'] = datetime.datetime.utcnow() - client.put(task) - - # [START multi_sort] - query = client.query(kind='Task') - query.order = [ - '-priority', - 'created' - ] - # [END multi_sort] - - return list(query.fetch()) - - -def keys_only_query(client): - # Create the entity that we're going to query. - upsert(client) - - # [START keys_only_query] - query = client.query() - query.keys_only() - # [END keys_only_query] - - # [START run_keys_only_query] - keys = list([entity.key for entity in query.fetch(limit=10)]) - # [END run_keys_only_query] - - return keys - - -def distinct_query(client): - # Create the entity that we're going to query. - upsert(client) - - # [START distinct_query] - query = client.query(kind='Task') - query.group_by = ['type', 'priority'] - query.order = ['type', 'priority'] - query.projection = ['type', 'priority'] - # [END distinct_query] - - return list(query.fetch()) - - -def distinct_on_query(client): - # Create the entity that we're going to query. - upsert(client) - - # [START distinct_on_query] - query = client.query(kind='Task') - query.group_by = ['type'] - query.order = ['type', 'priority'] - # [END distinct_on_query] - - return list(query.fetch()) - - -def kindless_query(client): - # Create the entity that we're going to query. - upsert(client) - - last_seen_key = client.key('Task', 'a') - - # [START kindless_query] - query = client.query() - query.add_filter('__key__', '>', last_seen_key) - # [END kindless_query] - - return list(query.fetch()) - - -def inequality_range(client): - # [START inequality_range] - start_date = datetime.datetime(1990, 1, 1) - end_date = datetime.datetime(2000, 1, 1) - query = client.query(kind='Task') - query.add_filter( - 'created', '>', start_date) - query.add_filter( - 'created', '<', end_date) - # [END inequality_range] - - return list(query.fetch()) - - -def inequality_invalid(client): - try: - # [START inequality_invalid] - start_date = datetime.datetime(1990, 1, 1) - query = client.query(kind='Task') - query.add_filter( - 'created', '>', start_date) - query.add_filter( - 'priority', '>', 3) - # [END inequality_invalid] - - return list(query.fetch()) - - except gcloud.exceptions.BadRequest: - pass - - -def equal_and_inequality_range(client): - # [START equal_and_inequality_range] - start_date = datetime.datetime(1990, 1, 1) - end_date = datetime.datetime(2000, 12, 31, 23, 59, 59) - query = client.query(kind='Task') - query.add_filter('priority', '=', 4) - query.add_filter('done', '=', False) - query.add_filter( - 'created', '>', start_date) - query.add_filter( - 'created', '<', end_date) - # [END equal_and_inequality_range] - - return list(query.fetch()) - - -def inequality_sort(client): - # [START inequality_sort] - query = client.query(kind='Task') - query.add_filter('priority', '>', 3) - query.order = ['priority', 'created'] - # [END inequality_sort] - - return list(query.fetch()) - - -def inequality_sort_invalid_not_same(client): - try: - # [START inequality_sort_invalid_not_same] - query = client.query(kind='Task') - query.add_filter('priority', '>', 3) - query.order = ['created'] - # [END inequality_sort_invalid_not_same] - - return list(query.fetch()) - - except gcloud.exceptions.BadRequest: - pass - - -def inequality_sort_invalid_not_first(client): - try: - # [START inequality_sort_invalid_not_first] - query = client.query(kind='Task') - query.add_filter('priority', '>', 3) - query.order = ['created', 'priority'] - # [END inequality_sort_invalid_not_first] - - return list(query.fetch()) - - except gcloud.exceptions.BadRequest: - pass - - -def array_value_inequality_range(client): - # [START array_value_inequality_range] - query = client.query(kind='Task') - query.add_filter('tag', '>', 'learn') - query.add_filter('tag', '<', 'math') - # [END array_value_inequality_range] - - return list(query.fetch()) - - -def array_value_equality(client): - # [START array_value_equality] - query = client.query(kind='Task') - query.add_filter('tag', '=', 'fun') - query.add_filter('tag', '=', 'programming') - # [END array_value_equality] - - return list(query.fetch()) - - -def exploding_properties(client): - # [START exploding_properties] - task = datastore.Entity(client.key('Task')) - task.update({ - 'tags': [ - 'fun', - 'programming', - 'learn' - ], - 'collaborators': [ - 'alice', - 'bob', - 'charlie' - ], - 'created': datetime.datetime.utcnow() - }) - # [END exploding_properties] - - return task - - -def transactional_update(client): - # Create the entities we're going to manipulate - account1 = datastore.Entity(client.key('Account')) - account1['balance'] = 100 - account2 = datastore.Entity(client.key('Account')) - account2['balance'] = 100 - client.put_multi([account1, account2]) - - # [START transactional_update] - def transfer_funds(client, from_key, to_key, amount): - with client.transaction(): - from_account, to_account = client.get_multi([from_key, to_key]) - - from_account['balance'] -= amount - to_account['balance'] += amount - - client.put_multi([from_account, to_account]) - # [END transactional_update] - - # [START transactional_retry] - for _ in range(5): - try: - transfer_funds(client, account1.key, account2.key, 50) - except gcloud.exceptions.Conflict: - continue - # [END transactional_retry] - - return account1.key, account2.key - - -def transactional_get_or_create(client): - # [START transactional_get_or_create] - with client.transaction(): - key = client.key('Task', datetime.datetime.utcnow().isoformat()) - - task = client.get(key) - - if not task: - task = datastore.Entity(key) - task.update({ - 'description': 'Example task' - }) - client.put(task) - - return task - # [END transactional_get_or_create] - - -def transactional_single_entity_group_read_only(client): - client.put_multi([ - datastore.Entity(key=client.key('TaskList', 'default')), - datastore.Entity(key=client.key('TaskList', 'default', 'Task', 1)) - ]) - - # [START transactional_single_entity_group_read_only] - with client.transaction(): - task_list_key = client.key('TaskList', 'default') - - task_list = client.get(task_list_key) - - query = client.query(kind='Task', ancestor=task_list_key) - tasks_in_list = list(query.fetch()) - - return task_list, tasks_in_list - # [END transactional_single_entity_group_read_only] - - -def namespace_run_query(client): - # Create an entity in another namespace. - task = datastore.Entity( - client.key('Task', 'sample-task', namespace='google')) - client.put(task) - - # [START namespace_run_query] - # All namespaces - query = client.query(kind='__namespace__') - query.keys_only() - - all_namespaces = [entity.key.id_or_name for entity in query.fetch()] - - # Filtered namespaces - start_namespace = client.key('__namespace__', 'g') - end_namespace = client.key('__namespace__', 'h') - query = client.query(kind='__namespace__') - query.add_filter( - '__key__', '>=', start_namespace) - query.add_filter( - '__key__', '<', end_namespace) - - filtered_namespaces = [entity.key.id_or_name for entity in query.fetch()] - # [END namespace_run_query] - - return all_namespaces, filtered_namespaces - - -def kind_run_query(client): - # Create the entity that we're going to query. - upsert(client) - - # [START kind_run_query] - query = client.query(kind='__kind__') - query.keys_only() - - kinds = [entity.key.id_or_name for entity in query.fetch()] - # [END kind_run_query] - - return kinds - - -def property_run_query(client): - # Create the entity that we're going to query. - upsert(client) - - # [START property_run_query] - query = client.query(kind='__property__') - query.keys_only() - - properties_by_kind = defaultdict(list) - - for entity in query.fetch(): - kind = entity.key.parent.name - property_ = entity.key.name - - properties_by_kind[kind].append(property_) - # [END property_run_query] - - return properties_by_kind - - -def property_by_kind_run_query(client): - # Create the entity that we're going to query. - upsert(client) - - # [START property_by_kind_run_query] - ancestor = client.key('__kind__', 'Task') - query = client.query(kind='__property__', ancestor=ancestor) - - representations_by_property = {} - - for entity in query.fetch(): - property_name = entity.key.name - property_types = entity['property_representation'] - - representations_by_property[property_name] = property_types - # [END property_by_kind_run_query] - - return representations_by_property - - -def eventual_consistent_query(client): - # [START eventual_consistent_query] - # Read consistency cannot be specified in gcloud-python. - # [END eventual_consistent_query] - pass - - -def main(project_id): - client = datastore.Client(project_id) - - for name, function in globals().iteritems(): - if name in ('main', 'defaultdict') or not callable(function): - continue - - print(name) - pprint(function(client)) - print('\n-----------------\n') - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description='Demonstrates datastore API operations.') - parser.add_argument('project_id', help='Your cloud project ID.') - - args = parser.parse_args() - - main(args.project_id) diff --git a/datastore/api/snippets_test.py b/datastore/api/snippets_test.py deleted file mode 100644 index 974578f9ef4..00000000000 --- a/datastore/api/snippets_test.py +++ /dev/null @@ -1,279 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import time - -from gcloud import datastore -from gcp.testing.flaky import flaky -import pytest -import snippets - - -class CleanupClient(datastore.Client): - def __init__(self, *args, **kwargs): - super(CleanupClient, self).__init__(*args, **kwargs) - self.entities_to_delete = [] - self.keys_to_delete = [] - - def cleanup(self): - with self.batch(): - self.delete_multi( - [x.key for x in self.entities_to_delete] + - self.keys_to_delete) - - -# This is pretty hacky, but make datastore wait 1s after any -# put operation to in order to account for eventual consistency. -class WaitingClient(CleanupClient): - def put_multi(self, *args, **kwargs): - result = super(WaitingClient, self).put_multi(*args, **kwargs) - time.sleep(1) - return result - - -@pytest.yield_fixture -def client(cloud_config): - client = CleanupClient(cloud_config.project) - yield client - client.cleanup() - - -@pytest.yield_fixture -def waiting_client(cloud_config): - client = WaitingClient(cloud_config.project) - yield client - client.cleanup() - - -@flaky -class TestDatastoreSnippets: - # These tests mostly just test the absence of exceptions. - def test_incomplete_key(self, client): - assert snippets.incomplete_key(client) - - def test_named_key(self, client): - assert snippets.named_key(client) - - def test_key_with_parent(self, client): - assert snippets.key_with_parent(client) - - def test_key_with_multilevel_parent(self, client): - assert snippets.key_with_multilevel_parent(client) - - def test_basic_entity(self, client): - assert snippets.basic_entity(client) - - def test_entity_with_parent(self, client): - assert snippets.entity_with_parent(client) - - def test_properties(self, client): - assert snippets.properties(client) - - def test_array_value(self, client): - assert snippets.array_value(client) - - def test_upsert(self, client): - task = snippets.upsert(client) - client.entities_to_delete.append(task) - assert task - - def test_insert(self, client): - task = snippets.insert(client) - client.entities_to_delete.append(task) - assert task - - def test_update(self, client): - task = snippets.insert(client) - client.entities_to_delete.append(task) - assert task - - def test_lookup(self, client): - task = snippets.lookup(client) - client.entities_to_delete.append(task) - assert task - - def test_delete(self, client): - snippets.delete(client) - - def test_batch_upsert(self, client): - tasks = snippets.batch_upsert(client) - client.entities_to_delete.extend(tasks) - assert tasks - - def test_batch_lookup(self, client): - tasks = snippets.batch_lookup(client) - client.entities_to_delete.extend(tasks) - assert tasks - - def test_batch_delete(self, client): - snippets.batch_delete(client) - - def test_unindexed_property_query(self, waiting_client): - tasks = snippets.unindexed_property_query(waiting_client) - waiting_client.entities_to_delete.extend(tasks) - assert tasks - - def test_basic_query(self, waiting_client): - tasks = snippets.basic_query(waiting_client) - waiting_client.entities_to_delete.extend(tasks) - assert tasks - - def test_projection_query(self, waiting_client): - priorities, percents = snippets.projection_query(waiting_client) - waiting_client.entities_to_delete.extend( - waiting_client.query(kind='Task').fetch()) - assert priorities - assert percents - - def test_ancestor_query(self, client): - tasks = snippets.ancestor_query(client) - client.entities_to_delete.extend(tasks) - assert tasks - - def test_run_query(self, client): - snippets.run_query(client) - - def test_cursor_paging(self, waiting_client): - for n in range(6): - waiting_client.entities_to_delete.append( - snippets.insert(waiting_client)) - - page_one, cursor_one, page_two, cursor_two = snippets.cursor_paging( - waiting_client) - - assert len(page_one) == 5 - assert len(page_two) == 1 - assert cursor_one - assert cursor_two - - def test_property_filter(self, waiting_client): - tasks = snippets.property_filter(waiting_client) - waiting_client.entities_to_delete.extend(tasks) - assert tasks - - def test_composite_filter(self, waiting_client): - tasks = snippets.composite_filter(waiting_client) - waiting_client.entities_to_delete.extend(tasks) - assert tasks - - def test_key_filter(self, waiting_client): - tasks = snippets.key_filter(waiting_client) - waiting_client.entities_to_delete.extend(tasks) - assert tasks - - def test_ascending_sort(self, waiting_client): - tasks = snippets.ascending_sort(waiting_client) - waiting_client.entities_to_delete.extend(tasks) - assert tasks - - def test_descending_sort(self, waiting_client): - tasks = snippets.descending_sort(waiting_client) - waiting_client.entities_to_delete.extend(tasks) - assert tasks - - def test_multi_sort(self, waiting_client): - tasks = snippets.multi_sort(waiting_client) - waiting_client.entities_to_delete.extend(tasks) - assert tasks - - def test_keys_only_query(self, waiting_client): - keys = snippets.keys_only_query(waiting_client) - waiting_client.entities_to_delete.extend( - waiting_client.query(kind='Task').fetch()) - assert keys - - def test_distinct_query(self, waiting_client): - tasks = snippets.distinct_query(waiting_client) - waiting_client.entities_to_delete.extend(tasks) - assert tasks - - def test_distinct_on_query(self, waiting_client): - tasks = snippets.distinct_on_query(waiting_client) - waiting_client.entities_to_delete.extend(tasks) - assert tasks - - def test_kindless_query(self, client): - tasks = snippets.kindless_query(client) - assert tasks - - def test_inequality_range(self, client): - snippets.inequality_range(client) - - def test_inequality_invalid(self, client): - snippets.inequality_invalid(client) - - def test_equal_and_inequality_range(self, client): - snippets.equal_and_inequality_range(client) - - def test_inequality_sort(self, client): - snippets.inequality_sort(client) - - def test_inequality_sort_invalid_not_same(self, client): - snippets.inequality_sort_invalid_not_same(client) - - def test_inequality_sort_invalid_not_first(self, client): - snippets.inequality_sort_invalid_not_first(client) - - def test_array_value_inequality_range(self, client): - snippets.array_value_inequality_range(client) - - def test_array_value_equality(self, client): - snippets.array_value_equality(client) - - def test_exploding_properties(self, client): - task = snippets.exploding_properties(client) - assert task - - def test_transactional_update(self, client): - keys = snippets.transactional_update(client) - client.keys_to_delete.extend(keys) - - def test_transactional_get_or_create(self, client): - task = snippets.transactional_get_or_create(client) - client.entities_to_delete.append(task) - assert task - - def transactional_single_entity_group_read_only(self, client): - task_list, tasks_in_list = \ - snippets.transactional_single_entity_group_read_only(client) - client.entities_to_delete.append(task_list) - client.entities_to_delete.extend(tasks_in_list) - assert task_list - assert tasks_in_list - - def test_namespace_run_query(self, waiting_client): - all_namespaces, filtered_namespaces = snippets.namespace_run_query( - waiting_client) - assert all_namespaces - assert filtered_namespaces - assert 'google' in filtered_namespaces - - def test_kind_run_query(self, waiting_client): - kinds = snippets.kind_run_query(waiting_client) - waiting_client.entities_to_delete.extend( - waiting_client.query(kind='Task').fetch()) - assert kinds - assert 'Task' in kinds - - def test_property_run_query(self, waiting_client): - kinds = snippets.property_run_query(waiting_client) - waiting_client.entities_to_delete.extend( - waiting_client.query(kind='Task').fetch()) - assert kinds - assert 'Task' in kinds - - def test_property_by_kind_run_query(self, waiting_client): - reprs = snippets.property_by_kind_run_query(waiting_client) - waiting_client.entities_to_delete.extend( - waiting_client.query(kind='Task').fetch()) - assert reprs diff --git a/datastore/api/tasks_test.py b/datastore/api/tasks_test.py deleted file mode 100644 index d8901d8f244..00000000000 --- a/datastore/api/tasks_test.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from gcloud import datastore -from gcp.testing.flaky import flaky -import pytest -import tasks - - -@pytest.yield_fixture -def client(cloud_config): - client = datastore.Client(cloud_config.project) - - yield client - - # Delete anything created during the test. - with client.batch(): - client.delete_multi( - [x.key for x in client.query(kind='Task').fetch()]) - - -@flaky -def test_create_client(cloud_config): - tasks.create_client(cloud_config.project) - - -@flaky -def test_add_task(client): - task_key = tasks.add_task(client, 'Test task') - task = client.get(task_key) - assert task - assert task['description'] == 'Test task' - - -@flaky -def test_mark_done(client): - task_key = tasks.add_task(client, 'Test task') - tasks.mark_done(client, task_key.id) - task = client.get(task_key) - assert task - assert task['done'] - - -@flaky -def test_list_tasks(client): - task1_key = tasks.add_task(client, 'Test task 1') - task2_key = tasks.add_task(client, 'Test task 2') - task_list = tasks.list_tasks(client) - assert [x.key for x in task_list] == [task1_key, task2_key] - - -@flaky -def test_delete_task(client): - task_key = tasks.add_task(client, 'Test task 1') - tasks.delete_task(client, task_key.id) - assert client.get(task_key) is None - - -@flaky -def test_format_tasks(client): - task1_key = tasks.add_task(client, 'Test task 1') - tasks.add_task(client, 'Test task 2') - tasks.mark_done(client, task1_key.id) - - output = tasks.format_tasks(tasks.list_tasks(client)) - - assert 'Test task 1' in output - assert 'Test task 2' in output - assert 'done' in output - assert 'created' in output diff --git a/datastore/cloud-client/README.rst b/datastore/cloud-client/README.rst new file mode 100644 index 00000000000..70454fdec82 --- /dev/null +++ b/datastore/cloud-client/README.rst @@ -0,0 +1,140 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Datastore Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=datastore/cloud-client/README.rst + + +This directory contains samples for Google Cloud Datastore. `Google Cloud Datastore`_ is a NoSQL document database built for automatic scaling, high performance, and ease of application development. + + + + +.. _Google Cloud Datastore: https://cloud.google.com/datastore/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Quickstart ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=datastore/cloud-client/quickstart.py,datastore/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python quickstart.py + + +Tasks example app ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=datastore/cloud-client/tasks.py,datastore/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python tasks.py + + +Snippets ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=datastore/cloud-client/snippets.py,datastore/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python snippets.py + + usage: snippets.py [-h] project_id + + Demonstrates datastore API operations. + + positional arguments: + project_id Your cloud project ID. + + optional arguments: + -h, --help show this help message and exit + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/datastore/cloud-client/README.rst.in b/datastore/cloud-client/README.rst.in new file mode 100644 index 00000000000..aeda020c25a --- /dev/null +++ b/datastore/cloud-client/README.rst.in @@ -0,0 +1,26 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Datastore + short_name: Cloud Datastore + url: https://cloud.google.com/datastore/docs + description: > + `Google Cloud Datastore`_ is a NoSQL document database built for automatic + scaling, high performance, and ease of application development. + +setup: +- auth +- install_deps + +samples: +- name: Quickstart + file: quickstart.py +- name: Tasks example app + file: tasks.py +- name: Snippets + file: snippets.py + show_help: true + +cloud_client_library: true + +folder: datastore/cloud-client \ No newline at end of file diff --git a/datastore/cloud-client/index.yaml b/datastore/cloud-client/index.yaml new file mode 100644 index 00000000000..054f39db6d9 --- /dev/null +++ b/datastore/cloud-client/index.yaml @@ -0,0 +1,29 @@ +indexes: +- kind: Task + properties: + - name: done + - name: priority + direction: desc +- kind: Task + properties: + - name: priority + - name: percent_complete +- kind: Task + properties: + - name: priority + direction: desc + - name: created +- kind: Task + properties: + - name: priority + - name: created +- kind: Task + properties: + - name: category + - name: priority +- kind: Task + properties: + - name: priority + - name: done + - name: created + direction: desc diff --git a/datastore/cloud-client/quickstart.py b/datastore/cloud-client/quickstart.py new file mode 100644 index 00000000000..37e1ee77ea4 --- /dev/null +++ b/datastore/cloud-client/quickstart.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def run_quickstart(): + # [START datastore_quickstart] + # Imports the Google Cloud client library + from google.cloud import datastore + + # Instantiates a client + datastore_client = datastore.Client() + + # The kind for the new entity + kind = 'Task' + # The name/ID for the new entity + name = 'sampletask1' + # The Cloud Datastore key for the new entity + task_key = datastore_client.key(kind, name) + + # Prepares the new entity + task = datastore.Entity(key=task_key) + task['description'] = 'Buy milk' + + # Saves the entity + datastore_client.put(task) + + print('Saved {}: {}'.format(task.key.name, task['description'])) + # [END datastore_quickstart] + + +if __name__ == '__main__': + run_quickstart() diff --git a/datastore/cloud-client/quickstart_test.py b/datastore/cloud-client/quickstart_test.py new file mode 100644 index 00000000000..8545dfa04de --- /dev/null +++ b/datastore/cloud-client/quickstart_test.py @@ -0,0 +1,21 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import quickstart + + +def test_quickstart(capsys): + quickstart.run_quickstart() + out, _ = capsys.readouterr() + assert 'Saved' in out diff --git a/datastore/cloud-client/requirements.txt b/datastore/cloud-client/requirements.txt new file mode 100644 index 00000000000..ff31c9011c4 --- /dev/null +++ b/datastore/cloud-client/requirements.txt @@ -0,0 +1 @@ +google-cloud-datastore==1.7.3 diff --git a/datastore/cloud-client/snippets.py b/datastore/cloud-client/snippets.py new file mode 100644 index 00000000000..3354749d81a --- /dev/null +++ b/datastore/cloud-client/snippets.py @@ -0,0 +1,811 @@ +# Copyright 2016, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +from collections import defaultdict +import datetime +from pprint import pprint + +from google.cloud import datastore +import google.cloud.exceptions + + +def incomplete_key(client): + # [START datastore_incomplete_key] + key = client.key('Task') + # [END datastore_incomplete_key] + + return key + + +def named_key(client): + # [START datastore_named_key] + key = client.key('Task', 'sample_task') + # [END datastore_named_key] + + return key + + +def key_with_parent(client): + # [START datastore_key_with_parent] + key = client.key('TaskList', 'default', 'Task', 'sample_task') + # Alternatively + parent_key = client.key('TaskList', 'default') + key = client.key('Task', 'sample_task', parent=parent_key) + # [END datastore_key_with_parent] + + return key + + +def key_with_multilevel_parent(client): + # [START datastore_key_with_multilevel_parent] + key = client.key( + 'User', 'alice', + 'TaskList', 'default', + 'Task', 'sample_task') + # [END datastore_key_with_multilevel_parent] + + return key + + +def basic_entity(client): + # [START datastore_basic_entity] + task = datastore.Entity(client.key('Task')) + task.update({ + 'category': 'Personal', + 'done': False, + 'priority': 4, + 'description': 'Learn Cloud Datastore' + }) + # [END datastore_basic_entity] + + return task + + +def entity_with_parent(client): + # [START datastore_entity_with_parent] + key_with_parent = client.key( + 'TaskList', 'default', 'Task', 'sample_task') + + task = datastore.Entity(key=key_with_parent) + + task.update({ + 'category': 'Personal', + 'done': False, + 'priority': 4, + 'description': 'Learn Cloud Datastore' + }) + # [END datastore_entity_with_parent] + + return task + + +def properties(client): + key = client.key('Task') + # [START datastore_properties] + task = datastore.Entity( + key, + exclude_from_indexes=['description']) + task.update({ + 'category': 'Personal', + 'description': 'Learn Cloud Datastore', + 'created': datetime.datetime.utcnow(), + 'done': False, + 'priority': 4, + 'percent_complete': 10.5, + }) + # [END datastore_properties] + + return task + + +def array_value(client): + key = client.key('Task') + # [START datastore_array_value] + task = datastore.Entity(key) + task.update({ + 'tags': [ + 'fun', + 'programming' + ], + 'collaborators': [ + 'alice', + 'bob' + ] + }) + # [END datastore_array_value] + + return task + + +def upsert(client): + # [START datastore_upsert] + complete_key = client.key('Task', 'sample_task') + + task = datastore.Entity(key=complete_key) + + task.update({ + 'category': 'Personal', + 'done': False, + 'priority': 4, + 'description': 'Learn Cloud Datastore' + }) + + client.put(task) + # [END datastore_upsert] + + return task + + +def insert(client): + # [START datastore_insert] + with client.transaction(): + incomplete_key = client.key('Task') + + task = datastore.Entity(key=incomplete_key) + + task.update({ + 'category': 'Personal', + 'done': False, + 'priority': 4, + 'description': 'Learn Cloud Datastore' + }) + + client.put(task) + # [END datastore_insert] + + return task + + +def update(client): + # Create the entity we're going to update. + upsert(client) + + # [START datastore_update] + with client.transaction(): + key = client.key('Task', 'sample_task') + task = client.get(key) + + task['done'] = True + + client.put(task) + # [END datastore_update] + + return task + + +def lookup(client): + # Create the entity that we're going to look up. + upsert(client) + + # [START datastore_lookup] + key = client.key('Task', 'sample_task') + task = client.get(key) + # [END datastore_lookup] + + return task + + +def delete(client): + # Create the entity we're going to delete. + upsert(client) + + # [START datastore_delete] + key = client.key('Task', 'sample_task') + client.delete(key) + # [END datastore_delete] + + return key + + +def batch_upsert(client): + task1 = datastore.Entity(client.key('Task', 1)) + + task1.update({ + 'category': 'Personal', + 'done': False, + 'priority': 4, + 'description': 'Learn Cloud Datastore' + }) + + task2 = datastore.Entity(client.key('Task', 2)) + + task2.update({ + 'category': 'Work', + 'done': False, + 'priority': 8, + 'description': 'Integrate Cloud Datastore' + }) + + # [START datastore_batch_upsert] + client.put_multi([task1, task2]) + # [END datastore_batch_upsert] + + return task1, task2 + + +def batch_lookup(client): + # Create the entities we will lookup. + batch_upsert(client) + + keys = [ + client.key('Task', 1), + client.key('Task', 2) + ] + + # [START datastore_batch_lookup] + tasks = client.get_multi(keys) + # [END datastore_batch_lookup] + + return tasks + + +def batch_delete(client): + # Create the entities we will delete. + batch_upsert(client) + + keys = [ + client.key('Task', 1), + client.key('Task', 2) + ] + + # [START datastore_batch_delete] + client.delete_multi(keys) + # [END datastore_batch_delete] + + return keys + + +def unindexed_property_query(client): + # Create the entity that we're going to query. + upsert(client) + + # [START datastore_unindexed_property_query] + query = client.query(kind='Task') + query.add_filter('description', '=', 'Learn Cloud Datastore') + # [END datastore_unindexed_property_query] + + return list(query.fetch()) + + +def basic_query(client): + # Create the entity that we're going to query. + upsert(client) + + # [START datastore_basic_query] + query = client.query(kind='Task') + query.add_filter('done', '=', False) + query.add_filter('priority', '>=', 4) + query.order = ['-priority'] + # [END datastore_basic_query] + + return list(query.fetch()) + + +def projection_query(client): + # Create the entity that we're going to query. + task = datastore.Entity(client.key('Task')) + task.update({ + 'category': 'Personal', + 'done': False, + 'priority': 4, + 'description': 'Learn Cloud Datastore', + 'percent_complete': 0.5 + }) + client.put(task) + + # [START datastore_projection_query] + query = client.query(kind='Task') + query.projection = ['priority', 'percent_complete'] + # [END datastore_projection_query] + + # [START datastore_run_query_projection] + priorities = [] + percent_completes = [] + + for task in query.fetch(): + priorities.append(task['priority']) + percent_completes.append(task['percent_complete']) + # [END datastore_run_query_projection] + + return priorities, percent_completes + + +def ancestor_query(client): + task = datastore.Entity( + client.key('TaskList', 'default', 'Task')) + task.update({ + 'category': 'Personal', + 'description': 'Learn Cloud Datastore', + }) + client.put(task) + + # [START datastore_ancestor_query] + # Query filters are omitted in this example as any ancestor queries with a + # non-key filter require a composite index. + ancestor = client.key('TaskList', 'default') + query = client.query(kind='Task', ancestor=ancestor) + # [END datastore_ancestor_query] + + return list(query.fetch()) + + +def run_query(client): + # [START datastore_run_query] + query = client.query() + results = list(query.fetch()) + # [END datastore_run_query] + + return results + + +def limit(client): + # [START datastore_limit] + query = client.query() + tasks = list(query.fetch(limit=5)) + # [END datastore_limit] + + return tasks + + +def cursor_paging(client): + # [START datastore_cursor_paging] + + def get_one_page_of_tasks(cursor=None): + query = client.query(kind='Task') + query_iter = query.fetch(start_cursor=cursor, limit=5) + page = next(query_iter.pages) + + tasks = list(page) + next_cursor = query_iter.next_page_token + + return tasks, next_cursor + # [END datastore_cursor_paging] + + page_one, cursor_one = get_one_page_of_tasks() + page_two, cursor_two = get_one_page_of_tasks(cursor=cursor_one) + return page_one, cursor_one, page_two, cursor_two + + +def property_filter(client): + # Create the entity that we're going to query. + upsert(client) + + # [START datastore_property_filter] + query = client.query(kind='Task') + query.add_filter('done', '=', False) + # [END datastore_property_filter] + + return list(query.fetch()) + + +def composite_filter(client): + # Create the entity that we're going to query. + upsert(client) + + # [START datastore_composite_filter] + query = client.query(kind='Task') + query.add_filter('done', '=', False) + query.add_filter('priority', '=', 4) + # [END datastore_composite_filter] + + return list(query.fetch()) + + +def key_filter(client): + # Create the entity that we're going to query. + upsert(client) + + # [START datastore_key_filter] + query = client.query(kind='Task') + first_key = client.key('Task', 'first_task') + # key_filter(key, op) translates to add_filter('__key__', op, key). + query.key_filter(first_key, '>') + # [END datastore_key_filter] + + return list(query.fetch()) + + +def ascending_sort(client): + # Create the entity that we're going to query. + task = upsert(client) + task['created'] = datetime.datetime.utcnow() + client.put(task) + + # [START datastore_ascending_sort] + query = client.query(kind='Task') + query.order = ['created'] + # [END datastore_ascending_sort] + + return list(query.fetch()) + + +def descending_sort(client): + # Create the entity that we're going to query. + task = upsert(client) + task['created'] = datetime.datetime.utcnow() + client.put(task) + + # [START datastore_descending_sort] + query = client.query(kind='Task') + query.order = ['-created'] + # [END datastore_descending_sort] + + return list(query.fetch()) + + +def multi_sort(client): + # Create the entity that we're going to query. + task = upsert(client) + task['created'] = datetime.datetime.utcnow() + client.put(task) + + # [START datastore_multi_sort] + query = client.query(kind='Task') + query.order = [ + '-priority', + 'created' + ] + # [END datastore_multi_sort] + + return list(query.fetch()) + + +def keys_only_query(client): + # Create the entity that we're going to query. + upsert(client) + + # [START datastore_keys_only_query] + query = client.query() + query.keys_only() + # [END datastore_keys_only_query] + + keys = list([entity.key for entity in query.fetch(limit=10)]) + + return keys + + +def distinct_on_query(client): + # Create the entity that we're going to query. + upsert(client) + + # [START datastore_distinct_on_query] + query = client.query(kind='Task') + query.distinct_on = ['category'] + query.order = ['category', 'priority'] + # [END datastore_distinct_on_query] + + return list(query.fetch()) + + +def kindless_query(client): + # Create the entity that we're going to query. + upsert(client) + + last_seen_key = client.key('Task', 'a') + + # [START datastore_kindless_query] + query = client.query() + query.key_filter(last_seen_key, '>') + # [END datastore_kindless_query] + + return list(query.fetch()) + + +def inequality_range(client): + # [START datastore_inequality_range] + start_date = datetime.datetime(1990, 1, 1) + end_date = datetime.datetime(2000, 1, 1) + query = client.query(kind='Task') + query.add_filter( + 'created', '>', start_date) + query.add_filter( + 'created', '<', end_date) + # [END datastore_inequality_range] + + return list(query.fetch()) + + +def inequality_invalid(client): + try: + # [START datastore_inequality_invalid] + start_date = datetime.datetime(1990, 1, 1) + query = client.query(kind='Task') + query.add_filter( + 'created', '>', start_date) + query.add_filter( + 'priority', '>', 3) + # [END datastore_inequality_invalid] + + return list(query.fetch()) + + except (google.cloud.exceptions.BadRequest, + google.cloud.exceptions.GrpcRendezvous): + pass + + +def equal_and_inequality_range(client): + # [START datastore_equal_and_inequality_range] + start_date = datetime.datetime(1990, 1, 1) + end_date = datetime.datetime(2000, 12, 31, 23, 59, 59) + query = client.query(kind='Task') + query.add_filter('priority', '=', 4) + query.add_filter('done', '=', False) + query.add_filter( + 'created', '>', start_date) + query.add_filter( + 'created', '<', end_date) + # [END datastore_equal_and_inequality_range] + + return list(query.fetch()) + + +def inequality_sort(client): + # [START datastore_inequality_sort] + query = client.query(kind='Task') + query.add_filter('priority', '>', 3) + query.order = ['priority', 'created'] + # [END datastore_inequality_sort] + + return list(query.fetch()) + + +def inequality_sort_invalid_not_same(client): + try: + # [START datastore_inequality_sort_invalid_not_same] + query = client.query(kind='Task') + query.add_filter('priority', '>', 3) + query.order = ['created'] + # [END datastore_inequality_sort_invalid_not_same] + + return list(query.fetch()) + + except (google.cloud.exceptions.BadRequest, + google.cloud.exceptions.GrpcRendezvous): + pass + + +def inequality_sort_invalid_not_first(client): + try: + # [START datastore_inequality_sort_invalid_not_first] + query = client.query(kind='Task') + query.add_filter('priority', '>', 3) + query.order = ['created', 'priority'] + # [END datastore_inequality_sort_invalid_not_first] + + return list(query.fetch()) + + except (google.cloud.exceptions.BadRequest, + google.cloud.exceptions.GrpcRendezvous): + pass + + +def array_value_inequality_range(client): + # [START datastore_array_value_inequality_range] + query = client.query(kind='Task') + query.add_filter('tag', '>', 'learn') + query.add_filter('tag', '<', 'math') + # [END datastore_array_value_inequality_range] + + return list(query.fetch()) + + +def array_value_equality(client): + # [START datastore_array_value_equality] + query = client.query(kind='Task') + query.add_filter('tag', '=', 'fun') + query.add_filter('tag', '=', 'programming') + # [END datastore_array_value_equality] + + return list(query.fetch()) + + +def exploding_properties(client): + # [START datastore_exploding_properties] + task = datastore.Entity(client.key('Task')) + task.update({ + 'tags': [ + 'fun', + 'programming', + 'learn' + ], + 'collaborators': [ + 'alice', + 'bob', + 'charlie' + ], + 'created': datetime.datetime.utcnow() + }) + # [END datastore_exploding_properties] + + return task + + +def transactional_update(client): + # Create the entities we're going to manipulate + account1 = datastore.Entity(client.key('Account')) + account1['balance'] = 100 + account2 = datastore.Entity(client.key('Account')) + account2['balance'] = 100 + client.put_multi([account1, account2]) + + # [START datastore_transactional_update] + def transfer_funds(client, from_key, to_key, amount): + with client.transaction(): + from_account = client.get(from_key) + to_account = client.get(to_key) + + from_account['balance'] -= amount + to_account['balance'] += amount + + client.put_multi([from_account, to_account]) + # [END datastore_transactional_update] + + # [START datastore_transactional_retry] + for _ in range(5): + try: + transfer_funds(client, account1.key, account2.key, 50) + break + except google.cloud.exceptions.Conflict: + continue + else: + print('Transaction failed.') + # [END datastore_transactional_retry] + + return account1.key, account2.key + + +def transactional_get_or_create(client): + # [START datastore_transactional_get_or_create] + with client.transaction(): + key = client.key('Task', datetime.datetime.utcnow().isoformat()) + + task = client.get(key) + + if not task: + task = datastore.Entity(key) + task.update({ + 'description': 'Example task' + }) + client.put(task) + + return task + # [END datastore_transactional_get_or_create] + + +def transactional_single_entity_group_read_only(client): + client.put_multi([ + datastore.Entity(key=client.key('TaskList', 'default')), + datastore.Entity(key=client.key('TaskList', 'default', 'Task', 1)) + ]) + + # [START datastore_transactional_single_entity_group_read_only] + with client.transaction(read_only=True): + task_list_key = client.key('TaskList', 'default') + + task_list = client.get(task_list_key) + + query = client.query(kind='Task', ancestor=task_list_key) + tasks_in_list = list(query.fetch()) + + return task_list, tasks_in_list + # [END datastore_transactional_single_entity_group_read_only] + + +def namespace_run_query(client): + # Create an entity in another namespace. + task = datastore.Entity( + client.key('Task', 'sample-task', namespace='google')) + client.put(task) + + # [START datastore_namespace_run_query] + # All namespaces + query = client.query(kind='__namespace__') + query.keys_only() + + all_namespaces = [entity.key.id_or_name for entity in query.fetch()] + + # Filtered namespaces + start_namespace = client.key('__namespace__', 'g') + end_namespace = client.key('__namespace__', 'h') + query = client.query(kind='__namespace__') + query.key_filter(start_namespace, '>=') + query.key_filter(end_namespace, '<') + + filtered_namespaces = [entity.key.id_or_name for entity in query.fetch()] + # [END datastore_namespace_run_query] + + return all_namespaces, filtered_namespaces + + +def kind_run_query(client): + # Create the entity that we're going to query. + upsert(client) + + # [START datastore_kind_run_query] + query = client.query(kind='__kind__') + query.keys_only() + + kinds = [entity.key.id_or_name for entity in query.fetch()] + # [END datastore_kind_run_query] + + return kinds + + +def property_run_query(client): + # Create the entity that we're going to query. + upsert(client) + + # [START datastore_property_run_query] + query = client.query(kind='__property__') + query.keys_only() + + properties_by_kind = defaultdict(list) + + for entity in query.fetch(): + kind = entity.key.parent.name + property_ = entity.key.name + + properties_by_kind[kind].append(property_) + # [END datastore_property_run_query] + + return properties_by_kind + + +def property_by_kind_run_query(client): + # Create the entity that we're going to query. + upsert(client) + + # [START datastore_property_by_kind_run_query] + ancestor = client.key('__kind__', 'Task') + query = client.query(kind='__property__', ancestor=ancestor) + + representations_by_property = {} + + for entity in query.fetch(): + property_name = entity.key.name + property_types = entity['property_representation'] + + representations_by_property[property_name] = property_types + # [END datastore_property_by_kind_run_query] + + return representations_by_property + + +def eventual_consistent_query(client): + # [START datastore_eventual_consistent_query] + # Read consistency cannot be specified in google-cloud-python. + # [END datastore_eventual_consistent_query] + pass + + +def main(project_id): + client = datastore.Client(project_id) + + for name, function in globals().iteritems(): + if name in ('main', 'defaultdict') or not callable(function): + continue + + print(name) + pprint(function(client)) + print('\n-----------------\n') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Demonstrates datastore API operations.') + parser.add_argument('project_id', help='Your cloud project ID.') + + args = parser.parse_args() + + main(args.project_id) diff --git a/datastore/cloud-client/snippets_test.py b/datastore/cloud-client/snippets_test.py new file mode 100644 index 00000000000..193439f7b8c --- /dev/null +++ b/datastore/cloud-client/snippets_test.py @@ -0,0 +1,278 @@ +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from gcp_devrel.testing import eventually_consistent +from gcp_devrel.testing.flaky import flaky +from google.cloud import datastore +import pytest + +import snippets + +PROJECT = os.environ['GCLOUD_PROJECT'] + + +class CleanupClient(datastore.Client): + def __init__(self, *args, **kwargs): + super(CleanupClient, self).__init__(*args, **kwargs) + self.entities_to_delete = [] + self.keys_to_delete = [] + + def cleanup(self): + with self.batch(): + self.delete_multi( + list(set([x.key for x in self.entities_to_delete])) + + list(set(self.keys_to_delete))) + + +@pytest.yield_fixture +def client(): + client = CleanupClient(PROJECT) + yield client + client.cleanup() + + +@flaky +class TestDatastoreSnippets: + # These tests mostly just test the absence of exceptions. + def test_incomplete_key(self, client): + assert snippets.incomplete_key(client) + + def test_named_key(self, client): + assert snippets.named_key(client) + + def test_key_with_parent(self, client): + assert snippets.key_with_parent(client) + + def test_key_with_multilevel_parent(self, client): + assert snippets.key_with_multilevel_parent(client) + + def test_basic_entity(self, client): + assert snippets.basic_entity(client) + + def test_entity_with_parent(self, client): + assert snippets.entity_with_parent(client) + + def test_properties(self, client): + assert snippets.properties(client) + + def test_array_value(self, client): + assert snippets.array_value(client) + + def test_upsert(self, client): + task = snippets.upsert(client) + client.entities_to_delete.append(task) + assert task + + def test_insert(self, client): + task = snippets.insert(client) + client.entities_to_delete.append(task) + assert task + + def test_update(self, client): + task = snippets.insert(client) + client.entities_to_delete.append(task) + assert task + + def test_lookup(self, client): + task = snippets.lookup(client) + client.entities_to_delete.append(task) + assert task + + def test_delete(self, client): + snippets.delete(client) + + def test_batch_upsert(self, client): + tasks = snippets.batch_upsert(client) + client.entities_to_delete.extend(tasks) + assert tasks + + def test_batch_lookup(self, client): + tasks = snippets.batch_lookup(client) + client.entities_to_delete.extend(tasks) + assert tasks + + def test_batch_delete(self, client): + snippets.batch_delete(client) + + @eventually_consistent.mark + def test_unindexed_property_query(self, client): + tasks = snippets.unindexed_property_query(client) + client.entities_to_delete.extend(tasks) + assert tasks + + @eventually_consistent.mark + def test_basic_query(self, client): + tasks = snippets.basic_query(client) + client.entities_to_delete.extend(tasks) + assert tasks + + @eventually_consistent.mark + def test_projection_query(self, client): + priorities, percents = snippets.projection_query(client) + client.entities_to_delete.extend( + client.query(kind='Task').fetch()) + assert priorities + assert percents + + def test_ancestor_query(self, client): + tasks = snippets.ancestor_query(client) + client.entities_to_delete.extend(tasks) + assert tasks + + def test_run_query(self, client): + snippets.run_query(client) + + def test_cursor_paging(self, client): + for n in range(6): + client.entities_to_delete.append( + snippets.insert(client)) + + @eventually_consistent.call + def _(): + results = snippets.cursor_paging(client) + page_one, cursor_one, page_two, cursor_two = results + + assert len(page_one) == 5 + assert len(page_two) + assert cursor_one + + @eventually_consistent.mark + def test_property_filter(self, client): + tasks = snippets.property_filter(client) + client.entities_to_delete.extend(tasks) + assert tasks + + @eventually_consistent.mark + def test_composite_filter(self, client): + tasks = snippets.composite_filter(client) + client.entities_to_delete.extend(tasks) + assert tasks + + @eventually_consistent.mark + def test_key_filter(self, client): + tasks = snippets.key_filter(client) + client.entities_to_delete.extend(tasks) + assert tasks + + @eventually_consistent.mark + def test_ascending_sort(self, client): + tasks = snippets.ascending_sort(client) + client.entities_to_delete.extend(tasks) + assert tasks + + @eventually_consistent.mark + def test_descending_sort(self, client): + tasks = snippets.descending_sort(client) + client.entities_to_delete.extend(tasks) + assert tasks + + @eventually_consistent.mark + def test_multi_sort(self, client): + tasks = snippets.multi_sort(client) + client.entities_to_delete.extend(tasks) + assert tasks + + @eventually_consistent.mark + def test_keys_only_query(self, client): + keys = snippets.keys_only_query(client) + client.entities_to_delete.extend( + client.query(kind='Task').fetch()) + assert keys + + @eventually_consistent.mark + def test_distinct_on_query(self, client): + tasks = snippets.distinct_on_query(client) + client.entities_to_delete.extend(tasks) + assert tasks + + def test_kindless_query(self, client): + tasks = snippets.kindless_query(client) + assert tasks + + def test_inequality_range(self, client): + snippets.inequality_range(client) + + def test_inequality_invalid(self, client): + snippets.inequality_invalid(client) + + def test_equal_and_inequality_range(self, client): + snippets.equal_and_inequality_range(client) + + def test_inequality_sort(self, client): + snippets.inequality_sort(client) + + def test_inequality_sort_invalid_not_same(self, client): + snippets.inequality_sort_invalid_not_same(client) + + def test_inequality_sort_invalid_not_first(self, client): + snippets.inequality_sort_invalid_not_first(client) + + def test_array_value_inequality_range(self, client): + snippets.array_value_inequality_range(client) + + def test_array_value_equality(self, client): + snippets.array_value_equality(client) + + def test_exploding_properties(self, client): + task = snippets.exploding_properties(client) + assert task + + def test_transactional_update(self, client): + keys = snippets.transactional_update(client) + client.keys_to_delete.extend(keys) + + def test_transactional_get_or_create(self, client): + task = snippets.transactional_get_or_create(client) + client.entities_to_delete.append(task) + assert task + + def transactional_single_entity_group_read_only(self, client): + task_list, tasks_in_list = \ + snippets.transactional_single_entity_group_read_only(client) + client.entities_to_delete.append(task_list) + client.entities_to_delete.extend(tasks_in_list) + assert task_list + assert tasks_in_list + + @eventually_consistent.mark + def test_namespace_run_query(self, client): + all_namespaces, filtered_namespaces = snippets.namespace_run_query( + client) + assert all_namespaces + assert filtered_namespaces + assert 'google' in filtered_namespaces + + @eventually_consistent.mark + def test_kind_run_query(self, client): + kinds = snippets.kind_run_query(client) + client.entities_to_delete.extend( + client.query(kind='Task').fetch()) + assert kinds + assert 'Task' in kinds + + @eventually_consistent.mark + def test_property_run_query(self, client): + kinds = snippets.property_run_query(client) + client.entities_to_delete.extend( + client.query(kind='Task').fetch()) + assert kinds + assert 'Task' in kinds + + @eventually_consistent.mark + def test_property_by_kind_run_query(self, client): + reprs = snippets.property_by_kind_run_query(client) + client.entities_to_delete.extend( + client.query(kind='Task').fetch()) + assert reprs diff --git a/datastore/api/tasks.py b/datastore/cloud-client/tasks.py similarity index 90% rename from datastore/api/tasks.py rename to datastore/cloud-client/tasks.py index 72d8fc2bf3b..cf801037ce4 100644 --- a/datastore/api/tasks.py +++ b/datastore/cloud-client/tasks.py @@ -14,16 +14,16 @@ import argparse import datetime -# [START build_service] -from gcloud import datastore +# [START datastore_build_service] +from google.cloud import datastore def create_client(project_id): return datastore.Client(project_id) -# [END build_service] +# [END datastore_build_service] -# [START add_entity] +# [START datastore_add_entity] def add_task(client, description): key = client.key('Task') @@ -39,10 +39,10 @@ def add_task(client, description): client.put(task) return task.key -# [END add_entity] +# [END datastore_add_entity] -# [START update_entity] +# [START datastore_update_entity] def mark_done(client, task_id): with client.transaction(): key = client.key('Task', task_id) @@ -55,26 +55,25 @@ def mark_done(client, task_id): task['done'] = True client.put(task) -# [END update_entity] +# [END datastore_update_entity] -# [START retrieve_entities] +# [START datastore_retrieve_entities] def list_tasks(client): query = client.query(kind='Task') query.order = ['created'] return list(query.fetch()) -# [END retrieve_entities] +# [END datastore_retrieve_entities] -# [START delete_entity] +# [START datastore_delete_entity] def delete_task(client, task_id): key = client.key('Task', task_id) client.delete(key) -# [END delete_entity] +# [END datastore_delete_entity] -# [START format_results] def format_tasks(tasks): lines = [] for task in tasks: @@ -87,7 +86,6 @@ def format_tasks(tasks): task.key.id, task['description'], status)) return '\n'.join(lines) -# [END format_results] def new_command(client, args): diff --git a/datastore/cloud-client/tasks_test.py b/datastore/cloud-client/tasks_test.py new file mode 100644 index 00000000000..e8fa3629178 --- /dev/null +++ b/datastore/cloud-client/tasks_test.py @@ -0,0 +1,91 @@ +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from gcp_devrel.testing import eventually_consistent +from gcp_devrel.testing.flaky import flaky +from google.cloud import datastore +import pytest + +import tasks + +PROJECT = os.environ['GCLOUD_PROJECT'] + + +@pytest.yield_fixture +def client(): + client = datastore.Client(PROJECT) + + yield client + + # Delete anything created during the test. + with client.batch(): + client.delete_multi( + [x.key for x in client.query(kind='Task').fetch()]) + + +@flaky +def test_create_client(): + tasks.create_client(PROJECT) + + +@flaky +def test_add_task(client): + task_key = tasks.add_task(client, 'Test task') + task = client.get(task_key) + assert task + assert task['description'] == 'Test task' + + +@flaky +def test_mark_done(client): + task_key = tasks.add_task(client, 'Test task') + tasks.mark_done(client, task_key.id) + task = client.get(task_key) + assert task + assert task['done'] + + +@flaky +def test_list_tasks(client): + task1_key = tasks.add_task(client, 'Test task 1') + task2_key = tasks.add_task(client, 'Test task 2') + + @eventually_consistent.call + def _(): + task_list = tasks.list_tasks(client) + assert [x.key for x in task_list] == [task1_key, task2_key] + + +@flaky +def test_delete_task(client): + task_key = tasks.add_task(client, 'Test task 1') + tasks.delete_task(client, task_key.id) + assert client.get(task_key) is None + + +@flaky +def test_format_tasks(client): + task1_key = tasks.add_task(client, 'Test task 1') + tasks.add_task(client, 'Test task 2') + tasks.mark_done(client, task1_key.id) + + @eventually_consistent.call + def _(): + output = tasks.format_tasks(tasks.list_tasks(client)) + + assert 'Test task 1' in output + assert 'Test task 2' in output + assert 'done' in output + assert 'created' in output diff --git a/dialogflow/cloud-client/README.rst b/dialogflow/cloud-client/README.rst new file mode 100644 index 00000000000..bcbe73777cf --- /dev/null +++ b/dialogflow/cloud-client/README.rst @@ -0,0 +1,632 @@ +.. This file is automatically generated. Do not edit this file directly. + +Dialogflow Enterprise Edition API Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=/README.rst + + +This directory contains samples for Dialogflow Enterprise Edition API. The `Dialogflow Enterprise Edition API`_ enables you to create conversational experiences across devices and platforms. + + + + +.. _Dialogflow Enterprise Edition API: https://cloud.google.com/dialogflow-enterprise/docs/ + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone the repository and change directory to the sample directory. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Detect Intent Text ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=/detect_intent_texts.py,/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python detect_intent_texts.py + + usage: detect_intent_texts.py [-h] --project-id PROJECT_ID + [--session-id SESSION_ID] + [--language-code LANGUAGE_CODE] + texts [texts ...] + + DialogFlow API Detect Intent Python sample with text inputs. + + Examples: + python detect_intent_texts.py -h + python detect_intent_texts.py --project-id PROJECT_ID --session-id SESSION_ID "hello" "book a meeting room" "Mountain View" + python detect_intent_texts.py --project-id PROJECT_ID --session-id SESSION_ID "tomorrow" "10 AM" "2 hours" "10 people" "A" "yes" + + positional arguments: + texts Text inputs. + + optional arguments: + -h, --help show this help message and exit + --project-id PROJECT_ID + Project/agent id. Required. + --session-id SESSION_ID + Identifier of the DetectIntent session. Defaults to a + random UUID. + --language-code LANGUAGE_CODE + Language code of the query. Defaults to "en-US". + + + +Detect Intent Audio ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=/detect_intent_audio.py,/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python detect_intent_audio.py + + usage: detect_intent_audio.py [-h] --project-id PROJECT_ID + [--session-id SESSION_ID] + [--language-code LANGUAGE_CODE] + --audio-file-path AUDIO_FILE_PATH + + DialogFlow API Detect Intent Python sample with audio file. + + Examples: + python detect_intent_audio.py -h + python detect_intent_audio.py --project-id PROJECT_ID --session-id SESSION_ID --audio-file-path resources/book_a_room.wav + python detect_intent_audio.py --project-id PROJECT_ID --session-id SESSION_ID --audio-file-path resources/mountain_view.wav + python detect_intent_audio.py --project-id PROJECT_ID --session-id SESSION_ID --audio-file-path resources/today.wav + + optional arguments: + -h, --help show this help message and exit + --project-id PROJECT_ID + Project/agent id. Required. + --session-id SESSION_ID + Identifier of the DetectIntent session. Defaults to a + random UUID. + --language-code LANGUAGE_CODE + Language code of the query. Defaults to "en-US". + --audio-file-path AUDIO_FILE_PATH + Path to the audio file. + + + +Detect Intent Stream ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=/detect_intent_stream.py,/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python detect_intent_stream.py + + usage: detect_intent_stream.py [-h] --project-id PROJECT_ID + [--session-id SESSION_ID] + [--language-code LANGUAGE_CODE] + --audio-file-path AUDIO_FILE_PATH + + DialogFlow API Detect Intent Python sample with audio files processed + as an audio stream. + + Examples: + python detect_intent_stream.py -h + python detect_intent_stream.py --project-id PROJECT_ID --session-id SESSION_ID --audio-file-path resources/book_a_room.wav + python detect_intent_stream.py --project-id PROJECT_ID --session-id SESSION_ID --audio-file-path resources/mountain_view.wav + + optional arguments: + -h, --help show this help message and exit + --project-id PROJECT_ID + Project/agent id. Required. + --session-id SESSION_ID + Identifier of the DetectIntent session. Defaults to a + random UUID. + --language-code LANGUAGE_CODE + Language code of the query. Defaults to "en-US". + --audio-file-path AUDIO_FILE_PATH + Path to the audio file. + + + +Detect Intent Knowledge Base ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=/detect_intent_knowledge.py,/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python detect_intent_knowledge.py + + usage: detect_intent_knowledge.py [-h] --project-id PROJECT_ID + [--session-id SESSION_ID] + [--language-code LANGUAGE_CODE] + --knowledge-base-id KNOWLEDGE_ID + texts [texts ...] + + Dialogflow API Detect Intent Python sample with text inputs. + + Examples: + python detect_intent_knowledge.py -h + python detect_intent_knowledge.py --project-id PROJECT_ID --session-id SESSION_ID --knowledge-base-id KNOWLEDGE_ID "hello" "how do I reset my password?" + + positional arguments: + texts Text inputs. + + optional arguments: + -h, --help show this help message and exit + --project-id PROJECT_ID + Project/agent id. Required. + --session-id SESSION_ID + ID of the DetectIntent session. Defaults to a random + UUID. + --language-code LANGUAGE_CODE + Language code of the query. Defaults to "en-US". + --knowledge-base-id KNOWLEDGE_ID + The id of the Knowledge Base to query against, e.g., OTE5NjYzMTkxNDA2NzI2MzQ4OA + + + +Detect Intent with Model Selection ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=/detect_intent_with_model_selection.py,/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python detect_intent_with_model_selection.py + + usage: detect_intent_with_model_selection.py [-h] --project-id PROJECT_ID + [--session-id SESSION_ID] + [--language-code LANGUAGE_CODE] + --audio-file-path AUDIO_FILE_PATH + + Dialogflow API Beta Detect Intent Python sample with model selection. + + Examples: + python detect_intent_with_model_selection.py -h + python detect_intent_with_model_selection.py --project-id PROJECT_ID --session-id SESSION_ID --audio-file-path resources/book_a_room.wav + + optional arguments: + -h, --help show this help message and exit + --project-id PROJECT_ID + Project/agent id. Required. + --session-id SESSION_ID + Identifier of the DetectIntent session. Defaults to a + random UUID. + --language-code LANGUAGE_CODE + Language code of the query. Defaults to "en-US". + --audio-file-path AUDIO_FILE_PATH + Path to the audio file. + + + +Detect Intent with Sentiment Analysis ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=/detect_intent_with_sentiment_analysis.py,/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python detect_intent_with_sentiment_analysis.py + + usage: detect_intent_with_sentiment_analysis.py [-h] --project-id PROJECT_ID + [--session-id SESSION_ID] + [--language-code LANGUAGE_CODE] + texts [texts ...] + + Dialogflow API Beta Detect Intent Python sample with sentiment analysis. + + Examples: + python detect_intent_with_sentiment_analysis.py -h + python detect_intent_with_sentiment_analysis.py --project-id PROJECT_ID --session-id SESSION_ID "hello" "book a meeting room" "Mountain View" + + positional arguments: + texts Text inputs. + + optional arguments: + -h, --help show this help message and exit + --project-id PROJECT_ID + Project/agent id. Required. + --session-id SESSION_ID + Identifier of the DetectIntent session. Defaults to a + random UUID. + --language-code LANGUAGE_CODE + Language code of the query. Defaults to "en-US". + + + +Detect Intent with Text to Speech Response ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=/detect_intent_with_texttospeech_response.py,/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python detect_intent_with_texttospeech_response.py + + usage: detect_intent_with_texttospeech_response.py [-h] --project-id + PROJECT_ID + [--session-id SESSION_ID] + [--language-code LANGUAGE_CODE] + texts [texts ...] + + Dialogflow API Beta Detect Intent Python sample with an audio response. + + Examples: + python detect_intent_with_texttospeech_response.py -h + python detect_intent_with_texttospeech_response.py --project-id PROJECT_ID --session-id SESSION_ID "hello" + + positional arguments: + texts Text inputs. + + optional arguments: + -h, --help show this help message and exit + --project-id PROJECT_ID + Project/agent id. Required. + --session-id SESSION_ID + Identifier of the DetectIntent session. Defaults to a + random UUID. + --language-code LANGUAGE_CODE + Language code of the query. Defaults to "en-US". + + + +Intent Management ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=/intent_management.py,/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python intent_management.py + + usage: intent_management.py [-h] --project-id PROJECT_ID + {list,create,delete} ... + + DialogFlow API Intent Python sample showing how to manage intents. + + Examples: + python intent_management.py -h + python intent_management.py --project-id PROJECT_ID list + python intent_management.py --project-id PROJECT_ID create "room.cancellation - yes" --training-phrases-parts "cancel" "cancellation" --message-texts "Are you sure you want to cancel?" "Cancelled." + python intent_management.py --project-id PROJECT_ID delete 74892d81-7901-496a-bb0a-c769eda5180e + + positional arguments: + {list,create,delete} + list + create Create an intent of the given intent type. + delete Delete intent with the given intent type and intent + value. + + optional arguments: + -h, --help show this help message and exit + --project-id PROJECT_ID + Project/agent id. Required. + + + +Entity Type Management ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=/entity_type_management.py,/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python entity_type_management.py + + usage: entity_type_management.py [-h] --project-id PROJECT_ID + {list,create,delete} ... + + DialogFlow API EntityType Python sample showing how to manage entity types. + + Examples: + python entity_type_management.py -h + python entity_type_management.py --project-id PROJECT_ID list + python entity_type_management.py --project-id PROJECT_ID create employee + python entity_type_management.py --project-id PROJECT_ID delete e57238e2-e692-44ea-9216-6be1b2332e2a + + positional arguments: + {list,create,delete} + list + create Create an entity type with the given display name. + delete Delete entity type with the given entity type name. + + optional arguments: + -h, --help show this help message and exit + --project-id PROJECT_ID + Project/agent id. Required. + + + +Entity Management ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=/entity_management.py,/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python entity_management.py + + usage: entity_management.py [-h] --project-id PROJECT_ID + {list,create,delete} ... + + DialogFlow API Entity Python sample showing how to manage entities. + + Examples: + python entity_management.py -h + python entity_management.py --project-id PROJECT_ID list --entity-type-id e57238e2-e692-44ea-9216-6be1b2332e2a + python entity_management.py --project-id PROJECT_ID create new_room --synonyms basement cellar --entity-type-id e57238e2-e692-44ea-9216-6be1b2332e2a + python entity_management.py --project-id PROJECT_ID delete new_room --entity-type-id e57238e2-e692-44ea-9216-6be1b2332e2a + + positional arguments: + {list,create,delete} + list + create Create an entity of the given entity type. + delete Delete entity with the given entity type and entity + value. + + optional arguments: + -h, --help show this help message and exit + --project-id PROJECT_ID + Project/agent id. Required. + + + +Session Entity Type Management ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=/session_entity_type_management.py,/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python session_entity_type_management.py + + usage: session_entity_type_management.py [-h] --project-id PROJECT_ID + {list,create,delete} ... + + DialogFlow API SessionEntityType Python sample showing how to manage + session entity types. + + Examples: + python session_entity_type_management.py -h + python session_entity_type_management.py --project-id PROJECT_ID list --session-id SESSION_ID + python session_entity_type_management.py --project-id PROJECT_ID create --session-id SESSION_ID --entity-type-display-name room --entity-values C D E F + python session_entity_type_management.py --project-id PROJECT_ID delete --session-id SESSION_ID --entity-type-display-name room + + positional arguments: + {list,create,delete} + list + create Create a session entity type with the given display + name. + delete Delete session entity type with the given entity type + display name. + + optional arguments: + -h, --help show this help message and exit + --project-id PROJECT_ID + Project/agent id. Required. + + + +Knowledge Base Management ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=/knowledge_base_management.py,/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python knowledge_base_management.py + + usage: knowledge_base_management.py [-h] --project-id PROJECT_ID + {list,create,get,delete} ... + + Dialogflow API Python sample showing how to manage Knowledge bases. + + Examples: + python knowledge_base_management.py -h + python knowledge_base_management.py --project-id PROJECT_ID list + python knowledge_base_management.py --project-id PROJECT_ID create --display-name DISPLAY_NAME + python knowledge_base_management.py --project-id PROJECT_ID get --knowledge-base-id knowledge_base_id + python knowledge_base_management.py --project-id PROJECT_ID delete --knowledge-base-id knowledge_base_id + + positional arguments: + {list,create,get,delete} + list List all Knowledge bases that belong to the project. + create Create a new Knowledge base. + get Get a Knowledge base by its id. + delete Delete a Knowledge base by its id. + + optional arguments: + -h, --help show this help message and exit + --project-id PROJECT_ID + Project/agent id. + + + +Document Management ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=/document_management.py,/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python document_management.py + + usage: document_management.py [-h] --project-id PROJECT_ID --knowledge-base-id + KNOWLEDGE_BASE_ID + {list,create,get,delete} ... + + Dialogflow API Python sample showing how to manage Knowledge Documents. + + Examples: + python document_management.py -h + python document_management.py --project-id PROJECT_ID --knowledge-base-id knowledge_base_id list + python document_management.py --project-id PROJECT_ID --knowledge-base-id knowledge_base_id create --display-name DISPLAY_NAME --mime-type MIME_TYPE --knowledge-type KNOWLEDGE_TYPE --content-uri CONTENT_URI + python document_management.py --project-id PROJECT_ID --knowledge-base-id knowledge_base_id get --document-id DOCUMENT_ID + python document_management.py --project-id PROJECT_ID --knowledge-base-id knowledge_base_id delete --document-id DOCUMENT_ID + + positional arguments: + {list,create,get,delete} + list List all Documents that belong to a certain Knowledge + base. + create Create a Document for a certain Knowledge base. Please note that it will be initially disabled until you enable it. + get Get a Document by its id and the Knowledge base id. + delete Delete a Document by its id and the Knowledge baseid. + + optional arguments: + -h, --help show this help message and exit + --project-id PROJECT_ID + Project id. Required. + --knowledge-base-id KNOWLEDGE_BASE_ID + The id of the Knowledge Base that the Document belongs + to, e.g., OTE5NjYzMTkxNDA2NzI2MzQ4OA + --mime_type The mime_type of the Document. e.g. text/csv, text/html, + text/plain, text/pdf etc. + + --knowledge_type The Knowledge type of the Document. e.g. FAQ, EXTRACTIVE_QA. + + --content_uri Uri of the document, e.g. gs://path/mydoc.csv, + http://mypage.com/faq.html. + + + + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ diff --git a/dialogflow/cloud-client/README.rst.in b/dialogflow/cloud-client/README.rst.in new file mode 100644 index 00000000000..250055c43d6 --- /dev/null +++ b/dialogflow/cloud-client/README.rst.in @@ -0,0 +1,55 @@ +# This file is used to generate README.rst + +product: + name: Dialogflow Enterprise Edition API + short_name: Dialogflow API + url: https://cloud.google.com/dialogflow-enterprise/docs/ + description: > + The `Dialogflow Enterprise Edition API`_ enables you to create conversational experiences across devices and platforms. + +setup: +- auth +- install_deps + +samples: +- name: Detect Intent Text + file: detect_intent_texts.py + show_help: True +- name: Detect Intent Audio + file: detect_intent_audio.py + show_help: True +- name: Detect Intent Stream + file: detect_intent_stream.py + show_help: True +- name: Detect Intent Knowledge Base + file: detect_intent_knowledge.py + show_help: True +- name: Detect Intent with Model Selection + file: detect_intent_with_model_selection.py + show_help: True +- name: Detect Intent with Sentiment Analysis + file: detect_intent_with_sentiment_analysis.py + show_help: True +- name: Detect Intent with Text to Speech Response + file: detect_intent_with_texttospeech_response.py + show_help: True +- name: Intent Management + file: intent_management.py + show_help: True +- name: Entity Type Management + file: entity_type_management.py + show_help: True +- name: Entity Management + file: entity_management.py + show_help: True +- name: Session Entity Type Management + file: session_entity_type_management.py + show_help: True +- name: Knowledge Base Management + file: knowledge_base_management.py + show_help: True +- name: Document Management + file: document_management.py + show_help: True + +cloud_client_library: true diff --git a/dialogflow/cloud-client/context_management.py b/dialogflow/cloud-client/context_management.py new file mode 100644 index 00000000000..d2a75bd711a --- /dev/null +++ b/dialogflow/cloud-client/context_management.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python + +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DialogFlow API Context Python sample showing how to manage session +contexts. + +Examples: + python context_management.py -h + python context_management.py --project-id PROJECT_ID \ + list --session-id SESSION_ID + python context_management.py --project-id PROJECT_ID \ + create --session-id SESSION_ID --context-id CONTEXT_ID + python context_management.py --project-id PROJECT_ID \ + delete --session-id SESSION_ID --context-id CONTEXT_ID +""" + +import argparse + + +# [START dialogflow_list_contexts] +def list_contexts(project_id, session_id): + import dialogflow_v2 as dialogflow + contexts_client = dialogflow.ContextsClient() + + session_path = contexts_client.session_path(project_id, session_id) + + contexts = contexts_client.list_contexts(session_path) + + print('Contexts for session {}:\n'.format(session_path)) + for context in contexts: + print('Context name: {}'.format(context.name)) + print('Lifespan count: {}'.format(context.lifespan_count)) + print('Fields:') + for field, value in context.parameters.fields.items(): + if value.string_value: + print('\t{}: {}'.format(field, value)) +# [END dialogflow_list_contexts] + + +# [START dialogflow_create_context] +def create_context(project_id, session_id, context_id, lifespan_count): + import dialogflow_v2 as dialogflow + contexts_client = dialogflow.ContextsClient() + + session_path = contexts_client.session_path(project_id, session_id) + context_name = contexts_client.context_path( + project_id, session_id, context_id) + + context = dialogflow.types.Context( + name=context_name, lifespan_count=lifespan_count) + + response = contexts_client.create_context(session_path, context) + + print('Context created: \n{}'.format(response)) +# [END dialogflow_create_context] + + +# [START dialogflow_delete_context] +def delete_context(project_id, session_id, context_id): + import dialogflow_v2 as dialogflow + contexts_client = dialogflow.ContextsClient() + + context_name = contexts_client.context_path( + project_id, session_id, context_id) + + contexts_client.delete_context(context_name) +# [END dialogflow_delete_context] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--project-id', + help='Project/agent id. Required.', + required=True) + + subparsers = parser.add_subparsers(dest='command') + + list_parser = subparsers.add_parser( + 'list', help=list_contexts.__doc__) + list_parser.add_argument( + '--session-id', + required=True) + + create_parser = subparsers.add_parser( + 'create', help=create_context.__doc__) + create_parser.add_argument( + '--session-id', + required=True) + create_parser.add_argument( + '--context-id', + help='The id of the context.', + required=True) + create_parser.add_argument( + '--lifespan-count', + help='The lifespan_count of the context. Defaults to 1.', + default=1) + + delete_parser = subparsers.add_parser( + 'delete', help=delete_context.__doc__) + delete_parser.add_argument( + '--session-id', + required=True) + delete_parser.add_argument( + '--context-id', + help='The id of the context.', + required=True) + + args = parser.parse_args() + + if args.command == 'list': + list_contexts(args.project_id, args.session_id, ) + elif args.command == 'create': + create_context( + args.project_id, args.session_id, args.context_id, + args.lifespan_count) + elif args.command == 'delete': + delete_context(args.project_id, args.session_id, args.context_id) diff --git a/dialogflow/cloud-client/context_management_test.py b/dialogflow/cloud-client/context_management_test.py new file mode 100644 index 00000000000..8460a9c65fb --- /dev/null +++ b/dialogflow/cloud-client/context_management_test.py @@ -0,0 +1,44 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import os + +import context_management +import detect_intent_texts + +PROJECT_ID = os.getenv('GCLOUD_PROJECT') +SESSION_ID = 'fake_session_for_testing' +CONTEXT_ID = 'fake_context_for_testing' + + +def test_create_context(capsys): + # Calling detect intent to create a session. + detect_intent_texts.detect_intent_texts( + PROJECT_ID, SESSION_ID, ['hi'], 'en-US') + + context_management.create_context(PROJECT_ID, SESSION_ID, CONTEXT_ID, 1) + context_management.list_contexts(PROJECT_ID, SESSION_ID) + + out, _ = capsys.readouterr() + assert CONTEXT_ID in out + + +def test_delete_context(capsys): + context_management.delete_context(PROJECT_ID, SESSION_ID, CONTEXT_ID) + context_management.list_contexts(PROJECT_ID, SESSION_ID) + + out, _ = capsys.readouterr() + assert CONTEXT_ID not in out diff --git a/dialogflow/cloud-client/detect_intent_audio.py b/dialogflow/cloud-client/detect_intent_audio.py new file mode 100644 index 00000000000..70fc370edd7 --- /dev/null +++ b/dialogflow/cloud-client/detect_intent_audio.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python + +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DialogFlow API Detect Intent Python sample with audio file. + +Examples: + python detect_intent_audio.py -h + python detect_intent_audio.py --project-id PROJECT_ID \ + --session-id SESSION_ID --audio-file-path resources/book_a_room.wav + python detect_intent_audio.py --project-id PROJECT_ID \ + --session-id SESSION_ID --audio-file-path resources/mountain_view.wav + python detect_intent_audio.py --project-id PROJECT_ID \ + --session-id SESSION_ID --audio-file-path resources/today.wav +""" + +import argparse +import uuid + + +# [START dialogflow_detect_intent_audio] +def detect_intent_audio(project_id, session_id, audio_file_path, + language_code): + """Returns the result of detect intent with an audio file as input. + + Using the same `session_id` between requests allows continuation + of the conversation.""" + import dialogflow_v2 as dialogflow + + session_client = dialogflow.SessionsClient() + + # Note: hard coding audio_encoding and sample_rate_hertz for simplicity. + audio_encoding = dialogflow.enums.AudioEncoding.AUDIO_ENCODING_LINEAR_16 + sample_rate_hertz = 16000 + + session = session_client.session_path(project_id, session_id) + print('Session path: {}\n'.format(session)) + + with open(audio_file_path, 'rb') as audio_file: + input_audio = audio_file.read() + + audio_config = dialogflow.types.InputAudioConfig( + audio_encoding=audio_encoding, language_code=language_code, + sample_rate_hertz=sample_rate_hertz) + query_input = dialogflow.types.QueryInput(audio_config=audio_config) + + response = session_client.detect_intent( + session=session, query_input=query_input, + input_audio=input_audio) + + print('=' * 20) + print('Query text: {}'.format(response.query_result.query_text)) + print('Detected intent: {} (confidence: {})\n'.format( + response.query_result.intent.display_name, + response.query_result.intent_detection_confidence)) + print('Fulfillment text: {}\n'.format( + response.query_result.fulfillment_text)) +# [END dialogflow_detect_intent_audio] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--project-id', + help='Project/agent id. Required.', + required=True) + parser.add_argument( + '--session-id', + help='Identifier of the DetectIntent session. ' + 'Defaults to a random UUID.', + default=str(uuid.uuid4())) + parser.add_argument( + '--language-code', + help='Language code of the query. Defaults to "en-US".', + default='en-US') + parser.add_argument( + '--audio-file-path', + help='Path to the audio file.', + required=True) + + args = parser.parse_args() + + detect_intent_audio( + args.project_id, args.session_id, args.audio_file_path, + args.language_code) diff --git a/dialogflow/cloud-client/detect_intent_audio_test.py b/dialogflow/cloud-client/detect_intent_audio_test.py new file mode 100644 index 00000000000..5f67f6ef149 --- /dev/null +++ b/dialogflow/cloud-client/detect_intent_audio_test.py @@ -0,0 +1,35 @@ +# Copyright 2017, Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import os + +from detect_intent_audio import detect_intent_audio + +DIRNAME = os.path.realpath(os.path.dirname(__file__)) +PROJECT_ID = os.getenv('GCLOUD_PROJECT') +SESSION_ID = 'fake_session_for_testing' +AUDIOS = [ + '{0}/resources/book_a_room.wav'.format(DIRNAME), + '{0}/resources/mountain_view.wav'.format(DIRNAME), + '{0}/resources/today.wav'.format(DIRNAME), +] + + +def test_detect_intent_audio(capsys): + for audio_file_path in AUDIOS: + detect_intent_audio(PROJECT_ID, SESSION_ID, audio_file_path, 'en-US') + out, _ = capsys.readouterr() + + assert 'Fulfillment text: What time will the meeting start?' in out diff --git a/dialogflow/cloud-client/detect_intent_knowledge.py b/dialogflow/cloud-client/detect_intent_knowledge.py new file mode 100644 index 00000000000..edfd4b517ec --- /dev/null +++ b/dialogflow/cloud-client/detect_intent_knowledge.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Dialogflow API Detect Knowledge Base Intent Python sample with text inputs. + +Examples: + python detect_intent_knowledge.py -h + python detect_intent_knowledge.py --project-id PROJECT_ID \ + --session-id SESSION_ID --knowledge-base-id KNOWLEDGE_BASE_ID \ + "hello" "how do I reset my password?" +""" + +import argparse +import uuid + + +# [START dialogflow_detect_intent_knowledge] +def detect_intent_knowledge(project_id, session_id, language_code, + knowledge_base_id, texts): + """Returns the result of detect intent with querying Knowledge Connector. + + Args: + project_id: The GCP project linked with the agent you are going to query. + session_id: Id of the session, using the same `session_id` between requests + allows continuation of the conversation. + language_code: Language of the queries. + knowledge_base_id: The Knowledge base's id to query against. + texts: A list of text queries to send. + """ + import dialogflow_v2beta1 as dialogflow + session_client = dialogflow.SessionsClient() + + session_path = session_client.session_path(project_id, session_id) + print('Session path: {}\n'.format(session_path)) + + for text in texts: + text_input = dialogflow.types.TextInput( + text=text, language_code=language_code) + + query_input = dialogflow.types.QueryInput(text=text_input) + + knowledge_base_path = dialogflow.knowledge_bases_client \ + .KnowledgeBasesClient \ + .knowledge_base_path(project_id, knowledge_base_id) + + query_params = dialogflow.types.QueryParameters( + knowledge_base_names=[knowledge_base_path]) + + response = session_client.detect_intent( + session=session_path, query_input=query_input, + query_params=query_params) + + print('=' * 20) + print('Query text: {}'.format(response.query_result.query_text)) + print('Detected intent: {} (confidence: {})\n'.format( + response.query_result.intent.display_name, + response.query_result.intent_detection_confidence)) + print('Fulfillment text: {}\n'.format( + response.query_result.fulfillment_text)) + print('Knowledge results:') + knowledge_answers = response.query_result.knowledge_answers + for answers in knowledge_answers.answers: + print(' - Answer: {}'.format(answers.answer)) + print(' - Confidence: {}'.format( + answers.match_confidence)) +# [END dialogflow_detect_intent_knowledge] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--project-id', help='Project/agent id. Required.', required=True) + parser.add_argument( + '--session-id', + help='ID of the DetectIntent session. ' + 'Defaults to a random UUID.', + default=str(uuid.uuid4())) + parser.add_argument( + '--language-code', + help='Language code of the query. Defaults to "en-US".', + default='en-US') + parser.add_argument( + '--knowledge-base-id', + help='The id of the Knowledge Base to query against', + required=True) + parser.add_argument('texts', nargs='+', type=str, help='Text inputs.') + + args = parser.parse_args() + + detect_intent_knowledge(args.project_id, args.session_id, + args.language_code, args.knowledge_base_id, + args.texts) diff --git a/dialogflow/cloud-client/detect_intent_stream.py b/dialogflow/cloud-client/detect_intent_stream.py new file mode 100644 index 00000000000..535358fd9a2 --- /dev/null +++ b/dialogflow/cloud-client/detect_intent_stream.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python + +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DialogFlow API Detect Intent Python sample with audio files processed +as an audio stream. + +Examples: + python detect_intent_stream.py -h + python detect_intent_stream.py --project-id PROJECT_ID \ + --session-id SESSION_ID --audio-file-path resources/book_a_room.wav + python detect_intent_stream.py --project-id PROJECT_ID \ + --session-id SESSION_ID --audio-file-path resources/mountain_view.wav +""" + +import argparse +import uuid + + +# [START dialogflow_detect_intent_streaming] +def detect_intent_stream(project_id, session_id, audio_file_path, + language_code): + """Returns the result of detect intent with streaming audio as input. + + Using the same `session_id` between requests allows continuation + of the conversation.""" + import dialogflow_v2 as dialogflow + session_client = dialogflow.SessionsClient() + + # Note: hard coding audio_encoding and sample_rate_hertz for simplicity. + audio_encoding = dialogflow.enums.AudioEncoding.AUDIO_ENCODING_LINEAR_16 + sample_rate_hertz = 16000 + + session_path = session_client.session_path(project_id, session_id) + print('Session path: {}\n'.format(session_path)) + + def request_generator(audio_config, audio_file_path): + query_input = dialogflow.types.QueryInput(audio_config=audio_config) + + # The first request contains the configuration. + yield dialogflow.types.StreamingDetectIntentRequest( + session=session_path, query_input=query_input) + + # Here we are reading small chunks of audio data from a local + # audio file. In practice these chunks should come from + # an audio input device. + with open(audio_file_path, 'rb') as audio_file: + while True: + chunk = audio_file.read(4096) + if not chunk: + break + # The later requests contains audio data. + yield dialogflow.types.StreamingDetectIntentRequest( + input_audio=chunk) + + audio_config = dialogflow.types.InputAudioConfig( + audio_encoding=audio_encoding, language_code=language_code, + sample_rate_hertz=sample_rate_hertz) + + requests = request_generator(audio_config, audio_file_path) + responses = session_client.streaming_detect_intent(requests) + + print('=' * 20) + for response in responses: + print('Intermediate transcript: "{}".'.format( + response.recognition_result.transcript)) + + # Note: The result from the last response is the final transcript along + # with the detected content. + query_result = response.query_result + + print('=' * 20) + print('Query text: {}'.format(query_result.query_text)) + print('Detected intent: {} (confidence: {})\n'.format( + query_result.intent.display_name, + query_result.intent_detection_confidence)) + print('Fulfillment text: {}\n'.format( + query_result.fulfillment_text)) +# [END dialogflow_detect_intent_streaming] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--project-id', + help='Project/agent id. Required.', + required=True) + parser.add_argument( + '--session-id', + help='Identifier of the DetectIntent session. ' + 'Defaults to a random UUID.', + default=str(uuid.uuid4())) + parser.add_argument( + '--language-code', + help='Language code of the query. Defaults to "en-US".', + default='en-US') + parser.add_argument( + '--audio-file-path', + help='Path to the audio file.', + required=True) + + args = parser.parse_args() + + detect_intent_stream( + args.project_id, args.session_id, args.audio_file_path, + args.language_code) diff --git a/dialogflow/cloud-client/detect_intent_stream_test.py b/dialogflow/cloud-client/detect_intent_stream_test.py new file mode 100644 index 00000000000..f83ab07b5e6 --- /dev/null +++ b/dialogflow/cloud-client/detect_intent_stream_test.py @@ -0,0 +1,32 @@ +# Copyright 2017, Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import os + +from detect_intent_stream import detect_intent_stream + +PROJECT_ID = os.getenv('GCLOUD_PROJECT') +SESSION_ID = 'fake_session_for_testing' +AUDIO_FILE_PATH = '{0}/resources/book_a_room.wav'.format( + os.path.realpath(os.path.dirname(__file__)), +) + + +def test_detect_intent_stream(capsys): + detect_intent_stream(PROJECT_ID, SESSION_ID, AUDIO_FILE_PATH, 'en-US') + out, _ = capsys.readouterr() + + assert 'Intermediate transcript: "book"' in out + assert 'Fulfillment text: What time will the meeting start?' in out diff --git a/dialogflow/cloud-client/detect_intent_texts.py b/dialogflow/cloud-client/detect_intent_texts.py new file mode 100644 index 00000000000..fba07904ecb --- /dev/null +++ b/dialogflow/cloud-client/detect_intent_texts.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python + +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DialogFlow API Detect Intent Python sample with text inputs. + +Examples: + python detect_intent_texts.py -h + python detect_intent_texts.py --project-id PROJECT_ID \ + --session-id SESSION_ID \ + "hello" "book a meeting room" "Mountain View" + python detect_intent_texts.py --project-id PROJECT_ID \ + --session-id SESSION_ID \ + "tomorrow" "10 AM" "2 hours" "10 people" "A" "yes" +""" + +import argparse +import uuid + + +# [START dialogflow_detect_intent_text] +def detect_intent_texts(project_id, session_id, texts, language_code): + """Returns the result of detect intent with texts as inputs. + + Using the same `session_id` between requests allows continuation + of the conversation.""" + import dialogflow_v2 as dialogflow + session_client = dialogflow.SessionsClient() + + session = session_client.session_path(project_id, session_id) + print('Session path: {}\n'.format(session)) + + for text in texts: + text_input = dialogflow.types.TextInput( + text=text, language_code=language_code) + + query_input = dialogflow.types.QueryInput(text=text_input) + + response = session_client.detect_intent( + session=session, query_input=query_input) + + print('=' * 20) + print('Query text: {}'.format(response.query_result.query_text)) + print('Detected intent: {} (confidence: {})\n'.format( + response.query_result.intent.display_name, + response.query_result.intent_detection_confidence)) + print('Fulfillment text: {}\n'.format( + response.query_result.fulfillment_text)) +# [END dialogflow_detect_intent_text] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--project-id', + help='Project/agent id. Required.', + required=True) + parser.add_argument( + '--session-id', + help='Identifier of the DetectIntent session. ' + 'Defaults to a random UUID.', + default=str(uuid.uuid4())) + parser.add_argument( + '--language-code', + help='Language code of the query. Defaults to "en-US".', + default='en-US') + parser.add_argument( + 'texts', + nargs='+', + type=str, + help='Text inputs.') + + args = parser.parse_args() + + detect_intent_texts( + args.project_id, args.session_id, args.texts, args.language_code) diff --git a/dialogflow/cloud-client/detect_intent_texts_test.py b/dialogflow/cloud-client/detect_intent_texts_test.py new file mode 100644 index 00000000000..1995b9b3d26 --- /dev/null +++ b/dialogflow/cloud-client/detect_intent_texts_test.py @@ -0,0 +1,30 @@ +# Copyright 2017, Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import os + +from detect_intent_texts import detect_intent_texts + +PROJECT_ID = os.getenv('GCLOUD_PROJECT') +SESSION_ID = 'fake_session_for_testing' +TEXTS = ["hello", "book a meeting room", "Mountain View", + "tomorrow", "10 AM", "2 hours", "10 people", "A", "yes"] + + +def test_detect_intent_texts(capsys): + detect_intent_texts(PROJECT_ID, SESSION_ID, TEXTS, 'en-US') + out, _ = capsys.readouterr() + + assert 'Fulfillment text: All set!' in out diff --git a/dialogflow/cloud-client/detect_intent_with_model_selection.py b/dialogflow/cloud-client/detect_intent_with_model_selection.py new file mode 100644 index 00000000000..fc47b8e64d2 --- /dev/null +++ b/dialogflow/cloud-client/detect_intent_with_model_selection.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Dialogflow API Beta Detect Intent Python sample with model selection. + +Examples: + python detect_intent_with_model_selection.py -h + python detect_intent_with_model_selection.py --project-id PROJECT_ID \ + --session-id SESSION_ID --audio-file-path resources/book_a_room.wav +""" + +import argparse +import uuid + + +# [START dialogflow_detect_intent_with_model_selection] +def detect_intent_with_model_selection(project_id, session_id, audio_file_path, + language_code): + """Returns the result of detect intent with model selection on an audio file + as input + + Using the same `session_id` between requests allows continuation + of the conversation.""" + import dialogflow_v2beta1 as dialogflow + session_client = dialogflow.SessionsClient() + + # Note: hard coding audio_encoding and sample_rate_hertz for simplicity. + audio_encoding = dialogflow.enums.AudioEncoding.AUDIO_ENCODING_LINEAR_16 + sample_rate_hertz = 16000 + + session_path = session_client.session_path(project_id, session_id) + print('Session path: {}\n'.format(session_path)) + + with open(audio_file_path, 'rb') as audio_file: + input_audio = audio_file.read() + + # Which Speech model to select for the given request. + # Possible models: video, phone_call, command_and_search, default + model = 'phone_call' + + audio_config = dialogflow.types.InputAudioConfig( + audio_encoding=audio_encoding, language_code=language_code, + sample_rate_hertz=sample_rate_hertz, + model=model) + query_input = dialogflow.types.QueryInput(audio_config=audio_config) + + response = session_client.detect_intent( + session=session_path, query_input=query_input, + input_audio=input_audio) + + print('=' * 20) + print('Query text: {}'.format(response.query_result.query_text)) + print('Detected intent: {} (confidence: {})\n'.format( + response.query_result.intent.display_name, + response.query_result.intent_detection_confidence)) + print('Fulfillment text: {}\n'.format( + response.query_result.fulfillment_text)) +# [END dialogflow_detect_intent_with_model_selection] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--project-id', + help='Project/agent id. Required.', + required=True) + parser.add_argument( + '--session-id', + help='Identifier of the DetectIntent session. ' + 'Defaults to a random UUID.', + default=str(uuid.uuid4())) + parser.add_argument( + '--language-code', + help='Language code of the query. Defaults to "en-US".', + default='en-US') + parser.add_argument( + '--audio-file-path', + help='Path to the audio file.', + required=True) + + args = parser.parse_args() + + detect_intent_with_model_selection( + args.project_id, args.session_id, args.audio_file_path, + args.language_code) diff --git a/dialogflow/cloud-client/detect_intent_with_model_selection_test.py b/dialogflow/cloud-client/detect_intent_with_model_selection_test.py new file mode 100644 index 00000000000..12762289331 --- /dev/null +++ b/dialogflow/cloud-client/detect_intent_with_model_selection_test.py @@ -0,0 +1,37 @@ +# Copyright 2018, Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import os + +from detect_intent_with_model_selection import \ + detect_intent_with_model_selection + +DIRNAME = os.path.realpath(os.path.dirname(__file__)) +PROJECT_ID = os.getenv('GCLOUD_PROJECT') +SESSION_ID = 'fake_session_for_testing' +AUDIOS = [ + '{0}/resources/book_a_room.wav'.format(DIRNAME), + '{0}/resources/mountain_view.wav'.format(DIRNAME), + '{0}/resources/today.wav'.format(DIRNAME), +] + + +def test_detect_intent_audio_with_model_selection(capsys): + for audio_file_path in AUDIOS: + detect_intent_with_model_selection(PROJECT_ID, SESSION_ID, + audio_file_path, 'en-US') + out, _ = capsys.readouterr() + + assert 'Fulfillment text: What time will the meeting start?' in out diff --git a/dialogflow/cloud-client/detect_intent_with_sentiment_analysis.py b/dialogflow/cloud-client/detect_intent_with_sentiment_analysis.py new file mode 100644 index 00000000000..07905b57872 --- /dev/null +++ b/dialogflow/cloud-client/detect_intent_with_sentiment_analysis.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Dialogflow API Beta Detect Intent Python sample with sentiment analysis. + +Examples: + python detect_intent_with_sentiment_analysis.py -h + python detect_intent_with_sentiment_analysis.py --project-id PROJECT_ID \ + --session-id SESSION_ID \ + "hello" "book a meeting room" "Mountain View" +""" + +import argparse +import uuid + + +# [START dialogflow_detect_intent_with_sentiment_analysis] +def detect_intent_with_sentiment_analysis(project_id, session_id, texts, + language_code): + """Returns the result of detect intent with texts as inputs and analyzes the + sentiment of the query text. + + Using the same `session_id` between requests allows continuation + of the conversation.""" + import dialogflow_v2beta1 as dialogflow + session_client = dialogflow.SessionsClient() + + session_path = session_client.session_path(project_id, session_id) + print('Session path: {}\n'.format(session_path)) + + for text in texts: + text_input = dialogflow.types.TextInput( + text=text, language_code=language_code) + + query_input = dialogflow.types.QueryInput(text=text_input) + + # Enable sentiment analysis + sentiment_config = dialogflow.types.SentimentAnalysisRequestConfig( + analyze_query_text_sentiment=True) + + # Set the query parameters with sentiment analysis + query_params = dialogflow.types.QueryParameters( + sentiment_analysis_request_config=sentiment_config) + + response = session_client.detect_intent( + session=session_path, query_input=query_input, + query_params=query_params) + + print('=' * 20) + print('Query text: {}'.format(response.query_result.query_text)) + print('Detected intent: {} (confidence: {})\n'.format( + response.query_result.intent.display_name, + response.query_result.intent_detection_confidence)) + print('Fulfillment text: {}\n'.format( + response.query_result.fulfillment_text)) + # Score between -1.0 (negative sentiment) and 1.0 (positive sentiment). + print('Query Text Sentiment Score: {}\n'.format( + response.query_result.sentiment_analysis_result + .query_text_sentiment.score)) + print('Query Text Sentiment Magnitude: {}\n'.format( + response.query_result.sentiment_analysis_result + .query_text_sentiment.magnitude)) +# [END dialogflow_detect_intent_with_sentiment_analysis] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--project-id', + help='Project/agent id. Required.', + required=True) + parser.add_argument( + '--session-id', + help='Identifier of the DetectIntent session. ' + 'Defaults to a random UUID.', + default=str(uuid.uuid4())) + parser.add_argument( + '--language-code', + help='Language code of the query. Defaults to "en-US".', + default='en-US') + parser.add_argument( + 'texts', + nargs='+', + type=str, + help='Text inputs.') + + args = parser.parse_args() + + detect_intent_with_sentiment_analysis( + args.project_id, args.session_id, args.texts, args.language_code) diff --git a/dialogflow/cloud-client/detect_intent_with_sentiment_analysis_test.py b/dialogflow/cloud-client/detect_intent_with_sentiment_analysis_test.py new file mode 100644 index 00000000000..5e9041b177a --- /dev/null +++ b/dialogflow/cloud-client/detect_intent_with_sentiment_analysis_test.py @@ -0,0 +1,32 @@ +# Copyright 2018, Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import os + +from detect_intent_with_sentiment_analysis import \ + detect_intent_with_sentiment_analysis + +PROJECT_ID = os.getenv('GCLOUD_PROJECT') +SESSION_ID = 'fake_session_for_testing' +TEXTS = ["hello", "book a meeting room", "Mountain View", + "tomorrow", "10 AM", "2 hours", "10 people", "A", "yes"] + + +def test_detect_intent_with_sentiment_analysis(capsys): + detect_intent_with_sentiment_analysis(PROJECT_ID, SESSION_ID, TEXTS, + 'en-US') + out, _ = capsys.readouterr() + + assert 'Query Text Sentiment Score' in out diff --git a/dialogflow/cloud-client/detect_intent_with_texttospeech_response.py b/dialogflow/cloud-client/detect_intent_with_texttospeech_response.py new file mode 100644 index 00000000000..1e47f13208b --- /dev/null +++ b/dialogflow/cloud-client/detect_intent_with_texttospeech_response.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Dialogflow API Beta Detect Intent Python sample with an audio response. + +Examples: + python detect_intent_with_texttospeech_response_test.py -h + python detect_intent_with_texttospeech_response_test.py \ + --project-id PROJECT_ID --session-id SESSION_ID "hello" +""" + +import argparse +import uuid + + +# [START dialogflow_detect_intent_with_texttospeech_response] +def detect_intent_with_texttospeech_response(project_id, session_id, texts, + language_code): + """Returns the result of detect intent with texts as inputs and includes + the response in an audio format. + + Using the same `session_id` between requests allows continuation + of the conversation.""" + import dialogflow_v2beta1 as dialogflow + session_client = dialogflow.SessionsClient() + + session_path = session_client.session_path(project_id, session_id) + print('Session path: {}\n'.format(session_path)) + + for text in texts: + text_input = dialogflow.types.TextInput( + text=text, language_code=language_code) + + query_input = dialogflow.types.QueryInput(text=text_input) + + # Set the query parameters with sentiment analysis + output_audio_config = dialogflow.types.OutputAudioConfig( + audio_encoding=dialogflow.enums.OutputAudioEncoding + .OUTPUT_AUDIO_ENCODING_LINEAR_16) + + response = session_client.detect_intent( + session=session_path, query_input=query_input, + output_audio_config=output_audio_config) + + print('=' * 20) + print('Query text: {}'.format(response.query_result.query_text)) + print('Detected intent: {} (confidence: {})\n'.format( + response.query_result.intent.display_name, + response.query_result.intent_detection_confidence)) + print('Fulfillment text: {}\n'.format( + response.query_result.fulfillment_text)) + # The response's audio_content is binary. + with open('output.wav', 'wb') as out: + out.write(response.output_audio) + print('Audio content written to file "output.wav"') +# [END dialogflow_detect_intent_with_texttospeech_response] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--project-id', + help='Project/agent id. Required.', + required=True) + parser.add_argument( + '--session-id', + help='Identifier of the DetectIntent session. ' + 'Defaults to a random UUID.', + default=str(uuid.uuid4())) + parser.add_argument( + '--language-code', + help='Language code of the query. Defaults to "en-US".', + default='en-US') + parser.add_argument( + 'texts', + nargs='+', + type=str, + help='Text inputs.') + + args = parser.parse_args() + + detect_intent_with_texttospeech_response( + args.project_id, args.session_id, args.texts, args.language_code) diff --git a/dialogflow/cloud-client/detect_intent_with_texttospeech_response_test.py b/dialogflow/cloud-client/detect_intent_with_texttospeech_response_test.py new file mode 100644 index 00000000000..96419b2f229 --- /dev/null +++ b/dialogflow/cloud-client/detect_intent_with_texttospeech_response_test.py @@ -0,0 +1,35 @@ +# Copyright 2018, Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import os + +from detect_intent_with_texttospeech_response import \ + detect_intent_with_texttospeech_response + +PROJECT_ID = os.getenv('GCLOUD_PROJECT') +SESSION_ID = 'fake_session_for_testing' +TEXTS = ["hello"] + + +def test_detect_intent_with_sentiment_analysis(capsys): + detect_intent_with_texttospeech_response(PROJECT_ID, SESSION_ID, TEXTS, + 'en-US') + out, _ = capsys.readouterr() + + assert 'Audio content written to file' in out + statinfo = os.stat('output.wav') + assert statinfo.st_size > 0 + os.remove('output.wav') + assert not os.path.isfile('output.wav') diff --git a/dialogflow/cloud-client/document_management.py b/dialogflow/cloud-client/document_management.py new file mode 100644 index 00000000000..6145c9df8a6 --- /dev/null +++ b/dialogflow/cloud-client/document_management.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Dialogflow API Python sample showing how to manage Knowledge Documents. + +Examples: + python document_management.py -h + python document_management.py --project-id PROJECT_ID \ + --knowledge-base-id knowledge_base_id \ + list + python document_management.py --project-id PROJECT_ID \ + --knowledge-base-id knowledge_base_id \ + create --display-name DISPLAY_NAME --mime-type MIME_TYPE \ + --knowledge-type KNOWLEDGE_TYPE --content-uri CONTENT_URI + python document_management.py --project-id PROJECT_ID \ + --knowledge-base-id knowledge_base_id \ + get --document-id DOCUMENT_ID + python document_management.py --project-id PROJECT_ID \ + --knowledge-base-id knowledge_base_id \ + delete --document-id DOCUMENT_ID +""" + +import argparse + + +KNOWLEDGE_TYPES = ['KNOWLEDGE_TYPE_UNSPECIFIED', 'FAQ', 'EXTRACTIVE_QA'] + + +# [START dialogflow_list_document] +def list_documents(project_id, knowledge_base_id): + """Lists the Documents belonging to a Knowledge base. + + Args: + project_id: The GCP project linked with the agent. + knowledge_base_id: Id of the Knowledge base.""" + import dialogflow_v2beta1 as dialogflow + client = dialogflow.DocumentsClient() + knowledge_base_path = client.knowledge_base_path(project_id, + knowledge_base_id) + + print('Documents for Knowledge Id: {}'.format(knowledge_base_id)) + for document in client.list_documents(knowledge_base_path): + print(' - Display Name: {}'.format(document.display_name)) + print(' - Knowledge ID: {}'.format(document.name)) + print(' - MIME Type: {}'.format(document.mime_type)) + print(' - Knowledge Types:') + for knowledge_type in document.knowledge_types: + print(' - {}'.format(KNOWLEDGE_TYPES[knowledge_type])) + print(' - Source: {}\n'.format(document.content_uri)) +# [END dialogflow_list_document] + + +# [START dialogflow_create_document]] +def create_document(project_id, knowledge_base_id, display_name, mime_type, + knowledge_type, content_uri): + """Creates a Document. + + Args: + project_id: The GCP project linked with the agent. + knowledge_base_id: Id of the Knowledge base. + display_name: The display name of the Document. + mime_type: The mime_type of the Document. e.g. text/csv, text/html, + text/plain, text/pdf etc. + knowledge_type: The Knowledge type of the Document. e.g. FAQ, + EXTRACTIVE_QA. + content_uri: Uri of the document, e.g. gs://path/mydoc.csv, + http://mypage.com/faq.html.""" + import dialogflow_v2beta1 as dialogflow + client = dialogflow.DocumentsClient() + knowledge_base_path = client.knowledge_base_path(project_id, + knowledge_base_id) + + document = dialogflow.types.Document( + display_name=display_name, mime_type=mime_type, + content_uri=content_uri) + + document.knowledge_types.append( + dialogflow.types.Document.KnowledgeType.Value(knowledge_type)) + + response = client.create_document(knowledge_base_path, document) + print('Waiting for results...') + document = response.result(timeout=90) + print('Created Document:') + print(' - Display Name: {}'.format(document.display_name)) + print(' - Knowledge ID: {}'.format(document.name)) + print(' - MIME Type: {}'.format(document.mime_type)) + print(' - Knowledge Types:') + for knowledge_type in document.knowledge_types: + print(' - {}'.format(KNOWLEDGE_TYPES[knowledge_type])) + print(' - Source: {}\n'.format(document.content_uri)) +# [END dialogflow_create_document]] + + +# [START dialogflow_get_document]] +def get_document(project_id, knowledge_base_id, document_id): + """Gets a Document. + + Args: + project_id: The GCP project linked with the agent. + knowledge_base_id: Id of the Knowledge base. + document_id: Id of the Document.""" + import dialogflow_v2beta1 as dialogflow + client = dialogflow.DocumentsClient() + document_path = client.document_path(project_id, knowledge_base_id, + document_id) + + response = client.get_document(document_path) + print('Got Document:') + print(' - Display Name: {}'.format(response.display_name)) + print(' - Knowledge ID: {}'.format(response.name)) + print(' - MIME Type: {}'.format(response.mime_type)) + print(' - Knowledge Types:') + for knowledge_type in response.knowledge_types: + print(' - {}'.format(KNOWLEDGE_TYPES[knowledge_type])) + print(' - Source: {}\n'.format(response.content_uri)) +# [END dialogflow_get_document]] + + +# [START dialogflow_delete_document]] +def delete_document(project_id, knowledge_base_id, document_id): + """Deletes a Document. + + Args: + project_id: The GCP project linked with the agent. + knowledge_base_id: Id of the Knowledge base. + document_id: Id of the Document.""" + import dialogflow_v2beta1 as dialogflow + client = dialogflow.DocumentsClient() + document_path = client.document_path(project_id, knowledge_base_id, + document_id) + + response = client.delete_document(document_path) + print('operation running:\n {}'.format(response.operation)) + print('Waiting for results...') + print('Done.\n {}'.format(response.result())) +# [END dialogflow_delete_document]] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--project-id', help='Project id. Required.', required=True) + parser.add_argument( + '--knowledge-base-id', + help='The id of the Knowledge Base that the Document belongs to', + required=True) + + subparsers = parser.add_subparsers(dest='command') + + list_parser = subparsers.add_parser( + 'list', + help='List all Documents that belong to a certain Knowledge base.') + + create_parser = subparsers.add_parser( + 'create', help='Create a Document for a certain Knowledge base.') + create_parser.add_argument( + '--display-name', + help='A name of the Document, mainly used for display purpose, ' + 'can not be used to identify the Document.', + default=str('')) + create_parser.add_argument( + '--mime-type', + help='The mime-type of the Document, e.g. text/csv, text/html, ' + 'text/plain, text/pdf etc. ', + required=True) + create_parser.add_argument( + '--knowledge-type', + help='The knowledge-type of the Document, e.g. FAQ, EXTRACTIVE_QA.', + required=True) + create_parser.add_argument( + '--content-uri', + help='The uri of the Document, e.g. gs://path/mydoc.csv, ' + 'http://mypage.com/faq.html', + required=True) + + get_parser = subparsers.add_parser( + 'get', help='Get a Document by its id and the Knowledge base id.') + get_parser.add_argument( + '--document-id', help='The id of the Document', required=True) + + delete_parser = subparsers.add_parser( + 'delete', help='Delete a Document by its id and the Knowledge base' + 'id.') + delete_parser.add_argument( + '--document-id', + help='The id of the Document you want to delete', + required=True) + + args = parser.parse_args() + + if args.command == 'list': + list_documents(args.project_id, args.knowledge_base_id) + elif args.command == 'create': + create_document(args.project_id, args.knowledge_base_id, + args.display_name, args.mime_type, args.knowledge_type, + args.content_uri) + elif args.command == 'get': + get_document(args.project_id, args.knowledge_base_id, args.document_id) + elif args.command == 'delete': + delete_document(args.project_id, args.knowledge_base_id, + args.document_id) diff --git a/dialogflow/cloud-client/entity_management.py b/dialogflow/cloud-client/entity_management.py new file mode 100644 index 00000000000..2fff2fe12fe --- /dev/null +++ b/dialogflow/cloud-client/entity_management.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python + +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DialogFlow API Entity Python sample showing how to manage entities. + +Examples: + python entity_management.py -h + python entity_management.py --project-id PROJECT_ID \ + list --entity-type-id e57238e2-e692-44ea-9216-6be1b2332e2a + python entity_management.py --project-id PROJECT_ID \ + create new_room --synonyms basement cellar \ + --entity-type-id e57238e2-e692-44ea-9216-6be1b2332e2a + python entity_management.py --project-id PROJECT_ID \ + delete new_room \ + --entity-type-id e57238e2-e692-44ea-9216-6be1b2332e2a +""" + +import argparse + + +# [START dialogflow_list_entities] +def list_entities(project_id, entity_type_id): + import dialogflow_v2 as dialogflow + entity_types_client = dialogflow.EntityTypesClient() + + parent = entity_types_client.entity_type_path( + project_id, entity_type_id) + + entities = entity_types_client.get_entity_type(parent).entities + + for entity in entities: + print('Entity value: {}'.format(entity.value)) + print('Entity synonyms: {}\n'.format(entity.synonyms)) +# [END dialogflow_list_entities] + + +# [START dialogflow_create_entity] +def create_entity(project_id, entity_type_id, entity_value, synonyms): + """Create an entity of the given entity type.""" + import dialogflow_v2 as dialogflow + entity_types_client = dialogflow.EntityTypesClient() + + # Note: synonyms must be exactly [entity_value] if the + # entity_type's kind is KIND_LIST + synonyms = synonyms or [entity_value] + + entity_type_path = entity_types_client.entity_type_path( + project_id, entity_type_id) + + entity = dialogflow.types.EntityType.Entity() + entity.value = entity_value + entity.synonyms.extend(synonyms) + + response = entity_types_client.batch_create_entities( + entity_type_path, [entity]) + + print('Entity created: {}'.format(response)) +# [END dialogflow_create_entity] + + +# [START dialogflow_delete_entity] +def delete_entity(project_id, entity_type_id, entity_value): + """Delete entity with the given entity type and entity value.""" + import dialogflow_v2 as dialogflow + entity_types_client = dialogflow.EntityTypesClient() + + entity_type_path = entity_types_client.entity_type_path( + project_id, entity_type_id) + + entity_types_client.batch_delete_entities( + entity_type_path, [entity_value]) +# [END dialogflow_delete_entity] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--project-id', + help='Project/agent id. Required.', + required=True) + + subparsers = parser.add_subparsers(dest='command') + + list_parser = subparsers.add_parser( + 'list', help=list_entities.__doc__) + list_parser.add_argument( + '--entity-type-id', + help='The id of the entity_type.') + + create_parser = subparsers.add_parser( + 'create', help=create_entity.__doc__) + create_parser.add_argument( + 'entity_value', + help='The entity value to be added.') + create_parser.add_argument( + '--entity-type-id', + help='The id of the entity_type to which to add an entity.', + required=True) + create_parser.add_argument( + '--synonyms', + nargs='*', + help='The synonyms that will map to the provided entity value.', + default=[]) + + delete_parser = subparsers.add_parser( + 'delete', help=delete_entity.__doc__) + delete_parser.add_argument( + '--entity-type-id', + help='The id of the entity_type.', + required=True) + delete_parser.add_argument( + 'entity_value', + help='The value of the entity to delete.') + + args = parser.parse_args() + + if args.command == 'list': + list_entities(args.project_id, args.entity_type_id) + elif args.command == 'create': + create_entity( + args.project_id, args.entity_type_id, args.entity_value, + args.synonyms) + elif args.command == 'delete': + delete_entity( + args.project_id, args.entity_type_id, args.entity_value) diff --git a/dialogflow/cloud-client/entity_management_test.py b/dialogflow/cloud-client/entity_management_test.py new file mode 100644 index 00000000000..e2433d7a4e9 --- /dev/null +++ b/dialogflow/cloud-client/entity_management_test.py @@ -0,0 +1,90 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import os + +import entity_management +import entity_type_management + +PROJECT_ID = os.getenv('GCLOUD_PROJECT') +ENTITY_TYPE_DISPLAY_NAME = 'fake_entity_type_for_testing' +ENTITY_VALUE_1 = 'fake_entity_for_testing_1' +ENTITY_VALUE_2 = 'fake_entity_for_testing_2' +SYNONYMS = ['fake_synonym_for_testing_1', 'fake_synonym_for_testing_2'] + + +def test_create_entity_type(capsys): + entity_type_ids = entity_type_management._get_entity_type_ids( + PROJECT_ID, ENTITY_TYPE_DISPLAY_NAME) + + assert len(entity_type_ids) == 0 + + entity_type_management.create_entity_type( + PROJECT_ID, ENTITY_TYPE_DISPLAY_NAME, 'KIND_MAP') + out, _ = capsys.readouterr() + + assert 'display_name: "{}"'.format(ENTITY_TYPE_DISPLAY_NAME) in out + + entity_type_ids = entity_type_management._get_entity_type_ids( + PROJECT_ID, ENTITY_TYPE_DISPLAY_NAME) + + assert len(entity_type_ids) == 1 + + +def test_create_entity(capsys): + entity_type_id = entity_type_management._get_entity_type_ids( + PROJECT_ID, ENTITY_TYPE_DISPLAY_NAME)[0] + + entity_management.create_entity( + PROJECT_ID, entity_type_id, ENTITY_VALUE_1, []) + entity_management.create_entity( + PROJECT_ID, entity_type_id, ENTITY_VALUE_2, SYNONYMS) + + entity_management.list_entities(PROJECT_ID, entity_type_id) + out, _ = capsys.readouterr() + + assert 'Entity value: {}'.format(ENTITY_VALUE_1) in out + assert 'Entity value: {}'.format(ENTITY_VALUE_2) in out + for synonym in SYNONYMS: + assert synonym in out + + +def test_delete_entity(capsys): + entity_type_id = entity_type_management._get_entity_type_ids( + PROJECT_ID, ENTITY_TYPE_DISPLAY_NAME)[0] + + entity_management.delete_entity( + PROJECT_ID, entity_type_id, ENTITY_VALUE_1) + entity_management.delete_entity( + PROJECT_ID, entity_type_id, ENTITY_VALUE_2) + + entity_management.list_entities(PROJECT_ID, entity_type_id) + out, _ = capsys.readouterr() + + assert out == '' + + +def test_delete_entity_type(capsys): + entity_type_ids = entity_type_management._get_entity_type_ids( + PROJECT_ID, ENTITY_TYPE_DISPLAY_NAME) + + for entity_type_id in entity_type_ids: + entity_type_management.delete_entity_type(PROJECT_ID, entity_type_id) + + entity_type_ids = entity_type_management._get_entity_type_ids( + PROJECT_ID, ENTITY_TYPE_DISPLAY_NAME) + + assert len(entity_type_ids) == 0 diff --git a/dialogflow/cloud-client/entity_type_management.py b/dialogflow/cloud-client/entity_type_management.py new file mode 100644 index 00000000000..1f342f3a1b9 --- /dev/null +++ b/dialogflow/cloud-client/entity_type_management.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python + +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DialogFlow API EntityType Python sample showing how to manage entity types. + +Examples: + python entity_type_management.py -h + python entity_type_management.py --project-id PROJECT_ID list + python entity_type_management.py --project-id PROJECT_ID create employee + python entity_type_management.py --project-id PROJECT_ID delete \ + e57238e2-e692-44ea-9216-6be1b2332e2a +""" + +import argparse + + +# [START dialogflow_list_entity_types] +def list_entity_types(project_id): + import dialogflow_v2 as dialogflow + entity_types_client = dialogflow.EntityTypesClient() + + parent = entity_types_client.project_agent_path(project_id) + + entity_types = entity_types_client.list_entity_types(parent) + + for entity_type in entity_types: + print('Entity type name: {}'.format(entity_type.name)) + print('Entity type display name: {}'.format(entity_type.display_name)) + print('Number of entities: {}\n'.format(len(entity_type.entities))) +# [END dialogflow_list_entity_types] + + +# [START dialogflow_create_entity_type] +def create_entity_type(project_id, display_name, kind): + """Create an entity type with the given display name.""" + import dialogflow_v2 as dialogflow + entity_types_client = dialogflow.EntityTypesClient() + + parent = entity_types_client.project_agent_path(project_id) + entity_type = dialogflow.types.EntityType( + display_name=display_name, kind=kind) + + response = entity_types_client.create_entity_type(parent, entity_type) + + print('Entity type created: \n{}'.format(response)) +# [END dialogflow_create_entity_type] + + +# [START dialogflow_delete_entity_type] +def delete_entity_type(project_id, entity_type_id): + """Delete entity type with the given entity type name.""" + import dialogflow_v2 as dialogflow + entity_types_client = dialogflow.EntityTypesClient() + + entity_type_path = entity_types_client.entity_type_path( + project_id, entity_type_id) + + entity_types_client.delete_entity_type(entity_type_path) +# [END dialogflow_delete_entity_type] + + +# Helper to get entity_type_id from display name. +def _get_entity_type_ids(project_id, display_name): + import dialogflow_v2 as dialogflow + entity_types_client = dialogflow.EntityTypesClient() + + parent = entity_types_client.project_agent_path(project_id) + entity_types = entity_types_client.list_entity_types(parent) + entity_type_names = [ + entity_type.name for entity_type in entity_types + if entity_type.display_name == display_name] + + entity_type_ids = [ + entity_type_name.split('/')[-1] for entity_type_name + in entity_type_names] + + return entity_type_ids + + +if __name__ == '__main__': + import dialogflow_v2 + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--project-id', + help='Project/agent id. Required.', + required=True) + + subparsers = parser.add_subparsers(dest='command') + + list_parser = subparsers.add_parser( + 'list', help=list_entity_types.__doc__) + + create_parser = subparsers.add_parser( + 'create', help=create_entity_type.__doc__) + create_parser.add_argument( + 'display_name', + help='The display name of the entity.') + create_parser.add_argument( + '--kind', + help='The kind of entity. KIND_MAP (default) or KIND_LIST.', + default=dialogflow_v2.enums.EntityType.Kind.KIND_MAP) + + delete_parser = subparsers.add_parser( + 'delete', help=delete_entity_type.__doc__) + delete_parser.add_argument( + 'entity_type_id', + help='The id of the entity_type.') + + args = parser.parse_args() + + if args.command == 'list': + list_entity_types(args.project_id) + elif args.command == 'create': + create_entity_type(args.project_id, args.display_name, args.kind) + elif args.command == 'delete': + delete_entity_type(args.project_id, args.entity_type_id) diff --git a/dialogflow/cloud-client/intent_management.py b/dialogflow/cloud-client/intent_management.py new file mode 100644 index 00000000000..853e191c1a1 --- /dev/null +++ b/dialogflow/cloud-client/intent_management.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python + +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DialogFlow API Intent Python sample showing how to manage intents. + +Examples: + python intent_management.py -h + python intent_management.py --project-id PROJECT_ID list + python intent_management.py --project-id PROJECT_ID create \ + "room.cancellation - yes" \ + --training-phrases-parts "cancel" "cancellation" \ + --message-texts "Are you sure you want to cancel?" "Cancelled." + python intent_management.py --project-id PROJECT_ID delete \ + 74892d81-7901-496a-bb0a-c769eda5180e +""" + +import argparse + + +# [START dialogflow_list_intents] +def list_intents(project_id): + import dialogflow_v2 as dialogflow + intents_client = dialogflow.IntentsClient() + + parent = intents_client.project_agent_path(project_id) + + intents = intents_client.list_intents(parent) + + for intent in intents: + print('=' * 20) + print('Intent name: {}'.format(intent.name)) + print('Intent display_name: {}'.format(intent.display_name)) + print('Action: {}\n'.format(intent.action)) + print('Root followup intent: {}'.format( + intent.root_followup_intent_name)) + print('Parent followup intent: {}\n'.format( + intent.parent_followup_intent_name)) + + print('Input contexts:') + for input_context_name in intent.input_context_names: + print('\tName: {}'.format(input_context_name)) + + print('Output contexts:') + for output_context in intent.output_contexts: + print('\tName: {}'.format(output_context.name)) +# [END dialogflow_list_intents] + + +# [START dialogflow_create_intent] +def create_intent(project_id, display_name, training_phrases_parts, + message_texts): + """Create an intent of the given intent type.""" + import dialogflow_v2 as dialogflow + intents_client = dialogflow.IntentsClient() + + parent = intents_client.project_agent_path(project_id) + training_phrases = [] + for training_phrases_part in training_phrases_parts: + part = dialogflow.types.Intent.TrainingPhrase.Part( + text=training_phrases_part) + # Here we create a new training phrase for each provided part. + training_phrase = dialogflow.types.Intent.TrainingPhrase(parts=[part]) + training_phrases.append(training_phrase) + + text = dialogflow.types.Intent.Message.Text(text=message_texts) + message = dialogflow.types.Intent.Message(text=text) + + intent = dialogflow.types.Intent( + display_name=display_name, + training_phrases=training_phrases, + messages=[message]) + + response = intents_client.create_intent(parent, intent) + + print('Intent created: {}'.format(response)) +# [END dialogflow_create_intent] + + +# [START dialogflow_delete_intent] +def delete_intent(project_id, intent_id): + """Delete intent with the given intent type and intent value.""" + import dialogflow_v2 as dialogflow + intents_client = dialogflow.IntentsClient() + + intent_path = intents_client.intent_path(project_id, intent_id) + + intents_client.delete_intent(intent_path) +# [END dialogflow_delete_intent] + + +# Helper to get intent from display name. +def _get_intent_ids(project_id, display_name): + import dialogflow_v2 as dialogflow + intents_client = dialogflow.IntentsClient() + + parent = intents_client.project_agent_path(project_id) + intents = intents_client.list_intents(parent) + intent_names = [ + intent.name for intent in intents + if intent.display_name == display_name] + + intent_ids = [ + intent_name.split('/')[-1] for intent_name + in intent_names] + + return intent_ids + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--project-id', + help='Project/agent id. Required.', + required=True) + + subparsers = parser.add_subparsers(dest='command') + + list_parser = subparsers.add_parser( + 'list', help=list_intents.__doc__) + + create_parser = subparsers.add_parser( + 'create', help=create_intent.__doc__) + create_parser.add_argument( + 'display_name') + create_parser.add_argument( + '--training-phrases-parts', + nargs='*', + type=str, + help='Training phrases.', + default=[]) + create_parser.add_argument( + '--message-texts', + nargs='*', + type=str, + help='Message texts for the agent\'s response when the intent ' + 'is detected.', + default=[]) + + delete_parser = subparsers.add_parser( + 'delete', help=delete_intent.__doc__) + delete_parser.add_argument( + 'intent_id', + help='The id of the intent.') + + args = parser.parse_args() + + if args.command == 'list': + list_intents(args.project_id) + elif args.command == 'create': + create_intent( + args.project_id, args.display_name, args.training_phrases_parts, + args.message_texts, ) + elif args.command == 'delete': + delete_intent(args.project_id, args.intent_id) diff --git a/dialogflow/cloud-client/intent_management_test.py b/dialogflow/cloud-client/intent_management_test.py new file mode 100644 index 00000000000..53105fe0002 --- /dev/null +++ b/dialogflow/cloud-client/intent_management_test.py @@ -0,0 +1,68 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import os + +import intent_management + +PROJECT_ID = os.getenv('GCLOUD_PROJECT') +INTENT_DISPLAY_NAME = 'fake_display_name_for_testing' +MESSAGE_TEXTS = [ + 'fake_message_text_for_testing_1', + 'fake_message_text_for_testing_2' +] +TRAINING_PHRASE_PARTS = [ + 'fake_training_phrase_part_1', + 'fake_training_phease_part_2' +] + + +def test_create_intent(capsys): + intent_management.create_intent( + PROJECT_ID, INTENT_DISPLAY_NAME, TRAINING_PHRASE_PARTS, + MESSAGE_TEXTS) + + intent_ids = intent_management._get_intent_ids( + PROJECT_ID, INTENT_DISPLAY_NAME) + + assert len(intent_ids) == 1 + + intent_management.list_intents(PROJECT_ID) + + out, _ = capsys.readouterr() + + assert INTENT_DISPLAY_NAME in out + + for message_text in MESSAGE_TEXTS: + assert message_text in out + + +def test_delete_session_entity_type(capsys): + intent_ids = intent_management._get_intent_ids( + PROJECT_ID, INTENT_DISPLAY_NAME) + + for intent_id in intent_ids: + intent_management.delete_intent(PROJECT_ID, intent_id) + + intent_management.list_intents(PROJECT_ID) + out, _ = capsys.readouterr() + + assert INTENT_DISPLAY_NAME not in out + + intent_ids = intent_management._get_intent_ids( + PROJECT_ID, INTENT_DISPLAY_NAME) + + assert len(intent_ids) == 0 diff --git a/dialogflow/cloud-client/knowledge_base_management.py b/dialogflow/cloud-client/knowledge_base_management.py new file mode 100644 index 00000000000..b5ceab786cc --- /dev/null +++ b/dialogflow/cloud-client/knowledge_base_management.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Dialogflow API Python sample showing how to manage Knowledge bases. + +Examples: + python knowledge_base_management.py -h + python knowledge_base_management.py --project-id PROJECT_ID \ + list + python knowledge_base_management.py --project-id PROJECT_ID \ + create --display-name DISPLAY_NAME + python knowledge_base_management.py --project-id PROJECT_ID \ + get --knowledge-base-id knowledge_base_id + python knowledge_base_management.py --project-id PROJECT_ID \ + delete --knowledge-base-id knowledge_base_id +""" + +import argparse + + +# [START dialogflow_list_knowledge_base] +def list_knowledge_bases(project_id): + """Lists the Knowledge bases belonging to a project. + + Args: + project_id: The GCP project linked with the agent.""" + import dialogflow_v2beta1 as dialogflow + client = dialogflow.KnowledgeBasesClient() + project_path = client.project_path(project_id) + + print('Knowledge Bases for: {}'.format(project_id)) + for knowledge_base in client.list_knowledge_bases(project_path): + print(' - Display Name: {}'.format(knowledge_base.display_name)) + print(' - Knowledge ID: {}\n'.format(knowledge_base.name)) +# [END dialogflow_list_knowledge_base] + + +# [START dialogflow_create_knowledge_base] +def create_knowledge_base(project_id, display_name): + """Creates a Knowledge base. + + Args: + project_id: The GCP project linked with the agent. + display_name: The display name of the Knowledge base.""" + import dialogflow_v2beta1 as dialogflow + client = dialogflow.KnowledgeBasesClient() + project_path = client.project_path(project_id) + + knowledge_base = dialogflow.types.KnowledgeBase( + display_name=display_name) + + response = client.create_knowledge_base(project_path, knowledge_base) + + print('Knowledge Base created:\n') + print('Display Name: {}\n'.format(response.display_name)) + print('Knowledge ID: {}\n'.format(response.name)) +# [END dialogflow_create_knowledge_base] + + +# [START dialogflow_get_knowledge_base] +def get_knowledge_base(project_id, knowledge_base_id): + """Gets a specific Knowledge base. + + Args: + project_id: The GCP project linked with the agent. + knowledge_base_id: Id of the Knowledge base.""" + import dialogflow_v2beta1 as dialogflow + client = dialogflow.KnowledgeBasesClient() + knowledge_base_path = client.knowledge_base_path( + project_id, knowledge_base_id) + + response = client.get_knowledge_base(knowledge_base_path) + + print('Got Knowledge Base:') + print(' - Display Name: {}'.format(response.display_name)) + print(' - Knowledge ID: {}'.format(response.name)) +# [END dialogflow_get_knowledge_base] + + +# [START dialogflow_delete_knowledge_base] +def delete_knowledge_base(project_id, knowledge_base_id): + """Deletes a specific Knowledge base. + + Args: + project_id: The GCP project linked with the agent. + knowledge_base_id: Id of the Knowledge base.""" + import dialogflow_v2beta1 as dialogflow + client = dialogflow.KnowledgeBasesClient() + knowledge_base_path = client.knowledge_base_path( + project_id, knowledge_base_id) + + response = client.delete_knowledge_base(knowledge_base_path) + + print('Knowledge Base deleted.'.format(response)) +# [END dialogflow_delete_knowledge_base] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--project-id', help='Project/agent id.', required=True) + + subparsers = parser.add_subparsers(dest='command') + + list_parser = subparsers.add_parser( + 'list', help='List all Knowledge bases that belong to the project.') + + create_parser = subparsers.add_parser( + 'create', help='Create a new Knowledge base.') + create_parser.add_argument( + '--display-name', + help='A name of the Knowledge base, used for display purpose, ' + 'can not be used to identify the Knowledge base.', + default=str('')) + + get_parser = subparsers.add_parser( + 'get', help='Get a Knowledge base by its id.') + get_parser.add_argument( + '--knowledge-base-id', help='The id of the Knowledge base.', + required=True) + + delete_parser = subparsers.add_parser( + 'delete', help='Delete a Knowledge base by its id.') + delete_parser.add_argument( + '--knowledge-base-id', + help='The id of the Knowledge base you want to delete.', + required=True) + + args = parser.parse_args() + + if args.command == 'list': + list_knowledge_bases(args.project_id) + elif args.command == 'create': + create_knowledge_base(args.project_id, args.display_name) + elif args.command == 'get': + get_knowledge_base(args.project_id, args.knowledge_base_id) + elif args.command == 'delete': + delete_knowledge_base(args.project_id, args.knowledge_base_id) diff --git a/dialogflow/cloud-client/knowledge_base_management_test.py b/dialogflow/cloud-client/knowledge_base_management_test.py new file mode 100644 index 00000000000..d5e6f26cf31 --- /dev/null +++ b/dialogflow/cloud-client/knowledge_base_management_test.py @@ -0,0 +1,107 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import os + +import detect_intent_knowledge +import document_management +import knowledge_base_management + +PROJECT_ID = os.getenv('GCLOUD_PROJECT') +SESSION_ID = 'fake_session_for_testing' +TEXTS = ['Where is my data stored?'] + +KNOWLEDGE_BASE_NAME = 'fake_knowledge_base_name' +DOCUMENT_BASE_NAME = 'fake_document_name' + + +def test_create_knowledge_base(capsys): + # Check the knowledge base does not yet exists + knowledge_base_management.list_knowledge_bases(PROJECT_ID) + + out, _ = capsys.readouterr() + assert 'Display Name: {}'.format(KNOWLEDGE_BASE_NAME) not in out + + # Create a knowledge base + knowledge_base_management.create_knowledge_base(PROJECT_ID, + KNOWLEDGE_BASE_NAME) + + out, _ = capsys.readouterr() + assert 'Display Name: {}'.format(KNOWLEDGE_BASE_NAME) in out + + # List the knowledge base + knowledge_base_management.list_knowledge_bases(PROJECT_ID) + + out, _ = capsys.readouterr() + assert 'Display Name: {}'.format(KNOWLEDGE_BASE_NAME) in out + + knowledge_base_id = out.split('knowledgeBases/')[1].rstrip() + + # Get the knowledge base + knowledge_base_management.get_knowledge_base(PROJECT_ID, knowledge_base_id) + + out, _ = capsys.readouterr() + assert 'Display Name: {}'.format(KNOWLEDGE_BASE_NAME) in out + + # Create a Document + document_management.create_document( + PROJECT_ID, knowledge_base_id, DOCUMENT_BASE_NAME, 'text/html', 'FAQ', + 'https://cloud.google.com/storage/docs/faq') + + out, _ = capsys.readouterr() + assert 'Display Name: {}'.format(DOCUMENT_BASE_NAME) in out + + # List the Document + document_management.list_documents(PROJECT_ID, knowledge_base_id) + + out, _ = capsys.readouterr() + assert 'Display Name: {}'.format(DOCUMENT_BASE_NAME) in out + + document_id = out.split('documents/')[1].split(' - MIME Type:')[0].rstrip() + + # Get the Document + document_management.get_document(PROJECT_ID, knowledge_base_id, + document_id) + + out, _ = capsys.readouterr() + assert 'Display Name: {}'.format(DOCUMENT_BASE_NAME) in out + + # Detect intent with Knowledge Base + detect_intent_knowledge.detect_intent_knowledge( + PROJECT_ID, SESSION_ID, 'en-us', knowledge_base_id, TEXTS) + + out, _ = capsys.readouterr() + assert 'Knowledge results' in out + + # Delete the Document + document_management.delete_document(PROJECT_ID, knowledge_base_id, + document_id) + + # List the Document + document_management.list_documents(PROJECT_ID, knowledge_base_id) + + out, _ = capsys.readouterr() + assert 'Display Name: {}'.format(DOCUMENT_BASE_NAME) not in out + + # Delete the Knowledge Base + knowledge_base_management.delete_knowledge_base(PROJECT_ID, + knowledge_base_id) + + # List the knowledge base + knowledge_base_management.list_knowledge_bases(PROJECT_ID) + + out, _ = capsys.readouterr() + assert 'Display Name: {}'.format(KNOWLEDGE_BASE_NAME) not in out diff --git a/dialogflow/cloud-client/requirements.txt b/dialogflow/cloud-client/requirements.txt new file mode 100644 index 00000000000..5c384ee4622 --- /dev/null +++ b/dialogflow/cloud-client/requirements.txt @@ -0,0 +1 @@ +dialogflow==0.5.2 diff --git a/dialogflow/cloud-client/resources/230pm.wav b/dialogflow/cloud-client/resources/230pm.wav new file mode 100644 index 00000000000..7509eca784d Binary files /dev/null and b/dialogflow/cloud-client/resources/230pm.wav differ diff --git a/dialogflow/cloud-client/resources/RoomReservation.zip b/dialogflow/cloud-client/resources/RoomReservation.zip new file mode 100644 index 00000000000..7873fb628c8 Binary files /dev/null and b/dialogflow/cloud-client/resources/RoomReservation.zip differ diff --git a/dialogflow/cloud-client/resources/book_a_room.wav b/dialogflow/cloud-client/resources/book_a_room.wav new file mode 100644 index 00000000000..9124e927946 Binary files /dev/null and b/dialogflow/cloud-client/resources/book_a_room.wav differ diff --git a/dialogflow/cloud-client/resources/half_an_hour.wav b/dialogflow/cloud-client/resources/half_an_hour.wav new file mode 100644 index 00000000000..71010a871bb Binary files /dev/null and b/dialogflow/cloud-client/resources/half_an_hour.wav differ diff --git a/dialogflow/cloud-client/resources/mountain_view.wav b/dialogflow/cloud-client/resources/mountain_view.wav new file mode 100644 index 00000000000..1c5437f7cb5 Binary files /dev/null and b/dialogflow/cloud-client/resources/mountain_view.wav differ diff --git a/dialogflow/cloud-client/resources/today.wav b/dialogflow/cloud-client/resources/today.wav new file mode 100644 index 00000000000..d47ed78b351 Binary files /dev/null and b/dialogflow/cloud-client/resources/today.wav differ diff --git a/dialogflow/cloud-client/resources/two_people.wav b/dialogflow/cloud-client/resources/two_people.wav new file mode 100644 index 00000000000..5114ebbd310 Binary files /dev/null and b/dialogflow/cloud-client/resources/two_people.wav differ diff --git a/dialogflow/cloud-client/session_entity_type_management.py b/dialogflow/cloud-client/session_entity_type_management.py new file mode 100644 index 00000000000..2c852325338 --- /dev/null +++ b/dialogflow/cloud-client/session_entity_type_management.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python + +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DialogFlow API SessionEntityType Python sample showing how to manage +session entity types. + +Examples: + python session_entity_type_management.py -h + python session_entity_type_management.py --project-id PROJECT_ID list \ + --session-id SESSION_ID + python session_entity_type_management.py --project-id PROJECT_ID create \ + --session-id SESSION_ID \ + --entity-type-display-name room --entity-values C D E F + python session_entity_type_management.py --project-id PROJECT_ID delete \ + --session-id SESSION_ID \ + --entity-type-display-name room +""" + +import argparse + + +# [START dialogflow_list_session_entity_types] +def list_session_entity_types(project_id, session_id): + import dialogflow_v2 as dialogflow + session_entity_types_client = dialogflow.SessionEntityTypesClient() + + session_path = session_entity_types_client.session_path( + project_id, session_id) + + session_entity_types = ( + session_entity_types_client. + list_session_entity_types(session_path)) + + print('SessionEntityTypes for session {}:\n'.format(session_path)) + for session_entity_type in session_entity_types: + print('\tSessionEntityType name: {}'.format(session_entity_type.name)) + print('\tNumber of entities: {}\n'.format( + len(session_entity_type.entities))) +# [END dialogflow_list_session_entity_types] + + +# [START dialogflow_create_session_entity_type] +def create_session_entity_type(project_id, session_id, entity_values, + entity_type_display_name, entity_override_mode): + """Create a session entity type with the given display name.""" + import dialogflow_v2 as dialogflow + session_entity_types_client = dialogflow.SessionEntityTypesClient() + + session_path = session_entity_types_client.session_path( + project_id, session_id) + session_entity_type_name = ( + session_entity_types_client.session_entity_type_path( + project_id, session_id, entity_type_display_name)) + + # Here we use the entity value as the only synonym. + entities = [ + dialogflow.types.EntityType.Entity(value=value, synonyms=[value]) + for value in entity_values] + session_entity_type = dialogflow.types.SessionEntityType( + name=session_entity_type_name, + entity_override_mode=entity_override_mode, + entities=entities) + + response = session_entity_types_client.create_session_entity_type( + session_path, session_entity_type) + + print('SessionEntityType created: \n\n{}'.format(response)) +# [END dialogflow_create_session_entity_type] + + +# [START dialogflow_delete_session_entity_type] +def delete_session_entity_type(project_id, session_id, + entity_type_display_name): + """Delete session entity type with the given entity type display name.""" + import dialogflow_v2 as dialogflow + session_entity_types_client = dialogflow.SessionEntityTypesClient() + + session_entity_type_name = ( + session_entity_types_client.session_entity_type_path( + project_id, session_id, entity_type_display_name)) + + session_entity_types_client.delete_session_entity_type( + session_entity_type_name) +# [END dialogflow_delete_session_entity_type] + + +if __name__ == '__main__': + import dialogflow_v2 + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--project-id', + help='Project/agent id. Required.', + required=True) + + subparsers = parser.add_subparsers(dest='command') + + list_parser = subparsers.add_parser( + 'list', help=list_session_entity_types.__doc__) + list_parser.add_argument( + '--session-id', + required=True) + + create_parser = subparsers.add_parser( + 'create', help=create_session_entity_type.__doc__) + create_parser.add_argument( + '--session-id', + required=True) + create_parser.add_argument( + '--entity-type-display-name', + help='The DISPLAY NAME of the entity type to be overridden ' + 'in the session.', + required=True) + create_parser.add_argument( + '--entity-values', + nargs='*', + help='The entity values of the session entity type.', + required=True) + create_parser.add_argument( + '--entity-override-mode', + help='ENTITY_OVERRIDE_MODE_OVERRIDE (default) or ' + 'ENTITY_OVERRIDE_MODE_SUPPLEMENT', + default=(dialogflow_v2.enums.SessionEntityType.EntityOverrideMode. + ENTITY_OVERRIDE_MODE_OVERRIDE)) + + delete_parser = subparsers.add_parser( + 'delete', help=delete_session_entity_type.__doc__) + delete_parser.add_argument( + '--session-id', + required=True) + delete_parser.add_argument( + '--entity-type-display-name', + help='The DISPLAY NAME of the entity type.', + required=True) + + args = parser.parse_args() + + if args.command == 'list': + list_session_entity_types(args.project_id, args.session_id) + elif args.command == 'create': + create_session_entity_type( + args.project_id, args.session_id, args.entity_values, + args.entity_type_display_name, args.entity_override_mode) + elif args.command == 'delete': + delete_session_entity_type( + args.project_id, args.session_id, args.entity_type_display_name) diff --git a/dialogflow/cloud-client/session_entity_type_management_test.py b/dialogflow/cloud-client/session_entity_type_management_test.py new file mode 100644 index 00000000000..5931ea2ea81 --- /dev/null +++ b/dialogflow/cloud-client/session_entity_type_management_test.py @@ -0,0 +1,63 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import os + +import entity_type_management +import session_entity_type_management + +PROJECT_ID = os.getenv('GCLOUD_PROJECT') +SESSION_ID = 'fake_session_for_testing' +ENTITY_TYPE_DISPLAY_NAME = 'fake_display_name_for_testing' +ENTITY_VALUES = ['fake_entity_value_1', 'fake_entity_value_2'] + + +def test_create_session_entity_type(capsys): + # Create an entity type + entity_type_management.create_entity_type( + PROJECT_ID, ENTITY_TYPE_DISPLAY_NAME, 'KIND_MAP') + + session_entity_type_management.create_session_entity_type( + PROJECT_ID, SESSION_ID, ENTITY_VALUES, ENTITY_TYPE_DISPLAY_NAME, + 'ENTITY_OVERRIDE_MODE_SUPPLEMENT') + session_entity_type_management.list_session_entity_types( + PROJECT_ID, SESSION_ID) + + out, _ = capsys.readouterr() + + assert SESSION_ID in out + assert ENTITY_TYPE_DISPLAY_NAME in out + for entity_value in ENTITY_VALUES: + assert entity_value in out + + +def test_delete_session_entity_type(capsys): + session_entity_type_management.delete_session_entity_type( + PROJECT_ID, SESSION_ID, ENTITY_TYPE_DISPLAY_NAME) + session_entity_type_management.list_session_entity_types( + PROJECT_ID, SESSION_ID) + + out, _ = capsys.readouterr() + assert ENTITY_TYPE_DISPLAY_NAME not in out + for entity_value in ENTITY_VALUES: + assert entity_value not in out + + # Clean up entity type + entity_type_ids = entity_type_management._get_entity_type_ids( + PROJECT_ID, ENTITY_TYPE_DISPLAY_NAME) + for entity_type_id in entity_type_ids: + entity_type_management.delete_entity_type( + PROJECT_ID, entity_type_id) diff --git a/dlp/README.rst b/dlp/README.rst new file mode 100644 index 00000000000..ce8b8550024 --- /dev/null +++ b/dlp/README.rst @@ -0,0 +1,379 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Data Loss Prevention Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=dlp/README.rst + + +This directory contains samples for Google Data Loss Prevention. `Google Data Loss Prevention`_ provides programmatic access to a powerful detection engine for personally identifiable information and other privacy-sensitive data in unstructured data streams. + + + + +.. _Google Data Loss Prevention: https://cloud.google.com/dlp/docs/ + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +#. For running *_test.py files, install test dependencies + + .. code-block:: bash + + $ pip install -r requirements-test.txt + $ pytest inspect_content_test.py + +** *_test.py files are demo wrappers and make API calls. You may get rate limited for making high number of requests. ** + +Samples +------------------------------------------------------------------------------- + +Quickstart ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=dlp/quickstart.py,dlp/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python quickstart.py + + +Inspect Content ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=dlp/inspect_content.py,dlp/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python inspect_content.py + + usage: inspect_content.py [-h] {string,file,gcs,datastore,bigquery} ... + + Sample app that uses the Data Loss Prevention API to inspect a string, a local + file or a file on Google Cloud Storage. + + positional arguments: + {string,file,gcs,datastore,bigquery} + Select how to submit content to the API. + string Inspect a string. + file Inspect a local file. + gcs Inspect files on Google Cloud Storage. + datastore Inspect files on Google Datastore. + bigquery Inspect files on Google BigQuery. + + optional arguments: + -h, --help show this help message and exit + + + +Redact Content ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=dlp/redact.py,dlp/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python redact.py + + usage: redact.py [-h] [--project PROJECT] [--info_types INFO_TYPES] + [--min_likelihood {LIKELIHOOD_UNSPECIFIED,VERY_UNLIKELY,UNLIKELY,POSSIBLE,LIKELY,VERY_LIKELY}] + [--mime_type MIME_TYPE] + filename output_filename + + Sample app that uses the Data Loss Prevent API to redact the contents of a + string or an image file. + + positional arguments: + filename The path to the file to inspect. + output_filename The path to which the redacted image will be written. + + optional arguments: + -h, --help show this help message and exit + --project PROJECT The Google Cloud project id to use as a parent + resource. + --info_types INFO_TYPES + Strings representing info types to look for. A full + list of info categories and types is available from + the API. Examples include "FIRST_NAME", "LAST_NAME", + "EMAIL_ADDRESS". If unspecified, the three above + examples will be used. + --min_likelihood {LIKELIHOOD_UNSPECIFIED,VERY_UNLIKELY,UNLIKELY,POSSIBLE,LIKELY,VERY_LIKELY} + A string representing the minimum likelihood threshold + that constitutes a match. + --mime_type MIME_TYPE + The MIME type of the file. If not specified, the type + is inferred via the Python standard library's + mimetypes module. + + + +Metadata ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=dlp/metadata.py,dlp/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python metadata.py + + usage: metadata.py [-h] [--language_code LANGUAGE_CODE] [--filter FILTER] + + Sample app that queries the Data Loss Prevention API for supported categories + and info types. + + optional arguments: + -h, --help show this help message and exit + --language_code LANGUAGE_CODE + The BCP-47 language code to use, e.g. 'en-US'. + --filter FILTER An optional filter to only return info types supported + by certain parts of the API. Defaults to + "supported_by=INSPECT". + + + +Jobs ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=dlp/jobs.py,dlp/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python jobs.py + + usage: jobs.py [-h] {list,delete} ... + + Sample app to list and delete DLP jobs using the Data Loss Prevent API. + + positional arguments: + {list,delete} Select how to submit content to the API. + list List Data Loss Prevention API jobs corresponding to a given + filter. + delete Delete results of a Data Loss Prevention API job. + + optional arguments: + -h, --help show this help message and exit + + + +Templates ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=dlp/templates.py,dlp/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python templates.py + + usage: templates.py [-h] {create,list,delete} ... + + Sample app that sets up Data Loss Prevention API inspect templates. + + positional arguments: + {create,list,delete} Select which action to perform. + create Create a template. + list List all templates. + delete Delete a template. + + optional arguments: + -h, --help show this help message and exit + + + +Triggers ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=dlp/triggers.py,dlp/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python triggers.py + + usage: triggers.py [-h] {create,list,delete} ... + + Sample app that sets up Data Loss Prevention API automation triggers. + + positional arguments: + {create,list,delete} Select which action to perform. + create Create a trigger. + list List all triggers. + delete Delete a trigger. + + optional arguments: + -h, --help show this help message and exit + + + +Risk Analysis ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=dlp/risk.py,dlp/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python risk.py + + usage: risk.py [-h] {numerical,categorical,k_anonymity,l_diversity,k_map} ... + + Sample app that uses the Data Loss Prevent API to perform risk anaylsis. + + positional arguments: + {numerical,categorical,k_anonymity,l_diversity,k_map} + Select how to submit content to the API. + numerical + categorical + k_anonymity Computes the k-anonymity of a column set in a Google + BigQuerytable. + l_diversity Computes the l-diversity of a column set in a Google + BigQuerytable. + k_map Computes the k-map risk estimation of a column set in + a GoogleBigQuery table. + + optional arguments: + -h, --help show this help message and exit + + + +DeID ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=dlp/deid.py,dlp/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python deid.py + + usage: deid.py [-h] {deid_mask,deid_fpe,reid_fpe,deid_date_shift} ... + + Uses of the Data Loss Prevention API for deidentifying sensitive data. + + positional arguments: + {deid_mask,deid_fpe,reid_fpe,deid_date_shift} + Select how to submit content to the API. + deid_mask Deidentify sensitive data in a string by masking it + with a character. + deid_fpe Deidentify sensitive data in a string using Format + Preserving Encryption (FPE). + reid_fpe Reidentify sensitive data in a string using Format + Preserving Encryption (FPE). + deid_date_shift Deidentify dates in a CSV file by pseudorandomly + shifting them. + + optional arguments: + -h, --help show this help message and exit + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/dlp/README.rst.in b/dlp/README.rst.in new file mode 100644 index 00000000000..8a143392b17 --- /dev/null +++ b/dlp/README.rst.in @@ -0,0 +1,46 @@ +# This file is used to generate README.rst + +product: + name: Google Data Loss Prevention + short_name: Data Loss Prevention + url: https://cloud.google.com/dlp/docs/ + description: > + `Google Data Loss Prevention`_ provides programmatic access to a powerful + detection engine for personally identifiable information and other + privacy-sensitive data in unstructured data streams. + +setup: +- auth +- install_deps + +samples: +- name: Quickstart + file: quickstart.py +- name: Inspect Content + file: inspect_content.py + show_help: true +- name: Redact Content + file: redact.py + show_help: true +- name: Metadata + file: metadata.py + show_help: true +- name: Jobs + file: jobs.py + show_help: true +- name: Templates + file: templates.py + show_help: true +- name: Triggers + file: triggers.py + show_help: true +- name: Risk Analysis + file: risk.py + show_help: true +- name: DeID + file: deid.py + show_help: true + +cloud_client_library: true + +folder: dlp diff --git a/dlp/deid.py b/dlp/deid.py new file mode 100644 index 00000000000..9c97f8e620e --- /dev/null +++ b/dlp/deid.py @@ -0,0 +1,584 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Uses of the Data Loss Prevention API for deidentifying sensitive data.""" + +from __future__ import print_function + +import argparse + + +# [START dlp_deidentify_masking] +def deidentify_with_mask(project, string, info_types, masking_character=None, + number_to_mask=0): + """Uses the Data Loss Prevention API to deidentify sensitive data in a + string by masking it with a character. + Args: + project: The Google Cloud project id to use as a parent resource. + item: The string to deidentify (will be treated as text). + masking_character: The character to mask matching sensitive data with. + number_to_mask: The maximum number of sensitive characters to mask in + a match. If omitted or set to zero, the API will default to no + maximum. + Returns: + None; the response from the API is printed to the terminal. + """ + + # Import the client library + import google.cloud.dlp + + # Instantiate a client + dlp = google.cloud.dlp.DlpServiceClient() + + # Convert the project id into a full resource id. + parent = dlp.project_path(project) + + # Construct inspect configuration dictionary + inspect_config = { + 'info_types': [{'name': info_type} for info_type in info_types] + } + + # Construct deidentify configuration dictionary + deidentify_config = { + 'info_type_transformations': { + 'transformations': [ + { + 'primitive_transformation': { + 'character_mask_config': { + 'masking_character': masking_character, + 'number_to_mask': number_to_mask + } + } + } + ] + } + } + + # Construct item + item = {'value': string} + + # Call the API + response = dlp.deidentify_content( + parent, inspect_config=inspect_config, + deidentify_config=deidentify_config, item=item) + + # Print out the results. + print(response.item.value) +# [END dlp_deidentify_masking] + + +# [START dlp_deidentify_fpe] +def deidentify_with_fpe(project, string, info_types, alphabet=None, + surrogate_type=None, key_name=None, wrapped_key=None): + """Uses the Data Loss Prevention API to deidentify sensitive data in a + string using Format Preserving Encryption (FPE). + Args: + project: The Google Cloud project id to use as a parent resource. + item: The string to deidentify (will be treated as text). + alphabet: The set of characters to replace sensitive ones with. For + more information, see https://cloud.google.com/dlp/docs/reference/ + rest/v2beta2/organizations.deidentifyTemplates#ffxcommonnativealphabet + surrogate_type: The name of the surrogate custom info type to use. Only + necessary if you want to reverse the deidentification process. Can + be essentially any arbitrary string, as long as it doesn't appear + in your dataset otherwise. + key_name: The name of the Cloud KMS key used to encrypt ('wrap') the + AES-256 key. Example: + key_name = 'projects/YOUR_GCLOUD_PROJECT/locations/YOUR_LOCATION/ + keyRings/YOUR_KEYRING_NAME/cryptoKeys/YOUR_KEY_NAME' + wrapped_key: The encrypted ('wrapped') AES-256 key to use. This key + should be encrypted using the Cloud KMS key specified by key_name. + Returns: + None; the response from the API is printed to the terminal. + """ + # Import the client library + import google.cloud.dlp + + # Instantiate a client + dlp = google.cloud.dlp.DlpServiceClient() + + # Convert the project id into a full resource id. + parent = dlp.project_path(project) + + # The wrapped key is base64-encoded, but the library expects a binary + # string, so decode it here. + import base64 + wrapped_key = base64.b64decode(wrapped_key) + + # Construct FPE configuration dictionary + crypto_replace_ffx_fpe_config = { + 'crypto_key': { + 'kms_wrapped': { + 'wrapped_key': wrapped_key, + 'crypto_key_name': key_name + } + }, + 'common_alphabet': alphabet + } + + # Add surrogate type + if surrogate_type: + crypto_replace_ffx_fpe_config['surrogate_info_type'] = { + 'name': surrogate_type + } + + # Construct inspect configuration dictionary + inspect_config = { + 'info_types': [{'name': info_type} for info_type in info_types] + } + + # Construct deidentify configuration dictionary + deidentify_config = { + 'info_type_transformations': { + 'transformations': [ + { + 'primitive_transformation': { + 'crypto_replace_ffx_fpe_config': + crypto_replace_ffx_fpe_config + } + } + ] + } + } + + # Convert string to item + item = {'value': string} + + # Call the API + response = dlp.deidentify_content( + parent, inspect_config=inspect_config, + deidentify_config=deidentify_config, item=item) + + # Print results + print(response.item.value) +# [END dlp_deidentify_fpe] + + +# [START dlp_reidentify_fpe] +def reidentify_with_fpe(project, string, alphabet=None, + surrogate_type=None, key_name=None, wrapped_key=None): + """Uses the Data Loss Prevention API to reidentify sensitive data in a + string that was encrypted by Format Preserving Encryption (FPE). + Args: + project: The Google Cloud project id to use as a parent resource. + item: The string to deidentify (will be treated as text). + alphabet: The set of characters to replace sensitive ones with. For + more information, see https://cloud.google.com/dlp/docs/reference/ + rest/v2beta2/organizations.deidentifyTemplates#ffxcommonnativealphabet + surrogate_type: The name of the surrogate custom info type to used + during the encryption process. + key_name: The name of the Cloud KMS key used to encrypt ('wrap') the + AES-256 key. Example: + keyName = 'projects/YOUR_GCLOUD_PROJECT/locations/YOUR_LOCATION/ + keyRings/YOUR_KEYRING_NAME/cryptoKeys/YOUR_KEY_NAME' + wrapped_key: The encrypted ('wrapped') AES-256 key to use. This key + should be encrypted using the Cloud KMS key specified by key_name. + Returns: + None; the response from the API is printed to the terminal. + """ + # Import the client library + import google.cloud.dlp + + # Instantiate a client + dlp = google.cloud.dlp.DlpServiceClient() + + # Convert the project id into a full resource id. + parent = dlp.project_path(project) + + # The wrapped key is base64-encoded, but the library expects a binary + # string, so decode it here. + import base64 + wrapped_key = base64.b64decode(wrapped_key) + + # Construct Deidentify Config + reidentify_config = { + 'info_type_transformations': { + 'transformations': [ + { + 'primitive_transformation': { + 'crypto_replace_ffx_fpe_config': { + 'crypto_key': { + 'kms_wrapped': { + 'wrapped_key': wrapped_key, + 'crypto_key_name': key_name + } + }, + 'common_alphabet': alphabet, + 'surrogate_info_type': { + 'name': surrogate_type + } + } + } + } + ] + } + } + + inspect_config = { + 'custom_info_types': [ + { + 'info_type': { + 'name': surrogate_type + }, + 'surrogate_type': { + } + } + ] + } + + # Convert string to item + item = {'value': string} + + # Call the API + response = dlp.reidentify_content( + parent, + inspect_config=inspect_config, + reidentify_config=reidentify_config, + item=item) + + # Print results + print(response.item.value) +# [END dlp_reidentify_fpe] + + +# [START dlp_deidentify_date_shift] +def deidentify_with_date_shift(project, input_csv_file=None, + output_csv_file=None, date_fields=None, + lower_bound_days=None, upper_bound_days=None, + context_field_id=None, wrapped_key=None, + key_name=None): + """Uses the Data Loss Prevention API to deidentify dates in a CSV file by + pseudorandomly shifting them. + Args: + project: The Google Cloud project id to use as a parent resource. + input_csv_file: The path to the CSV file to deidentify. The first row + of the file must specify column names, and all other rows must + contain valid values. + output_csv_file: The path to save the date-shifted CSV file. + date_fields: The list of (date) fields in the CSV file to date shift. + Example: ['birth_date', 'register_date'] + lower_bound_days: The maximum number of days to shift a date backward + upper_bound_days: The maximum number of days to shift a date forward + context_field_id: (Optional) The column to determine date shift amount + based on. If this is not specified, a random shift amount will be + used for every row. If this is specified, then 'wrappedKey' and + 'keyName' must also be set. Example: + contextFieldId = [{ 'name': 'user_id' }] + key_name: (Optional) The name of the Cloud KMS key used to encrypt + ('wrap') the AES-256 key. Example: + key_name = 'projects/YOUR_GCLOUD_PROJECT/locations/YOUR_LOCATION/ + keyRings/YOUR_KEYRING_NAME/cryptoKeys/YOUR_KEY_NAME' + wrapped_key: (Optional) The encrypted ('wrapped') AES-256 key to use. + This key should be encrypted using the Cloud KMS key specified by + key_name. + Returns: + None; the response from the API is printed to the terminal. + """ + # Import the client library + import google.cloud.dlp + + # Instantiate a client + dlp = google.cloud.dlp.DlpServiceClient() + + # Convert the project id into a full resource id. + parent = dlp.project_path(project) + + # Convert date field list to Protobuf type + def map_fields(field): + return {'name': field} + + if date_fields: + date_fields = map(map_fields, date_fields) + else: + date_fields = [] + + # Read and parse the CSV file + import csv + from datetime import datetime + f = [] + with open(input_csv_file, 'r') as csvfile: + reader = csv.reader(csvfile) + for row in reader: + f.append(row) + + # Helper function for converting CSV rows to Protobuf types + def map_headers(header): + return {'name': header} + + def map_data(value): + try: + date = datetime.strptime(value, '%m/%d/%Y') + return { + 'date_value': { + 'year': date.year, + 'month': date.month, + 'day': date.day + } + } + except ValueError: + return {'string_value': value} + + def map_rows(row): + return {'values': map(map_data, row)} + + # Using the helper functions, convert CSV rows to protobuf-compatible + # dictionaries. + csv_headers = map(map_headers, f[0]) + csv_rows = map(map_rows, f[1:]) + + # Construct the table dict + table_item = { + 'table': { + 'headers': csv_headers, + 'rows': csv_rows + } + } + + # Construct date shift config + date_shift_config = { + 'lower_bound_days': lower_bound_days, + 'upper_bound_days': upper_bound_days + } + + # If using a Cloud KMS key, add it to the date_shift_config. + # The wrapped key is base64-encoded, but the library expects a binary + # string, so decode it here. + if context_field_id and key_name and wrapped_key: + import base64 + date_shift_config['context'] = {'name': context_field_id} + date_shift_config['crypto_key'] = { + 'kms_wrapped': { + 'wrapped_key': base64.b64decode(wrapped_key), + 'crypto_key_name': key_name + } + } + elif context_field_id or key_name or wrapped_key: + raise ValueError("""You must set either ALL or NONE of + [context_field_id, key_name, wrapped_key]!""") + + # Construct Deidentify Config + deidentify_config = { + 'record_transformations': { + 'field_transformations': [ + { + 'fields': date_fields, + 'primitive_transformation': { + 'date_shift_config': date_shift_config + } + } + ] + } + } + + # Write to CSV helper methods + def write_header(header): + return header.name + + def write_data(data): + return data.string_value or '%s/%s/%s' % (data.date_value.month, + data.date_value.day, + data.date_value.year) + + # Call the API + response = dlp.deidentify_content( + parent, deidentify_config=deidentify_config, item=table_item) + + # Write results to CSV file + with open(output_csv_file, 'w') as csvfile: + write_file = csv.writer(csvfile, delimiter=',') + write_file.writerow(map(write_header, response.item.table.headers)) + for row in response.item.table.rows: + write_file.writerow(map(write_data, row.values)) + # Print status + print('Successfully saved date-shift output to {}'.format( + output_csv_file)) +# [END dlp_deidentify_date_shift] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers( + dest='content', help='Select how to submit content to the API.') + subparsers.required = True + + mask_parser = subparsers.add_parser( + 'deid_mask', + help='Deidentify sensitive data in a string by masking it with a ' + 'character.') + mask_parser.add_argument( + '--info_types', action='append', + help='Strings representing info types to look for. A full list of ' + 'info categories and types is available from the API. Examples ' + 'include "FIRST_NAME", "LAST_NAME", "EMAIL_ADDRESS". ' + 'If unspecified, the three above examples will be used.', + default=['FIRST_NAME', 'LAST_NAME', 'EMAIL_ADDRESS']) + mask_parser.add_argument( + 'project', + help='The Google Cloud project id to use as a parent resource.') + mask_parser.add_argument('item', help='The string to deidentify.') + mask_parser.add_argument( + '-n', '--number_to_mask', + type=int, + default=0, + help='The maximum number of sensitive characters to mask in a match. ' + 'If omitted the request or set to 0, the API will mask any mathcing ' + 'characters.') + mask_parser.add_argument( + '-m', '--masking_character', + help='The character to mask matching sensitive data with.') + + fpe_parser = subparsers.add_parser( + 'deid_fpe', + help='Deidentify sensitive data in a string using Format Preserving ' + 'Encryption (FPE).') + fpe_parser.add_argument( + '--info_types', action='append', + help='Strings representing info types to look for. A full list of ' + 'info categories and types is available from the API. Examples ' + 'include "FIRST_NAME", "LAST_NAME", "EMAIL_ADDRESS". ' + 'If unspecified, the three above examples will be used.', + default=['FIRST_NAME', 'LAST_NAME', 'EMAIL_ADDRESS']) + fpe_parser.add_argument( + 'project', + help='The Google Cloud project id to use as a parent resource.') + fpe_parser.add_argument( + 'item', + help='The string to deidentify. ' + 'Example: string = \'My SSN is 372819127\'') + fpe_parser.add_argument( + 'key_name', + help='The name of the Cloud KMS key used to encrypt (\'wrap\') the ' + 'AES-256 key. Example: ' + 'key_name = \'projects/YOUR_GCLOUD_PROJECT/locations/YOUR_LOCATION/' + 'keyRings/YOUR_KEYRING_NAME/cryptoKeys/YOUR_KEY_NAME\'') + fpe_parser.add_argument( + 'wrapped_key', + help='The encrypted (\'wrapped\') AES-256 key to use. This key should ' + 'be encrypted using the Cloud KMS key specified by key_name.') + fpe_parser.add_argument( + '-a', '--alphabet', default='ALPHA_NUMERIC', + help='The set of characters to replace sensitive ones with. Commonly ' + 'used subsets of the alphabet include "NUMERIC", "HEXADECIMAL", ' + '"UPPER_CASE_ALPHA_NUMERIC", "ALPHA_NUMERIC", ' + '"FFX_COMMON_NATIVE_ALPHABET_UNSPECIFIED"') + fpe_parser.add_argument( + '-s', '--surrogate_type', + help='The name of the surrogate custom info type to use. Only ' + 'necessary if you want to reverse the deidentification process. Can ' + 'be essentially any arbitrary string, as long as it doesn\'t appear ' + 'in your dataset otherwise.') + + reid_parser = subparsers.add_parser( + 'reid_fpe', + help='Reidentify sensitive data in a string using Format Preserving ' + 'Encryption (FPE).') + reid_parser.add_argument( + 'project', + help='The Google Cloud project id to use as a parent resource.') + reid_parser.add_argument( + 'item', + help='The string to deidentify. ' + 'Example: string = \'My SSN is 372819127\'') + reid_parser.add_argument( + 'surrogate_type', + help='The name of the surrogate custom info type to use. Only ' + 'necessary if you want to reverse the deidentification process. Can ' + 'be essentially any arbitrary string, as long as it doesn\'t appear ' + 'in your dataset otherwise.') + reid_parser.add_argument( + 'key_name', + help='The name of the Cloud KMS key used to encrypt (\'wrap\') the ' + 'AES-256 key. Example: ' + 'key_name = \'projects/YOUR_GCLOUD_PROJECT/locations/YOUR_LOCATION/' + 'keyRings/YOUR_KEYRING_NAME/cryptoKeys/YOUR_KEY_NAME\'') + reid_parser.add_argument( + 'wrapped_key', + help='The encrypted (\'wrapped\') AES-256 key to use. This key should ' + 'be encrypted using the Cloud KMS key specified by key_name.') + reid_parser.add_argument( + '-a', '--alphabet', default='ALPHA_NUMERIC', + help='The set of characters to replace sensitive ones with. Commonly ' + 'used subsets of the alphabet include "NUMERIC", "HEXADECIMAL", ' + '"UPPER_CASE_ALPHA_NUMERIC", "ALPHA_NUMERIC", ' + '"FFX_COMMON_NATIVE_ALPHABET_UNSPECIFIED"') + + date_shift_parser = subparsers.add_parser( + 'deid_date_shift', + help='Deidentify dates in a CSV file by pseudorandomly shifting them.') + date_shift_parser.add_argument( + 'project', + help='The Google Cloud project id to use as a parent resource.') + date_shift_parser.add_argument( + 'input_csv_file', + help='The path to the CSV file to deidentify. The first row of the ' + 'file must specify column names, and all other rows must contain ' + 'valid values.') + date_shift_parser.add_argument( + 'output_csv_file', + help='The path to save the date-shifted CSV file.') + date_shift_parser.add_argument( + 'lower_bound_days', type=int, + help='The maximum number of days to shift a date backward') + date_shift_parser.add_argument( + 'upper_bound_days', type=int, + help='The maximum number of days to shift a date forward') + date_shift_parser.add_argument( + 'date_fields', nargs='+', + help='The list of date fields in the CSV file to date shift. Example: ' + '[\'birth_date\', \'register_date\']') + date_shift_parser.add_argument( + '--context_field_id', + help='(Optional) The column to determine date shift amount based on. ' + 'If this is not specified, a random shift amount will be used for ' + 'every row. If this is specified, then \'wrappedKey\' and \'keyName\' ' + 'must also be set.') + date_shift_parser.add_argument( + '--key_name', + help='(Optional) The name of the Cloud KMS key used to encrypt ' + '(\'wrap\') the AES-256 key. Example: ' + 'key_name = \'projects/YOUR_GCLOUD_PROJECT/locations/YOUR_LOCATION/' + 'keyRings/YOUR_KEYRING_NAME/cryptoKeys/YOUR_KEY_NAME\'') + date_shift_parser.add_argument( + '--wrapped_key', + help='(Optional) The encrypted (\'wrapped\') AES-256 key to use. This ' + 'key should be encrypted using the Cloud KMS key specified by' + 'key_name.') + + args = parser.parse_args() + + if args.content == 'deid_mask': + deidentify_with_mask(args.project, args.item, args.info_types, + masking_character=args.masking_character, + number_to_mask=args.number_to_mask) + elif args.content == 'deid_fpe': + deidentify_with_fpe(args.project, args.item, args.info_types, + alphabet=args.alphabet, + wrapped_key=args.wrapped_key, + key_name=args.key_name, + surrogate_type=args.surrogate_type) + elif args.content == 'reid_fpe': + reidentify_with_fpe(args.project, args.item, + surrogate_type=args.surrogate_type, + wrapped_key=args.wrapped_key, + key_name=args.key_name, alphabet=args.alphabet) + elif args.content == 'deid_date_shift': + deidentify_with_date_shift(args.project, + input_csv_file=args.input_csv_file, + output_csv_file=args.output_csv_file, + lower_bound_days=args.lower_bound_days, + upper_bound_days=args.upper_bound_days, + date_fields=args.date_fields, + context_field_id=args.context_field_id, + wrapped_key=args.wrapped_key, + key_name=args.key_name) diff --git a/dlp/deid_test.py b/dlp/deid_test.py new file mode 100644 index 00000000000..e381f4502ef --- /dev/null +++ b/dlp/deid_test.py @@ -0,0 +1,171 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import tempfile + +import pytest + +import deid + +HARMFUL_STRING = 'My SSN is 372819127' +HARMLESS_STRING = 'My favorite color is blue' +GCLOUD_PROJECT = os.getenv('GCLOUD_PROJECT') +WRAPPED_KEY = ('CiQAz0hX4+go8fJwn80Fr8pVImwx+tmZdqU7JL+7TN/S5JxBU9gSSQDhFHpFVy' + 'uzJps0YH9ls480mU+JLG7jI/0lL04i6XJRWqmI6gUSZRUtECYcLH5gXK4SXHlL' + 'rotx7Chxz/4z7SIpXFOBY61z0/U=') +KEY_NAME = ('projects/python-docs-samples-tests/locations/global/keyRings/' + 'dlp-test/cryptoKeys/dlp-test') +SURROGATE_TYPE = 'SSN_TOKEN' +CSV_FILE = os.path.join(os.path.dirname(__file__), 'resources/dates.csv') +DATE_SHIFTED_AMOUNT = 30 +DATE_FIELDS = ['birth_date', 'register_date'] +CSV_CONTEXT_FIELD = 'name' + + +@pytest.fixture(scope='module') +def tempdir(): + tempdir = tempfile.mkdtemp() + yield tempdir + shutil.rmtree(tempdir) + + +def test_deidentify_with_mask(capsys): + deid.deidentify_with_mask(GCLOUD_PROJECT, HARMFUL_STRING, + ['US_SOCIAL_SECURITY_NUMBER']) + + out, _ = capsys.readouterr() + assert 'My SSN is *********' in out + + +def test_deidentify_with_mask_ignore_insensitive_data(capsys): + deid.deidentify_with_mask(GCLOUD_PROJECT, HARMLESS_STRING, + ['US_SOCIAL_SECURITY_NUMBER']) + + out, _ = capsys.readouterr() + assert HARMLESS_STRING in out + + +def test_deidentify_with_mask_masking_character_specified(capsys): + deid.deidentify_with_mask( + GCLOUD_PROJECT, + HARMFUL_STRING, + ['US_SOCIAL_SECURITY_NUMBER'], + masking_character='#') + + out, _ = capsys.readouterr() + assert 'My SSN is #########' in out + + +def test_deidentify_with_mask_masking_number_specified(capsys): + deid.deidentify_with_mask(GCLOUD_PROJECT, HARMFUL_STRING, + ['US_SOCIAL_SECURITY_NUMBER'], + number_to_mask=7) + + out, _ = capsys.readouterr() + assert 'My SSN is *******27' in out + + +def test_deidentify_with_fpe(capsys): + deid.deidentify_with_fpe( + GCLOUD_PROJECT, + HARMFUL_STRING, + ['US_SOCIAL_SECURITY_NUMBER'], + alphabet='NUMERIC', + wrapped_key=WRAPPED_KEY, + key_name=KEY_NAME) + + out, _ = capsys.readouterr() + assert 'My SSN is' in out + assert '372819127' not in out + + +def test_deidentify_with_fpe_uses_surrogate_info_types(capsys): + deid.deidentify_with_fpe( + GCLOUD_PROJECT, + HARMFUL_STRING, + ['US_SOCIAL_SECURITY_NUMBER'], + alphabet='NUMERIC', + wrapped_key=WRAPPED_KEY, + key_name=KEY_NAME, + surrogate_type=SURROGATE_TYPE) + + out, _ = capsys.readouterr() + assert 'My SSN is SSN_TOKEN' in out + assert '372819127' not in out + + +def test_deidentify_with_fpe_ignores_insensitive_data(capsys): + deid.deidentify_with_fpe( + GCLOUD_PROJECT, + HARMLESS_STRING, + ['US_SOCIAL_SECURITY_NUMBER'], + alphabet='NUMERIC', + wrapped_key=WRAPPED_KEY, + key_name=KEY_NAME) + + out, _ = capsys.readouterr() + assert HARMLESS_STRING in out + + +def test_deidentify_with_date_shift(tempdir, capsys): + output_filepath = os.path.join(tempdir, 'dates-shifted.csv') + + deid.deidentify_with_date_shift( + GCLOUD_PROJECT, + input_csv_file=CSV_FILE, + output_csv_file=output_filepath, + lower_bound_days=DATE_SHIFTED_AMOUNT, + upper_bound_days=DATE_SHIFTED_AMOUNT, + date_fields=DATE_FIELDS) + + out, _ = capsys.readouterr() + + assert 'Successful' in out + + +def test_deidentify_with_date_shift_using_context_field(tempdir, capsys): + output_filepath = os.path.join(tempdir, 'dates-shifted.csv') + + deid.deidentify_with_date_shift( + GCLOUD_PROJECT, + input_csv_file=CSV_FILE, + output_csv_file=output_filepath, + lower_bound_days=DATE_SHIFTED_AMOUNT, + upper_bound_days=DATE_SHIFTED_AMOUNT, + date_fields=DATE_FIELDS, + context_field_id=CSV_CONTEXT_FIELD, + wrapped_key=WRAPPED_KEY, + key_name=KEY_NAME) + + out, _ = capsys.readouterr() + + assert 'Successful' in out + + +def test_reidentify_with_fpe(capsys): + labeled_fpe_string = 'My SSN is SSN_TOKEN(9):731997681' + + deid.reidentify_with_fpe( + GCLOUD_PROJECT, + labeled_fpe_string, + surrogate_type=SURROGATE_TYPE, + wrapped_key=WRAPPED_KEY, + key_name=KEY_NAME, + alphabet='NUMERIC') + + out, _ = capsys.readouterr() + + assert '731997681' not in out diff --git a/dlp/inspect_content.py b/dlp/inspect_content.py new file mode 100644 index 00000000000..a741e0ee734 --- /dev/null +++ b/dlp/inspect_content.py @@ -0,0 +1,1133 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sample app that uses the Data Loss Prevention API to inspect a string, a +local file or a file on Google Cloud Storage.""" + +from __future__ import print_function + +import argparse +import os +import json + + +# [START dlp_inspect_string] +def inspect_string(project, content_string, info_types, + custom_dictionaries=None, custom_regexes=None, + min_likelihood=None, max_findings=None, include_quote=True): + """Uses the Data Loss Prevention API to analyze strings for protected data. + Args: + project: The Google Cloud project id to use as a parent resource. + content_string: The string to inspect. + info_types: A list of strings representing info types to look for. + A full list of info type categories can be fetched from the API. + min_likelihood: A string representing the minimum likelihood threshold + that constitutes a match. One of: 'LIKELIHOOD_UNSPECIFIED', + 'VERY_UNLIKELY', 'UNLIKELY', 'POSSIBLE', 'LIKELY', 'VERY_LIKELY'. + max_findings: The maximum number of findings to report; 0 = no maximum. + include_quote: Boolean for whether to display a quote of the detected + information in the results. + Returns: + None; the response from the API is printed to the terminal. + """ + + # Import the client library. + import google.cloud.dlp + + # Instantiate a client. + dlp = google.cloud.dlp.DlpServiceClient() + + # Prepare info_types by converting the list of strings into a list of + # dictionaries (protos are also accepted). + info_types = [{'name': info_type} for info_type in info_types] + + # Prepare custom_info_types by parsing the dictionary word lists and + # regex patterns. + if custom_dictionaries is None: + custom_dictionaries = [] + dictionaries = [{ + 'info_type': {'name': 'CUSTOM_DICTIONARY_{}'.format(i)}, + 'dictionary': { + 'word_list': {'words': custom_dict.split(',')} + } + } for i, custom_dict in enumerate(custom_dictionaries)] + if custom_regexes is None: + custom_regexes = [] + regexes = [{ + 'info_type': {'name': 'CUSTOM_REGEX_{}'.format(i)}, + 'regex': {'pattern': custom_regex} + } for i, custom_regex in enumerate(custom_regexes)] + custom_info_types = dictionaries + regexes + + # Construct the configuration dictionary. Keys which are None may + # optionally be omitted entirely. + inspect_config = { + 'info_types': info_types, + 'custom_info_types': custom_info_types, + 'min_likelihood': min_likelihood, + 'include_quote': include_quote, + 'limits': {'max_findings_per_request': max_findings}, + } + + # Construct the `item`. + item = {'value': content_string} + + # Convert the project id into a full resource id. + parent = dlp.project_path(project) + + # Call the API. + response = dlp.inspect_content(parent, inspect_config, item) + + # Print out the results. + if response.result.findings: + for finding in response.result.findings: + try: + if finding.quote: + print('Quote: {}'.format(finding.quote)) + except AttributeError: + pass + print('Info type: {}'.format(finding.info_type.name)) + print('Likelihood: {}'.format(finding.likelihood)) + else: + print('No findings.') +# [END dlp_inspect_string] + +# [START dlp_inspect_table] + + +def inspect_table(project, data, info_types, + custom_dictionaries=None, custom_regexes=None, + min_likelihood=None, max_findings=None, include_quote=True): + """Uses the Data Loss Prevention API to analyze strings for protected data. + Args: + project: The Google Cloud project id to use as a parent resource. + data: Json string representing table data. + info_types: A list of strings representing info types to look for. + A full list of info type categories can be fetched from the API. + min_likelihood: A string representing the minimum likelihood threshold + that constitutes a match. One of: 'LIKELIHOOD_UNSPECIFIED', + 'VERY_UNLIKELY', 'UNLIKELY', 'POSSIBLE', 'LIKELY', 'VERY_LIKELY'. + max_findings: The maximum number of findings to report; 0 = no maximum. + include_quote: Boolean for whether to display a quote of the detected + information in the results. + Returns: + None; the response from the API is printed to the terminal. + Example: + data = { + "header":[ + "email", + "phone number" + ], + "rows":[ + [ + "robertfrost@xyz.com", + "4232342345" + ], + [ + "johndoe@pqr.com", + "4253458383" + ] + ] + } + + >> $ python inspect_content.py table \ + '{"header": ["email", "phone number"], + "rows": [["robertfrost@xyz.com", "4232342345"], + ["johndoe@pqr.com", "4253458383"]]}' + >> Quote: robertfrost@xyz.com + Info type: EMAIL_ADDRESS + Likelihood: 4 + Quote: johndoe@pqr.com + Info type: EMAIL_ADDRESS + Likelihood: 4 + """ + + # Import the client library. + import google.cloud.dlp + + # Instantiate a client. + dlp = google.cloud.dlp.DlpServiceClient() + + # Prepare info_types by converting the list of strings into a list of + # dictionaries (protos are also accepted). + info_types = [{'name': info_type} for info_type in info_types] + + # Prepare custom_info_types by parsing the dictionary word lists and + # regex patterns. + if custom_dictionaries is None: + custom_dictionaries = [] + dictionaries = [{ + 'info_type': {'name': 'CUSTOM_DICTIONARY_{}'.format(i)}, + 'dictionary': { + 'word_list': {'words': custom_dict.split(',')} + } + } for i, custom_dict in enumerate(custom_dictionaries)] + if custom_regexes is None: + custom_regexes = [] + regexes = [{ + 'info_type': {'name': 'CUSTOM_REGEX_{}'.format(i)}, + 'regex': {'pattern': custom_regex} + } for i, custom_regex in enumerate(custom_regexes)] + custom_info_types = dictionaries + regexes + + # Construct the configuration dictionary. Keys which are None may + # optionally be omitted entirely. + inspect_config = { + 'info_types': info_types, + 'custom_info_types': custom_info_types, + 'min_likelihood': min_likelihood, + 'include_quote': include_quote, + 'limits': {'max_findings_per_request': max_findings}, + } + + # Construct the `table`. For more details on the table schema, please see + # https://cloud.google.com/dlp/docs/reference/rest/v2/ContentItem#Table + headers = [{"name": val} for val in data["header"]] + rows = [] + for row in data["rows"]: + rows.append({ + "values": [{"string_value": cell_val} for cell_val in row] + }) + + table = {} + table["headers"] = headers + table["rows"] = rows + item = {"table": table} + # Convert the project id into a full resource id. + parent = dlp.project_path(project) + + # Call the API. + response = dlp.inspect_content(parent, inspect_config, item) + + # Print out the results. + if response.result.findings: + for finding in response.result.findings: + try: + if finding.quote: + print('Quote: {}'.format(finding.quote)) + except AttributeError: + pass + print('Info type: {}'.format(finding.info_type.name)) + print('Likelihood: {}'.format(finding.likelihood)) + else: + print('No findings.') +# [END dlp_inspect_table] + +# [START dlp_inspect_file] + + +def inspect_file(project, filename, info_types, min_likelihood=None, + custom_dictionaries=None, custom_regexes=None, + max_findings=None, include_quote=True, mime_type=None): + """Uses the Data Loss Prevention API to analyze a file for protected data. + Args: + project: The Google Cloud project id to use as a parent resource. + filename: The path to the file to inspect. + info_types: A list of strings representing info types to look for. + A full list of info type categories can be fetched from the API. + min_likelihood: A string representing the minimum likelihood threshold + that constitutes a match. One of: 'LIKELIHOOD_UNSPECIFIED', + 'VERY_UNLIKELY', 'UNLIKELY', 'POSSIBLE', 'LIKELY', 'VERY_LIKELY'. + max_findings: The maximum number of findings to report; 0 = no maximum. + include_quote: Boolean for whether to display a quote of the detected + information in the results. + mime_type: The MIME type of the file. If not specified, the type is + inferred via the Python standard library's mimetypes module. + Returns: + None; the response from the API is printed to the terminal. + """ + + import mimetypes + + # Import the client library. + import google.cloud.dlp + + # Instantiate a client. + dlp = google.cloud.dlp.DlpServiceClient() + + # Prepare info_types by converting the list of strings into a list of + # dictionaries (protos are also accepted). + if not info_types: + info_types = ['FIRST_NAME', 'LAST_NAME', 'EMAIL_ADDRESS'] + info_types = [{'name': info_type} for info_type in info_types] + + # Prepare custom_info_types by parsing the dictionary word lists and + # regex patterns. + if custom_dictionaries is None: + custom_dictionaries = [] + dictionaries = [{ + 'info_type': {'name': 'CUSTOM_DICTIONARY_{}'.format(i)}, + 'dictionary': { + 'word_list': {'words': custom_dict.split(',')} + } + } for i, custom_dict in enumerate(custom_dictionaries)] + if custom_regexes is None: + custom_regexes = [] + regexes = [{ + 'info_type': {'name': 'CUSTOM_REGEX_{}'.format(i)}, + 'regex': {'pattern': custom_regex} + } for i, custom_regex in enumerate(custom_regexes)] + custom_info_types = dictionaries + regexes + + # Construct the configuration dictionary. Keys which are None may + # optionally be omitted entirely. + inspect_config = { + 'info_types': info_types, + 'custom_info_types': custom_info_types, + 'min_likelihood': min_likelihood, + 'limits': {'max_findings_per_request': max_findings}, + } + + # If mime_type is not specified, guess it from the filename. + if mime_type is None: + mime_guess = mimetypes.MimeTypes().guess_type(filename) + mime_type = mime_guess[0] + + # Select the content type index from the list of supported types. + supported_content_types = { + None: 0, # "Unspecified" + 'image/jpeg': 1, + 'image/bmp': 2, + 'image/png': 3, + 'image/svg': 4, + 'text/plain': 5, + } + content_type_index = supported_content_types.get(mime_type, 0) + + # Construct the item, containing the file's byte data. + with open(filename, mode='rb') as f: + item = {'byte_item': {'type': content_type_index, 'data': f.read()}} + + # Convert the project id into a full resource id. + parent = dlp.project_path(project) + + # Call the API. + response = dlp.inspect_content(parent, inspect_config, item) + + # Print out the results. + if response.result.findings: + for finding in response.result.findings: + try: + print('Quote: {}'.format(finding.quote)) + except AttributeError: + pass + print('Info type: {}'.format(finding.info_type.name)) + print('Likelihood: {}'.format(finding.likelihood)) + else: + print('No findings.') +# [END dlp_inspect_file] + + +# [START dlp_inspect_gcs] +def inspect_gcs_file(project, bucket, filename, topic_id, subscription_id, + info_types, custom_dictionaries=None, + custom_regexes=None, min_likelihood=None, + max_findings=None, timeout=300): + """Uses the Data Loss Prevention API to analyze a file on GCS. + Args: + project: The Google Cloud project id to use as a parent resource. + bucket: The name of the GCS bucket containing the file, as a string. + filename: The name of the file in the bucket, including the path, as a + string; e.g. 'images/myfile.png'. + topic_id: The id of the Cloud Pub/Sub topic to which the API will + broadcast job completion. The topic must already exist. + subscription_id: The id of the Cloud Pub/Sub subscription to listen on + while waiting for job completion. The subscription must already + exist and be subscribed to the topic. + info_types: A list of strings representing info types to look for. + A full list of info type categories can be fetched from the API. + min_likelihood: A string representing the minimum likelihood threshold + that constitutes a match. One of: 'LIKELIHOOD_UNSPECIFIED', + 'VERY_UNLIKELY', 'UNLIKELY', 'POSSIBLE', 'LIKELY', 'VERY_LIKELY'. + max_findings: The maximum number of findings to report; 0 = no maximum. + timeout: The number of seconds to wait for a response from the API. + Returns: + None; the response from the API is printed to the terminal. + """ + + # Import the client library. + import google.cloud.dlp + + # This sample additionally uses Cloud Pub/Sub to receive results from + # potentially long-running operations. + import google.cloud.pubsub + + # This sample also uses threading.Event() to wait for the job to finish. + import threading + + # Instantiate a client. + dlp = google.cloud.dlp.DlpServiceClient() + + # Prepare info_types by converting the list of strings into a list of + # dictionaries (protos are also accepted). + if not info_types: + info_types = ['FIRST_NAME', 'LAST_NAME', 'EMAIL_ADDRESS'] + info_types = [{'name': info_type} for info_type in info_types] + + # Prepare custom_info_types by parsing the dictionary word lists and + # regex patterns. + if custom_dictionaries is None: + custom_dictionaries = [] + dictionaries = [{ + 'info_type': {'name': 'CUSTOM_DICTIONARY_{}'.format(i)}, + 'dictionary': { + 'word_list': {'words': custom_dict.split(',')} + } + } for i, custom_dict in enumerate(custom_dictionaries)] + if custom_regexes is None: + custom_regexes = [] + regexes = [{ + 'info_type': {'name': 'CUSTOM_REGEX_{}'.format(i)}, + 'regex': {'pattern': custom_regex} + } for i, custom_regex in enumerate(custom_regexes)] + custom_info_types = dictionaries + regexes + + # Construct the configuration dictionary. Keys which are None may + # optionally be omitted entirely. + inspect_config = { + 'info_types': info_types, + 'custom_info_types': custom_info_types, + 'min_likelihood': min_likelihood, + 'limits': {'max_findings_per_request': max_findings}, + } + + # Construct a storage_config containing the file's URL. + url = 'gs://{}/{}'.format(bucket, filename) + storage_config = { + 'cloud_storage_options': { + 'file_set': {'url': url} + } + } + + # Convert the project id into a full resource id. + parent = dlp.project_path(project) + + # Tell the API where to send a notification when the job is complete. + actions = [{ + 'pub_sub': {'topic': '{}/topics/{}'.format(parent, topic_id)} + }] + + # Construct the inspect_job, which defines the entire inspect content task. + inspect_job = { + 'inspect_config': inspect_config, + 'storage_config': storage_config, + 'actions': actions, + } + + operation = dlp.create_dlp_job(parent, inspect_job=inspect_job) + + # Create a Pub/Sub client and find the subscription. The subscription is + # expected to already be listening to the topic. + subscriber = google.cloud.pubsub.SubscriberClient() + subscription_path = subscriber.subscription_path( + project, subscription_id) + + # Set up a callback to acknowledge a message. This closes around an event + # so that it can signal that it is done and the main thread can continue. + job_done = threading.Event() + + def callback(message): + try: + if (message.attributes['DlpJobName'] == operation.name): + # This is the message we're looking for, so acknowledge it. + message.ack() + + # Now that the job is done, fetch the results and print them. + job = dlp.get_dlp_job(operation.name) + if job.inspect_details.result.info_type_stats: + for finding in job.inspect_details.result.info_type_stats: + print('Info type: {}; Count: {}'.format( + finding.info_type.name, finding.count)) + else: + print('No findings.') + + # Signal to the main thread that we can exit. + job_done.set() + else: + # This is not the message we're looking for. + message.drop() + except Exception as e: + # Because this is executing in a thread, an exception won't be + # noted unless we print it manually. + print(e) + raise + + subscriber.subscribe(subscription_path, callback=callback) + finished = job_done.wait(timeout=timeout) + if not finished: + print('No event received before the timeout. Please verify that the ' + 'subscription provided is subscribed to the topic provided.') + +# [END dlp_inspect_gcs] + + +# [START dlp_inspect_datastore] +def inspect_datastore(project, datastore_project, kind, + topic_id, subscription_id, info_types, + custom_dictionaries=None, custom_regexes=None, + namespace_id=None, min_likelihood=None, + max_findings=None, timeout=300): + """Uses the Data Loss Prevention API to analyze Datastore data. + Args: + project: The Google Cloud project id to use as a parent resource. + datastore_project: The Google Cloud project id of the target Datastore. + kind: The kind of the Datastore entity to inspect, e.g. 'Person'. + topic_id: The id of the Cloud Pub/Sub topic to which the API will + broadcast job completion. The topic must already exist. + subscription_id: The id of the Cloud Pub/Sub subscription to listen on + while waiting for job completion. The subscription must already + exist and be subscribed to the topic. + info_types: A list of strings representing info types to look for. + A full list of info type categories can be fetched from the API. + namespace_id: The namespace of the Datastore document, if applicable. + min_likelihood: A string representing the minimum likelihood threshold + that constitutes a match. One of: 'LIKELIHOOD_UNSPECIFIED', + 'VERY_UNLIKELY', 'UNLIKELY', 'POSSIBLE', 'LIKELY', 'VERY_LIKELY'. + max_findings: The maximum number of findings to report; 0 = no maximum. + timeout: The number of seconds to wait for a response from the API. + Returns: + None; the response from the API is printed to the terminal. + """ + + # Import the client library. + import google.cloud.dlp + + # This sample additionally uses Cloud Pub/Sub to receive results from + # potentially long-running operations. + import google.cloud.pubsub + + # This sample also uses threading.Event() to wait for the job to finish. + import threading + + # Instantiate a client. + dlp = google.cloud.dlp.DlpServiceClient() + + # Prepare info_types by converting the list of strings into a list of + # dictionaries (protos are also accepted). + if not info_types: + info_types = ['FIRST_NAME', 'LAST_NAME', 'EMAIL_ADDRESS'] + info_types = [{'name': info_type} for info_type in info_types] + + # Prepare custom_info_types by parsing the dictionary word lists and + # regex patterns. + if custom_dictionaries is None: + custom_dictionaries = [] + dictionaries = [{ + 'info_type': {'name': 'CUSTOM_DICTIONARY_{}'.format(i)}, + 'dictionary': { + 'word_list': {'words': custom_dict.split(',')} + } + } for i, custom_dict in enumerate(custom_dictionaries)] + if custom_regexes is None: + custom_regexes = [] + regexes = [{ + 'info_type': {'name': 'CUSTOM_REGEX_{}'.format(i)}, + 'regex': {'pattern': custom_regex} + } for i, custom_regex in enumerate(custom_regexes)] + custom_info_types = dictionaries + regexes + + # Construct the configuration dictionary. Keys which are None may + # optionally be omitted entirely. + inspect_config = { + 'info_types': info_types, + 'custom_info_types': custom_info_types, + 'min_likelihood': min_likelihood, + 'limits': {'max_findings_per_request': max_findings}, + } + + # Construct a storage_config containing the target Datastore info. + storage_config = { + 'datastore_options': { + 'partition_id': { + 'project_id': datastore_project, + 'namespace_id': namespace_id, + }, + 'kind': { + 'name': kind + }, + } + } + + # Convert the project id into a full resource id. + parent = dlp.project_path(project) + + # Tell the API where to send a notification when the job is complete. + actions = [{ + 'pub_sub': {'topic': '{}/topics/{}'.format(parent, topic_id)} + }] + + # Construct the inspect_job, which defines the entire inspect content task. + inspect_job = { + 'inspect_config': inspect_config, + 'storage_config': storage_config, + 'actions': actions, + } + + operation = dlp.create_dlp_job(parent, inspect_job=inspect_job) + + # Create a Pub/Sub client and find the subscription. The subscription is + # expected to already be listening to the topic. + subscriber = google.cloud.pubsub.SubscriberClient() + subscription_path = subscriber.subscription_path( + project, subscription_id) + + # Set up a callback to acknowledge a message. This closes around an event + # so that it can signal that it is done and the main thread can continue. + job_done = threading.Event() + + def callback(message): + try: + if (message.attributes['DlpJobName'] == operation.name): + # This is the message we're looking for, so acknowledge it. + message.ack() + + # Now that the job is done, fetch the results and print them. + job = dlp.get_dlp_job(operation.name) + if job.inspect_details.result.info_type_stats: + for finding in job.inspect_details.result.info_type_stats: + print('Info type: {}; Count: {}'.format( + finding.info_type.name, finding.count)) + else: + print('No findings.') + + # Signal to the main thread that we can exit. + job_done.set() + else: + # This is not the message we're looking for. + message.drop() + except Exception as e: + # Because this is executing in a thread, an exception won't be + # noted unless we print it manually. + print(e) + raise + + # Register the callback and wait on the event. + subscriber.subscribe(subscription_path, callback=callback) + + finished = job_done.wait(timeout=timeout) + if not finished: + print('No event received before the timeout. Please verify that the ' + 'subscription provided is subscribed to the topic provided.') + +# [END dlp_inspect_datastore] + + +# [START dlp_inspect_bigquery] +def inspect_bigquery(project, bigquery_project, dataset_id, table_id, + topic_id, subscription_id, info_types, + custom_dictionaries=None, custom_regexes=None, + min_likelihood=None, max_findings=None, timeout=300): + """Uses the Data Loss Prevention API to analyze BigQuery data. + Args: + project: The Google Cloud project id to use as a parent resource. + bigquery_project: The Google Cloud project id of the target table. + dataset_id: The id of the target BigQuery dataset. + table_id: The id of the target BigQuery table. + topic_id: The id of the Cloud Pub/Sub topic to which the API will + broadcast job completion. The topic must already exist. + subscription_id: The id of the Cloud Pub/Sub subscription to listen on + while waiting for job completion. The subscription must already + exist and be subscribed to the topic. + info_types: A list of strings representing info types to look for. + A full list of info type categories can be fetched from the API. + namespace_id: The namespace of the Datastore document, if applicable. + min_likelihood: A string representing the minimum likelihood threshold + that constitutes a match. One of: 'LIKELIHOOD_UNSPECIFIED', + 'VERY_UNLIKELY', 'UNLIKELY', 'POSSIBLE', 'LIKELY', 'VERY_LIKELY'. + max_findings: The maximum number of findings to report; 0 = no maximum. + timeout: The number of seconds to wait for a response from the API. + Returns: + None; the response from the API is printed to the terminal. + """ + + # Import the client library. + import google.cloud.dlp + + # This sample additionally uses Cloud Pub/Sub to receive results from + # potentially long-running operations. + import google.cloud.pubsub + + # This sample also uses threading.Event() to wait for the job to finish. + import threading + + # Instantiate a client. + dlp = google.cloud.dlp.DlpServiceClient() + + # Prepare info_types by converting the list of strings into a list of + # dictionaries (protos are also accepted). + if not info_types: + info_types = ['FIRST_NAME', 'LAST_NAME', 'EMAIL_ADDRESS'] + info_types = [{'name': info_type} for info_type in info_types] + + # Prepare custom_info_types by parsing the dictionary word lists and + # regex patterns. + if custom_dictionaries is None: + custom_dictionaries = [] + dictionaries = [{ + 'info_type': {'name': 'CUSTOM_DICTIONARY_{}'.format(i)}, + 'dictionary': { + 'word_list': {'words': custom_dict.split(',')} + } + } for i, custom_dict in enumerate(custom_dictionaries)] + if custom_regexes is None: + custom_regexes = [] + regexes = [{ + 'info_type': {'name': 'CUSTOM_REGEX_{}'.format(i)}, + 'regex': {'pattern': custom_regex} + } for i, custom_regex in enumerate(custom_regexes)] + custom_info_types = dictionaries + regexes + + # Construct the configuration dictionary. Keys which are None may + # optionally be omitted entirely. + inspect_config = { + 'info_types': info_types, + 'custom_info_types': custom_info_types, + 'min_likelihood': min_likelihood, + 'limits': {'max_findings_per_request': max_findings}, + } + + # Construct a storage_config containing the target Bigquery info. + storage_config = { + 'big_query_options': { + 'table_reference': { + 'project_id': bigquery_project, + 'dataset_id': dataset_id, + 'table_id': table_id, + } + } + } + + # Convert the project id into a full resource id. + parent = dlp.project_path(project) + + # Tell the API where to send a notification when the job is complete. + actions = [{ + 'pub_sub': {'topic': '{}/topics/{}'.format(parent, topic_id)} + }] + + # Construct the inspect_job, which defines the entire inspect content task. + inspect_job = { + 'inspect_config': inspect_config, + 'storage_config': storage_config, + 'actions': actions, + } + + operation = dlp.create_dlp_job(parent, inspect_job=inspect_job) + + # Create a Pub/Sub client and find the subscription. The subscription is + # expected to already be listening to the topic. + subscriber = google.cloud.pubsub.SubscriberClient() + subscription_path = subscriber.subscription_path( + project, subscription_id) + + # Set up a callback to acknowledge a message. This closes around an event + # so that it can signal that it is done and the main thread can continue. + job_done = threading.Event() + + def callback(message): + try: + if (message.attributes['DlpJobName'] == operation.name): + # This is the message we're looking for, so acknowledge it. + message.ack() + + # Now that the job is done, fetch the results and print them. + job = dlp.get_dlp_job(operation.name) + if job.inspect_details.result.info_type_stats: + for finding in job.inspect_details.result.info_type_stats: + print('Info type: {}; Count: {}'.format( + finding.info_type.name, finding.count)) + else: + print('No findings.') + + # Signal to the main thread that we can exit. + job_done.set() + else: + # This is not the message we're looking for. + message.drop() + except Exception as e: + # Because this is executing in a thread, an exception won't be + # noted unless we print it manually. + print(e) + raise + + # Register the callback and wait on the event. + subscriber.subscribe(subscription_path, callback=callback) + finished = job_done.wait(timeout=timeout) + if not finished: + print('No event received before the timeout. Please verify that the ' + 'subscription provided is subscribed to the topic provided.') + +# [END dlp_inspect_bigquery] + + +if __name__ == '__main__': + default_project = os.environ.get('GCLOUD_PROJECT') + + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers( + dest='content', help='Select how to submit content to the API.') + subparsers.required = True + + parser_string = subparsers.add_parser('string', help='Inspect a string.') + parser_string.add_argument('item', help='The string to inspect.') + parser_string.add_argument( + '--project', + help='The Google Cloud project id to use as a parent resource.', + default=default_project) + parser_string.add_argument( + '--info_types', action='append', + help='Strings representing info types to look for. A full list of ' + 'info categories and types is available from the API. Examples ' + 'include "FIRST_NAME", "LAST_NAME", "EMAIL_ADDRESS". ' + 'If unspecified, the three above examples will be used.', + default=['FIRST_NAME', 'LAST_NAME', 'EMAIL_ADDRESS']) + parser_string.add_argument( + '--custom_dictionaries', action='append', + help='Strings representing comma-delimited lists of dictionary words' + ' to search for as custom info types. Each string is a comma ' + 'delimited list of words representing a distinct dictionary.', + default=None) + parser_string.add_argument( + '--custom_regexes', action='append', + help='Strings representing regex patterns to search for as custom ' + ' info types.', + default=None) + parser_string.add_argument( + '--min_likelihood', + choices=['LIKELIHOOD_UNSPECIFIED', 'VERY_UNLIKELY', 'UNLIKELY', + 'POSSIBLE', 'LIKELY', 'VERY_LIKELY'], + help='A string representing the minimum likelihood threshold that ' + 'constitutes a match.') + parser_string.add_argument( + '--max_findings', type=int, + help='The maximum number of findings to report; 0 = no maximum.') + parser_string.add_argument( + '--include_quote', type=bool, + help='A boolean for whether to display a quote of the detected ' + 'information in the results.', + default=True) + + parser_table = subparsers.add_parser('table', help='Inspect a table.') + parser_table.add_argument( + 'data', help='Json string representing a table.', type=json.loads) + parser_table.add_argument( + '--project', + help='The Google Cloud project id to use as a parent resource.', + default=default_project) + parser_table.add_argument( + '--info_types', action='append', + help='Strings representing info types to look for. A full list of ' + 'info categories and types is available from the API. Examples ' + 'include "FIRST_NAME", "LAST_NAME", "EMAIL_ADDRESS". ' + 'If unspecified, the three above examples will be used.', + default=['FIRST_NAME', 'LAST_NAME', 'EMAIL_ADDRESS']) + parser_table.add_argument( + '--custom_dictionaries', action='append', + help='Strings representing comma-delimited lists of dictionary words' + ' to search for as custom info types. Each string is a comma ' + 'delimited list of words representing a distinct dictionary.', + default=None) + parser_table.add_argument( + '--custom_regexes', action='append', + help='Strings representing regex patterns to search for as custom ' + ' info types.', + default=None) + parser_table.add_argument( + '--min_likelihood', + choices=['LIKELIHOOD_UNSPECIFIED', 'VERY_UNLIKELY', 'UNLIKELY', + 'POSSIBLE', 'LIKELY', 'VERY_LIKELY'], + help='A string representing the minimum likelihood threshold that ' + 'constitutes a match.') + parser_table.add_argument( + '--max_findings', type=int, + help='The maximum number of findings to report; 0 = no maximum.') + parser_table.add_argument( + '--include_quote', type=bool, + help='A boolean for whether to display a quote of the detected ' + 'information in the results.', + default=True) + + parser_file = subparsers.add_parser('file', help='Inspect a local file.') + parser_file.add_argument( + 'filename', help='The path to the file to inspect.') + parser_file.add_argument( + '--project', + help='The Google Cloud project id to use as a parent resource.', + default=default_project) + parser_file.add_argument( + '--info_types', action='append', + help='Strings representing info types to look for. A full list of ' + 'info categories and types is available from the API. Examples ' + 'include "FIRST_NAME", "LAST_NAME", "EMAIL_ADDRESS". ' + 'If unspecified, the three above examples will be used.', + default=['FIRST_NAME', 'LAST_NAME', 'EMAIL_ADDRESS']) + parser_file.add_argument( + '--custom_dictionaries', action='append', + help='Strings representing comma-delimited lists of dictionary words' + ' to search for as custom info types. Each string is a comma ' + 'delimited list of words representing a distinct dictionary.', + default=None) + parser_file.add_argument( + '--custom_regexes', action='append', + help='Strings representing regex patterns to search for as custom ' + ' info types.', + default=None) + parser_file.add_argument( + '--min_likelihood', + choices=['LIKELIHOOD_UNSPECIFIED', 'VERY_UNLIKELY', 'UNLIKELY', + 'POSSIBLE', 'LIKELY', 'VERY_LIKELY'], + help='A string representing the minimum likelihood threshold that ' + 'constitutes a match.') + parser_file.add_argument( + '--max_findings', type=int, + help='The maximum number of findings to report; 0 = no maximum.') + parser_file.add_argument( + '--include_quote', type=bool, + help='A boolean for whether to display a quote of the detected ' + 'information in the results.', + default=True) + parser_file.add_argument( + '--mime_type', + help='The MIME type of the file. If not specified, the type is ' + 'inferred via the Python standard library\'s mimetypes module.') + + parser_gcs = subparsers.add_parser( + 'gcs', help='Inspect files on Google Cloud Storage.') + parser_gcs.add_argument( + 'bucket', help='The name of the GCS bucket containing the file.') + parser_gcs.add_argument( + 'filename', + help='The name of the file in the bucket, including the path, e.g. ' + '"images/myfile.png". Wildcards are permitted.') + parser_gcs.add_argument( + 'topic_id', + help='The id of the Cloud Pub/Sub topic to use to report that the job ' + 'is complete, e.g. "dlp-sample-topic".') + parser_gcs.add_argument( + 'subscription_id', + help='The id of the Cloud Pub/Sub subscription to monitor for job ' + 'completion, e.g. "dlp-sample-subscription". The subscription must ' + 'already be subscribed to the topic. See the test files or the Cloud ' + 'Pub/Sub sample files for examples on how to create the subscription.') + parser_gcs.add_argument( + '--project', + help='The Google Cloud project id to use as a parent resource.', + default=default_project) + parser_gcs.add_argument( + '--info_types', action='append', + help='Strings representing info types to look for. A full list of ' + 'info categories and types is available from the API. Examples ' + 'include "FIRST_NAME", "LAST_NAME", "EMAIL_ADDRESS". ' + 'If unspecified, the three above examples will be used.', + default=['FIRST_NAME', 'LAST_NAME', 'EMAIL_ADDRESS']) + parser_gcs.add_argument( + '--custom_dictionaries', action='append', + help='Strings representing comma-delimited lists of dictionary words' + ' to search for as custom info types. Each string is a comma ' + 'delimited list of words representing a distinct dictionary.', + default=None) + parser_gcs.add_argument( + '--custom_regexes', action='append', + help='Strings representing regex patterns to search for as custom ' + ' info types.', + default=None) + parser_gcs.add_argument( + '--min_likelihood', + choices=['LIKELIHOOD_UNSPECIFIED', 'VERY_UNLIKELY', 'UNLIKELY', + 'POSSIBLE', 'LIKELY', 'VERY_LIKELY'], + help='A string representing the minimum likelihood threshold that ' + 'constitutes a match.') + parser_gcs.add_argument( + '--max_findings', type=int, + help='The maximum number of findings to report; 0 = no maximum.') + parser_gcs.add_argument( + '--timeout', type=int, + help='The maximum number of seconds to wait for a response from the ' + 'API. The default is 300 seconds.', + default=300) + + parser_datastore = subparsers.add_parser( + 'datastore', help='Inspect files on Google Datastore.') + parser_datastore.add_argument( + 'datastore_project', + help='The Google Cloud project id of the target Datastore.') + parser_datastore.add_argument( + 'kind', + help='The kind of the Datastore entity to inspect, e.g. "Person".') + parser_datastore.add_argument( + 'topic_id', + help='The id of the Cloud Pub/Sub topic to use to report that the job ' + 'is complete, e.g. "dlp-sample-topic".') + parser_datastore.add_argument( + 'subscription_id', + help='The id of the Cloud Pub/Sub subscription to monitor for job ' + 'completion, e.g. "dlp-sample-subscription". The subscription must ' + 'already be subscribed to the topic. See the test files or the Cloud ' + 'Pub/Sub sample files for examples on how to create the subscription.') + parser_datastore.add_argument( + '--project', + help='The Google Cloud project id to use as a parent resource.', + default=default_project) + parser_datastore.add_argument( + '--info_types', action='append', + help='Strings representing info types to look for. A full list of ' + 'info categories and types is available from the API. Examples ' + 'include "FIRST_NAME", "LAST_NAME", "EMAIL_ADDRESS". ' + 'If unspecified, the three above examples will be used.', + default=['FIRST_NAME', 'LAST_NAME', 'EMAIL_ADDRESS']) + parser_datastore.add_argument( + '--custom_dictionaries', action='append', + help='Strings representing comma-delimited lists of dictionary words' + ' to search for as custom info types. Each string is a comma ' + 'delimited list of words representing a distinct dictionary.', + default=None) + parser_datastore.add_argument( + '--custom_regexes', action='append', + help='Strings representing regex patterns to search for as custom ' + ' info types.', + default=None) + parser_datastore.add_argument( + '--namespace_id', + help='The Datastore namespace to use, if applicable.') + parser_datastore.add_argument( + '--min_likelihood', + choices=['LIKELIHOOD_UNSPECIFIED', 'VERY_UNLIKELY', 'UNLIKELY', + 'POSSIBLE', 'LIKELY', 'VERY_LIKELY'], + help='A string representing the minimum likelihood threshold that ' + 'constitutes a match.') + parser_datastore.add_argument( + '--max_findings', type=int, + help='The maximum number of findings to report; 0 = no maximum.') + parser_datastore.add_argument( + '--timeout', type=int, + help='The maximum number of seconds to wait for a response from the ' + 'API. The default is 300 seconds.', + default=300) + + parser_bigquery = subparsers.add_parser( + 'bigquery', help='Inspect files on Google BigQuery.') + parser_bigquery.add_argument( + 'bigquery_project', + help='The Google Cloud project id of the target table.') + parser_bigquery.add_argument( + 'dataset_id', + help='The ID of the target BigQuery dataset.') + parser_bigquery.add_argument( + 'table_id', + help='The ID of the target BigQuery table.') + parser_bigquery.add_argument( + 'topic_id', + help='The id of the Cloud Pub/Sub topic to use to report that the job ' + 'is complete, e.g. "dlp-sample-topic".') + parser_bigquery.add_argument( + 'subscription_id', + help='The id of the Cloud Pub/Sub subscription to monitor for job ' + 'completion, e.g. "dlp-sample-subscription". The subscription must ' + 'already be subscribed to the topic. See the test files or the Cloud ' + 'Pub/Sub sample files for examples on how to create the subscription.') + parser_bigquery.add_argument( + '--project', + help='The Google Cloud project id to use as a parent resource.', + default=default_project) + parser_bigquery.add_argument( + '--info_types', action='append', + help='Strings representing info types to look for. A full list of ' + 'info categories and types is available from the API. Examples ' + 'include "FIRST_NAME", "LAST_NAME", "EMAIL_ADDRESS". ' + 'If unspecified, the three above examples will be used.', + default=['FIRST_NAME', 'LAST_NAME', 'EMAIL_ADDRESS']) + parser_bigquery.add_argument( + '--custom_dictionaries', action='append', + help='Strings representing comma-delimited lists of dictionary words' + ' to search for as custom info types. Each string is a comma ' + 'delimited list of words representing a distinct dictionary.', + default=None) + parser_bigquery.add_argument( + '--custom_regexes', action='append', + help='Strings representing regex patterns to search for as custom ' + ' info types.', + default=None) + parser_bigquery.add_argument( + '--min_likelihood', + choices=['LIKELIHOOD_UNSPECIFIED', 'VERY_UNLIKELY', 'UNLIKELY', + 'POSSIBLE', 'LIKELY', 'VERY_LIKELY'], + help='A string representing the minimum likelihood threshold that ' + 'constitutes a match.') + parser_bigquery.add_argument( + '--max_findings', type=int, + help='The maximum number of findings to report; 0 = no maximum.') + parser_bigquery.add_argument( + '--timeout', type=int, + help='The maximum number of seconds to wait for a response from the ' + 'API. The default is 300 seconds.', + default=300) + + args = parser.parse_args() + + if args.content == 'string': + inspect_string( + args.project, args.item, args.info_types, + custom_dictionaries=args.custom_dictionaries, + custom_regexes=args.custom_regexes, + min_likelihood=args.min_likelihood, + max_findings=args.max_findings, + include_quote=args.include_quote) + elif args.content == 'table': + inspect_table( + args.project, args.data, args.info_types, + custom_dictionaries=args.custom_dictionaries, + custom_regexes=args.custom_regexes, + min_likelihood=args.min_likelihood, + max_findings=args.max_findings, + include_quote=args.include_quote) + elif args.content == 'file': + inspect_file( + args.project, args.filename, args.info_types, + custom_dictionaries=args.custom_dictionaries, + custom_regexes=args.custom_regexes, + min_likelihood=args.min_likelihood, + max_findings=args.max_findings, + include_quote=args.include_quote, + mime_type=args.mime_type) + elif args.content == 'gcs': + inspect_gcs_file( + args.project, args.bucket, args.filename, + args.topic_id, args.subscription_id, + args.info_types, + custom_dictionaries=args.custom_dictionaries, + custom_regexes=args.custom_regexes, + min_likelihood=args.min_likelihood, + max_findings=args.max_findings, + timeout=args.timeout) + elif args.content == 'datastore': + inspect_datastore( + args.project, args.datastore_project, args.kind, + args.topic_id, args.subscription_id, + args.info_types, + custom_dictionaries=args.custom_dictionaries, + custom_regexes=args.custom_regexes, + namespace_id=args.namespace_id, + min_likelihood=args.min_likelihood, + max_findings=args.max_findings, + timeout=args.timeout) + elif args.content == 'bigquery': + inspect_bigquery( + args.project, args.bigquery_project, args.dataset_id, + args.table_id, args.topic_id, args.subscription_id, + args.info_types, + custom_dictionaries=args.custom_dictionaries, + custom_regexes=args.custom_regexes, + min_likelihood=args.min_likelihood, + max_findings=args.max_findings, + timeout=args.timeout) diff --git a/dlp/inspect_content_test.py b/dlp/inspect_content_test.py new file mode 100644 index 00000000000..d5dd84a6f2f --- /dev/null +++ b/dlp/inspect_content_test.py @@ -0,0 +1,413 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from gcp_devrel.testing import eventually_consistent +from gcp_devrel.testing.flaky import flaky +import google.api_core.exceptions +import google.cloud.bigquery +import google.cloud.datastore +import google.cloud.exceptions +import google.cloud.pubsub +import google.cloud.storage + +import pytest +import inspect_content + + +GCLOUD_PROJECT = os.getenv('GCLOUD_PROJECT') +TEST_BUCKET_NAME = GCLOUD_PROJECT + '-dlp-python-client-test' +RESOURCE_DIRECTORY = os.path.join(os.path.dirname(__file__), 'resources') +RESOURCE_FILE_NAMES = ['test.txt', 'test.png', 'harmless.txt', 'accounts.txt'] +TOPIC_ID = 'dlp-test' +SUBSCRIPTION_ID = 'dlp-test-subscription' +DATASTORE_KIND = 'DLP test kind' +BIGQUERY_DATASET_ID = 'dlp_test_dataset' +BIGQUERY_TABLE_ID = 'dlp_test_table' + + +@pytest.fixture(scope='module') +def bucket(): + # Creates a GCS bucket, uploads files required for the test, and tears down + # the entire bucket afterwards. + + client = google.cloud.storage.Client() + try: + bucket = client.get_bucket(TEST_BUCKET_NAME) + except google.cloud.exceptions.NotFound: + bucket = client.create_bucket(TEST_BUCKET_NAME) + + # Upoad the blobs and keep track of them in a list. + blobs = [] + for name in RESOURCE_FILE_NAMES: + path = os.path.join(RESOURCE_DIRECTORY, name) + blob = bucket.blob(name) + blob.upload_from_filename(path) + blobs.append(blob) + + # Yield the object to the test; lines after this execute as a teardown. + yield bucket + + # Delete the files. + for blob in blobs: + blob.delete() + + # Attempt to delete the bucket; this will only work if it is empty. + bucket.delete() + + +@pytest.fixture(scope='module') +def topic_id(): + # Creates a pubsub topic, and tears it down. + publisher = google.cloud.pubsub.PublisherClient() + topic_path = publisher.topic_path(GCLOUD_PROJECT, TOPIC_ID) + try: + publisher.create_topic(topic_path) + except google.api_core.exceptions.AlreadyExists: + pass + + yield TOPIC_ID + + publisher.delete_topic(topic_path) + + +@pytest.fixture(scope='module') +def subscription_id(topic_id): + # Subscribes to a topic. + subscriber = google.cloud.pubsub.SubscriberClient() + topic_path = subscriber.topic_path(GCLOUD_PROJECT, topic_id) + subscription_path = subscriber.subscription_path( + GCLOUD_PROJECT, SUBSCRIPTION_ID) + try: + subscriber.create_subscription(subscription_path, topic_path) + except google.api_core.exceptions.AlreadyExists: + pass + + yield SUBSCRIPTION_ID + + subscriber.delete_subscription(subscription_path) + + +@pytest.fixture(scope='module') +def datastore_project(): + # Adds test Datastore data, yields the project ID and then tears down. + datastore_client = google.cloud.datastore.Client() + + kind = DATASTORE_KIND + name = 'DLP test object' + key = datastore_client.key(kind, name) + item = google.cloud.datastore.Entity(key=key) + item['payload'] = 'My name is Gary Smith and my email is gary@example.com' + + datastore_client.put(item) + + yield GCLOUD_PROJECT + + datastore_client.delete(key) + + +@pytest.fixture(scope='module') +def bigquery_project(): + # Adds test Bigquery data, yields the project ID and then tears down. + bigquery_client = google.cloud.bigquery.Client() + + dataset_ref = bigquery_client.dataset(BIGQUERY_DATASET_ID) + dataset = google.cloud.bigquery.Dataset(dataset_ref) + try: + dataset = bigquery_client.create_dataset(dataset) + except google.api_core.exceptions.Conflict: + dataset = bigquery_client.get_dataset(dataset) + + table_ref = dataset_ref.table(BIGQUERY_TABLE_ID) + table = google.cloud.bigquery.Table(table_ref) + + # DO NOT SUBMIT: trim this down once we find out what works + table.schema = ( + google.cloud.bigquery.SchemaField('Name', 'STRING'), + google.cloud.bigquery.SchemaField('Comment', 'STRING'), + ) + + try: + table = bigquery_client.create_table(table) + except google.api_core.exceptions.Conflict: + table = bigquery_client.get_table(table) + + rows_to_insert = [ + (u'Gary Smith', u'My email is gary@example.com',) + ] + + bigquery_client.insert_rows(table, rows_to_insert) + + yield GCLOUD_PROJECT + + bigquery_client.delete_dataset(dataset_ref, delete_contents=True) + + +def test_inspect_string(capsys): + test_string = 'My name is Gary Smith and my email is gary@example.com' + + inspect_content.inspect_string( + GCLOUD_PROJECT, + test_string, + ['FIRST_NAME', 'EMAIL_ADDRESS'], + include_quote=True) + + out, _ = capsys.readouterr() + assert 'Info type: FIRST_NAME' in out + assert 'Info type: EMAIL_ADDRESS' in out + + +def test_inspect_table(capsys): + test_tabular_data = { + "header": [ + "email", + "phone number" + ], + "rows": [ + [ + "robertfrost@xyz.com", + "4232342345" + ], + [ + "johndoe@pqr.com", + "4253458383" + ] + ] + } + + inspect_content.inspect_table( + GCLOUD_PROJECT, + test_tabular_data, + ['PHONE_NUMBER', 'EMAIL_ADDRESS'], + include_quote=True) + + out, _ = capsys.readouterr() + assert 'Info type: PHONE_NUMBER' in out + assert 'Info type: EMAIL_ADDRESS' in out + + +def test_inspect_string_with_custom_info_types(capsys): + test_string = 'My name is Gary Smith and my email is gary@example.com' + dictionaries = ['Gary Smith'] + regexes = ['\\w+@\\w+.com'] + + inspect_content.inspect_string( + GCLOUD_PROJECT, + test_string, + [], + custom_dictionaries=dictionaries, + custom_regexes=regexes, + include_quote=True) + + out, _ = capsys.readouterr() + assert 'Info type: CUSTOM_DICTIONARY_0' in out + assert 'Info type: CUSTOM_REGEX_0' in out + + +def test_inspect_string_no_results(capsys): + test_string = 'Nothing to see here' + + inspect_content.inspect_string( + GCLOUD_PROJECT, + test_string, + ['FIRST_NAME', 'EMAIL_ADDRESS'], + include_quote=True) + + out, _ = capsys.readouterr() + assert 'No findings' in out + + +def test_inspect_file(capsys): + test_filepath = os.path.join(RESOURCE_DIRECTORY, 'test.txt') + + inspect_content.inspect_file( + GCLOUD_PROJECT, + test_filepath, + ['FIRST_NAME', 'EMAIL_ADDRESS'], + include_quote=True) + + out, _ = capsys.readouterr() + assert 'Info type: EMAIL_ADDRESS' in out + + +def test_inspect_file_with_custom_info_types(capsys): + test_filepath = os.path.join(RESOURCE_DIRECTORY, 'test.txt') + dictionaries = ['gary@somedomain.com'] + regexes = ['\\(\\d{3}\\) \\d{3}-\\d{4}'] + + inspect_content.inspect_file( + GCLOUD_PROJECT, + test_filepath, + [], + custom_dictionaries=dictionaries, + custom_regexes=regexes, + include_quote=True) + + out, _ = capsys.readouterr() + assert 'Info type: CUSTOM_DICTIONARY_0' in out + assert 'Info type: CUSTOM_REGEX_0' in out + + +def test_inspect_file_no_results(capsys): + test_filepath = os.path.join(RESOURCE_DIRECTORY, 'harmless.txt') + + inspect_content.inspect_file( + GCLOUD_PROJECT, + test_filepath, + ['FIRST_NAME', 'EMAIL_ADDRESS'], + include_quote=True) + + out, _ = capsys.readouterr() + assert 'No findings' in out + + +def test_inspect_image_file(capsys): + test_filepath = os.path.join(RESOURCE_DIRECTORY, 'test.png') + + inspect_content.inspect_file( + GCLOUD_PROJECT, + test_filepath, + ['FIRST_NAME', 'EMAIL_ADDRESS', 'PHONE_NUMBER'], + include_quote=True) + + out, _ = capsys.readouterr() + assert 'Info type: PHONE_NUMBER' in out + + +@flaky +def test_inspect_gcs_file(bucket, topic_id, subscription_id, capsys): + inspect_content.inspect_gcs_file( + GCLOUD_PROJECT, + bucket.name, + 'test.txt', + topic_id, + subscription_id, + ['FIRST_NAME', 'EMAIL_ADDRESS', 'PHONE_NUMBER']) + + out, _ = capsys.readouterr() + assert 'Info type: EMAIL_ADDRESS' in out + + +@flaky +def test_inspect_gcs_file_with_custom_info_types(bucket, topic_id, + subscription_id, capsys): + dictionaries = ['gary@somedomain.com'] + regexes = ['\\(\\d{3}\\) \\d{3}-\\d{4}'] + + inspect_content.inspect_gcs_file( + GCLOUD_PROJECT, + bucket.name, + 'test.txt', + topic_id, + subscription_id, + [], + custom_dictionaries=dictionaries, + custom_regexes=regexes) + + out, _ = capsys.readouterr() + assert 'Info type: CUSTOM_DICTIONARY_0' in out + assert 'Info type: CUSTOM_REGEX_0' in out + + +@flaky +def test_inspect_gcs_file_no_results( + bucket, topic_id, subscription_id, capsys): + inspect_content.inspect_gcs_file( + GCLOUD_PROJECT, + bucket.name, + 'harmless.txt', + topic_id, + subscription_id, + ['FIRST_NAME', 'EMAIL_ADDRESS', 'PHONE_NUMBER']) + + out, _ = capsys.readouterr() + assert 'No findings' in out + + +@pytest.mark.skip(reason='nondeterministically failing') +def test_inspect_gcs_image_file(bucket, topic_id, subscription_id, capsys): + inspect_content.inspect_gcs_file( + GCLOUD_PROJECT, + bucket.name, + 'test.png', + topic_id, + subscription_id, + ['FIRST_NAME', 'EMAIL_ADDRESS', 'PHONE_NUMBER']) + + out, _ = capsys.readouterr() + assert 'Info type: EMAIL_ADDRESS' in out + + +@flaky +def test_inspect_gcs_multiple_files(bucket, topic_id, subscription_id, capsys): + inspect_content.inspect_gcs_file( + GCLOUD_PROJECT, + bucket.name, + '*', + topic_id, + subscription_id, + ['FIRST_NAME', 'EMAIL_ADDRESS', 'PHONE_NUMBER']) + + out, _ = capsys.readouterr() + assert 'Info type: EMAIL_ADDRESS' in out + assert 'Info type: PHONE_NUMBER' in out + + +@flaky +def test_inspect_datastore( + datastore_project, topic_id, subscription_id, capsys): + @eventually_consistent.call + def _(): + inspect_content.inspect_datastore( + GCLOUD_PROJECT, + datastore_project, + DATASTORE_KIND, + topic_id, + subscription_id, + ['FIRST_NAME', 'EMAIL_ADDRESS', 'PHONE_NUMBER']) + + out, _ = capsys.readouterr() + assert 'Info type: EMAIL_ADDRESS' in out + + +@flaky +def test_inspect_datastore_no_results( + datastore_project, topic_id, subscription_id, capsys): + inspect_content.inspect_datastore( + GCLOUD_PROJECT, + datastore_project, + DATASTORE_KIND, + topic_id, + subscription_id, + ['PHONE_NUMBER']) + + out, _ = capsys.readouterr() + assert 'No findings' in out + + +@pytest.mark.skip(reason='unknown issue') +def test_inspect_bigquery( + bigquery_project, topic_id, subscription_id, capsys): + inspect_content.inspect_bigquery( + GCLOUD_PROJECT, + bigquery_project, + BIGQUERY_DATASET_ID, + BIGQUERY_TABLE_ID, + topic_id, + subscription_id, + ['FIRST_NAME', 'EMAIL_ADDRESS', 'PHONE_NUMBER']) + + out, _ = capsys.readouterr() + assert 'Info type: FIRST_NAME' in out diff --git a/dlp/jobs.py b/dlp/jobs.py new file mode 100644 index 00000000000..43c9c34a3dc --- /dev/null +++ b/dlp/jobs.py @@ -0,0 +1,158 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sample app to list and delete DLP jobs using the Data Loss Prevent API. """ + +from __future__ import print_function + +import argparse + + +# [START dlp_list_jobs] +def list_dlp_jobs(project, filter_string=None, job_type=None): + """Uses the Data Loss Prevention API to lists DLP jobs that match the + specified filter in the request. + Args: + project: The project id to use as a parent resource. + filter: (Optional) Allows filtering. + Supported syntax: + * Filter expressions are made up of one or more restrictions. + * Restrictions can be combined by 'AND' or 'OR' logical operators. + A sequence of restrictions implicitly uses 'AND'. + * A restriction has the form of ' '. + * Supported fields/values for inspect jobs: + - `state` - PENDING|RUNNING|CANCELED|FINISHED|FAILED + - `inspected_storage` - DATASTORE|CLOUD_STORAGE|BIGQUERY + - `trigger_name` - The resource name of the trigger that + created job. + * Supported fields for risk analysis jobs: + - `state` - RUNNING|CANCELED|FINISHED|FAILED + * The operator must be '=' or '!='. + Examples: + * inspected_storage = cloud_storage AND state = done + * inspected_storage = cloud_storage OR inspected_storage = bigquery + * inspected_storage = cloud_storage AND + (state = done OR state = canceled) + type: (Optional) The type of job. Defaults to 'INSPECT'. + Choices: + DLP_JOB_TYPE_UNSPECIFIED + INSPECT_JOB: The job inspected content for sensitive data. + RISK_ANALYSIS_JOB: The job executed a Risk Analysis computation. + + Returns: + None; the response from the API is printed to the terminal. + """ + + # Import the client library. + import google.cloud.dlp + + # Instantiate a client. + dlp = google.cloud.dlp.DlpServiceClient() + + # Convert the project id into a full resource id. + parent = dlp.project_path(project) + + # Job type dictionary + job_type_to_int = { + 'DLP_JOB_TYPE_UNSPECIFIED': + google.cloud.dlp.enums.DlpJobType.DLP_JOB_TYPE_UNSPECIFIED, + 'INSPECT_JOB': google.cloud.dlp.enums.DlpJobType.INSPECT_JOB, + 'RISK_ANALYSIS_JOB': + google.cloud.dlp.enums.DlpJobType.RISK_ANALYSIS_JOB + } + # If job type is specified, convert job type to number through enums. + if job_type: + job_type = job_type_to_int[job_type] + + # Call the API to get a list of jobs. + response = dlp.list_dlp_jobs( + parent, + filter_=filter_string, + type_=job_type) + + # Iterate over results. + for job in response: + print('Job: %s; status: %s' % (job.name, job.JobState.Name(job.state))) +# [END dlp_list_jobs] + + +# [START dlp_delete_job] +def delete_dlp_job(project, job_name): + """Uses the Data Loss Prevention API to delete a long-running DLP job. + Args: + project: The project id to use as a parent resource. + job_name: The name of the DlpJob resource to be deleted. + + Returns: + None; the response from the API is printed to the terminal. + """ + + # Import the client library. + import google.cloud.dlp + + # Instantiate a client. + dlp = google.cloud.dlp.DlpServiceClient() + + # Convert the project id and job name into a full resource id. + name = dlp.dlp_job_path(project, job_name) + + # Call the API to delete job. + dlp.delete_dlp_job(name) + + print('Successfully deleted %s' % job_name) +# [END dlp_delete_job] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers( + dest='content', help='Select how to submit content to the API.') + subparsers.required = True + + list_parser = subparsers.add_parser( + 'list', + help='List Data Loss Prevention API jobs corresponding to a given ' + 'filter.') + list_parser.add_argument( + 'project', + help='The project id to use as a parent resource.') + list_parser.add_argument( + '-f', '--filter', + help='Filter expressions are made up of one or more restrictions.') + list_parser.add_argument( + '-t', '--type', + choices=['DLP_JOB_TYPE_UNSPECIFIED', 'INSPECT_JOB', + 'RISK_ANALYSIS_JOB'], + help='The type of job. API defaults to "INSPECT"') + + delete_parser = subparsers.add_parser( + 'delete', + help='Delete results of a Data Loss Prevention API job.') + delete_parser.add_argument( + 'project', + help='The project id to use as a parent resource.') + delete_parser.add_argument( + 'job_name', + help='The name of the DlpJob resource to be deleted. ' + 'Example: X-#####') + + args = parser.parse_args() + + if args.content == 'list': + list_dlp_jobs( + args.project, + filter_string=args.filter, + job_type=args.type) + elif args.content == 'delete': + delete_dlp_job(args.project, args.job_name) diff --git a/dlp/jobs_test.py b/dlp/jobs_test.py new file mode 100644 index 00000000000..8f47fb4d428 --- /dev/null +++ b/dlp/jobs_test.py @@ -0,0 +1,80 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytest + +import jobs + +GCLOUD_PROJECT = os.getenv('GCLOUD_PROJECT') +TEST_COLUMN_NAME = 'zip_code' +TEST_TABLE_PROJECT_ID = 'bigquery-public-data' +TEST_DATASET_ID = 'san_francisco' +TEST_TABLE_ID = 'bikeshare_trips' + + +@pytest.fixture(scope='session') +def test_job_name(): + import google.cloud.dlp + dlp = google.cloud.dlp.DlpServiceClient() + + parent = dlp.project_path(GCLOUD_PROJECT) + + # Construct job request + risk_job = { + 'privacy_metric': { + 'categorical_stats_config': { + 'field': { + 'name': TEST_COLUMN_NAME + } + } + }, + 'source_table': { + 'project_id': TEST_TABLE_PROJECT_ID, + 'dataset_id': TEST_DATASET_ID, + 'table_id': TEST_TABLE_ID + } + } + + response = dlp.create_dlp_job(parent, risk_job=risk_job) + full_path = response.name + # API expects only job name, not full project path + job_name = full_path[full_path.rfind('/')+1:] + return job_name + + +def test_list_dlp_jobs(capsys): + jobs.list_dlp_jobs(GCLOUD_PROJECT) + + out, _ = capsys.readouterr() + assert 'Job: projects/' in out + + +def test_list_dlp_jobs_with_filter(capsys): + jobs.list_dlp_jobs(GCLOUD_PROJECT, filter_string='state=DONE') + + out, _ = capsys.readouterr() + assert 'Job: projects/' in out + + +def test_list_dlp_jobs_with_job_type(capsys): + jobs.list_dlp_jobs(GCLOUD_PROJECT, job_type='INSPECT_JOB') + + out, _ = capsys.readouterr() + assert 'Job: projects/' in out + + +def test_delete_dlp_job(test_job_name, capsys): + jobs.delete_dlp_job(GCLOUD_PROJECT, test_job_name) diff --git a/dlp/metadata.py b/dlp/metadata.py new file mode 100644 index 00000000000..de05fa36852 --- /dev/null +++ b/dlp/metadata.py @@ -0,0 +1,63 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sample app that queries the Data Loss Prevention API for supported +categories and info types.""" + +from __future__ import print_function + +import argparse + + +# [START dlp_list_info_types] +def list_info_types(language_code=None, result_filter=None): + """List types of sensitive information within a category. + Args: + language_code: The BCP-47 language code to use, e.g. 'en-US'. + filter: An optional filter to only return info types supported by + certain parts of the API. Defaults to "supported_by=INSPECT". + Returns: + None; the response from the API is printed to the terminal. + """ + # Import the client library + import google.cloud.dlp + + # Instantiate a client. + dlp = google.cloud.dlp.DlpServiceClient() + + # Make the API call. + response = dlp.list_info_types(language_code, result_filter) + + # Print the results to the console. + print('Info types:') + for info_type in response.info_types: + print('{name}: {display_name}'.format( + name=info_type.name, display_name=info_type.display_name)) +# [END dlp_list_info_types] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + '--language_code', + help='The BCP-47 language code to use, e.g. \'en-US\'.') + parser.add_argument( + '--filter', + help='An optional filter to only return info types supported by ' + 'certain parts of the API. Defaults to "supported_by=INSPECT".') + + args = parser.parse_args() + + list_info_types( + language_code=args.language_code, result_filter=args.filter) diff --git a/dlp/metadata_test.py b/dlp/metadata_test.py new file mode 100644 index 00000000000..a7e3bb9dcce --- /dev/null +++ b/dlp/metadata_test.py @@ -0,0 +1,22 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import metadata + + +def test_fetch_info_types(capsys): + metadata.list_info_types() + + out, _ = capsys.readouterr() + assert 'EMAIL_ADDRESS' in out diff --git a/dlp/quickstart.py b/dlp/quickstart.py new file mode 100644 index 00000000000..736d59ddd8f --- /dev/null +++ b/dlp/quickstart.py @@ -0,0 +1,94 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sample app that queries the Data Loss Prevention API for supported +categories and info types.""" + +from __future__ import print_function + +import sys +import argparse + + +def quickstart(project_id): + """Demonstrates use of the Data Loss Prevention API client library.""" + + # [START dlp_quickstart] + # Import the client library + import google.cloud.dlp + + # Instantiate a client. + dlp_client = google.cloud.dlp.DlpServiceClient() + + # The string to inspect + content = 'Robert Frost' + + # Construct the item to inspect. + item = {'value': content} + + # The info types to search for in the content. Required. + info_types = [{'name': 'FIRST_NAME'}, {'name': 'LAST_NAME'}] + + # The minimum likelihood to constitute a match. Optional. + min_likelihood = 'LIKELIHOOD_UNSPECIFIED' + + # The maximum number of findings to report (0 = server maximum). Optional. + max_findings = 0 + + # Whether to include the matching string in the results. Optional. + include_quote = True + + # Construct the configuration dictionary. Keys which are None may + # optionally be omitted entirely. + inspect_config = { + 'info_types': info_types, + 'min_likelihood': min_likelihood, + 'include_quote': include_quote, + 'limits': {'max_findings_per_request': max_findings}, + } + + # Convert the project id into a full resource id. + parent = dlp_client.project_path(project_id) + + # Call the API. + response = dlp_client.inspect_content(parent, inspect_config, item) + + # Print out the results. + if response.result.findings: + for finding in response.result.findings: + try: + print('Quote: {}'.format(finding.quote)) + except AttributeError: + pass + print('Info type: {}'.format(finding.info_type.name)) + # Convert likelihood value to string respresentation. + likelihood = (google.cloud.dlp.types.Finding.DESCRIPTOR + .fields_by_name['likelihood'] + .enum_type.values_by_number[finding.likelihood] + .name) + print('Likelihood: {}'.format(likelihood)) + else: + print('No findings.') + # [END dlp_quickstart] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + "project_id", help="Enter your GCP project id.", type=str) + args = parser.parse_args() + if len(sys.argv) == 1: + parser.print_usage() + sys.exit(1) + quickstart(args.project_id) diff --git a/dlp/quickstart_test.py b/dlp/quickstart_test.py new file mode 100644 index 00000000000..19c215fdbb0 --- /dev/null +++ b/dlp/quickstart_test.py @@ -0,0 +1,36 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import google.cloud.dlp +import mock + +import quickstart + + +GCLOUD_PROJECT = os.getenv('GCLOUD_PROJECT') + + +def test_quickstart(capsys): + # Mock out project_path to use the test runner's project ID. + with mock.patch.object( + google.cloud.dlp.DlpServiceClient, + 'project_path', + return_value='projects/{}'.format(GCLOUD_PROJECT)): + quickstart.quickstart(GCLOUD_PROJECT) + + out, _ = capsys.readouterr() + assert 'FIRST_NAME' in out + assert 'LAST_NAME' in out diff --git a/dlp/redact.py b/dlp/redact.py new file mode 100644 index 00000000000..490f5524289 --- /dev/null +++ b/dlp/redact.py @@ -0,0 +1,146 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sample app that uses the Data Loss Prevent API to redact the contents of +an image file.""" + +from __future__ import print_function + +import argparse +# [START dlp_redact_image] +import mimetypes +# [END dlp_redact_image] +import os + + +# [START dlp_redact_image] + +def redact_image(project, filename, output_filename, + info_types, min_likelihood=None, mime_type=None): + """Uses the Data Loss Prevention API to redact protected data in an image. + Args: + project: The Google Cloud project id to use as a parent resource. + filename: The path to the file to inspect. + output_filename: The path to which the redacted image will be written. + info_types: A list of strings representing info types to look for. + A full list of info type categories can be fetched from the API. + min_likelihood: A string representing the minimum likelihood threshold + that constitutes a match. One of: 'LIKELIHOOD_UNSPECIFIED', + 'VERY_UNLIKELY', 'UNLIKELY', 'POSSIBLE', 'LIKELY', 'VERY_LIKELY'. + mime_type: The MIME type of the file. If not specified, the type is + inferred via the Python standard library's mimetypes module. + Returns: + None; the response from the API is printed to the terminal. + """ + # Import the client library + import google.cloud.dlp + + # Instantiate a client. + dlp = google.cloud.dlp.DlpServiceClient() + + # Prepare info_types by converting the list of strings into a list of + # dictionaries (protos are also accepted). + info_types = [{'name': info_type} for info_type in info_types] + + # Prepare image_redaction_configs, a list of dictionaries. Each dictionary + # contains an info_type and optionally the color used for the replacement. + # The color is omitted in this sample, so the default (black) will be used. + image_redaction_configs = [] + + if info_types is not None: + for info_type in info_types: + image_redaction_configs.append({'info_type': info_type}) + + # Construct the configuration dictionary. Keys which are None may + # optionally be omitted entirely. + inspect_config = { + 'min_likelihood': min_likelihood, + 'info_types': info_types, + } + + # If mime_type is not specified, guess it from the filename. + if mime_type is None: + mime_guess = mimetypes.MimeTypes().guess_type(filename) + mime_type = mime_guess[0] or 'application/octet-stream' + + # Select the content type index from the list of supported types. + supported_content_types = { + None: 0, # "Unspecified" + 'image/jpeg': 1, + 'image/bmp': 2, + 'image/png': 3, + 'image/svg': 4, + 'text/plain': 5, + } + content_type_index = supported_content_types.get(mime_type, 0) + + # Construct the byte_item, containing the file's byte data. + with open(filename, mode='rb') as f: + byte_item = {'type': content_type_index, 'data': f.read()} + + # Convert the project id into a full resource id. + parent = dlp.project_path(project) + + # Call the API. + response = dlp.redact_image( + parent, inspect_config=inspect_config, + image_redaction_configs=image_redaction_configs, + byte_item=byte_item) + + # Write out the results. + with open(output_filename, mode='wb') as f: + f.write(response.redacted_image) + print("Wrote {byte_count} to {filename}".format( + byte_count=len(response.redacted_image), filename=output_filename)) +# [END dlp_redact_image] + + +if __name__ == '__main__': + default_project = os.environ.get('GCLOUD_PROJECT') + + parser = argparse.ArgumentParser(description=__doc__) + + parser.add_argument( + 'filename', help='The path to the file to inspect.') + parser.add_argument( + 'output_filename', + help='The path to which the redacted image will be written.') + parser.add_argument( + '--project', + help='The Google Cloud project id to use as a parent resource.', + default=default_project) + parser.add_argument( + '--info_types', action='append', + help='Strings representing info types to look for. A full list of ' + 'info categories and types is available from the API. Examples ' + 'include "FIRST_NAME", "LAST_NAME", "EMAIL_ADDRESS". ' + 'If unspecified, the three above examples will be used.', + default=['FIRST_NAME', 'LAST_NAME', 'EMAIL_ADDRESS']) + parser.add_argument( + '--min_likelihood', + choices=['LIKELIHOOD_UNSPECIFIED', 'VERY_UNLIKELY', 'UNLIKELY', + 'POSSIBLE', 'LIKELY', 'VERY_LIKELY'], + help='A string representing the minimum likelihood threshold that ' + 'constitutes a match.') + parser.add_argument( + '--mime_type', + help='The MIME type of the file. If not specified, the type is ' + 'inferred via the Python standard library\'s mimetypes module.') + + args = parser.parse_args() + + redact_image( + args.project, args.filename, args.output_filename, + args.info_types, min_likelihood=args.min_likelihood, + mime_type=args.mime_type) diff --git a/dlp/redact_test.py b/dlp/redact_test.py new file mode 100644 index 00000000000..50eb826b051 --- /dev/null +++ b/dlp/redact_test.py @@ -0,0 +1,45 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import tempfile + +import pytest + +import redact + +GCLOUD_PROJECT = os.getenv('GCLOUD_PROJECT') +RESOURCE_DIRECTORY = os.path.join(os.path.dirname(__file__), 'resources') + + +@pytest.fixture(scope='module') +def tempdir(): + tempdir = tempfile.mkdtemp() + yield tempdir + shutil.rmtree(tempdir) + + +def test_redact_image_file(tempdir, capsys): + test_filepath = os.path.join(RESOURCE_DIRECTORY, 'test.png') + output_filepath = os.path.join(tempdir, 'redacted.png') + + redact.redact_image( + GCLOUD_PROJECT, + test_filepath, + output_filepath, + ['FIRST_NAME', 'EMAIL_ADDRESS']) + + out, _ = capsys.readouterr() + assert output_filepath in out diff --git a/dlp/requirements.txt b/dlp/requirements.txt new file mode 100644 index 00000000000..ec8e9f3590d --- /dev/null +++ b/dlp/requirements.txt @@ -0,0 +1,5 @@ +google-cloud-dlp==0.10.0 +google-cloud-storage==1.13.2 +google-cloud-pubsub==0.39.1 +google-cloud-datastore==1.7.3 +google-cloud-bigquery==1.9.0 diff --git a/dlp/resources/accounts.txt b/dlp/resources/accounts.txt new file mode 100644 index 00000000000..2763cd0ab82 --- /dev/null +++ b/dlp/resources/accounts.txt @@ -0,0 +1 @@ +My credit card number is 1234 5678 9012 3456, and my CVV is 789. \ No newline at end of file diff --git a/dlp/resources/dates.csv b/dlp/resources/dates.csv new file mode 100644 index 00000000000..056fccb328e --- /dev/null +++ b/dlp/resources/dates.csv @@ -0,0 +1,5 @@ +name,birth_date,register_date,credit_card +Ann,01/01/1970,07/21/1996,4532908762519852 +James,03/06/1988,04/09/2001,4301261899725540 +Dan,08/14/1945,11/15/2011,4620761856015295 +Laura,11/03/1992,01/04/2017,4564981067258901 \ No newline at end of file diff --git a/dlp/resources/harmless.txt b/dlp/resources/harmless.txt new file mode 100644 index 00000000000..5666de37ab2 --- /dev/null +++ b/dlp/resources/harmless.txt @@ -0,0 +1 @@ +This file is mostly harmless. diff --git a/dlp/resources/test.png b/dlp/resources/test.png new file mode 100644 index 00000000000..8f32c825884 Binary files /dev/null and b/dlp/resources/test.png differ diff --git a/dlp/resources/test.txt b/dlp/resources/test.txt new file mode 100644 index 00000000000..c2ee3815bc9 --- /dev/null +++ b/dlp/resources/test.txt @@ -0,0 +1 @@ +My phone number is (223) 456-7890 and my email address is gary@somedomain.com. \ No newline at end of file diff --git a/dlp/risk.py b/dlp/risk.py new file mode 100644 index 00000000000..273cfd1548d --- /dev/null +++ b/dlp/risk.py @@ -0,0 +1,820 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sample app that uses the Data Loss Prevent API to perform risk anaylsis.""" + +from __future__ import print_function + +import argparse + + +# [START dlp_numerical_stats] +def numerical_risk_analysis(project, table_project_id, dataset_id, table_id, + column_name, topic_id, subscription_id, + timeout=300): + """Uses the Data Loss Prevention API to compute risk metrics of a column + of numerical data in a Google BigQuery table. + Args: + project: The Google Cloud project id to use as a parent resource. + table_project_id: The Google Cloud project id where the BigQuery table + is stored. + dataset_id: The id of the dataset to inspect. + table_id: The id of the table to inspect. + column_name: The name of the column to compute risk metrics for. + topic_id: The name of the Pub/Sub topic to notify once the job + completes. + subscription_id: The name of the Pub/Sub subscription to use when + listening for job completion notifications. + timeout: The number of seconds to wait for a response from the API. + + Returns: + None; the response from the API is printed to the terminal. + """ + + # Import the client library. + import google.cloud.dlp + + # This sample additionally uses Cloud Pub/Sub to receive results from + # potentially long-running operations. + import google.cloud.pubsub + + def callback(message): + if (message.attributes['DlpJobName'] == operation.name): + # This is the message we're looking for, so acknowledge it. + message.ack() + + # Now that the job is done, fetch the results and print them. + job = dlp.get_dlp_job(operation.name) + results = job.risk_details.numerical_stats_result + print('Value Range: [{}, {}]'.format( + results.min_value.integer_value, + results.max_value.integer_value)) + prev_value = None + for percent, result in enumerate(results.quantile_values): + value = result.integer_value + if prev_value != value: + print('Value at {}% quantile: {}'.format( + percent, value)) + prev_value = value + subscription.set_result(None) + else: + # This is not the message we're looking for. + message.drop() + + # Instantiate a client. + dlp = google.cloud.dlp.DlpServiceClient() + + # Convert the project id into a full resource id. + parent = dlp.project_path(project) + + # Location info of the BigQuery table. + source_table = { + 'project_id': table_project_id, + 'dataset_id': dataset_id, + 'table_id': table_id + } + + # Tell the API where to send a notification when the job is complete. + actions = [{ + 'pub_sub': {'topic': '{}/topics/{}'.format(parent, topic_id)} + }] + + # Configure risk analysis job + # Give the name of the numeric column to compute risk metrics for + risk_job = { + 'privacy_metric': { + 'numerical_stats_config': { + 'field': { + 'name': column_name + } + } + }, + 'source_table': source_table, + 'actions': actions + } + + # Create a Pub/Sub client and find the subscription. The subscription is + # expected to already be listening to the topic. + subscriber = google.cloud.pubsub.SubscriberClient() + subscription_path = subscriber.subscription_path( + project, subscription_id) + subscription = subscriber.subscribe(subscription_path, callback) + + # Call API to start risk analysis job + operation = dlp.create_dlp_job(parent, risk_job=risk_job) + + try: + subscription.result(timeout=timeout) + except TimeoutError: + print('No event received before the timeout. Please verify that the ' + 'subscription provided is subscribed to the topic provided.') + subscription.close() +# [END dlp_numerical_stats] + + +# [START dlp_categorical_stats] +def categorical_risk_analysis(project, table_project_id, dataset_id, table_id, + column_name, topic_id, subscription_id, + timeout=300): + """Uses the Data Loss Prevention API to compute risk metrics of a column + of categorical data in a Google BigQuery table. + Args: + project: The Google Cloud project id to use as a parent resource. + table_project_id: The Google Cloud project id where the BigQuery table + is stored. + dataset_id: The id of the dataset to inspect. + table_id: The id of the table to inspect. + column_name: The name of the column to compute risk metrics for. + topic_id: The name of the Pub/Sub topic to notify once the job + completes. + subscription_id: The name of the Pub/Sub subscription to use when + listening for job completion notifications. + timeout: The number of seconds to wait for a response from the API. + + Returns: + None; the response from the API is printed to the terminal. + """ + + # Import the client library. + import google.cloud.dlp + + # This sample additionally uses Cloud Pub/Sub to receive results from + # potentially long-running operations. + import google.cloud.pubsub + + def callback(message): + if (message.attributes['DlpJobName'] == operation.name): + # This is the message we're looking for, so acknowledge it. + message.ack() + + # Now that the job is done, fetch the results and print them. + job = dlp.get_dlp_job(operation.name) + histogram_buckets = (job.risk_details + .categorical_stats_result + .value_frequency_histogram_buckets) + # Print bucket stats + for i, bucket in enumerate(histogram_buckets): + print('Bucket {}:'.format(i)) + print(' Most common value occurs {} time(s)'.format( + bucket.value_frequency_upper_bound)) + print(' Least common value occurs {} time(s)'.format( + bucket.value_frequency_lower_bound)) + print(' {} unique values total.'.format( + bucket.bucket_size)) + for value in bucket.bucket_values: + print(' Value {} occurs {} time(s)'.format( + value.value.integer_value, value.count)) + subscription.set_result(None) + else: + # This is not the message we're looking for. + message.drop() + + # Instantiate a client. + dlp = google.cloud.dlp.DlpServiceClient() + + # Convert the project id into a full resource id. + parent = dlp.project_path(project) + + # Location info of the BigQuery table. + source_table = { + 'project_id': table_project_id, + 'dataset_id': dataset_id, + 'table_id': table_id + } + + # Tell the API where to send a notification when the job is complete. + actions = [{ + 'pub_sub': {'topic': '{}/topics/{}'.format(parent, topic_id)} + }] + + # Configure risk analysis job + # Give the name of the numeric column to compute risk metrics for + risk_job = { + 'privacy_metric': { + 'categorical_stats_config': { + 'field': { + 'name': column_name + } + } + }, + 'source_table': source_table, + 'actions': actions + } + + # Create a Pub/Sub client and find the subscription. The subscription is + # expected to already be listening to the topic. + subscriber = google.cloud.pubsub.SubscriberClient() + subscription_path = subscriber.subscription_path( + project, subscription_id) + subscription = subscriber.subscribe(subscription_path, callback) + + # Call API to start risk analysis job + operation = dlp.create_dlp_job(parent, risk_job=risk_job) + + try: + subscription.result(timeout=timeout) + except TimeoutError: + print('No event received before the timeout. Please verify that the ' + 'subscription provided is subscribed to the topic provided.') + subscription.close() +# [END dlp_categorical_stats] + + +# [START dlp_k_anonymity] +def k_anonymity_analysis(project, table_project_id, dataset_id, table_id, + topic_id, subscription_id, quasi_ids, timeout=300): + """Uses the Data Loss Prevention API to compute the k-anonymity of a + column set in a Google BigQuery table. + Args: + project: The Google Cloud project id to use as a parent resource. + table_project_id: The Google Cloud project id where the BigQuery table + is stored. + dataset_id: The id of the dataset to inspect. + table_id: The id of the table to inspect. + topic_id: The name of the Pub/Sub topic to notify once the job + completes. + subscription_id: The name of the Pub/Sub subscription to use when + listening for job completion notifications. + quasi_ids: A set of columns that form a composite key. + timeout: The number of seconds to wait for a response from the API. + + Returns: + None; the response from the API is printed to the terminal. + """ + + # Import the client library. + import google.cloud.dlp + + # This sample additionally uses Cloud Pub/Sub to receive results from + # potentially long-running operations. + import google.cloud.pubsub + + # Create helper function for unpacking values + def get_values(obj): + return int(obj.integer_value) + + def callback(message): + if (message.attributes['DlpJobName'] == operation.name): + # This is the message we're looking for, so acknowledge it. + message.ack() + + # Now that the job is done, fetch the results and print them. + job = dlp.get_dlp_job(operation.name) + histogram_buckets = (job.risk_details + .k_anonymity_result + .equivalence_class_histogram_buckets) + # Print bucket stats + for i, bucket in enumerate(histogram_buckets): + print('Bucket {}:'.format(i)) + if bucket.equivalence_class_size_lower_bound: + print(' Bucket size range: [{}, {}]'.format( + bucket.equivalence_class_size_lower_bound, + bucket.equivalence_class_size_upper_bound)) + for value_bucket in bucket.bucket_values: + print(' Quasi-ID values: {}'.format( + map(get_values, value_bucket.quasi_ids_values) + )) + print(' Class size: {}'.format( + value_bucket.equivalence_class_size)) + subscription.set_result(None) + else: + # This is not the message we're looking for. + message.drop() + + # Instantiate a client. + dlp = google.cloud.dlp.DlpServiceClient() + + # Convert the project id into a full resource id. + parent = dlp.project_path(project) + + # Location info of the BigQuery table. + source_table = { + 'project_id': table_project_id, + 'dataset_id': dataset_id, + 'table_id': table_id + } + + # Convert quasi id list to Protobuf type + def map_fields(field): + return {'name': field} + + quasi_ids = map(map_fields, quasi_ids) + + # Tell the API where to send a notification when the job is complete. + actions = [{ + 'pub_sub': {'topic': '{}/topics/{}'.format(parent, topic_id)} + }] + + # Configure risk analysis job + # Give the name of the numeric column to compute risk metrics for + risk_job = { + 'privacy_metric': { + 'k_anonymity_config': { + 'quasi_ids': quasi_ids + } + }, + 'source_table': source_table, + 'actions': actions + } + + # Create a Pub/Sub client and find the subscription. The subscription is + # expected to already be listening to the topic. + subscriber = google.cloud.pubsub.SubscriberClient() + subscription_path = subscriber.subscription_path( + project, subscription_id) + subscription = subscriber.subscribe(subscription_path, callback) + + # Call API to start risk analysis job + operation = dlp.create_dlp_job(parent, risk_job=risk_job) + + try: + subscription.result(timeout=timeout) + except TimeoutError: + print('No event received before the timeout. Please verify that the ' + 'subscription provided is subscribed to the topic provided.') + subscription.close() +# [END dlp_k_anonymity] + + +# [START dlp_l_diversity] +def l_diversity_analysis(project, table_project_id, dataset_id, table_id, + topic_id, subscription_id, sensitive_attribute, + quasi_ids, timeout=300): + """Uses the Data Loss Prevention API to compute the l-diversity of a + column set in a Google BigQuery table. + Args: + project: The Google Cloud project id to use as a parent resource. + table_project_id: The Google Cloud project id where the BigQuery table + is stored. + dataset_id: The id of the dataset to inspect. + table_id: The id of the table to inspect. + topic_id: The name of the Pub/Sub topic to notify once the job + completes. + subscription_id: The name of the Pub/Sub subscription to use when + listening for job completion notifications. + sensitive_attribute: The column to measure l-diversity relative to. + quasi_ids: A set of columns that form a composite key. + timeout: The number of seconds to wait for a response from the API. + + Returns: + None; the response from the API is printed to the terminal. + """ + + # Import the client library. + import google.cloud.dlp + + # This sample additionally uses Cloud Pub/Sub to receive results from + # potentially long-running operations. + import google.cloud.pubsub + + # Create helper function for unpacking values + def get_values(obj): + return int(obj.integer_value) + + def callback(message): + if (message.attributes['DlpJobName'] == operation.name): + # This is the message we're looking for, so acknowledge it. + message.ack() + + # Now that the job is done, fetch the results and print them. + job = dlp.get_dlp_job(operation.name) + histogram_buckets = ( + job.risk_details + .l_diversity_result + .sensitive_value_frequency_histogram_buckets) + # Print bucket stats + for i, bucket in enumerate(histogram_buckets): + print('Bucket {}:'.format(i)) + print(' Bucket size range: [{}, {}]'.format( + bucket.sensitive_value_frequency_lower_bound, + bucket.sensitive_value_frequency_upper_bound)) + for value_bucket in bucket.bucket_values: + print(' Quasi-ID values: {}'.format( + map(get_values, value_bucket.quasi_ids_values))) + print(' Class size: {}'.format( + value_bucket.equivalence_class_size)) + for value in value_bucket.top_sensitive_values: + print((' Sensitive value {} occurs {} time(s)' + .format(value.value, value.count))) + subscription.set_result(None) + else: + # This is not the message we're looking for. + message.drop() + + # Instantiate a client. + dlp = google.cloud.dlp.DlpServiceClient() + + # Convert the project id into a full resource id. + parent = dlp.project_path(project) + + # Location info of the BigQuery table. + source_table = { + 'project_id': table_project_id, + 'dataset_id': dataset_id, + 'table_id': table_id + } + + # Convert quasi id list to Protobuf type + def map_fields(field): + return {'name': field} + + quasi_ids = map(map_fields, quasi_ids) + + # Tell the API where to send a notification when the job is complete. + actions = [{ + 'pub_sub': {'topic': '{}/topics/{}'.format(parent, topic_id)} + }] + + # Configure risk analysis job + # Give the name of the numeric column to compute risk metrics for + risk_job = { + 'privacy_metric': { + 'l_diversity_config': { + 'quasi_ids': quasi_ids, + 'sensitive_attribute': { + 'name': sensitive_attribute + } + } + }, + 'source_table': source_table, + 'actions': actions + } + + # Create a Pub/Sub client and find the subscription. The subscription is + # expected to already be listening to the topic. + subscriber = google.cloud.pubsub.SubscriberClient() + subscription_path = subscriber.subscription_path( + project, subscription_id) + subscription = subscriber.subscribe(subscription_path, callback) + + # Call API to start risk analysis job + operation = dlp.create_dlp_job(parent, risk_job=risk_job) + + try: + subscription.result(timeout=timeout) + except TimeoutError: + print('No event received before the timeout. Please verify that the ' + 'subscription provided is subscribed to the topic provided.') + subscription.close() +# [END dlp_l_diversity] + + +# [START dlp_k_map] +def k_map_estimate_analysis(project, table_project_id, dataset_id, table_id, + topic_id, subscription_id, quasi_ids, info_types, + region_code='US', timeout=300): + """Uses the Data Loss Prevention API to compute the k-map risk estimation + of a column set in a Google BigQuery table. + Args: + project: The Google Cloud project id to use as a parent resource. + table_project_id: The Google Cloud project id where the BigQuery table + is stored. + dataset_id: The id of the dataset to inspect. + table_id: The id of the table to inspect. + column_name: The name of the column to compute risk metrics for. + topic_id: The name of the Pub/Sub topic to notify once the job + completes. + subscription_id: The name of the Pub/Sub subscription to use when + listening for job completion notifications. + quasi_ids: A set of columns that form a composite key and optionally + their reidentification distributions. + info_types: Type of information of the quasi_id in order to provide a + statistical model of population. + region_code: The ISO 3166-1 region code that the data is representative + of. Can be omitted if using a region-specific infoType (such as + US_ZIP_5) + timeout: The number of seconds to wait for a response from the API. + + Returns: + None; the response from the API is printed to the terminal. + """ + + # Import the client library. + import google.cloud.dlp + + # This sample additionally uses Cloud Pub/Sub to receive results from + # potentially long-running operations. + import google.cloud.pubsub + + # Create helper function for unpacking values + def get_values(obj): + return int(obj.integer_value) + + def callback(message): + if (message.attributes['DlpJobName'] == operation.name): + # This is the message we're looking for, so acknowledge it. + message.ack() + + # Now that the job is done, fetch the results and print them. + job = dlp.get_dlp_job(operation.name) + histogram_buckets = (job.risk_details + .k_map_estimation_result + .k_map_estimation_histogram) + # Print bucket stats + for i, bucket in enumerate(histogram_buckets): + print('Bucket {}:'.format(i)) + print(' Anonymity range: [{}, {}]'.format( + bucket.min_anonymity, bucket.max_anonymity)) + print(' Size: {}'.format(bucket.bucket_size)) + for value_bucket in bucket.bucket_values: + print(' Values: {}'.format( + map(get_values, value_bucket.quasi_ids_values))) + print(' Estimated k-map anonymity: {}'.format( + value_bucket.estimated_anonymity)) + subscription.set_result(None) + else: + # This is not the message we're looking for. + message.drop() + + # Instantiate a client. + dlp = google.cloud.dlp.DlpServiceClient() + + # Convert the project id into a full resource id. + parent = dlp.project_path(project) + + # Location info of the BigQuery table. + source_table = { + 'project_id': table_project_id, + 'dataset_id': dataset_id, + 'table_id': table_id + } + + # Check that numbers of quasi-ids and info types are equal + if len(quasi_ids) != len(info_types): + raise ValueError("""Number of infoTypes and number of quasi-identifiers + must be equal!""") + + # Convert quasi id list to Protobuf type + def map_fields(quasi_id, info_type): + return {'field': {'name': quasi_id}, 'info_type': {'name': info_type}} + + quasi_ids = map(map_fields, quasi_ids, info_types) + + # Tell the API where to send a notification when the job is complete. + actions = [{ + 'pub_sub': {'topic': '{}/topics/{}'.format(parent, topic_id)} + }] + + # Configure risk analysis job + # Give the name of the numeric column to compute risk metrics for + risk_job = { + 'privacy_metric': { + 'k_map_estimation_config': { + 'quasi_ids': quasi_ids, + 'region_code': region_code + } + }, + 'source_table': source_table, + 'actions': actions + } + + # Create a Pub/Sub client and find the subscription. The subscription is + # expected to already be listening to the topic. + subscriber = google.cloud.pubsub.SubscriberClient() + subscription_path = subscriber.subscription_path( + project, subscription_id) + subscription = subscriber.subscribe(subscription_path, callback) + + # Call API to start risk analysis job + operation = dlp.create_dlp_job(parent, risk_job=risk_job) + + try: + subscription.result(timeout=timeout) + except TimeoutError: + print('No event received before the timeout. Please verify that the ' + 'subscription provided is subscribed to the topic provided.') + subscription.close() +# [END dlp_k_map] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers( + dest='content', help='Select how to submit content to the API.') + subparsers.required = True + + numerical_parser = subparsers.add_parser( + 'numerical', + help='') + numerical_parser.add_argument( + 'project', + help='The Google Cloud project id to use as a parent resource.') + numerical_parser.add_argument( + 'table_project_id', + help='The Google Cloud project id where the BigQuery table is stored.') + numerical_parser.add_argument( + 'dataset_id', + help='The id of the dataset to inspect.') + numerical_parser.add_argument( + 'table_id', + help='The id of the table to inspect.') + numerical_parser.add_argument( + 'column_name', + help='The name of the column to compute risk metrics for.') + numerical_parser.add_argument( + 'topic_id', + help='The name of the Pub/Sub topic to notify once the job completes.') + numerical_parser.add_argument( + 'subscription_id', + help='The name of the Pub/Sub subscription to use when listening for' + 'job completion notifications.') + numerical_parser.add_argument( + '--timeout', type=int, + help='The number of seconds to wait for a response from the API.') + + categorical_parser = subparsers.add_parser( + 'categorical', + help='') + categorical_parser.add_argument( + 'project', + help='The Google Cloud project id to use as a parent resource.') + categorical_parser.add_argument( + 'table_project_id', + help='The Google Cloud project id where the BigQuery table is stored.') + categorical_parser.add_argument( + 'dataset_id', + help='The id of the dataset to inspect.') + categorical_parser.add_argument( + 'table_id', + help='The id of the table to inspect.') + categorical_parser.add_argument( + 'column_name', + help='The name of the column to compute risk metrics for.') + categorical_parser.add_argument( + 'topic_id', + help='The name of the Pub/Sub topic to notify once the job completes.') + categorical_parser.add_argument( + 'subscription_id', + help='The name of the Pub/Sub subscription to use when listening for' + 'job completion notifications.') + categorical_parser.add_argument( + '--timeout', type=int, + help='The number of seconds to wait for a response from the API.') + + k_anonymity_parser = subparsers.add_parser( + 'k_anonymity', + help='Computes the k-anonymity of a column set in a Google BigQuery' + 'table.') + k_anonymity_parser.add_argument( + 'project', + help='The Google Cloud project id to use as a parent resource.') + k_anonymity_parser.add_argument( + 'table_project_id', + help='The Google Cloud project id where the BigQuery table is stored.') + k_anonymity_parser.add_argument( + 'dataset_id', + help='The id of the dataset to inspect.') + k_anonymity_parser.add_argument( + 'table_id', + help='The id of the table to inspect.') + k_anonymity_parser.add_argument( + 'topic_id', + help='The name of the Pub/Sub topic to notify once the job completes.') + k_anonymity_parser.add_argument( + 'subscription_id', + help='The name of the Pub/Sub subscription to use when listening for' + 'job completion notifications.') + k_anonymity_parser.add_argument( + 'quasi_ids', nargs='+', + help='A set of columns that form a composite key.') + k_anonymity_parser.add_argument( + '--timeout', type=int, + help='The number of seconds to wait for a response from the API.') + + l_diversity_parser = subparsers.add_parser( + 'l_diversity', + help='Computes the l-diversity of a column set in a Google BigQuery' + 'table.') + l_diversity_parser.add_argument( + 'project', + help='The Google Cloud project id to use as a parent resource.') + l_diversity_parser.add_argument( + 'table_project_id', + help='The Google Cloud project id where the BigQuery table is stored.') + l_diversity_parser.add_argument( + 'dataset_id', + help='The id of the dataset to inspect.') + l_diversity_parser.add_argument( + 'table_id', + help='The id of the table to inspect.') + l_diversity_parser.add_argument( + 'topic_id', + help='The name of the Pub/Sub topic to notify once the job completes.') + l_diversity_parser.add_argument( + 'subscription_id', + help='The name of the Pub/Sub subscription to use when listening for' + 'job completion notifications.') + l_diversity_parser.add_argument( + 'sensitive_attribute', + help='The column to measure l-diversity relative to.') + l_diversity_parser.add_argument( + 'quasi_ids', nargs='+', + help='A set of columns that form a composite key.') + l_diversity_parser.add_argument( + '--timeout', type=int, + help='The number of seconds to wait for a response from the API.') + + k_map_parser = subparsers.add_parser( + 'k_map', + help='Computes the k-map risk estimation of a column set in a Google' + 'BigQuery table.') + k_map_parser.add_argument( + 'project', + help='The Google Cloud project id to use as a parent resource.') + k_map_parser.add_argument( + 'table_project_id', + help='The Google Cloud project id where the BigQuery table is stored.') + k_map_parser.add_argument( + 'dataset_id', + help='The id of the dataset to inspect.') + k_map_parser.add_argument( + 'table_id', + help='The id of the table to inspect.') + k_map_parser.add_argument( + 'topic_id', + help='The name of the Pub/Sub topic to notify once the job completes.') + k_map_parser.add_argument( + 'subscription_id', + help='The name of the Pub/Sub subscription to use when listening for' + 'job completion notifications.') + k_map_parser.add_argument( + 'quasi_ids', nargs='+', + help='A set of columns that form a composite key.') + k_map_parser.add_argument( + '-t', '--info-types', nargs='+', + help='Type of information of the quasi_id in order to provide a' + 'statistical model of population.', + required=True) + k_map_parser.add_argument( + '-r', '--region-code', default='US', + help='The ISO 3166-1 region code that the data is representative of.') + k_map_parser.add_argument( + '--timeout', type=int, + help='The number of seconds to wait for a response from the API.') + + args = parser.parse_args() + + if args.content == 'numerical': + numerical_risk_analysis( + args.project, + args.table_project_id, + args.dataset_id, + args.table_id, + args.column_name, + args.topic_id, + args.subscription_id, + timeout=args.timeout) + elif args.content == 'categorical': + categorical_risk_analysis( + args.project, + args.table_project_id, + args.dataset_id, + args.table_id, + args.column_name, + args.topic_id, + args.subscription_id, + timeout=args.timeout) + elif args.content == 'k_anonymity': + k_anonymity_analysis( + args.project, + args.table_project_id, + args.dataset_id, + args.table_id, + args.topic_id, + args.subscription_id, + args.quasi_ids, + timeout=args.timeout) + elif args.content == 'l_diversity': + l_diversity_analysis( + args.project, + args.table_project_id, + args.dataset_id, + args.table_id, + args.topic_id, + args.subscription_id, + args.sensitive_attribute, + args.quasi_ids, + timeout=args.timeout) + elif args.content == 'k_map': + k_map_estimate_analysis( + args.project, + args.table_project_id, + args.dataset_id, + args.table_id, + args.topic_id, + args.subscription_id, + args.quasi_ids, + args.info_types, + region_code=args.region_code, + timeout=args.timeout) diff --git a/dlp/risk_test.py b/dlp/risk_test.py new file mode 100644 index 00000000000..d89e83bcb49 --- /dev/null +++ b/dlp/risk_test.py @@ -0,0 +1,233 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from gcp_devrel.testing.flaky import flaky +import google.cloud.pubsub + +import pytest + +import risk + +GCLOUD_PROJECT = 'nodejs-docs-samples' +TABLE_PROJECT = 'nodejs-docs-samples' +TOPIC_ID = 'dlp-test' +SUBSCRIPTION_ID = 'dlp-test-subscription' +DATASET_ID = 'integration_tests_dlp' +UNIQUE_FIELD = 'Name' +REPEATED_FIELD = 'Mystery' +NUMERIC_FIELD = 'Age' +STRING_BOOLEAN_FIELD = 'Gender' + + +# Create new custom topic/subscription +@pytest.fixture(scope='module') +def topic_id(): + # Creates a pubsub topic, and tears it down. + publisher = google.cloud.pubsub.PublisherClient() + topic_path = publisher.topic_path(GCLOUD_PROJECT, TOPIC_ID) + try: + publisher.create_topic(topic_path) + except google.api_core.exceptions.AlreadyExists: + pass + + yield TOPIC_ID + + publisher.delete_topic(topic_path) + + +@pytest.fixture(scope='module') +def subscription_id(topic_id): + # Subscribes to a topic. + subscriber = google.cloud.pubsub.SubscriberClient() + topic_path = subscriber.topic_path(GCLOUD_PROJECT, topic_id) + subscription_path = subscriber.subscription_path( + GCLOUD_PROJECT, SUBSCRIPTION_ID) + try: + subscriber.create_subscription(subscription_path, topic_path) + except google.api_core.exceptions.AlreadyExists: + pass + + yield SUBSCRIPTION_ID + + subscriber.delete_subscription(subscription_path) + + +@flaky +def test_numerical_risk_analysis(topic_id, subscription_id, capsys): + risk.numerical_risk_analysis( + GCLOUD_PROJECT, + TABLE_PROJECT, + DATASET_ID, + 'harmful', + NUMERIC_FIELD, + topic_id, + subscription_id) + + out, _ = capsys.readouterr() + assert 'Value Range:' in out + + +@flaky +def test_categorical_risk_analysis_on_string_field( + topic_id, subscription_id, capsys): + risk.categorical_risk_analysis( + GCLOUD_PROJECT, + TABLE_PROJECT, + DATASET_ID, + 'harmful', + UNIQUE_FIELD, + topic_id, + subscription_id, timeout=180) + + out, _ = capsys.readouterr() + assert 'Most common value occurs' in out + + +@flaky +def test_categorical_risk_analysis_on_number_field( + topic_id, subscription_id, capsys): + risk.categorical_risk_analysis( + GCLOUD_PROJECT, + TABLE_PROJECT, + DATASET_ID, + 'harmful', + NUMERIC_FIELD, + topic_id, + subscription_id) + + out, _ = capsys.readouterr() + assert 'Most common value occurs' in out + + +@flaky +def test_k_anonymity_analysis_single_field(topic_id, subscription_id, capsys): + risk.k_anonymity_analysis( + GCLOUD_PROJECT, + TABLE_PROJECT, + DATASET_ID, + 'harmful', + topic_id, + subscription_id, + [NUMERIC_FIELD]) + + out, _ = capsys.readouterr() + assert 'Quasi-ID values:' in out + assert 'Class size:' in out + + +@flaky +def test_k_anonymity_analysis_multiple_fields(topic_id, subscription_id, + capsys): + risk.k_anonymity_analysis( + GCLOUD_PROJECT, + TABLE_PROJECT, + DATASET_ID, + 'harmful', + topic_id, + subscription_id, + [NUMERIC_FIELD, REPEATED_FIELD]) + + out, _ = capsys.readouterr() + assert 'Quasi-ID values:' in out + assert 'Class size:' in out + + +@flaky +def test_l_diversity_analysis_single_field(topic_id, subscription_id, capsys): + risk.l_diversity_analysis( + GCLOUD_PROJECT, + TABLE_PROJECT, + DATASET_ID, + 'harmful', + topic_id, + subscription_id, + UNIQUE_FIELD, + [NUMERIC_FIELD]) + + out, _ = capsys.readouterr() + assert 'Quasi-ID values:' in out + assert 'Class size:' in out + assert 'Sensitive value' in out + + +@flaky +def test_l_diversity_analysis_multiple_field( + topic_id, subscription_id, capsys): + risk.l_diversity_analysis( + GCLOUD_PROJECT, + TABLE_PROJECT, + DATASET_ID, + 'harmful', + topic_id, + subscription_id, + UNIQUE_FIELD, + [NUMERIC_FIELD, REPEATED_FIELD]) + + out, _ = capsys.readouterr() + assert 'Quasi-ID values:' in out + assert 'Class size:' in out + assert 'Sensitive value' in out + + +@flaky +def test_k_map_estimate_analysis_single_field( + topic_id, subscription_id, capsys): + risk.k_map_estimate_analysis( + GCLOUD_PROJECT, + TABLE_PROJECT, + DATASET_ID, + 'harmful', + topic_id, + subscription_id, + [NUMERIC_FIELD], + ['AGE']) + + out, _ = capsys.readouterr() + assert 'Anonymity range:' in out + assert 'Size:' in out + assert 'Values' in out + + +@flaky +def test_k_map_estimate_analysis_multiple_field( + topic_id, subscription_id, capsys): + risk.k_map_estimate_analysis( + GCLOUD_PROJECT, + TABLE_PROJECT, + DATASET_ID, + 'harmful', + topic_id, + subscription_id, + [NUMERIC_FIELD, STRING_BOOLEAN_FIELD], + ['AGE', 'GENDER']) + + out, _ = capsys.readouterr() + assert 'Anonymity range:' in out + assert 'Size:' in out + assert 'Values' in out + + +@flaky +def test_k_map_estimate_analysis_quasi_ids_info_types_equal( + topic_id, subscription_id): + with pytest.raises(ValueError): + risk.k_map_estimate_analysis( + GCLOUD_PROJECT, + TABLE_PROJECT, + DATASET_ID, + 'harmful', + topic_id, + subscription_id, + [NUMERIC_FIELD, STRING_BOOLEAN_FIELD], + ['AGE']) diff --git a/dlp/templates.py b/dlp/templates.py new file mode 100644 index 00000000000..7ebde2cef1b --- /dev/null +++ b/dlp/templates.py @@ -0,0 +1,229 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sample app that sets up Data Loss Prevention API inspect templates.""" + +from __future__ import print_function + +import argparse +import os +import time + + +# [START dlp_create_template] +def create_inspect_template(project, info_types, + template_id=None, display_name=None, + min_likelihood=None, max_findings=None, + include_quote=None): + """Creates a Data Loss Prevention API inspect template. + Args: + project: The Google Cloud project id to use as a parent resource. + info_types: A list of strings representing info types to look for. + A full list of info type categories can be fetched from the API. + template_id: The id of the template. If omitted, an id will be randomly + generated. + display_name: The optional display name of the template. + min_likelihood: A string representing the minimum likelihood threshold + that constitutes a match. One of: 'LIKELIHOOD_UNSPECIFIED', + 'VERY_UNLIKELY', 'UNLIKELY', 'POSSIBLE', 'LIKELY', 'VERY_LIKELY'. + max_findings: The maximum number of findings to report; 0 = no maximum. + include_quote: Boolean for whether to display a quote of the detected + information in the results. + Returns: + None; the response from the API is printed to the terminal. + """ + + # Import the client library + import google.cloud.dlp + + # Instantiate a client. + dlp = google.cloud.dlp.DlpServiceClient() + + # Prepare info_types by converting the list of strings into a list of + # dictionaries (protos are also accepted). + info_types = [{'name': info_type} for info_type in info_types] + + # Construct the configuration dictionary. Keys which are None may + # optionally be omitted entirely. + inspect_config = { + 'info_types': info_types, + 'min_likelihood': min_likelihood, + 'include_quote': include_quote, + 'limits': {'max_findings_per_request': max_findings}, + } + + inspect_template = { + 'inspect_config': inspect_config, + 'display_name': display_name, + } + + # Convert the project id into a full resource id. + parent = dlp.project_path(project) + + # Call the API. + response = dlp.create_inspect_template( + parent, inspect_template=inspect_template, template_id=template_id) + + print('Successfully created template {}'.format(response.name)) + +# [END dlp_create_template] + + +# [START dlp_list_templates] +def list_inspect_templates(project): + """Lists all Data Loss Prevention API inspect templates. + Args: + project: The Google Cloud project id to use as a parent resource. + Returns: + None; the response from the API is printed to the terminal. + """ + + # Import the client library + import google.cloud.dlp + + # Instantiate a client. + dlp = google.cloud.dlp.DlpServiceClient() + + # Convert the project id into a full resource id. + parent = dlp.project_path(project) + + # Call the API. + response = dlp.list_inspect_templates(parent) + + # Define a helper function to convert the API's "seconds since the epoch" + # time format into a human-readable string. + def human_readable_time(timestamp): + return str(time.localtime(timestamp.seconds)) + + for template in response: + print('Template {}:'.format(template.name)) + if template.display_name: + print(' Display Name: {}'.format(template.display_name)) + print(' Created: {}'.format( + human_readable_time(template.create_time))) + print(' Updated: {}'.format( + human_readable_time(template.update_time))) + + config = template.inspect_config + print(' InfoTypes: {}'.format(', '.join( + [it.name for it in config.info_types] + ))) + print(' Minimum likelihood: {}'.format(config.min_likelihood)) + print(' Include quotes: {}'.format(config.include_quote)) + print(' Max findings per request: {}'.format( + config.limits.max_findings_per_request)) + +# [END dlp_list_templates] + + +# [START dlp_delete_template] +def delete_inspect_template(project, template_id): + """Deletes a Data Loss Prevention API template. + Args: + project: The id of the Google Cloud project which owns the template. + template_id: The id of the template to delete. + Returns: + None; the response from the API is printed to the terminal. + """ + + # Import the client library + import google.cloud.dlp + + # Instantiate a client. + dlp = google.cloud.dlp.DlpServiceClient() + + # Convert the project id into a full resource id. + parent = dlp.project_path(project) + + # Combine the template id with the parent id. + template_resource = '{}/inspectTemplates/{}'.format(parent, template_id) + + # Call the API. + dlp.delete_inspect_template(template_resource) + + print('Template {} successfully deleted.'.format(template_resource)) + +# [END dlp_delete_template] + + +if __name__ == '__main__': + default_project = os.environ.get('GCLOUD_PROJECT') + + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers( + dest='action', help='Select which action to perform.') + subparsers.required = True + + parser_create = subparsers.add_parser('create', help='Create a template.') + parser_create.add_argument( + '--template_id', + help='The id of the template. If omitted, an id will be randomly ' + 'generated') + parser_create.add_argument( + '--display_name', + help='The optional display name of the template.') + parser_create.add_argument( + '--project', + help='The Google Cloud project id to use as a parent resource.', + default=default_project) + parser_create.add_argument( + '--info_types', action='append', + help='Strings representing info types to look for. A full list of ' + 'info categories and types is available from the API. Examples ' + 'include "FIRST_NAME", "LAST_NAME", "EMAIL_ADDRESS". ' + 'If unspecified, the three above examples will be used.', + default=['FIRST_NAME', 'LAST_NAME', 'EMAIL_ADDRESS']) + parser_create.add_argument( + '--min_likelihood', + choices=['LIKELIHOOD_UNSPECIFIED', 'VERY_UNLIKELY', 'UNLIKELY', + 'POSSIBLE', 'LIKELY', 'VERY_LIKELY'], + help='A string representing the minimum likelihood threshold that ' + 'constitutes a match.') + parser_create.add_argument( + '--max_findings', type=int, + help='The maximum number of findings to report; 0 = no maximum.') + parser_create.add_argument( + '--include_quote', type=bool, + help='A boolean for whether to display a quote of the detected ' + 'information in the results.', + default=True) + + parser_list = subparsers.add_parser('list', help='List all templates.') + parser_list.add_argument( + '--project', + help='The Google Cloud project id to use as a parent resource.', + default=default_project) + + parser_delete = subparsers.add_parser('delete', help='Delete a template.') + parser_delete.add_argument( + 'template_id', + help='The id of the template to delete.') + parser_delete.add_argument( + '--project', + help='The Google Cloud project id to use as a parent resource.', + default=default_project) + + args = parser.parse_args() + + if args.action == 'create': + create_inspect_template( + args.project, args.info_types, + template_id=args.template_id, display_name=args.display_name, + min_likelihood=args.min_likelihood, + max_findings=args.max_findings, include_quote=args.include_quote + ) + elif args.action == 'list': + list_inspect_templates(args.project) + elif args.action == 'delete': + delete_inspect_template(args.project, args.template_id) diff --git a/dlp/templates_test.py b/dlp/templates_test.py new file mode 100644 index 00000000000..776096719ef --- /dev/null +++ b/dlp/templates_test.py @@ -0,0 +1,57 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import google.api_core.exceptions +import google.cloud.storage + +import templates + + +GCLOUD_PROJECT = os.getenv('GCLOUD_PROJECT') +TEST_TEMPLATE_ID = 'test-template' + + +def test_create_list_and_delete_template(capsys): + try: + templates.create_inspect_template( + GCLOUD_PROJECT, ['FIRST_NAME', 'EMAIL_ADDRESS', 'PHONE_NUMBER'], + template_id=TEST_TEMPLATE_ID, + ) + except google.api_core.exceptions.InvalidArgument: + # Template already exists, perhaps due to a previous interrupted test. + templates.delete_inspect_template(GCLOUD_PROJECT, TEST_TEMPLATE_ID) + + out, _ = capsys.readouterr() + assert TEST_TEMPLATE_ID in out + + # Try again and move on. + templates.create_inspect_template( + GCLOUD_PROJECT, ['FIRST_NAME', 'EMAIL_ADDRESS', 'PHONE_NUMBER'], + template_id=TEST_TEMPLATE_ID, + ) + + out, _ = capsys.readouterr() + assert TEST_TEMPLATE_ID in out + + templates.list_inspect_templates(GCLOUD_PROJECT) + + out, _ = capsys.readouterr() + assert TEST_TEMPLATE_ID in out + + templates.delete_inspect_template(GCLOUD_PROJECT, TEST_TEMPLATE_ID) + + out, _ = capsys.readouterr() + assert TEST_TEMPLATE_ID in out diff --git a/dlp/triggers.py b/dlp/triggers.py new file mode 100644 index 00000000000..7126f0dcdc7 --- /dev/null +++ b/dlp/triggers.py @@ -0,0 +1,266 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sample app that sets up Data Loss Prevention API automation triggers.""" + +from __future__ import print_function + +import argparse +import os +import time + + +# [START dlp_create_trigger] +def create_trigger(project, bucket, scan_period_days, info_types, + trigger_id=None, display_name=None, description=None, + min_likelihood=None, max_findings=None, + auto_populate_timespan=False): + """Creates a scheduled Data Loss Prevention API inspect_content trigger. + Args: + project: The Google Cloud project id to use as a parent resource. + bucket: The name of the GCS bucket to scan. This sample scans all + files in the bucket using a wildcard. + scan_period_days: How often to repeat the scan, in days. + The minimum is 1 day. + info_types: A list of strings representing info types to look for. + A full list of info type categories can be fetched from the API. + trigger_id: The id of the trigger. If omitted, an id will be randomly + generated. + display_name: The optional display name of the trigger. + description: The optional description of the trigger. + min_likelihood: A string representing the minimum likelihood threshold + that constitutes a match. One of: 'LIKELIHOOD_UNSPECIFIED', + 'VERY_UNLIKELY', 'UNLIKELY', 'POSSIBLE', 'LIKELY', 'VERY_LIKELY'. + max_findings: The maximum number of findings to report; 0 = no maximum. + auto_populate_timespan: Automatically populates time span config start + and end times in order to scan new content only. + Returns: + None; the response from the API is printed to the terminal. + """ + + # Import the client library + import google.cloud.dlp + + # Instantiate a client. + dlp = google.cloud.dlp.DlpServiceClient() + + # Prepare info_types by converting the list of strings into a list of + # dictionaries (protos are also accepted). + info_types = [{'name': info_type} for info_type in info_types] + + # Construct the configuration dictionary. Keys which are None may + # optionally be omitted entirely. + inspect_config = { + 'info_types': info_types, + 'min_likelihood': min_likelihood, + 'limits': {'max_findings_per_request': max_findings}, + } + + # Construct a cloud_storage_options dictionary with the bucket's URL. + url = 'gs://{}/*'.format(bucket) + storage_config = { + 'cloud_storage_options': { + 'file_set': {'url': url} + }, + # Time-based configuration for each storage object. + 'timespan_config': { + # Auto-populate start and end times in order to scan new objects + # only. + 'enable_auto_population_of_timespan_config': auto_populate_timespan + }, + } + + # Construct the job definition. + job = { + 'inspect_config': inspect_config, + 'storage_config': storage_config, + } + + # Construct the schedule definition: + schedule = { + 'recurrence_period_duration': { + 'seconds': scan_period_days * 60 * 60 * 24, + } + } + + # Construct the trigger definition. + job_trigger = { + 'inspect_job': job, + 'display_name': display_name, + 'description': description, + 'triggers': [ + {'schedule': schedule} + ], + 'status': 'HEALTHY' + } + + # Convert the project id into a full resource id. + parent = dlp.project_path(project) + + # Call the API. + response = dlp.create_job_trigger( + parent, job_trigger=job_trigger, trigger_id=trigger_id) + + print('Successfully created trigger {}'.format(response.name)) + +# [END dlp_create_trigger] + + +# [START dlp_list_triggers] +def list_triggers(project): + """Lists all Data Loss Prevention API triggers. + Args: + project: The Google Cloud project id to use as a parent resource. + Returns: + None; the response from the API is printed to the terminal. + """ + + # Import the client library + import google.cloud.dlp + + # Instantiate a client. + dlp = google.cloud.dlp.DlpServiceClient() + + # Convert the project id into a full resource id. + parent = dlp.project_path(project) + + # Call the API. + response = dlp.list_job_triggers(parent) + + # Define a helper function to convert the API's "seconds since the epoch" + # time format into a human-readable string. + def human_readable_time(timestamp): + return str(time.localtime(timestamp.seconds)) + + for trigger in response: + print('Trigger {}:'.format(trigger.name)) + print(' Created: {}'.format(human_readable_time(trigger.create_time))) + print(' Updated: {}'.format(human_readable_time(trigger.update_time))) + if trigger.display_name: + print(' Display Name: {}'.format(trigger.display_name)) + if trigger.description: + print(' Description: {}'.format(trigger.discription)) + print(' Status: {}'.format(trigger.status)) + print(' Error count: {}'.format(len(trigger.errors))) + +# [END dlp_list_triggers] + + +# [START dlp_delete_trigger] +def delete_trigger(project, trigger_id): + """Deletes a Data Loss Prevention API trigger. + Args: + project: The id of the Google Cloud project which owns the trigger. + trigger_id: The id of the trigger to delete. + Returns: + None; the response from the API is printed to the terminal. + """ + + # Import the client library + import google.cloud.dlp + + # Instantiate a client. + dlp = google.cloud.dlp.DlpServiceClient() + + # Convert the project id into a full resource id. + parent = dlp.project_path(project) + + # Combine the trigger id with the parent id. + trigger_resource = '{}/jobTriggers/{}'.format(parent, trigger_id) + + # Call the API. + dlp.delete_job_trigger(trigger_resource) + + print('Trigger {} successfully deleted.'.format(trigger_resource)) + +# [END dlp_delete_triggers] + + +if __name__ == '__main__': + default_project = os.environ.get('GCLOUD_PROJECT') + + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers( + dest='action', help='Select which action to perform.') + subparsers.required = True + + parser_create = subparsers.add_parser('create', help='Create a trigger.') + parser_create.add_argument( + 'bucket', help='The name of the GCS bucket containing the file.') + parser_create.add_argument( + 'scan_period_days', type=int, + help='How often to repeat the scan, in days. The minimum is 1 day.') + parser_create.add_argument( + '--trigger_id', + help='The id of the trigger. If omitted, an id will be randomly ' + 'generated') + parser_create.add_argument( + '--display_name', + help='The optional display name of the trigger.') + parser_create.add_argument( + '--description', + help='The optional description of the trigger.') + parser_create.add_argument( + '--project', + help='The Google Cloud project id to use as a parent resource.', + default=default_project) + parser_create.add_argument( + '--info_types', action='append', + help='Strings representing info types to look for. A full list of ' + 'info categories and types is available from the API. Examples ' + 'include "FIRST_NAME", "LAST_NAME", "EMAIL_ADDRESS". ' + 'If unspecified, the three above examples will be used.', + default=['FIRST_NAME', 'LAST_NAME', 'EMAIL_ADDRESS']) + parser_create.add_argument( + '--min_likelihood', + choices=['LIKELIHOOD_UNSPECIFIED', 'VERY_UNLIKELY', 'UNLIKELY', + 'POSSIBLE', 'LIKELY', 'VERY_LIKELY'], + help='A string representing the minimum likelihood threshold that ' + 'constitutes a match.') + parser_create.add_argument( + '--max_findings', type=int, + help='The maximum number of findings to report; 0 = no maximum.') + parser_create.add_argument( + '--auto_populate_timespan', type=bool, + help='Limit scan to new content only.') + + parser_list = subparsers.add_parser('list', help='List all triggers.') + parser_list.add_argument( + '--project', + help='The Google Cloud project id to use as a parent resource.', + default=default_project) + + parser_delete = subparsers.add_parser('delete', help='Delete a trigger.') + parser_delete.add_argument( + 'trigger_id', + help='The id of the trigger to delete.') + parser_delete.add_argument( + '--project', + help='The Google Cloud project id to use as a parent resource.', + default=default_project) + + args = parser.parse_args() + + if args.action == 'create': + create_trigger( + args.project, args.bucket, args.scan_period_days, args.info_types, + trigger_id=args.trigger_id, display_name=args.display_name, + description=args.description, min_likelihood=args.min_likelihood, + max_findings=args.max_findings, + auto_populate_timespan=args.auto_populate_timespan, + ) + elif args.action == 'list': + list_triggers(args.project) + elif args.action == 'delete': + delete_trigger(args.project, args.trigger_id) diff --git a/dlp/triggers_test.py b/dlp/triggers_test.py new file mode 100644 index 00000000000..a24d58370a4 --- /dev/null +++ b/dlp/triggers_test.py @@ -0,0 +1,95 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import google.api_core.exceptions +import google.cloud.storage + +import pytest + +import triggers + + +GCLOUD_PROJECT = os.getenv('GCLOUD_PROJECT') +TEST_BUCKET_NAME = GCLOUD_PROJECT + '-dlp-python-client-test' +RESOURCE_DIRECTORY = os.path.join(os.path.dirname(__file__), 'resources') +RESOURCE_FILE_NAMES = ['test.txt', 'test.png', 'harmless.txt', 'accounts.txt'] +TEST_TRIGGER_ID = 'test-trigger' + + +@pytest.fixture(scope='module') +def bucket(): + # Creates a GCS bucket, uploads files required for the test, and tears down + # the entire bucket afterwards. + + client = google.cloud.storage.Client() + try: + bucket = client.get_bucket(TEST_BUCKET_NAME) + except google.cloud.exceptions.NotFound: + bucket = client.create_bucket(TEST_BUCKET_NAME) + + # Upoad the blobs and keep track of them in a list. + blobs = [] + for name in RESOURCE_FILE_NAMES: + path = os.path.join(RESOURCE_DIRECTORY, name) + blob = bucket.blob(name) + blob.upload_from_filename(path) + blobs.append(blob) + + # Yield the object to the test; lines after this execute as a teardown. + yield bucket + + # Delete the files. + for blob in blobs: + blob.delete() + + # Attempt to delete the bucket; this will only work if it is empty. + bucket.delete() + + +def test_create_list_and_delete_trigger(bucket, capsys): + try: + triggers.create_trigger( + GCLOUD_PROJECT, bucket.name, 7, + ['FIRST_NAME', 'EMAIL_ADDRESS', 'PHONE_NUMBER'], + trigger_id=TEST_TRIGGER_ID, + ) + except google.api_core.exceptions.InvalidArgument: + # Trigger already exists, perhaps due to a previous interrupted test. + triggers.delete_trigger(GCLOUD_PROJECT, TEST_TRIGGER_ID) + + out, _ = capsys.readouterr() + assert TEST_TRIGGER_ID in out + + # Try again and move on. + triggers.create_trigger( + GCLOUD_PROJECT, bucket.name, 7, + ['FIRST_NAME', 'EMAIL_ADDRESS', 'PHONE_NUMBER'], + trigger_id=TEST_TRIGGER_ID, + auto_populate_timespan=True, + ) + + out, _ = capsys.readouterr() + assert TEST_TRIGGER_ID in out + + triggers.list_triggers(GCLOUD_PROJECT) + + out, _ = capsys.readouterr() + assert TEST_TRIGGER_ID in out + + triggers.delete_trigger(GCLOUD_PROJECT, TEST_TRIGGER_ID) + + out, _ = capsys.readouterr() + assert TEST_TRIGGER_ID in out diff --git a/dns/api/README.rst b/dns/api/README.rst new file mode 100644 index 00000000000..1069a05dec3 --- /dev/null +++ b/dns/api/README.rst @@ -0,0 +1,97 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud DNS Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=dns/api/README.rst + + +This directory contains samples for Google Cloud DNS. `Google Cloud DNS`_ allows you publish your domain names using Google's infrastructure for production-quality, high-volume DNS services. Google's global network of anycast name servers provide reliable, low-latency authoritative name lookups for your domains from anywhere in the world. + + + + +.. _Google Cloud DNS: https://cloud.google.com/dns/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Snippets ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=dns/api/main.py,dns/api/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python main.py + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/dns/api/README.rst.in b/dns/api/README.rst.in new file mode 100644 index 00000000000..25c6d852d3f --- /dev/null +++ b/dns/api/README.rst.in @@ -0,0 +1,24 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud DNS + short_name: Cloud DNS + url: https://cloud.google.com/dns/docs + description: > + `Google Cloud DNS`_ allows you publish your domain names using Google's + infrastructure for production-quality, high-volume DNS services. + Google's global network of anycast name servers provide reliable, + low-latency authoritative name lookups for your domains from anywhere + in the world. + +setup: +- auth +- install_deps + +samples: +- name: Snippets + file: main.py + +cloud_client_library: true + +folder: dns/api \ No newline at end of file diff --git a/dns/api/main.py b/dns/api/main.py new file mode 100644 index 00000000000..f18517d2dea --- /dev/null +++ b/dns/api/main.py @@ -0,0 +1,164 @@ +# Copyright 2016, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +from google.cloud import dns +from google.cloud.exceptions import NotFound + + +# [START create_zone] +def create_zone(project_id, name, dns_name, description): + client = dns.Client(project=project_id) + zone = client.zone( + name, # examplezonename + dns_name=dns_name, # example.com. + description=description) + zone.create() + return zone +# [END create_zone] + + +# [START get_zone] +def get_zone(project_id, name): + client = dns.Client(project=project_id) + zone = client.zone(name=name) + + try: + zone.reload() + return zone + except NotFound: + return None +# [END get_zone] + + +# [START list_zones] +def list_zones(project_id): + client = dns.Client(project=project_id) + zones = client.list_zones() + return [zone.name for zone in zones] +# [END list_zones] + + +# [START delete_zone] +def delete_zone(project_id, name): + client = dns.Client(project=project_id) + zone = client.zone(name) + zone.delete() +# [END delete_zone] + + +# [START list_resource_records] +def list_resource_records(project_id, zone_name): + client = dns.Client(project=project_id) + zone = client.zone(zone_name) + + records = zone.list_resource_record_sets() + + return [(record.name, record.record_type, record.ttl, record.rrdatas) + for record in records] +# [END list_resource_records] + + +# [START changes] +def list_changes(project_id, zone_name): + client = dns.Client(project=project_id) + zone = client.zone(zone_name) + + changes = zone.list_changes() + + return [(change.started, change.status) for change in changes] +# [END changes] + + +def create_command(args): + """Adds a zone with the given name, DNS name, and description.""" + zone = create_zone( + args.project_id, args.name, args.dns_name, args.description) + print('Zone {} added.'.format(zone.name)) + + +def get_command(args): + """Gets a zone by name.""" + zone = get_zone(args.project_id, args.name) + if not zone: + print('Zone not found.') + else: + print('Zone: {}, {}, {}'.format( + zone.name, zone.dns_name, zone.description)) + + +def list_command(args): + """Lists all zones.""" + print(list_zones(args.project_id)) + + +def delete_command(args): + """Deletes a zone.""" + delete_zone(args.project_id, args.name) + print('Zone {} deleted.'.format(args.name)) + + +def list_resource_records_command(args): + """List all resource records for a zone.""" + records = list_resource_records(args.project_id, args.name) + for record in records: + print(record) + + +def changes_command(args): + """List all changes records for a zone.""" + changes = list_changes(args.project_id, args.name) + for change in changes: + print(change) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers() + + parser.add_argument('--project-id', help='Your cloud project ID.') + + create_parser = subparsers.add_parser( + 'create', help=create_command.__doc__) + create_parser.set_defaults(func=create_command) + create_parser.add_argument('name', help='New zone name, e.g. "azonename".') + create_parser.add_argument( + 'dns_name', help='New zone dns name, e.g. "example.com."') + create_parser.add_argument('description', help='New zone description.') + + get_parser = subparsers.add_parser('get', help=get_command.__doc__) + get_parser.add_argument('name', help='Zone name, e.g. "azonename".') + get_parser.set_defaults(func=get_command) + + list_parser = subparsers.add_parser('list', help=list_command.__doc__) + list_parser.set_defaults(func=list_command) + + delete_parser = subparsers.add_parser( + 'delete', help=delete_command.__doc__) + delete_parser.add_argument('name', help='Zone name, e.g. "azonename".') + delete_parser.set_defaults(func=delete_command) + + list_rr_parser = subparsers.add_parser( + 'list-resource-records', help=list_resource_records_command.__doc__) + list_rr_parser.add_argument('name', help='Zone name, e.g. "azonename".') + list_rr_parser.set_defaults(func=list_resource_records_command) + + changes_parser = subparsers.add_parser( + 'changes', help=changes_command.__doc__) + changes_parser.add_argument('name', help='Zone name, e.g. "azonename".') + changes_parser.set_defaults(func=changes_command) + + args = parser.parse_args() + + args.func(args) diff --git a/dns/api/main_test.py b/dns/api/main_test.py new file mode 100644 index 00000000000..8846c5c2b70 --- /dev/null +++ b/dns/api/main_test.py @@ -0,0 +1,104 @@ +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from gcp_devrel.testing.flaky import flaky +from google.cloud import dns +from google.cloud.exceptions import NotFound + +import pytest + +import main + +PROJECT = os.environ['GCLOUD_PROJECT'] +TEST_ZONE_NAME = 'test-zone' +TEST_ZONE_DNS_NAME = 'theadora.is.' +TEST_ZONE_DESCRIPTION = 'Test zone' + + +@pytest.yield_fixture +def client(): + client = dns.Client(PROJECT) + + yield client + + # Delete anything created during the test. + for zone in client.list_zones(): + try: + zone.delete() + except NotFound: # May have been in process + pass + + +@pytest.yield_fixture +def zone(client): + zone = client.zone(TEST_ZONE_NAME, TEST_ZONE_DNS_NAME) + zone.description = TEST_ZONE_DESCRIPTION + zone.create() + + yield zone + + if zone.exists(): + try: + zone.delete() + except NotFound: # May have been under way + pass + + +@flaky +def test_create_zone(client): + zone = main.create_zone( + PROJECT, + TEST_ZONE_NAME, + TEST_ZONE_DNS_NAME, + TEST_ZONE_DESCRIPTION) + + assert zone.name == TEST_ZONE_NAME + assert zone.dns_name == TEST_ZONE_DNS_NAME + assert zone.description == TEST_ZONE_DESCRIPTION + + +@flaky +def test_get_zone(client, zone): + zone = main.get_zone(PROJECT, TEST_ZONE_NAME) + + assert zone.name == TEST_ZONE_NAME + assert zone.dns_name == TEST_ZONE_DNS_NAME + assert zone.description == TEST_ZONE_DESCRIPTION + + +@flaky +def test_list_zones(client, zone): + zones = main.list_zones(PROJECT) + + assert TEST_ZONE_NAME in zones + + +@flaky +def test_list_resource_records(client, zone): + records = main.list_resource_records(PROJECT, TEST_ZONE_NAME) + + assert records + + +@flaky +def test_list_changes(client, zone): + changes = main.list_changes(PROJECT, TEST_ZONE_NAME) + + assert changes + + +@flaky +def test_delete_zone(client, zone): + main.delete_zone(PROJECT, TEST_ZONE_NAME) diff --git a/dns/api/requirements.txt b/dns/api/requirements.txt new file mode 100644 index 00000000000..42b3df79d91 --- /dev/null +++ b/dns/api/requirements.txt @@ -0,0 +1 @@ +google-cloud-dns==0.29.2 diff --git a/endpoints/bookstore-grpc-transcoding/Dockerfile b/endpoints/bookstore-grpc-transcoding/Dockerfile new file mode 100644 index 00000000000..d8a0773cb29 --- /dev/null +++ b/endpoints/bookstore-grpc-transcoding/Dockerfile @@ -0,0 +1,22 @@ +# The Google Cloud Platform Python runtime is based on Debian Jessie +# You can read more about the runtime at: +# https://github.com/GoogleCloudPlatform/python-runtime +FROM gcr.io/google_appengine/python + +# Create a virtualenv for dependencies. This isolates these packages from +# system-level packages. +RUN virtualenv -p python3.6 /env + +# Setting these environment variables are the same as running +# source /env/bin/activate. +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH + +ADD . /bookstore/ + +WORKDIR /bookstore +RUN pip install -r requirements.txt + +EXPOSE 8000 + +ENTRYPOINT ["python", "/bookstore/bookstore_server.py"] diff --git a/endpoints/bookstore-grpc-transcoding/README.md b/endpoints/bookstore-grpc-transcoding/README.md new file mode 100644 index 00000000000..9a9bc9f3ac0 --- /dev/null +++ b/endpoints/bookstore-grpc-transcoding/README.md @@ -0,0 +1,68 @@ +# Google Cloud Endpoints gRPC with Transcoding Bookstore in Python + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=endpoints/bookstore-grpc-transcoding/README.md + +This example demonstrates how to implement a gRPC server +with Google Cloud Endpoints and HTTP/JSON Transcoding. + +## Installing the dependencies using virtualenv: + + virtualenv bookstore-env + source bookstore-env/bin/activate + +Install all the Python dependencies: + + pip install -r requirements.txt + +Install the [gRPC libraries and tools](http://www.grpc.io/docs/quickstart/python.html#prerequisites) + +## Running the Server Locally + +To run the server: + + python bookstore_server.py + +The `-h` command line flag shows the various settings available. + +## Running the Client Locally + +To run the client: + + python bookstore_client.py + +As with the server, the `-h` command line flag shows the various settings +available. + +## Generating a JWT token from a service account file + +To run the script: + + python jwt_token_gen.py --file=account_file --audiences=audiences --issuer=issuer + +The output can be used as "--auth_token" for bookstore_client.py + +## Regenerating the Protocol Buffer and gRPC API stubs + +The bookstore gRPC API is defined by `bookstore.proto` +The API client stubs and server interfaces are generated by tools. + +To make it easier to get something up and running, we've included the generated +code in the sample distribution. To modify the sample or create your own gRPC +API definition, you'll need to update the generated code. To do this, once the +gRPC libraries and tools are installed, run: + + GOOGLEAPIS=/path/to/googleapis + git clone https://github.com/googleapis/googleapis $GOOGLEAPIS + + python -m grpc.tools.protoc \ + --include_imports \ + --include_source_info \ + --proto_path=. \ + --proto_path=$GOOGLEAPIS \ + --python_out=. \ + --grpc_python_out=. \ + --descriptor_set_out=api_descriptor.pb \ + bookstore.proto diff --git a/endpoints/bookstore-grpc-transcoding/api_config.yaml b/endpoints/bookstore-grpc-transcoding/api_config.yaml new file mode 100644 index 00000000000..b7f6ea0e5ef --- /dev/null +++ b/endpoints/bookstore-grpc-transcoding/api_config.yaml @@ -0,0 +1,45 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# A Bookstore example API configuration. +# +# Below, replace MY_PROJECT_ID with your Google Cloud Project ID. +# + +# The configuration schema is defined by service.proto file +# https://github.com/googleapis/googleapis/blob/master/google/api/service.proto +type: google.api.Service +config_version: 3 + +# +# Name of the service configuration. +# +name: bookstore.endpoints..cloud.goog + +# +# API title to appear in the user interface (Google Cloud Console). +# +title: Bookstore gRPC API +apis: +- name: endpoints.examples.bookstore.Bookstore + +# +# API usage restrictions. +# +usage: + rules: + # ListShelves methods can be called without an API Key. + - selector: endpoints.examples.bookstore.Bookstore.ListShelves + allow_unregistered_calls: true diff --git a/endpoints/bookstore-grpc-transcoding/api_config_auth.yaml b/endpoints/bookstore-grpc-transcoding/api_config_auth.yaml new file mode 100644 index 00000000000..099b7292a93 --- /dev/null +++ b/endpoints/bookstore-grpc-transcoding/api_config_auth.yaml @@ -0,0 +1,59 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# A Bookstore example API configuration. +# +# Below, replace MY_PROJECT_ID with your Google Cloud Project ID. +# + +# The configuration schema is defined by service.proto file +# https://github.com/googleapis/googleapis/blob/master/google/api/service.proto +type: google.api.Service +config_version: 3 + +# +# Name of the service configuration. +# +name: bookstore.endpoints..cloud.goog + +# +# API title to appear in the user interface (Google Cloud Console). +# +title: Bookstore gRPC API +apis: +- name: endpoints.examples.bookstore.Bookstore + +# +# API usage restrictions. +# +usage: + rules: + - selector: "*" + allow_unregistered_calls: true + +# +# Request authentication. +# +authentication: + providers: + - id: google_service_account + # Replace SERVICE-ACCOUNT-ID with your service account's email address. + issuer: SERVICE-ACCOUNT-ID + jwks_uri: https://www.googleapis.com/robot/v1/metadata/x509/SERVICE-ACCOUNT-ID + rules: + # This auth rule will apply to all methods. + - selector: "*" + requirements: + - provider_id: google_service_account diff --git a/endpoints/bookstore-grpc-transcoding/bookstore.proto b/endpoints/bookstore-grpc-transcoding/bookstore.proto new file mode 100644 index 00000000000..e7f655455ed --- /dev/null +++ b/endpoints/bookstore-grpc-transcoding/bookstore.proto @@ -0,0 +1,166 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// + +syntax = "proto3"; + +package endpoints.examples.bookstore; + +option java_multiple_files = true; +option java_outer_classname = "BookstoreProto"; +option java_package = "com.google.endpoints.examples.bookstore"; + + +import "google/api/annotations.proto"; +import "google/protobuf/empty.proto"; + +// A simple Bookstore API. +// +// The API manages shelves and books resources. Shelves contain books. +service Bookstore { + // Returns a list of all shelves in the bookstore. + rpc ListShelves(google.protobuf.Empty) returns (ListShelvesResponse) { + // Define HTTP mapping. + // Client example (Assuming your service is hosted at the given 'DOMAIN_NAME'): + // curl http://DOMAIN_NAME/v1/shelves + option (google.api.http) = { get: "/v1/shelves" }; + } + // Creates a new shelf in the bookstore. + rpc CreateShelf(CreateShelfRequest) returns (Shelf) { + // Client example: + // curl -d '{"theme":"Music"}' http://DOMAIN_NAME/v1/shelves + option (google.api.http) = { + post: "/v1/shelves" + body: "shelf" + }; + } + // Returns a specific bookstore shelf. + rpc GetShelf(GetShelfRequest) returns (Shelf) { + // Client example - returns the first shelf: + // curl http://DOMAIN_NAME/v1/shelves/1 + option (google.api.http) = { get: "/v1/shelves/{shelf}" }; + } + // Deletes a shelf, including all books that are stored on the shelf. + rpc DeleteShelf(DeleteShelfRequest) returns (google.protobuf.Empty) { + // Client example - deletes the second shelf: + // curl -X DELETE http://DOMAIN_NAME/v1/shelves/2 + option (google.api.http) = { delete: "/v1/shelves/{shelf}" }; + } + // Returns a list of books on a shelf. + rpc ListBooks(ListBooksRequest) returns (ListBooksResponse) { + // Client example - list the books from the first shelf: + // curl http://DOMAIN_NAME/v1/shelves/1/books + option (google.api.http) = { get: "/v1/shelves/{shelf}/books" }; + } + // Creates a new book. + rpc CreateBook(CreateBookRequest) returns (Book) { + // Client example - create a new book in the first shelf: + // curl -d '{"author":"foo","title":"bar"}' http://DOMAIN_NAME/v1/shelves/1/books + option (google.api.http) = { + post: "/v1/shelves/{shelf}/books" + body: "book" + }; + } + // Returns a specific book. + rpc GetBook(GetBookRequest) returns (Book) { + // Client example - get the first book from the second shelf: + // curl http://DOMAIN_NAME/v1/shelves/2/books/1 + option (google.api.http) = { get: "/v1/shelves/{shelf}/books/{book}" }; + } + // Deletes a book from a shelf. + rpc DeleteBook(DeleteBookRequest) returns (google.protobuf.Empty) { + // Client example - delete the first book from the first shelf: + // curl -X DELETE http://DOMAIN_NAME/v1/shelves/1/books/1 + option (google.api.http) = { delete: "/v1/shelves/{shelf}/books/{book}" }; + } +} + +// A shelf resource. +message Shelf { + // A unique shelf id. + int64 id = 1; + // A theme of the shelf (fiction, poetry, etc). + string theme = 2; +} + +// A book resource. +message Book { + // A unique book id. + int64 id = 1; + // An author of the book. + string author = 2; + // A book title. + string title = 3; +} + +// Response to ListShelves call. +message ListShelvesResponse { + // Shelves in the bookstore. + repeated Shelf shelves = 1; +} + +// Request message for CreateShelf method. +message CreateShelfRequest { + // The shelf resource to create. + Shelf shelf = 1; +} + +// Request message for GetShelf method. +message GetShelfRequest { + // The ID of the shelf resource to retrieve. + int64 shelf = 1; +} + +// Request message for DeleteShelf method. +message DeleteShelfRequest { + // The ID of the shelf to delete. + int64 shelf = 1; +} + +// Request message for ListBooks method. +message ListBooksRequest { + // ID of the shelf which books to list. + int64 shelf = 1; +} + +// Response message to ListBooks method. +message ListBooksResponse { + // The books on the shelf. + repeated Book books = 1; +} + +// Request message for CreateBook method. +message CreateBookRequest { + // The ID of the shelf on which to create a book. + int64 shelf = 1; + // A book resource to create on the shelf. + Book book = 2; +} + +// Request message for GetBook method. +message GetBookRequest { + // The ID of the shelf from which to retrieve a book. + int64 shelf = 1; + // The ID of the book to retrieve. + int64 book = 2; +} + +// Request message for DeleteBook method. +message DeleteBookRequest { + // The ID of the shelf from which to delete a book. + int64 shelf = 1; + // The ID of the book to delete. + int64 book = 2; +} diff --git a/endpoints/bookstore-grpc-transcoding/bookstore.py b/endpoints/bookstore-grpc-transcoding/bookstore.py new file mode 100644 index 00000000000..a32be906d47 --- /dev/null +++ b/endpoints/bookstore-grpc-transcoding/bookstore.py @@ -0,0 +1,76 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import threading + +import six + + +class ShelfInfo(object): + """The contents of a single shelf.""" + def __init__(self, shelf): + self._shelf = shelf + self._last_book_id = 0 + self._books = dict() + + +class Bookstore(object): + """An in-memory backend for storing Bookstore data.""" + + def __init__(self): + self._last_shelf_id = 0 + self._shelves = dict() + self._lock = threading.Lock() + + def list_shelf(self): + with self._lock: + return [s._shelf for (_, s) in six.iteritems(self._shelves)] + + def create_shelf(self, shelf): + with self._lock: + self._last_shelf_id += 1 + shelf_id = self._last_shelf_id + shelf.id = shelf_id + self._shelves[shelf_id] = ShelfInfo(shelf) + return (shelf, shelf_id) + + def get_shelf(self, shelf_id): + with self._lock: + return self._shelves[shelf_id]._shelf + + def delete_shelf(self, shelf_id): + with self._lock: + del self._shelves[shelf_id] + + def list_books(self, shelf_id): + with self._lock: + return [book for ( + _, book) in six.iteritems(self._shelves[shelf_id]._books)] + + def create_book(self, shelf_id, book): + with self._lock: + shelf_info = self._shelves[shelf_id] + shelf_info._last_book_id += 1 + book_id = shelf_info._last_book_id + book.id = book_id + shelf_info._books[book_id] = book + return book + + def get_book(self, shelf_id, book_id): + with self._lock: + return self._shelves[shelf_id]._books[book_id] + + def delete_book(self, shelf_id, book_id): + with self._lock: + del self._shelves[shelf_id]._books[book_id] diff --git a/endpoints/bookstore-grpc-transcoding/bookstore_client.py b/endpoints/bookstore-grpc-transcoding/bookstore_client.py new file mode 100644 index 00000000000..37d31e11183 --- /dev/null +++ b/endpoints/bookstore-grpc-transcoding/bookstore_client.py @@ -0,0 +1,56 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The Python gRPC Bookstore Client Example.""" + +import argparse + +from google.protobuf import empty_pb2 +import grpc + +import bookstore_pb2_grpc + + +def run(host, port, api_key, auth_token, timeout): + """Makes a basic ListShelves call against a gRPC Bookstore server.""" + + channel = grpc.insecure_channel('{}:{}'.format(host, port)) + + stub = bookstore_pb2_grpc.BookstoreStub(channel) + metadata = [] + if api_key: + metadata.append(('x-api-key', api_key)) + if auth_token: + metadata.append(('authorization', 'Bearer ' + auth_token)) + shelves = stub.ListShelves(empty_pb2.Empty(), timeout, metadata=metadata) + print('ListShelves: {}'.format(shelves)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--host', default='localhost', help='The host to connect to') + parser.add_argument( + '--port', type=int, default=8000, help='The port to connect to') + parser.add_argument( + '--timeout', type=int, default=10, help='The call timeout, in seconds') + parser.add_argument( + '--api_key', default=None, help='The API key to use for the call') + parser.add_argument( + '--auth_token', default=None, + help='The JWT auth token to use for the call') + args = parser.parse_args() + run(args.host, args.port, args.api_key, args.auth_token, args.timeout) diff --git a/endpoints/bookstore-grpc-transcoding/bookstore_pb2.py b/endpoints/bookstore-grpc-transcoding/bookstore_pb2.py new file mode 100644 index 00000000000..54f0e1fea1a --- /dev/null +++ b/endpoints/bookstore-grpc-transcoding/bookstore_pb2.py @@ -0,0 +1,597 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: bookstore.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 +from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='bookstore.proto', + package='endpoints.examples.bookstore', + syntax='proto3', + serialized_pb=_b('\n\x0f\x62ookstore.proto\x12\x1c\x65ndpoints.examples.bookstore\x1a\x1cgoogle/api/annotations.proto\x1a\x1bgoogle/protobuf/empty.proto\"\"\n\x05Shelf\x12\n\n\x02id\x18\x01 \x01(\x03\x12\r\n\x05theme\x18\x02 \x01(\t\"1\n\x04\x42ook\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x0e\n\x06\x61uthor\x18\x02 \x01(\t\x12\r\n\x05title\x18\x03 \x01(\t\"K\n\x13ListShelvesResponse\x12\x34\n\x07shelves\x18\x01 \x03(\x0b\x32#.endpoints.examples.bookstore.Shelf\"H\n\x12\x43reateShelfRequest\x12\x32\n\x05shelf\x18\x01 \x01(\x0b\x32#.endpoints.examples.bookstore.Shelf\" \n\x0fGetShelfRequest\x12\r\n\x05shelf\x18\x01 \x01(\x03\"#\n\x12\x44\x65leteShelfRequest\x12\r\n\x05shelf\x18\x01 \x01(\x03\"!\n\x10ListBooksRequest\x12\r\n\x05shelf\x18\x01 \x01(\x03\"F\n\x11ListBooksResponse\x12\x31\n\x05\x62ooks\x18\x01 \x03(\x0b\x32\".endpoints.examples.bookstore.Book\"T\n\x11\x43reateBookRequest\x12\r\n\x05shelf\x18\x01 \x01(\x03\x12\x30\n\x04\x62ook\x18\x02 \x01(\x0b\x32\".endpoints.examples.bookstore.Book\"-\n\x0eGetBookRequest\x12\r\n\x05shelf\x18\x01 \x01(\x03\x12\x0c\n\x04\x62ook\x18\x02 \x01(\x03\"0\n\x11\x44\x65leteBookRequest\x12\r\n\x05shelf\x18\x01 \x01(\x03\x12\x0c\n\x04\x62ook\x18\x02 \x01(\x03\x32\x98\x08\n\tBookstore\x12m\n\x0bListShelves\x12\x16.google.protobuf.Empty\x1a\x31.endpoints.examples.bookstore.ListShelvesResponse\"\x13\x82\xd3\xe4\x93\x02\r\x12\x0b/v1/shelves\x12\x80\x01\n\x0b\x43reateShelf\x12\x30.endpoints.examples.bookstore.CreateShelfRequest\x1a#.endpoints.examples.bookstore.Shelf\"\x1a\x82\xd3\xe4\x93\x02\x14\"\x0b/v1/shelves:\x05shelf\x12{\n\x08GetShelf\x12-.endpoints.examples.bookstore.GetShelfRequest\x1a#.endpoints.examples.bookstore.Shelf\"\x1b\x82\xd3\xe4\x93\x02\x15\x12\x13/v1/shelves/{shelf}\x12t\n\x0b\x44\x65leteShelf\x12\x30.endpoints.examples.bookstore.DeleteShelfRequest\x1a\x16.google.protobuf.Empty\"\x1b\x82\xd3\xe4\x93\x02\x15*\x13/v1/shelves/{shelf}\x12\x8f\x01\n\tListBooks\x12..endpoints.examples.bookstore.ListBooksRequest\x1a/.endpoints.examples.bookstore.ListBooksResponse\"!\x82\xd3\xe4\x93\x02\x1b\x12\x19/v1/shelves/{shelf}/books\x12\x8a\x01\n\nCreateBook\x12/.endpoints.examples.bookstore.CreateBookRequest\x1a\".endpoints.examples.bookstore.Book\"\'\x82\xd3\xe4\x93\x02!\"\x19/v1/shelves/{shelf}/books:\x04\x62ook\x12\x85\x01\n\x07GetBook\x12,.endpoints.examples.bookstore.GetBookRequest\x1a\".endpoints.examples.bookstore.Book\"(\x82\xd3\xe4\x93\x02\"\x12 /v1/shelves/{shelf}/books/{book}\x12\x7f\n\nDeleteBook\x12/.endpoints.examples.bookstore.DeleteBookRequest\x1a\x16.google.protobuf.Empty\"(\x82\xd3\xe4\x93\x02\"* /v1/shelves/{shelf}/books/{book}B;\n\'com.google.endpoints.examples.bookstoreB\x0e\x42ookstoreProtoP\x01\x62\x06proto3') + , + dependencies=[google_dot_api_dot_annotations__pb2.DESCRIPTOR,google_dot_protobuf_dot_empty__pb2.DESCRIPTOR,]) + + + + +_SHELF = _descriptor.Descriptor( + name='Shelf', + full_name='endpoints.examples.bookstore.Shelf', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='id', full_name='endpoints.examples.bookstore.Shelf.id', index=0, + number=1, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='theme', full_name='endpoints.examples.bookstore.Shelf.theme', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=108, + serialized_end=142, +) + + +_BOOK = _descriptor.Descriptor( + name='Book', + full_name='endpoints.examples.bookstore.Book', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='id', full_name='endpoints.examples.bookstore.Book.id', index=0, + number=1, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='author', full_name='endpoints.examples.bookstore.Book.author', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='title', full_name='endpoints.examples.bookstore.Book.title', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=144, + serialized_end=193, +) + + +_LISTSHELVESRESPONSE = _descriptor.Descriptor( + name='ListShelvesResponse', + full_name='endpoints.examples.bookstore.ListShelvesResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='shelves', full_name='endpoints.examples.bookstore.ListShelvesResponse.shelves', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=195, + serialized_end=270, +) + + +_CREATESHELFREQUEST = _descriptor.Descriptor( + name='CreateShelfRequest', + full_name='endpoints.examples.bookstore.CreateShelfRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='shelf', full_name='endpoints.examples.bookstore.CreateShelfRequest.shelf', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=272, + serialized_end=344, +) + + +_GETSHELFREQUEST = _descriptor.Descriptor( + name='GetShelfRequest', + full_name='endpoints.examples.bookstore.GetShelfRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='shelf', full_name='endpoints.examples.bookstore.GetShelfRequest.shelf', index=0, + number=1, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=346, + serialized_end=378, +) + + +_DELETESHELFREQUEST = _descriptor.Descriptor( + name='DeleteShelfRequest', + full_name='endpoints.examples.bookstore.DeleteShelfRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='shelf', full_name='endpoints.examples.bookstore.DeleteShelfRequest.shelf', index=0, + number=1, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=380, + serialized_end=415, +) + + +_LISTBOOKSREQUEST = _descriptor.Descriptor( + name='ListBooksRequest', + full_name='endpoints.examples.bookstore.ListBooksRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='shelf', full_name='endpoints.examples.bookstore.ListBooksRequest.shelf', index=0, + number=1, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=417, + serialized_end=450, +) + + +_LISTBOOKSRESPONSE = _descriptor.Descriptor( + name='ListBooksResponse', + full_name='endpoints.examples.bookstore.ListBooksResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='books', full_name='endpoints.examples.bookstore.ListBooksResponse.books', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=452, + serialized_end=522, +) + + +_CREATEBOOKREQUEST = _descriptor.Descriptor( + name='CreateBookRequest', + full_name='endpoints.examples.bookstore.CreateBookRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='shelf', full_name='endpoints.examples.bookstore.CreateBookRequest.shelf', index=0, + number=1, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='book', full_name='endpoints.examples.bookstore.CreateBookRequest.book', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=524, + serialized_end=608, +) + + +_GETBOOKREQUEST = _descriptor.Descriptor( + name='GetBookRequest', + full_name='endpoints.examples.bookstore.GetBookRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='shelf', full_name='endpoints.examples.bookstore.GetBookRequest.shelf', index=0, + number=1, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='book', full_name='endpoints.examples.bookstore.GetBookRequest.book', index=1, + number=2, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=610, + serialized_end=655, +) + + +_DELETEBOOKREQUEST = _descriptor.Descriptor( + name='DeleteBookRequest', + full_name='endpoints.examples.bookstore.DeleteBookRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='shelf', full_name='endpoints.examples.bookstore.DeleteBookRequest.shelf', index=0, + number=1, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='book', full_name='endpoints.examples.bookstore.DeleteBookRequest.book', index=1, + number=2, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=657, + serialized_end=705, +) + +_LISTSHELVESRESPONSE.fields_by_name['shelves'].message_type = _SHELF +_CREATESHELFREQUEST.fields_by_name['shelf'].message_type = _SHELF +_LISTBOOKSRESPONSE.fields_by_name['books'].message_type = _BOOK +_CREATEBOOKREQUEST.fields_by_name['book'].message_type = _BOOK +DESCRIPTOR.message_types_by_name['Shelf'] = _SHELF +DESCRIPTOR.message_types_by_name['Book'] = _BOOK +DESCRIPTOR.message_types_by_name['ListShelvesResponse'] = _LISTSHELVESRESPONSE +DESCRIPTOR.message_types_by_name['CreateShelfRequest'] = _CREATESHELFREQUEST +DESCRIPTOR.message_types_by_name['GetShelfRequest'] = _GETSHELFREQUEST +DESCRIPTOR.message_types_by_name['DeleteShelfRequest'] = _DELETESHELFREQUEST +DESCRIPTOR.message_types_by_name['ListBooksRequest'] = _LISTBOOKSREQUEST +DESCRIPTOR.message_types_by_name['ListBooksResponse'] = _LISTBOOKSRESPONSE +DESCRIPTOR.message_types_by_name['CreateBookRequest'] = _CREATEBOOKREQUEST +DESCRIPTOR.message_types_by_name['GetBookRequest'] = _GETBOOKREQUEST +DESCRIPTOR.message_types_by_name['DeleteBookRequest'] = _DELETEBOOKREQUEST +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +Shelf = _reflection.GeneratedProtocolMessageType('Shelf', (_message.Message,), dict( + DESCRIPTOR = _SHELF, + __module__ = 'bookstore_pb2' + # @@protoc_insertion_point(class_scope:endpoints.examples.bookstore.Shelf) + )) +_sym_db.RegisterMessage(Shelf) + +Book = _reflection.GeneratedProtocolMessageType('Book', (_message.Message,), dict( + DESCRIPTOR = _BOOK, + __module__ = 'bookstore_pb2' + # @@protoc_insertion_point(class_scope:endpoints.examples.bookstore.Book) + )) +_sym_db.RegisterMessage(Book) + +ListShelvesResponse = _reflection.GeneratedProtocolMessageType('ListShelvesResponse', (_message.Message,), dict( + DESCRIPTOR = _LISTSHELVESRESPONSE, + __module__ = 'bookstore_pb2' + # @@protoc_insertion_point(class_scope:endpoints.examples.bookstore.ListShelvesResponse) + )) +_sym_db.RegisterMessage(ListShelvesResponse) + +CreateShelfRequest = _reflection.GeneratedProtocolMessageType('CreateShelfRequest', (_message.Message,), dict( + DESCRIPTOR = _CREATESHELFREQUEST, + __module__ = 'bookstore_pb2' + # @@protoc_insertion_point(class_scope:endpoints.examples.bookstore.CreateShelfRequest) + )) +_sym_db.RegisterMessage(CreateShelfRequest) + +GetShelfRequest = _reflection.GeneratedProtocolMessageType('GetShelfRequest', (_message.Message,), dict( + DESCRIPTOR = _GETSHELFREQUEST, + __module__ = 'bookstore_pb2' + # @@protoc_insertion_point(class_scope:endpoints.examples.bookstore.GetShelfRequest) + )) +_sym_db.RegisterMessage(GetShelfRequest) + +DeleteShelfRequest = _reflection.GeneratedProtocolMessageType('DeleteShelfRequest', (_message.Message,), dict( + DESCRIPTOR = _DELETESHELFREQUEST, + __module__ = 'bookstore_pb2' + # @@protoc_insertion_point(class_scope:endpoints.examples.bookstore.DeleteShelfRequest) + )) +_sym_db.RegisterMessage(DeleteShelfRequest) + +ListBooksRequest = _reflection.GeneratedProtocolMessageType('ListBooksRequest', (_message.Message,), dict( + DESCRIPTOR = _LISTBOOKSREQUEST, + __module__ = 'bookstore_pb2' + # @@protoc_insertion_point(class_scope:endpoints.examples.bookstore.ListBooksRequest) + )) +_sym_db.RegisterMessage(ListBooksRequest) + +ListBooksResponse = _reflection.GeneratedProtocolMessageType('ListBooksResponse', (_message.Message,), dict( + DESCRIPTOR = _LISTBOOKSRESPONSE, + __module__ = 'bookstore_pb2' + # @@protoc_insertion_point(class_scope:endpoints.examples.bookstore.ListBooksResponse) + )) +_sym_db.RegisterMessage(ListBooksResponse) + +CreateBookRequest = _reflection.GeneratedProtocolMessageType('CreateBookRequest', (_message.Message,), dict( + DESCRIPTOR = _CREATEBOOKREQUEST, + __module__ = 'bookstore_pb2' + # @@protoc_insertion_point(class_scope:endpoints.examples.bookstore.CreateBookRequest) + )) +_sym_db.RegisterMessage(CreateBookRequest) + +GetBookRequest = _reflection.GeneratedProtocolMessageType('GetBookRequest', (_message.Message,), dict( + DESCRIPTOR = _GETBOOKREQUEST, + __module__ = 'bookstore_pb2' + # @@protoc_insertion_point(class_scope:endpoints.examples.bookstore.GetBookRequest) + )) +_sym_db.RegisterMessage(GetBookRequest) + +DeleteBookRequest = _reflection.GeneratedProtocolMessageType('DeleteBookRequest', (_message.Message,), dict( + DESCRIPTOR = _DELETEBOOKREQUEST, + __module__ = 'bookstore_pb2' + # @@protoc_insertion_point(class_scope:endpoints.examples.bookstore.DeleteBookRequest) + )) +_sym_db.RegisterMessage(DeleteBookRequest) + + +DESCRIPTOR.has_options = True +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\'com.google.endpoints.examples.bookstoreB\016BookstoreProtoP\001')) + +_BOOKSTORE = _descriptor.ServiceDescriptor( + name='Bookstore', + full_name='endpoints.examples.bookstore.Bookstore', + file=DESCRIPTOR, + index=0, + options=None, + serialized_start=708, + serialized_end=1756, + methods=[ + _descriptor.MethodDescriptor( + name='ListShelves', + full_name='endpoints.examples.bookstore.Bookstore.ListShelves', + index=0, + containing_service=None, + input_type=google_dot_protobuf_dot_empty__pb2._EMPTY, + output_type=_LISTSHELVESRESPONSE, + options=_descriptor._ParseOptions(descriptor_pb2.MethodOptions(), _b('\202\323\344\223\002\r\022\013/v1/shelves')), + ), + _descriptor.MethodDescriptor( + name='CreateShelf', + full_name='endpoints.examples.bookstore.Bookstore.CreateShelf', + index=1, + containing_service=None, + input_type=_CREATESHELFREQUEST, + output_type=_SHELF, + options=_descriptor._ParseOptions(descriptor_pb2.MethodOptions(), _b('\202\323\344\223\002\024\"\013/v1/shelves:\005shelf')), + ), + _descriptor.MethodDescriptor( + name='GetShelf', + full_name='endpoints.examples.bookstore.Bookstore.GetShelf', + index=2, + containing_service=None, + input_type=_GETSHELFREQUEST, + output_type=_SHELF, + options=_descriptor._ParseOptions(descriptor_pb2.MethodOptions(), _b('\202\323\344\223\002\025\022\023/v1/shelves/{shelf}')), + ), + _descriptor.MethodDescriptor( + name='DeleteShelf', + full_name='endpoints.examples.bookstore.Bookstore.DeleteShelf', + index=3, + containing_service=None, + input_type=_DELETESHELFREQUEST, + output_type=google_dot_protobuf_dot_empty__pb2._EMPTY, + options=_descriptor._ParseOptions(descriptor_pb2.MethodOptions(), _b('\202\323\344\223\002\025*\023/v1/shelves/{shelf}')), + ), + _descriptor.MethodDescriptor( + name='ListBooks', + full_name='endpoints.examples.bookstore.Bookstore.ListBooks', + index=4, + containing_service=None, + input_type=_LISTBOOKSREQUEST, + output_type=_LISTBOOKSRESPONSE, + options=_descriptor._ParseOptions(descriptor_pb2.MethodOptions(), _b('\202\323\344\223\002\033\022\031/v1/shelves/{shelf}/books')), + ), + _descriptor.MethodDescriptor( + name='CreateBook', + full_name='endpoints.examples.bookstore.Bookstore.CreateBook', + index=5, + containing_service=None, + input_type=_CREATEBOOKREQUEST, + output_type=_BOOK, + options=_descriptor._ParseOptions(descriptor_pb2.MethodOptions(), _b('\202\323\344\223\002!\"\031/v1/shelves/{shelf}/books:\004book')), + ), + _descriptor.MethodDescriptor( + name='GetBook', + full_name='endpoints.examples.bookstore.Bookstore.GetBook', + index=6, + containing_service=None, + input_type=_GETBOOKREQUEST, + output_type=_BOOK, + options=_descriptor._ParseOptions(descriptor_pb2.MethodOptions(), _b('\202\323\344\223\002\"\022 /v1/shelves/{shelf}/books/{book}')), + ), + _descriptor.MethodDescriptor( + name='DeleteBook', + full_name='endpoints.examples.bookstore.Bookstore.DeleteBook', + index=7, + containing_service=None, + input_type=_DELETEBOOKREQUEST, + output_type=google_dot_protobuf_dot_empty__pb2._EMPTY, + options=_descriptor._ParseOptions(descriptor_pb2.MethodOptions(), _b('\202\323\344\223\002\"* /v1/shelves/{shelf}/books/{book}')), + ), +]) +_sym_db.RegisterServiceDescriptor(_BOOKSTORE) + +DESCRIPTOR.services_by_name['Bookstore'] = _BOOKSTORE + +# @@protoc_insertion_point(module_scope) diff --git a/endpoints/bookstore-grpc-transcoding/bookstore_pb2_grpc.py b/endpoints/bookstore-grpc-transcoding/bookstore_pb2_grpc.py new file mode 100644 index 00000000000..d2bfc7a1af7 --- /dev/null +++ b/endpoints/bookstore-grpc-transcoding/bookstore_pb2_grpc.py @@ -0,0 +1,170 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +import grpc + +import bookstore_pb2 as bookstore__pb2 +from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 + + +class BookstoreStub(object): + """A simple Bookstore API. + + The API manages shelves and books resources. Shelves contain books. + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.ListShelves = channel.unary_unary( + '/endpoints.examples.bookstore.Bookstore/ListShelves', + request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + response_deserializer=bookstore__pb2.ListShelvesResponse.FromString, + ) + self.CreateShelf = channel.unary_unary( + '/endpoints.examples.bookstore.Bookstore/CreateShelf', + request_serializer=bookstore__pb2.CreateShelfRequest.SerializeToString, + response_deserializer=bookstore__pb2.Shelf.FromString, + ) + self.GetShelf = channel.unary_unary( + '/endpoints.examples.bookstore.Bookstore/GetShelf', + request_serializer=bookstore__pb2.GetShelfRequest.SerializeToString, + response_deserializer=bookstore__pb2.Shelf.FromString, + ) + self.DeleteShelf = channel.unary_unary( + '/endpoints.examples.bookstore.Bookstore/DeleteShelf', + request_serializer=bookstore__pb2.DeleteShelfRequest.SerializeToString, + response_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, + ) + self.ListBooks = channel.unary_unary( + '/endpoints.examples.bookstore.Bookstore/ListBooks', + request_serializer=bookstore__pb2.ListBooksRequest.SerializeToString, + response_deserializer=bookstore__pb2.ListBooksResponse.FromString, + ) + self.CreateBook = channel.unary_unary( + '/endpoints.examples.bookstore.Bookstore/CreateBook', + request_serializer=bookstore__pb2.CreateBookRequest.SerializeToString, + response_deserializer=bookstore__pb2.Book.FromString, + ) + self.GetBook = channel.unary_unary( + '/endpoints.examples.bookstore.Bookstore/GetBook', + request_serializer=bookstore__pb2.GetBookRequest.SerializeToString, + response_deserializer=bookstore__pb2.Book.FromString, + ) + self.DeleteBook = channel.unary_unary( + '/endpoints.examples.bookstore.Bookstore/DeleteBook', + request_serializer=bookstore__pb2.DeleteBookRequest.SerializeToString, + response_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, + ) + + +class BookstoreServicer(object): + """A simple Bookstore API. + + The API manages shelves and books resources. Shelves contain books. + """ + + def ListShelves(self, request, context): + """Returns a list of all shelves in the bookstore. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def CreateShelf(self, request, context): + """Creates a new shelf in the bookstore. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetShelf(self, request, context): + """Returns a specific bookstore shelf. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def DeleteShelf(self, request, context): + """Deletes a shelf, including all books that are stored on the shelf. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ListBooks(self, request, context): + """Returns a list of books on a shelf. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def CreateBook(self, request, context): + """Creates a new book. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetBook(self, request, context): + """Returns a specific book. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def DeleteBook(self, request, context): + """Deletes a book from a shelf. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_BookstoreServicer_to_server(servicer, server): + rpc_method_handlers = { + 'ListShelves': grpc.unary_unary_rpc_method_handler( + servicer.ListShelves, + request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, + response_serializer=bookstore__pb2.ListShelvesResponse.SerializeToString, + ), + 'CreateShelf': grpc.unary_unary_rpc_method_handler( + servicer.CreateShelf, + request_deserializer=bookstore__pb2.CreateShelfRequest.FromString, + response_serializer=bookstore__pb2.Shelf.SerializeToString, + ), + 'GetShelf': grpc.unary_unary_rpc_method_handler( + servicer.GetShelf, + request_deserializer=bookstore__pb2.GetShelfRequest.FromString, + response_serializer=bookstore__pb2.Shelf.SerializeToString, + ), + 'DeleteShelf': grpc.unary_unary_rpc_method_handler( + servicer.DeleteShelf, + request_deserializer=bookstore__pb2.DeleteShelfRequest.FromString, + response_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + ), + 'ListBooks': grpc.unary_unary_rpc_method_handler( + servicer.ListBooks, + request_deserializer=bookstore__pb2.ListBooksRequest.FromString, + response_serializer=bookstore__pb2.ListBooksResponse.SerializeToString, + ), + 'CreateBook': grpc.unary_unary_rpc_method_handler( + servicer.CreateBook, + request_deserializer=bookstore__pb2.CreateBookRequest.FromString, + response_serializer=bookstore__pb2.Book.SerializeToString, + ), + 'GetBook': grpc.unary_unary_rpc_method_handler( + servicer.GetBook, + request_deserializer=bookstore__pb2.GetBookRequest.FromString, + response_serializer=bookstore__pb2.Book.SerializeToString, + ), + 'DeleteBook': grpc.unary_unary_rpc_method_handler( + servicer.DeleteBook, + request_deserializer=bookstore__pb2.DeleteBookRequest.FromString, + response_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'endpoints.examples.bookstore.Bookstore', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) diff --git a/endpoints/bookstore-grpc-transcoding/bookstore_server.py b/endpoints/bookstore-grpc-transcoding/bookstore_server.py new file mode 100644 index 00000000000..97d85ad198e --- /dev/null +++ b/endpoints/bookstore-grpc-transcoding/bookstore_server.py @@ -0,0 +1,131 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The Python gRPC Bookstore Server Example.""" + +import argparse +from concurrent import futures +import time + +from google.protobuf import struct_pb2 +import grpc + +import bookstore +import bookstore_pb2 +import bookstore_pb2_grpc +import status + +_ONE_DAY_IN_SECONDS = 60 * 60 * 24 + + +class BookstoreServicer(bookstore_pb2_grpc.BookstoreServicer): + """Implements the bookstore API server.""" + def __init__(self, store): + self._store = store + + def ListShelves(self, unused_request, context): + with status.context(context): + response = bookstore_pb2.ListShelvesResponse() + response.shelves.extend(self._store.list_shelf()) + return response + + def CreateShelf(self, request, context): + with status.context(context): + shelf, _ = self._store.create_shelf(request.shelf) + return shelf + + def GetShelf(self, request, context): + with status.context(context): + return self._store.get_shelf(request.shelf) + + def DeleteShelf(self, request, context): + with status.context(context): + self._store.delete_shelf(request.shelf) + return struct_pb2.Value() + + def ListBooks(self, request, context): + with status.context(context): + response = bookstore_pb2.ListBooksResponse() + response.books.extend(self._store.list_books(request.shelf)) + return response + + def CreateBook(self, request, context): + with status.context(context): + return self._store.create_book(request.shelf, request.book) + + def GetBook(self, request, context): + with status.context(context): + return self._store.get_book(request.shelf, request.book) + + def DeleteBook(self, request, context): + with status.context(context): + self._store.delete_book(request.shelf, request.book) + return struct_pb2.Value() + + +def create_sample_bookstore(): + """Creates a Bookstore with some initial sample data.""" + store = bookstore.Bookstore() + + shelf = bookstore_pb2.Shelf() + shelf.theme = 'Fiction' + _, fiction = store.create_shelf(shelf) + + book = bookstore_pb2.Book() + book.title = 'README' + book.author = "Neal Stephenson" + store.create_book(fiction, book) + + shelf = bookstore_pb2.Shelf() + shelf.theme = 'Fantasy' + _, fantasy = store.create_shelf(shelf) + + book = bookstore_pb2.Book() + book.title = 'A Game of Thrones' + book.author = 'George R.R. Martin' + store.create_book(fantasy, book) + + return store + + +def serve(port, shutdown_grace_duration): + """Configures and runs the bookstore API server.""" + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + + store = create_sample_bookstore() + bookstore_pb2_grpc.add_BookstoreServicer_to_server( + BookstoreServicer(store), server) + server.add_insecure_port('[::]:{}'.format(port)) + server.start() + + try: + while True: + time.sleep(_ONE_DAY_IN_SECONDS) + except KeyboardInterrupt: + server.stop(shutdown_grace_duration) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--port', type=int, default=8000, help='The port to listen on') + parser.add_argument( + '--shutdown_grace_duration', type=int, default=5, + help='The shutdown grace duration, in seconds') + + args = parser.parse_args() + + serve(args.port, args.shutdown_grace_duration) diff --git a/endpoints/bookstore-grpc-transcoding/requirements.txt b/endpoints/bookstore-grpc-transcoding/requirements.txt new file mode 100644 index 00000000000..e7487cf5b3b --- /dev/null +++ b/endpoints/bookstore-grpc-transcoding/requirements.txt @@ -0,0 +1,5 @@ +grpcio>=1.10.0 +grpcio-tools>=1.10.0 +google-auth>=1.4.1 +six>=1.11 +googleapis-common-protos>=1.5.3 diff --git a/endpoints/bookstore-grpc-transcoding/status.py b/endpoints/bookstore-grpc-transcoding/status.py new file mode 100644 index 00000000000..3248d27d03a --- /dev/null +++ b/endpoints/bookstore-grpc-transcoding/status.py @@ -0,0 +1,28 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from contextlib import contextmanager + +import grpc + + +@contextmanager +def context(grpc_context): + """A context manager that automatically handles KeyError.""" + try: + yield + except KeyError as key_error: + grpc_context.code(grpc.StatusCode.NOT_FOUND) + grpc_context.details( + 'Unable to find the item keyed by {}'.format(key_error)) diff --git a/endpoints/bookstore-grpc/Dockerfile b/endpoints/bookstore-grpc/Dockerfile new file mode 100644 index 00000000000..d8a0773cb29 --- /dev/null +++ b/endpoints/bookstore-grpc/Dockerfile @@ -0,0 +1,22 @@ +# The Google Cloud Platform Python runtime is based on Debian Jessie +# You can read more about the runtime at: +# https://github.com/GoogleCloudPlatform/python-runtime +FROM gcr.io/google_appengine/python + +# Create a virtualenv for dependencies. This isolates these packages from +# system-level packages. +RUN virtualenv -p python3.6 /env + +# Setting these environment variables are the same as running +# source /env/bin/activate. +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH + +ADD . /bookstore/ + +WORKDIR /bookstore +RUN pip install -r requirements.txt + +EXPOSE 8000 + +ENTRYPOINT ["python", "/bookstore/bookstore_server.py"] diff --git a/endpoints/bookstore-grpc/README.md b/endpoints/bookstore-grpc/README.md new file mode 100644 index 00000000000..bcd49c635cd --- /dev/null +++ b/endpoints/bookstore-grpc/README.md @@ -0,0 +1,65 @@ +# Google Cloud Endpoints Bookstore App in Python + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=endpoints/bookstore-grpc/README.md + +## Installing the dependencies using virtualenv: + + virtualenv bookstore-env + source bookstore-env/bin/activate + +Install all the Python dependencies: + + pip install -r requirements.txt + +Install the [gRPC libraries and tools](http://www.grpc.io/docs/quickstart/python.html#prerequisites) + +## Running the Server Locally + +To run the server: + + python bookstore_server.py + +The `-h` command line flag shows the various settings available. + +## Running the Client Locally + +To run the client: + + python bookstore_client.py + +As with the server, the `-h` command line flag shows the various settings +available. + +## Generating a JWT token from a service account file + +To run the script: + + python jwt_token_gen.py --file=account_file --audiences=audiences --issuer=issuer + +The output can be used as "--auth_token" for bookstore_client.py + +## Regenerating the API stubs + +The bookstore gRPC API is defined by `bookstore.proto` +The API client stubs and server interfaces are generated by tools. + +To make it easier to get something up and running, we've included the generated +code in the sample distribution. To modify the sample or create your own gRPC +API definition, you'll need to update the generated code. To do this, once the +gRPC libraries and tools are installed, run: + + python -m grpc.tools.protoc \ + --include_imports \ + --include_source_info \ + --proto_path=. \ + --python_out=. \ + --grpc_python_out=. \ + --descriptor_set_out=api_descriptor.pb \ + bookstore.proto + +## Running the server with gRPC <-> HTTP/JSON Transcoding + +Follow the instructions for [Deploying a service using transcoding](https://cloud.google.com/endpoints/docs/transcoding#deploying_a_service_using_transcoding). diff --git a/endpoints/bookstore-grpc/api_config.yaml b/endpoints/bookstore-grpc/api_config.yaml new file mode 100644 index 00000000000..b7f6ea0e5ef --- /dev/null +++ b/endpoints/bookstore-grpc/api_config.yaml @@ -0,0 +1,45 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# A Bookstore example API configuration. +# +# Below, replace MY_PROJECT_ID with your Google Cloud Project ID. +# + +# The configuration schema is defined by service.proto file +# https://github.com/googleapis/googleapis/blob/master/google/api/service.proto +type: google.api.Service +config_version: 3 + +# +# Name of the service configuration. +# +name: bookstore.endpoints..cloud.goog + +# +# API title to appear in the user interface (Google Cloud Console). +# +title: Bookstore gRPC API +apis: +- name: endpoints.examples.bookstore.Bookstore + +# +# API usage restrictions. +# +usage: + rules: + # ListShelves methods can be called without an API Key. + - selector: endpoints.examples.bookstore.Bookstore.ListShelves + allow_unregistered_calls: true diff --git a/endpoints/bookstore-grpc/api_config_auth.yaml b/endpoints/bookstore-grpc/api_config_auth.yaml new file mode 100644 index 00000000000..099b7292a93 --- /dev/null +++ b/endpoints/bookstore-grpc/api_config_auth.yaml @@ -0,0 +1,59 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# A Bookstore example API configuration. +# +# Below, replace MY_PROJECT_ID with your Google Cloud Project ID. +# + +# The configuration schema is defined by service.proto file +# https://github.com/googleapis/googleapis/blob/master/google/api/service.proto +type: google.api.Service +config_version: 3 + +# +# Name of the service configuration. +# +name: bookstore.endpoints..cloud.goog + +# +# API title to appear in the user interface (Google Cloud Console). +# +title: Bookstore gRPC API +apis: +- name: endpoints.examples.bookstore.Bookstore + +# +# API usage restrictions. +# +usage: + rules: + - selector: "*" + allow_unregistered_calls: true + +# +# Request authentication. +# +authentication: + providers: + - id: google_service_account + # Replace SERVICE-ACCOUNT-ID with your service account's email address. + issuer: SERVICE-ACCOUNT-ID + jwks_uri: https://www.googleapis.com/robot/v1/metadata/x509/SERVICE-ACCOUNT-ID + rules: + # This auth rule will apply to all methods. + - selector: "*" + requirements: + - provider_id: google_service_account diff --git a/endpoints/bookstore-grpc/api_config_http.yaml b/endpoints/bookstore-grpc/api_config_http.yaml new file mode 100644 index 00000000000..be450b3fcbc --- /dev/null +++ b/endpoints/bookstore-grpc/api_config_http.yaml @@ -0,0 +1,119 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Example HTTP rules for Bookstore +# + +# The configuration schema is defined by service.proto file +# https://github.com/googleapis/googleapis/blob/master/google/api/service.proto +type: google.api.Service +config_version: 3 + +# +# Name of the service configuration. +# +name: bookstore.endpoints..cloud.goog + +# +# HTTP rules define translation from HTTP/REST/JSON to gRPC. With these rules +# HTTP/REST/JSON clients will be able to call the Bookstore service. +# +http: + rules: + # + # HTTP/REST/JSON clients can call the 'ListShelves' method of the Bookstore + # service using the GET HTTP verb and the '/shelves' URL path. The response + # will the JSON representation of the 'ListShelvesResponse' message. + # + # Client example (Assuming your service is hosted at the given 'DOMAIN_NAME'): + # curl http://DOMAIN_NAME/v1/shelves + # + - selector: endpoints.examples.bookstore.Bookstore.ListShelves + get: /v1/shelves + # + # 'CreateShelf' can be called using the POST HTTP verb and the '/shelves' URL + # path. The posted HTTP body is the JSON respresentation of the 'shelf' field + # of 'CreateShelfRequest' protobuf message. + # + # Client example: + # curl -d '{"theme":"Music"}' http://DOMAIN_NAME/v1/shelves + # + - selector: endpoints.examples.bookstore.Bookstore.CreateShelf + post: /v1/shelves + body: shelf + # + # 'GetShelf' is available via the GET HTTP verb and '/shelves/{shelf}' URL + # path, where {shelf} is the value of the 'shelf' field of 'GetShelfRequest' + # protobuf message. + # + # Client example - returns the first shelf: + # curl http://DOMAIN_NAME/v1/shelves/1 + # + - selector: endpoints.examples.bookstore.Bookstore.GetShelf + get: /v1/shelves/{shelf} + # + # 'DeleteShelf' can be called using the DELETE HTTP verb and + # '/shelves/{shelf}' URL path, where {shelf} is the value of the 'shelf' field + # of 'DeleteShelfRequest' protobuf message. + # + # Client example - deletes the second shelf: + # curl -X DELETE http://DOMAIN_NAME/v1/shelves/2 + # + - selector: endpoints.examples.bookstore.Bookstore.DeleteShelf + delete: /v1/shelves/{shelf} + # + # 'ListBooks' can be called using the GET HTTP verb and + # '/shelves/{shelf}/books' URL path, where {shelf} is the value of the 'shelf' + # field of 'ListBooksRequest' protobuf message. + # + # Client example - list the books from the first shelf: + # curl http://DOMAIN_NAME/v1/shelves/1/books + # + - selector: endpoints.examples.bookstore.Bookstore.ListBooks + get: /v1/shelves/{shelf}/books + # + # 'CreateBook' can be called using the POST HTTP verb and + # '/shelves/{shelf}/books' URL path, where {shelf} is the value of the 'shelf' + # field of 'CreateBookRequest' protobuf message and the posted HTTP body is + # the JSON representation of the 'book' field of 'CreateBookRequest' message. + # + # Client example - create a new book in the first shelf: + # curl -d '{"author":"foo","title":"bar"}' http://DOMAIN_NAME/v1/shelves/1/books + # + - selector: endpoints.examples.bookstore.Bookstore.CreateBook + post: /v1/shelves/{shelf}/books + body: book + # + # 'GetBook' can be called using the GET HTTP verb and + # '/shelves/{shelf}/books/{book}' URL path, where {shelf} is the value of the + # 'shelf' field of 'GetShelfRequest' protobuf message and {book} is the value + # of the 'book' field of the 'GetShelfRequest' message. + # + # Client example - get the first book from the second shelf: + # curl http://DOMAIN_NAME/v1/shelves/2/books/1 + # + - selector: endpoints.examples.bookstore.Bookstore.GetBook + get: /v1/shelves/{shelf}/books/{book} + # + # 'DeleteBook' can be called using DELETE HTTP verb and + # '/shelves/{shelf}/books/{book}' URL path, where {shelf} is the value of the + # 'shelf' field of 'DeleteShelfRequest' protobuf message and {book} is the + # value of the 'book' field of the 'DeleteShelfRequest' message. + # + # Client example - delete the first book from the first shelf: + # curl -X DELETE http://DOMAIN_NAME/v1/shelves/1/books/1 + # + - selector: endpoints.examples.bookstore.Bookstore.DeleteBook + delete: /v1/shelves/{shelf}/books/{book} diff --git a/endpoints/bookstore-grpc/bookstore.proto b/endpoints/bookstore-grpc/bookstore.proto new file mode 100644 index 00000000000..c3f685f1a0c --- /dev/null +++ b/endpoints/bookstore-grpc/bookstore.proto @@ -0,0 +1,126 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// + +syntax = "proto3"; + +package endpoints.examples.bookstore; + +option java_multiple_files = true; +option java_outer_classname = "BookstoreProto"; +option java_package = "com.google.endpoints.examples.bookstore"; + + +import "google/protobuf/empty.proto"; + +// A simple Bookstore API. +// +// The API manages shelves and books resources. Shelves contain books. +service Bookstore { + // Returns a list of all shelves in the bookstore. + rpc ListShelves(google.protobuf.Empty) returns (ListShelvesResponse) {} + // Creates a new shelf in the bookstore. + rpc CreateShelf(CreateShelfRequest) returns (Shelf) {} + // Returns a specific bookstore shelf. + rpc GetShelf(GetShelfRequest) returns (Shelf) {} + // Deletes a shelf, including all books that are stored on the shelf. + rpc DeleteShelf(DeleteShelfRequest) returns (google.protobuf.Empty) {} + // Returns a list of books on a shelf. + rpc ListBooks(ListBooksRequest) returns (ListBooksResponse) {} + // Creates a new book. + rpc CreateBook(CreateBookRequest) returns (Book) {} + // Returns a specific book. + rpc GetBook(GetBookRequest) returns (Book) {} + // Deletes a book from a shelf. + rpc DeleteBook(DeleteBookRequest) returns (google.protobuf.Empty) {} +} + +// A shelf resource. +message Shelf { + // A unique shelf id. + int64 id = 1; + // A theme of the shelf (fiction, poetry, etc). + string theme = 2; +} + +// A book resource. +message Book { + // A unique book id. + int64 id = 1; + // An author of the book. + string author = 2; + // A book title. + string title = 3; +} + +// Response to ListShelves call. +message ListShelvesResponse { + // Shelves in the bookstore. + repeated Shelf shelves = 1; +} + +// Request message for CreateShelf method. +message CreateShelfRequest { + // The shelf resource to create. + Shelf shelf = 1; +} + +// Request message for GetShelf method. +message GetShelfRequest { + // The ID of the shelf resource to retrieve. + int64 shelf = 1; +} + +// Request message for DeleteShelf method. +message DeleteShelfRequest { + // The ID of the shelf to delete. + int64 shelf = 1; +} + +// Request message for ListBooks method. +message ListBooksRequest { + // ID of the shelf which books to list. + int64 shelf = 1; +} + +// Response message to ListBooks method. +message ListBooksResponse { + // The books on the shelf. + repeated Book books = 1; +} + +// Request message for CreateBook method. +message CreateBookRequest { + // The ID of the shelf on which to create a book. + int64 shelf = 1; + // A book resource to create on the shelf. + Book book = 2; +} + +// Request message for GetBook method. +message GetBookRequest { + // The ID of the shelf from which to retrieve a book. + int64 shelf = 1; + // The ID of the book to retrieve. + int64 book = 2; +} + +// Request message for DeleteBook method. +message DeleteBookRequest { + // The ID of the shelf from which to delete a book. + int64 shelf = 1; + // The ID of the book to delete. + int64 book = 2; +} diff --git a/endpoints/bookstore-grpc/bookstore.py b/endpoints/bookstore-grpc/bookstore.py new file mode 100644 index 00000000000..a32be906d47 --- /dev/null +++ b/endpoints/bookstore-grpc/bookstore.py @@ -0,0 +1,76 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import threading + +import six + + +class ShelfInfo(object): + """The contents of a single shelf.""" + def __init__(self, shelf): + self._shelf = shelf + self._last_book_id = 0 + self._books = dict() + + +class Bookstore(object): + """An in-memory backend for storing Bookstore data.""" + + def __init__(self): + self._last_shelf_id = 0 + self._shelves = dict() + self._lock = threading.Lock() + + def list_shelf(self): + with self._lock: + return [s._shelf for (_, s) in six.iteritems(self._shelves)] + + def create_shelf(self, shelf): + with self._lock: + self._last_shelf_id += 1 + shelf_id = self._last_shelf_id + shelf.id = shelf_id + self._shelves[shelf_id] = ShelfInfo(shelf) + return (shelf, shelf_id) + + def get_shelf(self, shelf_id): + with self._lock: + return self._shelves[shelf_id]._shelf + + def delete_shelf(self, shelf_id): + with self._lock: + del self._shelves[shelf_id] + + def list_books(self, shelf_id): + with self._lock: + return [book for ( + _, book) in six.iteritems(self._shelves[shelf_id]._books)] + + def create_book(self, shelf_id, book): + with self._lock: + shelf_info = self._shelves[shelf_id] + shelf_info._last_book_id += 1 + book_id = shelf_info._last_book_id + book.id = book_id + shelf_info._books[book_id] = book + return book + + def get_book(self, shelf_id, book_id): + with self._lock: + return self._shelves[shelf_id]._books[book_id] + + def delete_book(self, shelf_id, book_id): + with self._lock: + del self._shelves[shelf_id]._books[book_id] diff --git a/endpoints/bookstore-grpc/bookstore_client.py b/endpoints/bookstore-grpc/bookstore_client.py new file mode 100644 index 00000000000..37d31e11183 --- /dev/null +++ b/endpoints/bookstore-grpc/bookstore_client.py @@ -0,0 +1,56 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The Python gRPC Bookstore Client Example.""" + +import argparse + +from google.protobuf import empty_pb2 +import grpc + +import bookstore_pb2_grpc + + +def run(host, port, api_key, auth_token, timeout): + """Makes a basic ListShelves call against a gRPC Bookstore server.""" + + channel = grpc.insecure_channel('{}:{}'.format(host, port)) + + stub = bookstore_pb2_grpc.BookstoreStub(channel) + metadata = [] + if api_key: + metadata.append(('x-api-key', api_key)) + if auth_token: + metadata.append(('authorization', 'Bearer ' + auth_token)) + shelves = stub.ListShelves(empty_pb2.Empty(), timeout, metadata=metadata) + print('ListShelves: {}'.format(shelves)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--host', default='localhost', help='The host to connect to') + parser.add_argument( + '--port', type=int, default=8000, help='The port to connect to') + parser.add_argument( + '--timeout', type=int, default=10, help='The call timeout, in seconds') + parser.add_argument( + '--api_key', default=None, help='The API key to use for the call') + parser.add_argument( + '--auth_token', default=None, + help='The JWT auth token to use for the call') + args = parser.parse_args() + run(args.host, args.port, args.api_key, args.auth_token, args.timeout) diff --git a/endpoints/bookstore-grpc/bookstore_pb2.py b/endpoints/bookstore-grpc/bookstore_pb2.py new file mode 100644 index 00000000000..e652efe28ce --- /dev/null +++ b/endpoints/bookstore-grpc/bookstore_pb2.py @@ -0,0 +1,596 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: bookstore.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='bookstore.proto', + package='endpoints.examples.bookstore', + syntax='proto3', + serialized_pb=_b('\n\x0f\x62ookstore.proto\x12\x1c\x65ndpoints.examples.bookstore\x1a\x1bgoogle/protobuf/empty.proto\"\"\n\x05Shelf\x12\n\n\x02id\x18\x01 \x01(\x03\x12\r\n\x05theme\x18\x02 \x01(\t\"1\n\x04\x42ook\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x0e\n\x06\x61uthor\x18\x02 \x01(\t\x12\r\n\x05title\x18\x03 \x01(\t\"K\n\x13ListShelvesResponse\x12\x34\n\x07shelves\x18\x01 \x03(\x0b\x32#.endpoints.examples.bookstore.Shelf\"H\n\x12\x43reateShelfRequest\x12\x32\n\x05shelf\x18\x01 \x01(\x0b\x32#.endpoints.examples.bookstore.Shelf\" \n\x0fGetShelfRequest\x12\r\n\x05shelf\x18\x01 \x01(\x03\"#\n\x12\x44\x65leteShelfRequest\x12\r\n\x05shelf\x18\x01 \x01(\x03\"!\n\x10ListBooksRequest\x12\r\n\x05shelf\x18\x01 \x01(\x03\"F\n\x11ListBooksResponse\x12\x31\n\x05\x62ooks\x18\x01 \x03(\x0b\x32\".endpoints.examples.bookstore.Book\"T\n\x11\x43reateBookRequest\x12\r\n\x05shelf\x18\x01 \x01(\x03\x12\x30\n\x04\x62ook\x18\x02 \x01(\x0b\x32\".endpoints.examples.bookstore.Book\"-\n\x0eGetBookRequest\x12\r\n\x05shelf\x18\x01 \x01(\x03\x12\x0c\n\x04\x62ook\x18\x02 \x01(\x03\"0\n\x11\x44\x65leteBookRequest\x12\r\n\x05shelf\x18\x01 \x01(\x03\x12\x0c\n\x04\x62ook\x18\x02 \x01(\x03\x32\x99\x06\n\tBookstore\x12Z\n\x0bListShelves\x12\x16.google.protobuf.Empty\x1a\x31.endpoints.examples.bookstore.ListShelvesResponse\"\x00\x12\x66\n\x0b\x43reateShelf\x12\x30.endpoints.examples.bookstore.CreateShelfRequest\x1a#.endpoints.examples.bookstore.Shelf\"\x00\x12`\n\x08GetShelf\x12-.endpoints.examples.bookstore.GetShelfRequest\x1a#.endpoints.examples.bookstore.Shelf\"\x00\x12Y\n\x0b\x44\x65leteShelf\x12\x30.endpoints.examples.bookstore.DeleteShelfRequest\x1a\x16.google.protobuf.Empty\"\x00\x12n\n\tListBooks\x12..endpoints.examples.bookstore.ListBooksRequest\x1a/.endpoints.examples.bookstore.ListBooksResponse\"\x00\x12\x63\n\nCreateBook\x12/.endpoints.examples.bookstore.CreateBookRequest\x1a\".endpoints.examples.bookstore.Book\"\x00\x12]\n\x07GetBook\x12,.endpoints.examples.bookstore.GetBookRequest\x1a\".endpoints.examples.bookstore.Book\"\x00\x12W\n\nDeleteBook\x12/.endpoints.examples.bookstore.DeleteBookRequest\x1a\x16.google.protobuf.Empty\"\x00\x42;\n\'com.google.endpoints.examples.bookstoreB\x0e\x42ookstoreProtoP\x01\x62\x06proto3') + , + dependencies=[google_dot_protobuf_dot_empty__pb2.DESCRIPTOR,]) + + + + +_SHELF = _descriptor.Descriptor( + name='Shelf', + full_name='endpoints.examples.bookstore.Shelf', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='id', full_name='endpoints.examples.bookstore.Shelf.id', index=0, + number=1, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='theme', full_name='endpoints.examples.bookstore.Shelf.theme', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=78, + serialized_end=112, +) + + +_BOOK = _descriptor.Descriptor( + name='Book', + full_name='endpoints.examples.bookstore.Book', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='id', full_name='endpoints.examples.bookstore.Book.id', index=0, + number=1, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='author', full_name='endpoints.examples.bookstore.Book.author', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='title', full_name='endpoints.examples.bookstore.Book.title', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=114, + serialized_end=163, +) + + +_LISTSHELVESRESPONSE = _descriptor.Descriptor( + name='ListShelvesResponse', + full_name='endpoints.examples.bookstore.ListShelvesResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='shelves', full_name='endpoints.examples.bookstore.ListShelvesResponse.shelves', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=165, + serialized_end=240, +) + + +_CREATESHELFREQUEST = _descriptor.Descriptor( + name='CreateShelfRequest', + full_name='endpoints.examples.bookstore.CreateShelfRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='shelf', full_name='endpoints.examples.bookstore.CreateShelfRequest.shelf', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=242, + serialized_end=314, +) + + +_GETSHELFREQUEST = _descriptor.Descriptor( + name='GetShelfRequest', + full_name='endpoints.examples.bookstore.GetShelfRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='shelf', full_name='endpoints.examples.bookstore.GetShelfRequest.shelf', index=0, + number=1, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=316, + serialized_end=348, +) + + +_DELETESHELFREQUEST = _descriptor.Descriptor( + name='DeleteShelfRequest', + full_name='endpoints.examples.bookstore.DeleteShelfRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='shelf', full_name='endpoints.examples.bookstore.DeleteShelfRequest.shelf', index=0, + number=1, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=350, + serialized_end=385, +) + + +_LISTBOOKSREQUEST = _descriptor.Descriptor( + name='ListBooksRequest', + full_name='endpoints.examples.bookstore.ListBooksRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='shelf', full_name='endpoints.examples.bookstore.ListBooksRequest.shelf', index=0, + number=1, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=387, + serialized_end=420, +) + + +_LISTBOOKSRESPONSE = _descriptor.Descriptor( + name='ListBooksResponse', + full_name='endpoints.examples.bookstore.ListBooksResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='books', full_name='endpoints.examples.bookstore.ListBooksResponse.books', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=422, + serialized_end=492, +) + + +_CREATEBOOKREQUEST = _descriptor.Descriptor( + name='CreateBookRequest', + full_name='endpoints.examples.bookstore.CreateBookRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='shelf', full_name='endpoints.examples.bookstore.CreateBookRequest.shelf', index=0, + number=1, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='book', full_name='endpoints.examples.bookstore.CreateBookRequest.book', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=494, + serialized_end=578, +) + + +_GETBOOKREQUEST = _descriptor.Descriptor( + name='GetBookRequest', + full_name='endpoints.examples.bookstore.GetBookRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='shelf', full_name='endpoints.examples.bookstore.GetBookRequest.shelf', index=0, + number=1, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='book', full_name='endpoints.examples.bookstore.GetBookRequest.book', index=1, + number=2, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=580, + serialized_end=625, +) + + +_DELETEBOOKREQUEST = _descriptor.Descriptor( + name='DeleteBookRequest', + full_name='endpoints.examples.bookstore.DeleteBookRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='shelf', full_name='endpoints.examples.bookstore.DeleteBookRequest.shelf', index=0, + number=1, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='book', full_name='endpoints.examples.bookstore.DeleteBookRequest.book', index=1, + number=2, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=627, + serialized_end=675, +) + +_LISTSHELVESRESPONSE.fields_by_name['shelves'].message_type = _SHELF +_CREATESHELFREQUEST.fields_by_name['shelf'].message_type = _SHELF +_LISTBOOKSRESPONSE.fields_by_name['books'].message_type = _BOOK +_CREATEBOOKREQUEST.fields_by_name['book'].message_type = _BOOK +DESCRIPTOR.message_types_by_name['Shelf'] = _SHELF +DESCRIPTOR.message_types_by_name['Book'] = _BOOK +DESCRIPTOR.message_types_by_name['ListShelvesResponse'] = _LISTSHELVESRESPONSE +DESCRIPTOR.message_types_by_name['CreateShelfRequest'] = _CREATESHELFREQUEST +DESCRIPTOR.message_types_by_name['GetShelfRequest'] = _GETSHELFREQUEST +DESCRIPTOR.message_types_by_name['DeleteShelfRequest'] = _DELETESHELFREQUEST +DESCRIPTOR.message_types_by_name['ListBooksRequest'] = _LISTBOOKSREQUEST +DESCRIPTOR.message_types_by_name['ListBooksResponse'] = _LISTBOOKSRESPONSE +DESCRIPTOR.message_types_by_name['CreateBookRequest'] = _CREATEBOOKREQUEST +DESCRIPTOR.message_types_by_name['GetBookRequest'] = _GETBOOKREQUEST +DESCRIPTOR.message_types_by_name['DeleteBookRequest'] = _DELETEBOOKREQUEST +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +Shelf = _reflection.GeneratedProtocolMessageType('Shelf', (_message.Message,), dict( + DESCRIPTOR = _SHELF, + __module__ = 'bookstore_pb2' + # @@protoc_insertion_point(class_scope:endpoints.examples.bookstore.Shelf) + )) +_sym_db.RegisterMessage(Shelf) + +Book = _reflection.GeneratedProtocolMessageType('Book', (_message.Message,), dict( + DESCRIPTOR = _BOOK, + __module__ = 'bookstore_pb2' + # @@protoc_insertion_point(class_scope:endpoints.examples.bookstore.Book) + )) +_sym_db.RegisterMessage(Book) + +ListShelvesResponse = _reflection.GeneratedProtocolMessageType('ListShelvesResponse', (_message.Message,), dict( + DESCRIPTOR = _LISTSHELVESRESPONSE, + __module__ = 'bookstore_pb2' + # @@protoc_insertion_point(class_scope:endpoints.examples.bookstore.ListShelvesResponse) + )) +_sym_db.RegisterMessage(ListShelvesResponse) + +CreateShelfRequest = _reflection.GeneratedProtocolMessageType('CreateShelfRequest', (_message.Message,), dict( + DESCRIPTOR = _CREATESHELFREQUEST, + __module__ = 'bookstore_pb2' + # @@protoc_insertion_point(class_scope:endpoints.examples.bookstore.CreateShelfRequest) + )) +_sym_db.RegisterMessage(CreateShelfRequest) + +GetShelfRequest = _reflection.GeneratedProtocolMessageType('GetShelfRequest', (_message.Message,), dict( + DESCRIPTOR = _GETSHELFREQUEST, + __module__ = 'bookstore_pb2' + # @@protoc_insertion_point(class_scope:endpoints.examples.bookstore.GetShelfRequest) + )) +_sym_db.RegisterMessage(GetShelfRequest) + +DeleteShelfRequest = _reflection.GeneratedProtocolMessageType('DeleteShelfRequest', (_message.Message,), dict( + DESCRIPTOR = _DELETESHELFREQUEST, + __module__ = 'bookstore_pb2' + # @@protoc_insertion_point(class_scope:endpoints.examples.bookstore.DeleteShelfRequest) + )) +_sym_db.RegisterMessage(DeleteShelfRequest) + +ListBooksRequest = _reflection.GeneratedProtocolMessageType('ListBooksRequest', (_message.Message,), dict( + DESCRIPTOR = _LISTBOOKSREQUEST, + __module__ = 'bookstore_pb2' + # @@protoc_insertion_point(class_scope:endpoints.examples.bookstore.ListBooksRequest) + )) +_sym_db.RegisterMessage(ListBooksRequest) + +ListBooksResponse = _reflection.GeneratedProtocolMessageType('ListBooksResponse', (_message.Message,), dict( + DESCRIPTOR = _LISTBOOKSRESPONSE, + __module__ = 'bookstore_pb2' + # @@protoc_insertion_point(class_scope:endpoints.examples.bookstore.ListBooksResponse) + )) +_sym_db.RegisterMessage(ListBooksResponse) + +CreateBookRequest = _reflection.GeneratedProtocolMessageType('CreateBookRequest', (_message.Message,), dict( + DESCRIPTOR = _CREATEBOOKREQUEST, + __module__ = 'bookstore_pb2' + # @@protoc_insertion_point(class_scope:endpoints.examples.bookstore.CreateBookRequest) + )) +_sym_db.RegisterMessage(CreateBookRequest) + +GetBookRequest = _reflection.GeneratedProtocolMessageType('GetBookRequest', (_message.Message,), dict( + DESCRIPTOR = _GETBOOKREQUEST, + __module__ = 'bookstore_pb2' + # @@protoc_insertion_point(class_scope:endpoints.examples.bookstore.GetBookRequest) + )) +_sym_db.RegisterMessage(GetBookRequest) + +DeleteBookRequest = _reflection.GeneratedProtocolMessageType('DeleteBookRequest', (_message.Message,), dict( + DESCRIPTOR = _DELETEBOOKREQUEST, + __module__ = 'bookstore_pb2' + # @@protoc_insertion_point(class_scope:endpoints.examples.bookstore.DeleteBookRequest) + )) +_sym_db.RegisterMessage(DeleteBookRequest) + + +DESCRIPTOR.has_options = True +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\'com.google.endpoints.examples.bookstoreB\016BookstoreProtoP\001')) + +_BOOKSTORE = _descriptor.ServiceDescriptor( + name='Bookstore', + full_name='endpoints.examples.bookstore.Bookstore', + file=DESCRIPTOR, + index=0, + options=None, + serialized_start=678, + serialized_end=1471, + methods=[ + _descriptor.MethodDescriptor( + name='ListShelves', + full_name='endpoints.examples.bookstore.Bookstore.ListShelves', + index=0, + containing_service=None, + input_type=google_dot_protobuf_dot_empty__pb2._EMPTY, + output_type=_LISTSHELVESRESPONSE, + options=None, + ), + _descriptor.MethodDescriptor( + name='CreateShelf', + full_name='endpoints.examples.bookstore.Bookstore.CreateShelf', + index=1, + containing_service=None, + input_type=_CREATESHELFREQUEST, + output_type=_SHELF, + options=None, + ), + _descriptor.MethodDescriptor( + name='GetShelf', + full_name='endpoints.examples.bookstore.Bookstore.GetShelf', + index=2, + containing_service=None, + input_type=_GETSHELFREQUEST, + output_type=_SHELF, + options=None, + ), + _descriptor.MethodDescriptor( + name='DeleteShelf', + full_name='endpoints.examples.bookstore.Bookstore.DeleteShelf', + index=3, + containing_service=None, + input_type=_DELETESHELFREQUEST, + output_type=google_dot_protobuf_dot_empty__pb2._EMPTY, + options=None, + ), + _descriptor.MethodDescriptor( + name='ListBooks', + full_name='endpoints.examples.bookstore.Bookstore.ListBooks', + index=4, + containing_service=None, + input_type=_LISTBOOKSREQUEST, + output_type=_LISTBOOKSRESPONSE, + options=None, + ), + _descriptor.MethodDescriptor( + name='CreateBook', + full_name='endpoints.examples.bookstore.Bookstore.CreateBook', + index=5, + containing_service=None, + input_type=_CREATEBOOKREQUEST, + output_type=_BOOK, + options=None, + ), + _descriptor.MethodDescriptor( + name='GetBook', + full_name='endpoints.examples.bookstore.Bookstore.GetBook', + index=6, + containing_service=None, + input_type=_GETBOOKREQUEST, + output_type=_BOOK, + options=None, + ), + _descriptor.MethodDescriptor( + name='DeleteBook', + full_name='endpoints.examples.bookstore.Bookstore.DeleteBook', + index=7, + containing_service=None, + input_type=_DELETEBOOKREQUEST, + output_type=google_dot_protobuf_dot_empty__pb2._EMPTY, + options=None, + ), +]) +_sym_db.RegisterServiceDescriptor(_BOOKSTORE) + +DESCRIPTOR.services_by_name['Bookstore'] = _BOOKSTORE + +# @@protoc_insertion_point(module_scope) diff --git a/endpoints/bookstore-grpc/bookstore_pb2_grpc.py b/endpoints/bookstore-grpc/bookstore_pb2_grpc.py new file mode 100644 index 00000000000..d2bfc7a1af7 --- /dev/null +++ b/endpoints/bookstore-grpc/bookstore_pb2_grpc.py @@ -0,0 +1,170 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +import grpc + +import bookstore_pb2 as bookstore__pb2 +from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 + + +class BookstoreStub(object): + """A simple Bookstore API. + + The API manages shelves and books resources. Shelves contain books. + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.ListShelves = channel.unary_unary( + '/endpoints.examples.bookstore.Bookstore/ListShelves', + request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + response_deserializer=bookstore__pb2.ListShelvesResponse.FromString, + ) + self.CreateShelf = channel.unary_unary( + '/endpoints.examples.bookstore.Bookstore/CreateShelf', + request_serializer=bookstore__pb2.CreateShelfRequest.SerializeToString, + response_deserializer=bookstore__pb2.Shelf.FromString, + ) + self.GetShelf = channel.unary_unary( + '/endpoints.examples.bookstore.Bookstore/GetShelf', + request_serializer=bookstore__pb2.GetShelfRequest.SerializeToString, + response_deserializer=bookstore__pb2.Shelf.FromString, + ) + self.DeleteShelf = channel.unary_unary( + '/endpoints.examples.bookstore.Bookstore/DeleteShelf', + request_serializer=bookstore__pb2.DeleteShelfRequest.SerializeToString, + response_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, + ) + self.ListBooks = channel.unary_unary( + '/endpoints.examples.bookstore.Bookstore/ListBooks', + request_serializer=bookstore__pb2.ListBooksRequest.SerializeToString, + response_deserializer=bookstore__pb2.ListBooksResponse.FromString, + ) + self.CreateBook = channel.unary_unary( + '/endpoints.examples.bookstore.Bookstore/CreateBook', + request_serializer=bookstore__pb2.CreateBookRequest.SerializeToString, + response_deserializer=bookstore__pb2.Book.FromString, + ) + self.GetBook = channel.unary_unary( + '/endpoints.examples.bookstore.Bookstore/GetBook', + request_serializer=bookstore__pb2.GetBookRequest.SerializeToString, + response_deserializer=bookstore__pb2.Book.FromString, + ) + self.DeleteBook = channel.unary_unary( + '/endpoints.examples.bookstore.Bookstore/DeleteBook', + request_serializer=bookstore__pb2.DeleteBookRequest.SerializeToString, + response_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, + ) + + +class BookstoreServicer(object): + """A simple Bookstore API. + + The API manages shelves and books resources. Shelves contain books. + """ + + def ListShelves(self, request, context): + """Returns a list of all shelves in the bookstore. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def CreateShelf(self, request, context): + """Creates a new shelf in the bookstore. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetShelf(self, request, context): + """Returns a specific bookstore shelf. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def DeleteShelf(self, request, context): + """Deletes a shelf, including all books that are stored on the shelf. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ListBooks(self, request, context): + """Returns a list of books on a shelf. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def CreateBook(self, request, context): + """Creates a new book. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetBook(self, request, context): + """Returns a specific book. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def DeleteBook(self, request, context): + """Deletes a book from a shelf. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_BookstoreServicer_to_server(servicer, server): + rpc_method_handlers = { + 'ListShelves': grpc.unary_unary_rpc_method_handler( + servicer.ListShelves, + request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, + response_serializer=bookstore__pb2.ListShelvesResponse.SerializeToString, + ), + 'CreateShelf': grpc.unary_unary_rpc_method_handler( + servicer.CreateShelf, + request_deserializer=bookstore__pb2.CreateShelfRequest.FromString, + response_serializer=bookstore__pb2.Shelf.SerializeToString, + ), + 'GetShelf': grpc.unary_unary_rpc_method_handler( + servicer.GetShelf, + request_deserializer=bookstore__pb2.GetShelfRequest.FromString, + response_serializer=bookstore__pb2.Shelf.SerializeToString, + ), + 'DeleteShelf': grpc.unary_unary_rpc_method_handler( + servicer.DeleteShelf, + request_deserializer=bookstore__pb2.DeleteShelfRequest.FromString, + response_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + ), + 'ListBooks': grpc.unary_unary_rpc_method_handler( + servicer.ListBooks, + request_deserializer=bookstore__pb2.ListBooksRequest.FromString, + response_serializer=bookstore__pb2.ListBooksResponse.SerializeToString, + ), + 'CreateBook': grpc.unary_unary_rpc_method_handler( + servicer.CreateBook, + request_deserializer=bookstore__pb2.CreateBookRequest.FromString, + response_serializer=bookstore__pb2.Book.SerializeToString, + ), + 'GetBook': grpc.unary_unary_rpc_method_handler( + servicer.GetBook, + request_deserializer=bookstore__pb2.GetBookRequest.FromString, + response_serializer=bookstore__pb2.Book.SerializeToString, + ), + 'DeleteBook': grpc.unary_unary_rpc_method_handler( + servicer.DeleteBook, + request_deserializer=bookstore__pb2.DeleteBookRequest.FromString, + response_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'endpoints.examples.bookstore.Bookstore', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) diff --git a/endpoints/bookstore-grpc/bookstore_server.py b/endpoints/bookstore-grpc/bookstore_server.py new file mode 100644 index 00000000000..97d85ad198e --- /dev/null +++ b/endpoints/bookstore-grpc/bookstore_server.py @@ -0,0 +1,131 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The Python gRPC Bookstore Server Example.""" + +import argparse +from concurrent import futures +import time + +from google.protobuf import struct_pb2 +import grpc + +import bookstore +import bookstore_pb2 +import bookstore_pb2_grpc +import status + +_ONE_DAY_IN_SECONDS = 60 * 60 * 24 + + +class BookstoreServicer(bookstore_pb2_grpc.BookstoreServicer): + """Implements the bookstore API server.""" + def __init__(self, store): + self._store = store + + def ListShelves(self, unused_request, context): + with status.context(context): + response = bookstore_pb2.ListShelvesResponse() + response.shelves.extend(self._store.list_shelf()) + return response + + def CreateShelf(self, request, context): + with status.context(context): + shelf, _ = self._store.create_shelf(request.shelf) + return shelf + + def GetShelf(self, request, context): + with status.context(context): + return self._store.get_shelf(request.shelf) + + def DeleteShelf(self, request, context): + with status.context(context): + self._store.delete_shelf(request.shelf) + return struct_pb2.Value() + + def ListBooks(self, request, context): + with status.context(context): + response = bookstore_pb2.ListBooksResponse() + response.books.extend(self._store.list_books(request.shelf)) + return response + + def CreateBook(self, request, context): + with status.context(context): + return self._store.create_book(request.shelf, request.book) + + def GetBook(self, request, context): + with status.context(context): + return self._store.get_book(request.shelf, request.book) + + def DeleteBook(self, request, context): + with status.context(context): + self._store.delete_book(request.shelf, request.book) + return struct_pb2.Value() + + +def create_sample_bookstore(): + """Creates a Bookstore with some initial sample data.""" + store = bookstore.Bookstore() + + shelf = bookstore_pb2.Shelf() + shelf.theme = 'Fiction' + _, fiction = store.create_shelf(shelf) + + book = bookstore_pb2.Book() + book.title = 'README' + book.author = "Neal Stephenson" + store.create_book(fiction, book) + + shelf = bookstore_pb2.Shelf() + shelf.theme = 'Fantasy' + _, fantasy = store.create_shelf(shelf) + + book = bookstore_pb2.Book() + book.title = 'A Game of Thrones' + book.author = 'George R.R. Martin' + store.create_book(fantasy, book) + + return store + + +def serve(port, shutdown_grace_duration): + """Configures and runs the bookstore API server.""" + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + + store = create_sample_bookstore() + bookstore_pb2_grpc.add_BookstoreServicer_to_server( + BookstoreServicer(store), server) + server.add_insecure_port('[::]:{}'.format(port)) + server.start() + + try: + while True: + time.sleep(_ONE_DAY_IN_SECONDS) + except KeyboardInterrupt: + server.stop(shutdown_grace_duration) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--port', type=int, default=8000, help='The port to listen on') + parser.add_argument( + '--shutdown_grace_duration', type=int, default=5, + help='The shutdown grace duration, in seconds') + + args = parser.parse_args() + + serve(args.port, args.shutdown_grace_duration) diff --git a/endpoints/bookstore-grpc/http_bookstore.proto b/endpoints/bookstore-grpc/http_bookstore.proto new file mode 100644 index 00000000000..e7f655455ed --- /dev/null +++ b/endpoints/bookstore-grpc/http_bookstore.proto @@ -0,0 +1,166 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// + +syntax = "proto3"; + +package endpoints.examples.bookstore; + +option java_multiple_files = true; +option java_outer_classname = "BookstoreProto"; +option java_package = "com.google.endpoints.examples.bookstore"; + + +import "google/api/annotations.proto"; +import "google/protobuf/empty.proto"; + +// A simple Bookstore API. +// +// The API manages shelves and books resources. Shelves contain books. +service Bookstore { + // Returns a list of all shelves in the bookstore. + rpc ListShelves(google.protobuf.Empty) returns (ListShelvesResponse) { + // Define HTTP mapping. + // Client example (Assuming your service is hosted at the given 'DOMAIN_NAME'): + // curl http://DOMAIN_NAME/v1/shelves + option (google.api.http) = { get: "/v1/shelves" }; + } + // Creates a new shelf in the bookstore. + rpc CreateShelf(CreateShelfRequest) returns (Shelf) { + // Client example: + // curl -d '{"theme":"Music"}' http://DOMAIN_NAME/v1/shelves + option (google.api.http) = { + post: "/v1/shelves" + body: "shelf" + }; + } + // Returns a specific bookstore shelf. + rpc GetShelf(GetShelfRequest) returns (Shelf) { + // Client example - returns the first shelf: + // curl http://DOMAIN_NAME/v1/shelves/1 + option (google.api.http) = { get: "/v1/shelves/{shelf}" }; + } + // Deletes a shelf, including all books that are stored on the shelf. + rpc DeleteShelf(DeleteShelfRequest) returns (google.protobuf.Empty) { + // Client example - deletes the second shelf: + // curl -X DELETE http://DOMAIN_NAME/v1/shelves/2 + option (google.api.http) = { delete: "/v1/shelves/{shelf}" }; + } + // Returns a list of books on a shelf. + rpc ListBooks(ListBooksRequest) returns (ListBooksResponse) { + // Client example - list the books from the first shelf: + // curl http://DOMAIN_NAME/v1/shelves/1/books + option (google.api.http) = { get: "/v1/shelves/{shelf}/books" }; + } + // Creates a new book. + rpc CreateBook(CreateBookRequest) returns (Book) { + // Client example - create a new book in the first shelf: + // curl -d '{"author":"foo","title":"bar"}' http://DOMAIN_NAME/v1/shelves/1/books + option (google.api.http) = { + post: "/v1/shelves/{shelf}/books" + body: "book" + }; + } + // Returns a specific book. + rpc GetBook(GetBookRequest) returns (Book) { + // Client example - get the first book from the second shelf: + // curl http://DOMAIN_NAME/v1/shelves/2/books/1 + option (google.api.http) = { get: "/v1/shelves/{shelf}/books/{book}" }; + } + // Deletes a book from a shelf. + rpc DeleteBook(DeleteBookRequest) returns (google.protobuf.Empty) { + // Client example - delete the first book from the first shelf: + // curl -X DELETE http://DOMAIN_NAME/v1/shelves/1/books/1 + option (google.api.http) = { delete: "/v1/shelves/{shelf}/books/{book}" }; + } +} + +// A shelf resource. +message Shelf { + // A unique shelf id. + int64 id = 1; + // A theme of the shelf (fiction, poetry, etc). + string theme = 2; +} + +// A book resource. +message Book { + // A unique book id. + int64 id = 1; + // An author of the book. + string author = 2; + // A book title. + string title = 3; +} + +// Response to ListShelves call. +message ListShelvesResponse { + // Shelves in the bookstore. + repeated Shelf shelves = 1; +} + +// Request message for CreateShelf method. +message CreateShelfRequest { + // The shelf resource to create. + Shelf shelf = 1; +} + +// Request message for GetShelf method. +message GetShelfRequest { + // The ID of the shelf resource to retrieve. + int64 shelf = 1; +} + +// Request message for DeleteShelf method. +message DeleteShelfRequest { + // The ID of the shelf to delete. + int64 shelf = 1; +} + +// Request message for ListBooks method. +message ListBooksRequest { + // ID of the shelf which books to list. + int64 shelf = 1; +} + +// Response message to ListBooks method. +message ListBooksResponse { + // The books on the shelf. + repeated Book books = 1; +} + +// Request message for CreateBook method. +message CreateBookRequest { + // The ID of the shelf on which to create a book. + int64 shelf = 1; + // A book resource to create on the shelf. + Book book = 2; +} + +// Request message for GetBook method. +message GetBookRequest { + // The ID of the shelf from which to retrieve a book. + int64 shelf = 1; + // The ID of the book to retrieve. + int64 book = 2; +} + +// Request message for DeleteBook method. +message DeleteBookRequest { + // The ID of the shelf from which to delete a book. + int64 shelf = 1; + // The ID of the book to delete. + int64 book = 2; +} diff --git a/endpoints/bookstore-grpc/jwt_token_gen.py b/endpoints/bookstore-grpc/jwt_token_gen.py new file mode 100644 index 00000000000..5138fc8100e --- /dev/null +++ b/endpoints/bookstore-grpc/jwt_token_gen.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Example of generating a JWT signed from a service account file.""" + +import argparse +import json +import time + +import google.auth.crypt +import google.auth.jwt + +"""Max lifetime of the token (one hour, in seconds).""" +MAX_TOKEN_LIFETIME_SECS = 3600 + + +def generate_jwt(service_account_file, issuer, audiences): + """Generates a signed JSON Web Token using a Google API Service Account.""" + with open(service_account_file, 'r') as fh: + service_account_info = json.load(fh) + + signer = google.auth.crypt.RSASigner.from_string( + service_account_info['private_key'], + service_account_info['private_key_id']) + + now = int(time.time()) + + payload = { + 'iat': now, + 'exp': now + MAX_TOKEN_LIFETIME_SECS, + # aud must match 'audience' in the security configuration in your + # swagger spec. It can be any string. + 'aud': audiences, + # iss must match 'issuer' in the security configuration in your + # swagger spec. It can be any string. + 'iss': issuer, + # sub and email are mapped to the user id and email respectively. + 'sub': issuer, + 'email': 'user@example.com' + } + + signed_jwt = google.auth.jwt.encode(signer, payload) + return signed_jwt + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('--file', + help='The path to your service account json file.') + parser.add_argument('--issuer', default='', help='issuer') + parser.add_argument('--audiences', default='', help='audiences') + + args = parser.parse_args() + + signed_jwt = generate_jwt(args.file, args.issuer, args.audiences) + print(signed_jwt) diff --git a/endpoints/bookstore-grpc/requirements.txt b/endpoints/bookstore-grpc/requirements.txt new file mode 100644 index 00000000000..e1c139acee8 --- /dev/null +++ b/endpoints/bookstore-grpc/requirements.txt @@ -0,0 +1,4 @@ +grpcio>=1.10.0 +grpcio-tools>=1.10.0 +google-auth>=1.4.1 +six>=1.11 diff --git a/endpoints/bookstore-grpc/status.py b/endpoints/bookstore-grpc/status.py new file mode 100644 index 00000000000..3248d27d03a --- /dev/null +++ b/endpoints/bookstore-grpc/status.py @@ -0,0 +1,28 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from contextlib import contextmanager + +import grpc + + +@contextmanager +def context(grpc_context): + """A context manager that automatically handles KeyError.""" + try: + yield + except KeyError as key_error: + grpc_context.code(grpc.StatusCode.NOT_FOUND) + grpc_context.details( + 'Unable to find the item keyed by {}'.format(key_error)) diff --git a/endpoints/getting-started-grpc/Dockerfile b/endpoints/getting-started-grpc/Dockerfile new file mode 100644 index 00000000000..facd78e7d43 --- /dev/null +++ b/endpoints/getting-started-grpc/Dockerfile @@ -0,0 +1,21 @@ +# The Google Cloud Platform Python runtime is based on Debian Jessie +# You can read more about the runtime at: +# https://github.com/GoogleCloudPlatform/python-runtime +FROM gcr.io/google_appengine/python + +# Create a virtualenv for dependencies. This isolates these packages from +# system-level packages. +RUN virtualenv /env + +# Setting these environment variables are the same as running +# source /env/bin/activate. +ENV VIRTUAL_ENV -p python3.5 /env +ENV PATH /env/bin:$PATH + +COPY requirements.txt /app/ +RUN pip install --requirement /app/requirements.txt +COPY . /app/ + +ENTRYPOINT [] + +CMD ["python", "/app/greeter_server.py"] diff --git a/endpoints/getting-started-grpc/README.md b/endpoints/getting-started-grpc/README.md new file mode 100644 index 00000000000..3caa0b06d6c --- /dev/null +++ b/endpoints/getting-started-grpc/README.md @@ -0,0 +1,172 @@ +# Endpoints Getting Started with gRPC & Python Quickstart + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=endpoints/getting-started-grpc/README.md + +It is assumed that you have a working Python environment and a Google +Cloud account and [SDK](https://cloud.google.com/sdk/) configured. + +1. Install dependencies using virtualenv: + + ```bash + virtualenv -p python3 env + source env/bin/activate + pip install -r requirements.txt + ``` + +1. Test running the code, optional: + + ```bash + # Run the server: + python greeter_server.py + + # Open another command line tab and enter the virtual environment: + source env/bin/activate + + # In the new command line tab, run the client: + python greeter_client.py + ``` + +1. The gRPC Services have already been generated. If you change the proto, or + just wish to regenerate these files, run: + + ```bash + python -m grpc.tools.protoc \ + --include_imports \ + --include_source_info \ + --proto_path=protos \ + --python_out=. \ + --grpc_python_out=. \ + --descriptor_set_out=api_descriptor.pb \ + helloworld.proto + ``` + +1. Edit, `api_config.yaml`. Replace `MY_PROJECT_ID` with your project id. + +1. Deploy your service config to Service Management: + + ```bash + gcloud endpoints services deploy api_descriptor.pb api_config.yaml + + # Set your project ID as a variable to make commands easier: + GCLOUD_PROJECT= + + ``` + +1. Also get an API key from the Console's API Manager for use in the + client later. [Get API Key](https://console.cloud.google.com/apis/credentials) + +1. Enable the Cloud Build API: + + ```bash + gcloud services enable cloudbuild.googleapis.com + ``` + +1. Build a docker image for your gRPC server, and store it in your Registry: + + ```bash + gcloud container builds submit --tag gcr.io/${GCLOUD_PROJECT}/python-grpc-hello:1.0 . + ``` + +1. Either deploy to GCE (below) or GKE (further down). + +## Google Compute Engine + +1. Enable the Compute Engine API: + + ```bash + gcloud services enable compute-component.googleapis.com + ``` + +1. Create your instance and ssh in: + + ```bash + gcloud compute instances create grpc-host --image-family gci-stable --image-project google-containers --tags=http-server + gcloud compute ssh grpc-host + ``` + +1. Set some variables to make commands easier: + + ```bash + GCLOUD_PROJECT=$(curl -s "http://metadata.google.internal/computeMetadata/v1/project/project-id" -H "Metadata-Flavor: Google") + SERVICE_NAME=hellogrpc.endpoints.${GCLOUD_PROJECT}.cloud.goog + ``` + +1. Pull your credentials to access Container Registry, and run your gRPC server + container: + + ```bash + /usr/share/google/dockercfg_update.sh + docker run --detach --name=grpc-hello gcr.io/${GCLOUD_PROJECT}/python-grpc-hello:1.0 + ``` + +1. Run the Endpoints proxy: + + ```bash + docker run --detach --name=esp \ + --publish=80:9000 \ + --link=grpc-hello:grpc-hello \ + gcr.io/endpoints-release/endpoints-runtime:1 \ + --service=${SERVICE_NAME} \ + --rollout_strategy=managed \ + --http2_port=9000 \ + --backend=grpc://grpc-hello:50051 + ``` + +1. Back on your local machine, get the external IP of your GCE instance: + + ```bash + gcloud compute instances list + ``` + +1. Run the client: + + ```bash + python greeter_client.py --host=:80 --api_key= + ``` + +1. Cleanup: + + ```bash + gcloud compute instances delete grpc-host + ``` + +### Google Kubernetes Engine + +1. Create a cluster. You can specify a different zone than us-central1-a if you + want: + + ```bash + gcloud container clusters create my-cluster --zone=us-central1-a + ``` + +1. Edit `deployment.yaml`. Replace `SERVICE_NAME` and `GCLOUD_PROJECT` with your values: + + `SERVICE_NAME` is equal to hellogrpc.endpoints.GCLOUD_PROJECT.cloud.goog, + replacing GCLOUD_PROJECT with your project ID. + +1. Deploy to GKE: + + ```bash + kubectl create -f ./deployment.yaml + ``` + +1. Get IP of load balancer, run until you see an External IP: + + ```bash + kubectl get svc grpc-hello + ``` + +1. Run the client: + + ```bash + python greeter_client.py --host=:80 --api_key= + ``` + +1. Cleanup: + + ```bash + gcloud container clusters delete my-cluster --zone=us-central1-a + ``` diff --git a/endpoints/getting-started-grpc/api_config.yaml b/endpoints/getting-started-grpc/api_config.yaml new file mode 100644 index 00000000000..eb521979979 --- /dev/null +++ b/endpoints/getting-started-grpc/api_config.yaml @@ -0,0 +1,36 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# An example API configuration. +# +# Below, replace MY_PROJECT_ID with your Google Cloud Project ID. +# + +# The configuration schema is defined by service.proto file +# https://github.com/googleapis/googleapis/blob/master/google/api/service.proto +type: google.api.Service +config_version: 3 + +# +# Name of the service configuration. +# +name: hellogrpc.endpoints.MY_PROJECT_ID.cloud.goog + +# +# API title to appear in the user interface (Google Cloud Console). +# +title: Hello gRPC API +apis: +- name: helloworld.Greeter diff --git a/endpoints/getting-started-grpc/api_descriptor.pb b/endpoints/getting-started-grpc/api_descriptor.pb new file mode 100644 index 00000000000..cad59570b9b Binary files /dev/null and b/endpoints/getting-started-grpc/api_descriptor.pb differ diff --git a/endpoints/getting-started-grpc/deployment.yaml b/endpoints/getting-started-grpc/deployment.yaml new file mode 100644 index 00000000000..78d1d2f494f --- /dev/null +++ b/endpoints/getting-started-grpc/deployment.yaml @@ -0,0 +1,54 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: grpc-hello +spec: + ports: + - port: 80 + targetPort: 9000 + protocol: TCP + name: http + selector: + app: grpc-hello + type: LoadBalancer +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: grpc-hello +spec: + replicas: 1 + template: + metadata: + labels: + app: grpc-hello + spec: + containers: + - name: esp + image: gcr.io/endpoints-release/endpoints-runtime:1 + args: [ + "--http2_port=9000", + "--backend=grpc://127.0.0.1:50051", + "--service=SERVICE_NAME", + "--rollout_strategy=managed", + ] + ports: + - containerPort: 9000 + - name: python-grpc-hello + image: gcr.io/GCLOUD_PROJECT/python-grpc-hello:1.0 + ports: + - containerPort: 50051 diff --git a/endpoints/getting-started-grpc/greeter_client.py b/endpoints/getting-started-grpc/greeter_client.py new file mode 100644 index 00000000000..bdbc1fc5ab7 --- /dev/null +++ b/endpoints/getting-started-grpc/greeter_client.py @@ -0,0 +1,63 @@ +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""The Python implementation of the GRPC helloworld.Greeter client.""" + +import argparse + +import grpc + +import helloworld_pb2 +import helloworld_pb2_grpc + + +def run(host, api_key): + channel = grpc.insecure_channel(host) + stub = helloworld_pb2_grpc.GreeterStub(channel) + metadata = [] + if api_key: + metadata.append(('x-api-key', api_key)) + response = stub.SayHello( + helloworld_pb2.HelloRequest(name='you'), metadata=metadata) + print('Greeter client received: ' + response.message) + response = stub.SayHelloAgain( + helloworld_pb2.HelloRequest(name='you'), metadata=metadata) + print('Greeter client received: ' + response.message) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--host', default='localhost:50051', help='The server host.') + parser.add_argument( + '--api_key', default=None, help='The API key to use for the call.') + args = parser.parse_args() + run(args.host, args.api_key) diff --git a/endpoints/getting-started-grpc/greeter_server.py b/endpoints/getting-started-grpc/greeter_server.py new file mode 100644 index 00000000000..07e2f9d1345 --- /dev/null +++ b/endpoints/getting-started-grpc/greeter_server.py @@ -0,0 +1,69 @@ +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""The Python implementation of the GRPC helloworld.Greeter server.""" + +from concurrent import futures +import time + +import grpc + +import helloworld_pb2 +import helloworld_pb2_grpc + +_ONE_DAY_IN_SECONDS = 60 * 60 * 24 + + +class Greeter(helloworld_pb2_grpc.GreeterServicer): + + def SayHello(self, request, context): + return helloworld_pb2.HelloReply(message='Hello, %s!' % request.name) + + def SayHelloAgain(self, request, context): + return helloworld_pb2.HelloReply( + message='Hello again, %s!' % request.name) + + +def serve(): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + helloworld_pb2_grpc.add_GreeterServicer_to_server(Greeter(), server) + server.add_insecure_port('[::]:50051') + server.start() + + # gRPC starts a new thread to service requests. Just make the main thread + # sleep. + try: + while True: + time.sleep(_ONE_DAY_IN_SECONDS) + except KeyboardInterrupt: + server.stop(grace=0) + + +if __name__ == '__main__': + serve() diff --git a/endpoints/getting-started-grpc/helloworld_pb2.py b/endpoints/getting-started-grpc/helloworld_pb2.py new file mode 100644 index 00000000000..6bdce677057 --- /dev/null +++ b/endpoints/getting-started-grpc/helloworld_pb2.py @@ -0,0 +1,141 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: helloworld.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='helloworld.proto', + package='helloworld', + syntax='proto3', + serialized_pb=_b('\n\x10helloworld.proto\x12\nhelloworld\"\x1c\n\x0cHelloRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x1d\n\nHelloReply\x12\x0f\n\x07message\x18\x01 \x01(\t2\x8e\x01\n\x07Greeter\x12>\n\x08SayHello\x12\x18.helloworld.HelloRequest\x1a\x16.helloworld.HelloReply\"\x00\x12\x43\n\rSayHelloAgain\x12\x18.helloworld.HelloRequest\x1a\x16.helloworld.HelloReply\"\x00\x62\x06proto3') +) + + + + +_HELLOREQUEST = _descriptor.Descriptor( + name='HelloRequest', + full_name='helloworld.HelloRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='helloworld.HelloRequest.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=32, + serialized_end=60, +) + + +_HELLOREPLY = _descriptor.Descriptor( + name='HelloReply', + full_name='helloworld.HelloReply', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='message', full_name='helloworld.HelloReply.message', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=62, + serialized_end=91, +) + +DESCRIPTOR.message_types_by_name['HelloRequest'] = _HELLOREQUEST +DESCRIPTOR.message_types_by_name['HelloReply'] = _HELLOREPLY +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +HelloRequest = _reflection.GeneratedProtocolMessageType('HelloRequest', (_message.Message,), dict( + DESCRIPTOR = _HELLOREQUEST, + __module__ = 'helloworld_pb2' + # @@protoc_insertion_point(class_scope:helloworld.HelloRequest) + )) +_sym_db.RegisterMessage(HelloRequest) + +HelloReply = _reflection.GeneratedProtocolMessageType('HelloReply', (_message.Message,), dict( + DESCRIPTOR = _HELLOREPLY, + __module__ = 'helloworld_pb2' + # @@protoc_insertion_point(class_scope:helloworld.HelloReply) + )) +_sym_db.RegisterMessage(HelloReply) + + + +_GREETER = _descriptor.ServiceDescriptor( + name='Greeter', + full_name='helloworld.Greeter', + file=DESCRIPTOR, + index=0, + options=None, + serialized_start=94, + serialized_end=236, + methods=[ + _descriptor.MethodDescriptor( + name='SayHello', + full_name='helloworld.Greeter.SayHello', + index=0, + containing_service=None, + input_type=_HELLOREQUEST, + output_type=_HELLOREPLY, + options=None, + ), + _descriptor.MethodDescriptor( + name='SayHelloAgain', + full_name='helloworld.Greeter.SayHelloAgain', + index=1, + containing_service=None, + input_type=_HELLOREQUEST, + output_type=_HELLOREPLY, + options=None, + ), +]) +_sym_db.RegisterServiceDescriptor(_GREETER) + +DESCRIPTOR.services_by_name['Greeter'] = _GREETER + +# @@protoc_insertion_point(module_scope) diff --git a/endpoints/getting-started-grpc/helloworld_pb2_grpc.py b/endpoints/getting-started-grpc/helloworld_pb2_grpc.py new file mode 100644 index 00000000000..d4375701a33 --- /dev/null +++ b/endpoints/getting-started-grpc/helloworld_pb2_grpc.py @@ -0,0 +1,63 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +import grpc + +import helloworld_pb2 as helloworld__pb2 + + +class GreeterStub(object): + """The greeting service definition. + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.SayHello = channel.unary_unary( + '/helloworld.Greeter/SayHello', + request_serializer=helloworld__pb2.HelloRequest.SerializeToString, + response_deserializer=helloworld__pb2.HelloReply.FromString, + ) + self.SayHelloAgain = channel.unary_unary( + '/helloworld.Greeter/SayHelloAgain', + request_serializer=helloworld__pb2.HelloRequest.SerializeToString, + response_deserializer=helloworld__pb2.HelloReply.FromString, + ) + + +class GreeterServicer(object): + """The greeting service definition. + """ + + def SayHello(self, request, context): + """Sends a greeting + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def SayHelloAgain(self, request, context): + """Sends another greeting + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_GreeterServicer_to_server(servicer, server): + rpc_method_handlers = { + 'SayHello': grpc.unary_unary_rpc_method_handler( + servicer.SayHello, + request_deserializer=helloworld__pb2.HelloRequest.FromString, + response_serializer=helloworld__pb2.HelloReply.SerializeToString, + ), + 'SayHelloAgain': grpc.unary_unary_rpc_method_handler( + servicer.SayHelloAgain, + request_deserializer=helloworld__pb2.HelloRequest.FromString, + response_serializer=helloworld__pb2.HelloReply.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'helloworld.Greeter', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) diff --git a/endpoints/getting-started-grpc/protos/helloworld.proto b/endpoints/getting-started-grpc/protos/helloworld.proto new file mode 100644 index 00000000000..68d8888e6e0 --- /dev/null +++ b/endpoints/getting-started-grpc/protos/helloworld.proto @@ -0,0 +1,50 @@ +// Copyright 2015, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package helloworld; + +// The greeting service definition. +service Greeter { + // Sends a greeting + rpc SayHello (HelloRequest) returns (HelloReply) {} + // Sends another greeting + rpc SayHelloAgain (HelloRequest) returns (HelloReply) {} +} + +// The request message containing the user's name. +message HelloRequest { + string name = 1; +} + +// The response message containing the greetings +message HelloReply { + string message = 1; +} diff --git a/endpoints/getting-started-grpc/requirements.txt b/endpoints/getting-started-grpc/requirements.txt new file mode 100644 index 00000000000..24efc6b4f48 --- /dev/null +++ b/endpoints/getting-started-grpc/requirements.txt @@ -0,0 +1,2 @@ +grpcio==1.18.0 +grpcio-tools==1.18.0 diff --git a/endpoints/getting-started/Dockerfile.custom b/endpoints/getting-started/Dockerfile.custom new file mode 100644 index 00000000000..9c83956363c --- /dev/null +++ b/endpoints/getting-started/Dockerfile.custom @@ -0,0 +1,16 @@ +# The Google App Engine python runtime is Debian Jessie with Python installed +# and various os-level packages to allow installation of popular Python +# libraries. The source is on github at: +# https://github.com/GoogleCloudPlatform/python-docker +FROM gcr.io/google_appengine/python + +RUN apt-get update && \ + apt-get install -y python2.7 python-pip && \ + apt-get clean && \ + rm /var/lib/apt/lists/*_* + +ADD . /app +WORKDIR /app + +RUN pip install -r requirements.txt +ENTRYPOINT ["gunicorn", "-b", ":8080", "main:app"] diff --git a/endpoints/getting-started/README.md b/endpoints/getting-started/README.md new file mode 100644 index 00000000000..e85b4804cbb --- /dev/null +++ b/endpoints/getting-started/README.md @@ -0,0 +1,170 @@ +# Google Cloud Endpoints & Python + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=endpoints/getting-started/README.md + +This sample demonstrates how to use Google Cloud Endpoints using Python. + +For a complete walkthrough showing how to run this sample in different +environments, see the +[Google Cloud Endpoints Quickstarts](https://cloud.google.com/endpoints/docs/quickstarts). + +This sample consists of two parts: + +1. The backend +2. The clients + +## Running locally + +### Running the backend + +Install all the dependencies: +```bash +$ virtualenv env +$ source env/bin/activate +$ pip install -r requirements.txt +``` + +Run the application: +```bash +$ python main.py +``` + +### Using the echo client + +With the app running locally, you can execute the simple echo client using: +```bash +$ python clients/echo-client.py http://localhost:8080 APIKEY helloworld +``` + +The `APIKEY` doesn't matter as the endpoint proxy is not running to do authentication. + +## Deploying to Production + +See the +[Google Cloud Endpoints Quickstarts](https://cloud.google.com/endpoints/docs/quickstarts). + +### Using the echo client + +With the project deployed, you'll need to create an API key to access the API. + +1. Open the Credentials page of the API Manager in the [Cloud Console](https://console.cloud.google.com/apis/credentials). +2. Click 'Create credentials'. +3. Select 'API Key'. +4. Choose 'Server Key' + +With the API key, you can use the echo client to access the API: +```bash +$ python clients/echo-client.py https://YOUR-PROJECT-ID.appspot.com YOUR-API-KEY helloworld +``` + +### Using the JWT client (with key file) + +The JWT client demonstrates how to use a service account to authenticate to endpoints with the service account's private key file. To use the client, you'll need both an API key (as described in the echo client section) and a service account. To create a service account: + +1. Open the Credentials page of the API Manager in the [Cloud Console](https://console.cloud.google.com/apis/credentials). +2. Click 'Create credentials'. +3. Select 'Service account key'. +4. In the 'Select service account' dropdown, select 'Create new service account'. +5. Choose 'JSON' for the key type. + +To use the service account for authentication: + +1. Update the `google_jwt`'s `x-google-jwks_uri` in `openapi.yaml` with your service account's email address. +2. Redeploy your application. + +Now you can use the JWT client to make requests to the API: +```bash +$ python clients/google-jwt-client.py https://YOUR-PROJECT-ID.appspot.com YOUR-API-KEY /path/to/service-account.json +``` + +### Using the ID Token client (with key file) + +The ID Token client demonstrates how to use user credentials to authenticate to endpoints. To use the client, you'll need both an API key (as described in the echo client section) and a OAuth2 client ID. To create a client ID: + +1. Open the Credentials page of the API Manager in the [Cloud Console](https://console.cloud.google.com/apis/credentials). +2. Click 'Create credentials'. +3. Select 'OAuth client ID'. +4. Choose 'Other' for the application type. + +To use the client ID for authentication: + +1. Update the `google_id_token`'s `x-google-audiences` in `openapi.yaml`with your client ID. +2. Redeploy your application. + +Now you can use the client ID to make requests to the API: +```bash +$ python clients/google-id-token-client.py https://YOUR-PROJECT-ID.appspot.com YOUR-API-KEY /path/to/client-id.json +``` + +### Using the App Engine default service account client (no key file needed) + +The App Engine default service account client demonstrates how to use the Google App Engine default service account to authenticate to endpoints. +We refer to the project that serves API requests as the server project. You also need to create a client project in the [Cloud Console](https://console.cloud.google.com). The client project is running Google App Engine standard application. + +To use the App Engine default service account for authentication: + +1. Update the `gae_default_service_account`'s `x-google-issuer` and `x-google-jwks_uri` in `openapi.yaml` with your client project ID. +2. Redeploy your server application. +3. Update clients/service_to_service_gae_default/main.py, replace 'YOUR-CLIENT-PROJECT-ID' and 'YOUR-SERVER-PROJECT-ID' with your client project ID and your server project ID. +4. Upload your application to Google App Engine by invoking the following command. Note that you need to provide project ID in the command because there are two projects (server and client projects) here and gcloud needs to know which project to pick. +```bash +$ gcloud app deploy app.yaml --project=YOUR-CLIENT-PROJECT-ID +``` + +Your client app is now deployed at https://YOUR-CLIENT-PROJECT-ID.appspot.com. When you access https://YOUR-CLIENT-PROJECT-ID.appspot.com, your client calls your server project API using +the client's service account. + +### Using the service account client (no key file needed) + +The service account client demonstrates how to use a non-default service account to authenticate to endpoints. +We refer to the project that serves API requests as the server project. You also need to create a client project in the [Cloud Console](https://console.cloud.google.com). +The client project is running Google App Engine standard application. + +In the example, we use Google Cloud Identity and Access Management (IAM) API to create a JSON Web Token (JWT) for a service account, and use it to call an Endpoints API. + +To use the client, you will need to enable "Service Account Actor" role for App Engine default service account: + +1. Go to [IAM page](https://console.cloud.google.com/iam-admin/iam) of your client project. +2. For App Engine default service account, from “Role(s)” drop-down menu, select “Project”-“Service Account Actor”, and Save. + +You also need to install Google API python library because the client code (main.py) uses googleapiclient, +which is a python library that needs to be uploaded to App Engine with your application code. After you run "pip install -t lib -r requirements", +Google API python client library should have already been installed under 'lib' directory. Additional information can be found +[here](https://cloud.google.com/appengine/docs/python/tools/using-libraries-python-27#requesting_a_library). + +To use the client for authentication: + +1. Update the `google_service_account`'s `x-google-issuer` and `x-google-jwks_uri` in `openapi.yaml` with your service account email. +2. Redeploy your server application. +3. Update clients/service_to_service_non_default/main.py by replacing 'YOUR-SERVICE-ACCOUNT-EMAIL', 'YOUR-SERVER-PROJECT-ID' and 'YOUR-CLIENT-PROJECT-ID' +with your service account email, your server project ID, and your client project ID, respectively. +4. Upload your application to Google App Engine by invoking the following command. Note that you need to provide project ID in the command because there are two projects (server and client projects) here and gcloud needs to know which project to pick. +```bash +$ gcloud app deploy app.yaml --project=YOUR-CLIENT-PROJECT-ID +``` + +Your client app is now deployed at https://YOUR-CLIENT-PROJECT-ID.appspot.com. When you access https://YOUR-CLIENT-PROJECT-ID.appspot.com, your client calls your server project API using +the client's service account. + +### Using the ID token client (no key file needed) + +This example demonstrates how to authenticate to endpoints from Google App Engine default service account using Google ID token. +In the example, we first create a JSON Web Token (JWT) using the App Engine default service account. We then request a Google +ID token using the JWT, and call an Endpoints API using the Google ID token. + +We refer to the project that serves API requests as the server project. You also need to create a client project in the [Cloud Console](https://console.cloud.google.com). +The client project is running Google App Engine standard application. + +To use the client for authentication: + +1. Update clients/service_to_service_google_id_token/main.py, replace 'YOUR-CLIENT-PROJECT-ID' and 'YOUR-SERVER-PROJECT-ID' with your client project ID and your server project ID. +2. Upload your application to Google App Engine by invoking the following command. Note that you need to provide project ID in the command because there are two projects (server and client projects) here and gcloud needs to know which project to pick. +```bash +$ gcloud app deploy app.yaml --project=YOUR-CLIENT-PROJECT-ID +``` + +Your client app is now deployed at https://YOUR-CLIENT-PROJECT-ID.appspot.com. When you access https://YOUR-CLIENT-PROJECT-ID.appspot.com, your client calls your server project API from +the client's service account using Google ID token. diff --git a/endpoints/getting-started/app.yaml b/endpoints/getting-started/app.yaml new file mode 100644 index 00000000000..8b52e5f869c --- /dev/null +++ b/endpoints/getting-started/app.yaml @@ -0,0 +1,14 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 + +# [START configuration] +endpoints_api_service: + # The following values are to be replaced by information from the output of + # 'gcloud endpoints services deploy openapi-appengine.yaml' command. + name: ENDPOINTS-SERVICE-NAME + rollout_strategy: managed + # [END configuration] diff --git a/endpoints/getting-started/clients/echo-client.py b/endpoints/getting-started/clients/echo-client.py new file mode 100755 index 00000000000..28f99a0d8d3 --- /dev/null +++ b/endpoints/getting-started/clients/echo-client.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Example of calling a simple Google Cloud Endpoint API.""" + +import argparse + +import requests +from six.moves import urllib + + +def make_request(host, api_key, message): + """Makes a request to the auth info endpoint for Google ID tokens.""" + url = urllib.parse.urljoin(host, 'echo') + params = { + 'key': api_key + } + body = { + 'message': message + } + + response = requests.post(url, params=params, json=body) + + response.raise_for_status() + return response.text + + +def main(host, api_key, message): + response = make_request(host, api_key, message) + print(response) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + 'host', help='Your API host, e.g. https://your-project.appspot.com.') + parser.add_argument( + 'api_key', help='Your API key.') + parser.add_argument( + 'message', + help='Message to echo.') + + args = parser.parse_args() + + main(args.host, args.api_key, args.message) diff --git a/endpoints/getting-started/clients/google-id-token-client.py b/endpoints/getting-started/clients/google-id-token-client.py new file mode 100644 index 00000000000..e4a1104a468 --- /dev/null +++ b/endpoints/getting-started/clients/google-id-token-client.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Example of calling a Google Cloud Endpoint API with an ID token obtained +using the Google OAuth2 flow.""" + +import argparse + +import google_auth_oauthlib.flow +import requests +from six.moves import urllib + + +def get_id_token(client_secrets_file, extra_args): + """Obtains credentials from the user using OAuth 2.0 and then returns the + ID token from those credentials.""" + + flow = google_auth_oauthlib.flow.InstalledAppFlow.from_client_secrets_file( + client_secrets_file, scopes=['openid', 'email', 'profile']) + + # Run the OAuth 2.0 flow to obtain credentials from the user. + flow.run_local_server() + + # The credentials have both an access token and an ID token. Cloud + # Endpoints uses the ID Token. + id_token = flow.oauth2session.token['id_token'] + + return id_token + + +def make_request(host, api_key, id_token): + """Makes a request to the auth info endpoint for Google ID tokens.""" + url = urllib.parse.urljoin(host, '/auth/info/googleidtoken') + params = { + 'key': api_key + } + headers = { + 'Authorization': 'Bearer {}'.format(id_token) + } + + response = requests.get(url, params=params, headers=headers) + + response.raise_for_status() + return response.text + + +def main(host, api_key, client_secrets_file, extra_args): + id_token = get_id_token(client_secrets_file, extra_args) + response = make_request(host, api_key, id_token) + print(response) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + 'host', help='Your API host, e.g. https://your-project.appspot.com.') + parser.add_argument( + 'api_key', help='Your API key.') + parser.add_argument( + 'client_secrets_file', + help='The path to your OAuth2 client secrets file.') + + args = parser.parse_args() + + main(args.host, args.api_key, args.client_secrets_file, args) diff --git a/endpoints/getting-started/clients/google-jwt-client.py b/endpoints/getting-started/clients/google-jwt-client.py new file mode 100644 index 00000000000..533816c6082 --- /dev/null +++ b/endpoints/getting-started/clients/google-jwt-client.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Example of calling a Google Cloud Endpoint API with a JWT signed by +a Google API Service Account.""" + +import argparse +import time + +import google.auth.crypt +import google.auth.jwt + +import requests + + +# [START endpoints_generate_jwt_sa] +def generate_jwt(sa_keyfile, + sa_email='account@project-id.iam.gserviceaccount.com', + audience='your-service-name', + expiry_length=3600): + + """Generates a signed JSON Web Token using a Google API Service Account.""" + + now = int(time.time()) + + # build payload + payload = { + 'iat': now, + # expires after 'expirary_length' seconds. + "exp": now + expiry_length, + # iss must match 'issuer' in the security configuration in your + # swagger spec (e.g. service account email). It can be any string. + 'iss': sa_email, + # aud must be either your Endpoints service name, or match the value + # specified as the 'x-google-audience' in the OpenAPI document. + 'aud': audience, + # sub and email should match the service account's email address + 'sub': sa_email, + 'email': sa_email + } + + # sign with keyfile + signer = google.auth.crypt.RSASigner.from_service_account_file(sa_keyfile) + jwt = google.auth.jwt.encode(signer, payload) + + return jwt +# [END endpoints_generate_jwt_sa] + + +# [START endpoints_jwt_request] +def make_jwt_request(signed_jwt, url='https://your-endpoint.com'): + """Makes an authorized request to the endpoint""" + headers = { + 'Authorization': 'Bearer {}'.format(signed_jwt), + 'content-type': 'application/json' + } + response = requests.get(url, headers=headers) + + response.raise_for_status() + return response.text +# [END endpoints_jwt_request] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + 'host', help='Your API host, e.g. https://your-project.appspot.com.') + parser.add_argument( + 'audience', help='The aud entry for the JWT') + parser.add_argument( + 'sa_path', + help='The path to your service account json file.') + parser.add_argument( + 'sa_email', + help='The email address for the service account.') + + args = parser.parse_args() + + expiry_length = 3600 + keyfile_jwt = generate_jwt(args.sa_path, + args.sa_email, + args.audience, + expiry_length) + print(make_jwt_request(keyfile_jwt, args.host)) diff --git a/endpoints/getting-started/clients/service_to_service_gae_default/app.yaml b/endpoints/getting-started/clients/service_to_service_gae_default/app.yaml new file mode 100644 index 00000000000..f041d384c05 --- /dev/null +++ b/endpoints/getting-started/clients/service_to_service_gae_default/app.yaml @@ -0,0 +1,7 @@ +runtime: python27 +api_version: 1 +threadsafe: true + +handlers: +- url: /.* + script: main.app diff --git a/endpoints/getting-started/clients/service_to_service_gae_default/main.py b/endpoints/getting-started/clients/service_to_service_gae_default/main.py new file mode 100644 index 00000000000..32059a52ad3 --- /dev/null +++ b/endpoints/getting-started/clients/service_to_service_gae_default/main.py @@ -0,0 +1,84 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0(the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Example of calling a Google Cloud Endpoint API with a JWT signed by +Google App Engine Default Service Account.""" + +import base64 +import httplib +import json +import time + +from google.appengine.api import app_identity +import webapp2 + +DEFAULT_SERVICE_ACCOUNT = 'YOUR-CLIENT-PROJECT-ID@appspot.gserviceaccount.com' +HOST = "YOUR-SERVER-PROJECT-ID.appspot.com" + + +def generate_jwt(): + """Generates a signed JSON Web Token using the Google App Engine default + service account.""" + now = int(time.time()) + + header_json = json.dumps({ + "typ": "JWT", + "alg": "RS256"}) + + payload_json = json.dumps({ + 'iat': now, + # expires after one hour. + "exp": now + 3600, + # iss is the Google App Engine default service account email. + 'iss': DEFAULT_SERVICE_ACCOUNT, + 'sub': DEFAULT_SERVICE_ACCOUNT, + # Typically, the audience is the hostname of your API. The aud + # defined here must match the audience in the security configuration + # in yourOpenAPI spec. + 'aud': 'echo.endpoints.sample.google.com', + "email": DEFAULT_SERVICE_ACCOUNT + }) + + header_and_payload = '{}.{}'.format( + base64.urlsafe_b64encode(header_json), + base64.urlsafe_b64encode(payload_json)) + (key_name, signature) = app_identity.sign_blob(header_and_payload) + signed_jwt = '{}.{}'.format( + header_and_payload, + base64.urlsafe_b64encode(signature)) + + return signed_jwt + + +def make_request(signed_jwt): + """Makes a request to the auth info endpoint for Google JWTs.""" + headers = {'Authorization': 'Bearer {}'.format(signed_jwt)} + conn = httplib.HTTPSConnection(HOST) + conn.request("GET", '/auth/info/googlejwt', None, headers) + res = conn.getresponse() + conn.close() + return res.read() + + +class MainPage(webapp2.RequestHandler): + def get(self): + self.response.headers['Content-Type'] = 'text/plain' + signed_jwt = generate_jwt() + res = make_request(signed_jwt) + self.response.write(res) + + +app = webapp2.WSGIApplication([ + ('/', MainPage), +], debug=True) diff --git a/endpoints/getting-started/clients/service_to_service_google_id_token/app.yaml b/endpoints/getting-started/clients/service_to_service_google_id_token/app.yaml new file mode 100644 index 00000000000..f041d384c05 --- /dev/null +++ b/endpoints/getting-started/clients/service_to_service_google_id_token/app.yaml @@ -0,0 +1,7 @@ +runtime: python27 +api_version: 1 +threadsafe: true + +handlers: +- url: /.* + script: main.app diff --git a/endpoints/getting-started/clients/service_to_service_google_id_token/main.py b/endpoints/getting-started/clients/service_to_service_google_id_token/main.py new file mode 100644 index 00000000000..b0033ff1589 --- /dev/null +++ b/endpoints/getting-started/clients/service_to_service_google_id_token/main.py @@ -0,0 +1,97 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0(the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Example of calling a Google Cloud Endpoint API from Google App Engine +Default Service Account using Google ID token.""" + +import base64 +import httplib +import json +import time +import urllib + +from google.appengine.api import app_identity +import webapp2 + +SERVICE_ACCOUNT_EMAIL = "YOUR-CLIENT-PROJECT-ID@appspot.gserviceaccount.com" +HOST = "YOUR-SERVER-PROJECT-ID.appspot.com" +TARGET_AUD = "https://YOUR-SERVER-PROJECT-ID.appspot.com" + + +def generate_jwt(): + """Generates a signed JSON Web Token using the Google App Engine default + service account.""" + now = int(time.time()) + + header_json = json.dumps({ + "typ": "JWT", + "alg": "RS256"}) + + payload_json = json.dumps({ + "iat": now, + # expires after one hour. + "exp": now + 3600, + # iss is the service account email. + "iss": SERVICE_ACCOUNT_EMAIL, + # target_audience is the URL of the target service. + "target_audience": TARGET_AUD, + # aud must be Google token endpoints URL. + "aud": "https://www.googleapis.com/oauth2/v4/token" + }) + + header_and_payload = '{}.{}'.format( + base64.urlsafe_b64encode(header_json), + base64.urlsafe_b64encode(payload_json)) + (key_name, signature) = app_identity.sign_blob(header_and_payload) + signed_jwt = '{}.{}'.format( + header_and_payload, + base64.urlsafe_b64encode(signature)) + + return signed_jwt + + +def get_id_token(): + """Request a Google ID token using a JWT.""" + params = urllib.urlencode({ + 'grant_type': 'urn:ietf:params:oauth:grant-type:jwt-bearer', + 'assertion': generate_jwt()}) + headers = {"Content-Type": "application/x-www-form-urlencoded"} + conn = httplib.HTTPSConnection("www.googleapis.com") + conn.request("POST", "/oauth2/v4/token", params, headers) + res = json.loads(conn.getresponse().read()) + conn.close() + return res['id_token'] + + +def make_request(token): + """Makes a request to the auth info endpoint for Google ID token.""" + headers = {'Authorization': 'Bearer {}'.format(token)} + conn = httplib.HTTPSConnection(HOST) + conn.request("GET", '/auth/info/googleidtoken', None, headers) + res = conn.getresponse() + conn.close() + return res.read() + + +class MainPage(webapp2.RequestHandler): + def get(self): + self.response.headers['Content-Type'] = 'text/plain' + token = get_id_token() + res = make_request(token) + self.response.write(res) + + +app = webapp2.WSGIApplication([ + ('/', MainPage), +], debug=True) diff --git a/endpoints/getting-started/clients/service_to_service_non_default/app.yaml b/endpoints/getting-started/clients/service_to_service_non_default/app.yaml new file mode 100644 index 00000000000..f041d384c05 --- /dev/null +++ b/endpoints/getting-started/clients/service_to_service_non_default/app.yaml @@ -0,0 +1,7 @@ +runtime: python27 +api_version: 1 +threadsafe: true + +handlers: +- url: /.* + script: main.app diff --git a/endpoints/getting-started/clients/service_to_service_non_default/appengine_config.py b/endpoints/getting-started/clients/service_to_service_non_default/appengine_config.py new file mode 100644 index 00000000000..f4489ff9680 --- /dev/null +++ b/endpoints/getting-started/clients/service_to_service_non_default/appengine_config.py @@ -0,0 +1,3 @@ +from google.appengine.ext import vendor + +vendor.add('lib') diff --git a/endpoints/getting-started/clients/service_to_service_non_default/main.py b/endpoints/getting-started/clients/service_to_service_non_default/main.py new file mode 100644 index 00000000000..361a99e1e6a --- /dev/null +++ b/endpoints/getting-started/clients/service_to_service_non_default/main.py @@ -0,0 +1,93 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0(the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Example of calling a Google Cloud Endpoint API with a JWT signed by a +Service Account.""" + +import base64 +import httplib +import json +import time + +import google.auth.app_engine +import googleapiclient.discovery +import webapp2 + +SERVICE_ACCOUNT_EMAIL = "YOUR-SERVICE-ACCOUNT-EMAIL" +HOST = "YOUR-SERVER-PROJECT-ID.appspot.com" +SERVICE_ACCOUNT = \ + "projects/YOUR-CLIENT-PROJECT-ID/serviceAccounts/YOUR-SERVICE-ACCOUNT-EMAIL" + + +def generate_jwt(): + """Generates a signed JSON Web Token using a service account.""" + credentials = google.auth.app_engine.Credentials( + scopes=['https://www.googleapis.com/auth/iam']) + service = googleapiclient.discovery.build( + serviceName='iam', version='v1', credentials=credentials) + + now = int(time.time()) + + header_json = json.dumps({ + "typ": "JWT", + "alg": "RS256"}) + + payload_json = json.dumps({ + 'iat': now, + # expires after one hour. + "exp": now + 3600, + # iss is the service account email. + 'iss': SERVICE_ACCOUNT_EMAIL, + 'sub': SERVICE_ACCOUNT_EMAIL, + # aud must match 'audience' in the security configuration in your + # swagger spec.It can be any string. + 'aud': 'echo.endpoints.sample.google.com', + "email": SERVICE_ACCOUNT_EMAIL + }) + + header_and_payload = '{}.{}'.format( + base64.urlsafe_b64encode(header_json), + base64.urlsafe_b64encode(payload_json)) + slist = service.projects().serviceAccounts().signBlob( + name=SERVICE_ACCOUNT, + body={'bytesToSign': base64.b64encode(header_and_payload)}) + res = slist.execute() + signature = base64.urlsafe_b64encode( + base64.decodestring(res['signature'])) + signed_jwt = '{}.{}'.format(header_and_payload, signature) + + return signed_jwt + + +def make_request(signed_jwt): + """Makes a request to the auth info endpoint for Google JWTs.""" + headers = {'Authorization': 'Bearer {}'.format(signed_jwt)} + conn = httplib.HTTPSConnection(HOST) + conn.request("GET", '/auth/info/googlejwt', None, headers) + res = conn.getresponse() + conn.close() + return res.read() + + +class MainPage(webapp2.RequestHandler): + def get(self): + self.response.headers['Content-Type'] = 'text/plain' + signed_jwt = generate_jwt() + res = make_request(signed_jwt) + self.response.write(res) + + +app = webapp2.WSGIApplication([ + ('/', MainPage), +], debug=True) diff --git a/endpoints/getting-started/clients/service_to_service_non_default/requirements.txt b/endpoints/getting-started/clients/service_to_service_non_default/requirements.txt new file mode 100644 index 00000000000..7e4359ce08d --- /dev/null +++ b/endpoints/getting-started/clients/service_to_service_non_default/requirements.txt @@ -0,0 +1,3 @@ +google-api-python-client==1.7.8 +google-auth==1.6.2 +google-auth-httplib2==0.0.3 diff --git a/endpoints/getting-started/deployment.yaml b/endpoints/getting-started/deployment.yaml new file mode 100644 index 00000000000..a0e77534126 --- /dev/null +++ b/endpoints/getting-started/deployment.yaml @@ -0,0 +1,56 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: esp-echo +spec: + ports: + - port: 80 + targetPort: 8081 + protocol: TCP + name: http + selector: + app: esp-echo + type: LoadBalancer +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: esp-echo +spec: + replicas: 1 + template: + metadata: + labels: + app: esp-echo + spec: + containers: + # [START esp] + - name: esp + image: gcr.io/endpoints-release/endpoints-runtime:1 + args: [ + "--http_port=8081", + "--backend=127.0.0.1:8080", + "--service=SERVICE_NAME", + "--rollout_strategy=managed", + ] + # [END esp] + ports: + - containerPort: 8081 + - name: echo + image: gcr.io/google-samples/echo-python:1.0 + ports: + - containerPort: 8080 diff --git a/endpoints/getting-started/k8s/esp_echo_http.yaml b/endpoints/getting-started/k8s/esp_echo_http.yaml new file mode 100644 index 00000000000..f178f1e658c --- /dev/null +++ b/endpoints/getting-started/k8s/esp_echo_http.yaml @@ -0,0 +1,70 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: esp-echo +spec: + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: esp-echo + # Change this type to NodePort if you use Minikube. + type: LoadBalancer +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: esp-echo +spec: + replicas: 1 + template: + metadata: + labels: + app: esp-echo + spec: + # [START secret-1] + volumes: + - name: service-account-creds + secret: + secretName: service-account-creds + # [END secret-1] + # [START service] + containers: + - name: esp + image: gcr.io/endpoints-release/endpoints-runtime:1 + args: [ + "--http_port", "8080", + "--backend", "127.0.0.1:8081", + "--service", "SERVICE_NAME", + "--rollout_strategy", "managed", + "--service_account_key", "/etc/nginx/creds/service-account-creds.json", + ] + # [END service] + ports: + - containerPort: 8080 + # [START secret-2] + volumeMounts: + - mountPath: /etc/nginx/creds + name: service-account-creds + readOnly: true + # [END secret-2] + - name: echo + image: gcr.io/endpoints-release/echo:latest + ports: + - containerPort: 8081 diff --git a/endpoints/getting-started/main.py b/endpoints/getting-started/main.py new file mode 100644 index 00000000000..8ce27ace860 --- /dev/null +++ b/endpoints/getting-started/main.py @@ -0,0 +1,98 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Google Cloud Endpoints sample application. + +Demonstrates how to create a simple echo API as well as how to deal with +various authentication methods. +""" + +import base64 +import json +import logging + +from flask import Flask, jsonify, request + +from flask_cors import cross_origin + +from six.moves import http_client + + +app = Flask(__name__) + + +def _base64_decode(encoded_str): + # Add paddings manually if necessary. + num_missed_paddings = 4 - len(encoded_str) % 4 + if num_missed_paddings != 4: + encoded_str += b'=' * num_missed_paddings + return base64.b64decode(encoded_str).decode('utf-8') + + +@app.route('/echo', methods=['POST']) +def echo(): + """Simple echo service.""" + message = request.get_json().get('message', '') + return jsonify({'message': message}) + + +# [START endpoints_auth_info_backend] +def auth_info(): + """Retrieves the authenication information from Google Cloud Endpoints.""" + encoded_info = request.headers.get('X-Endpoint-API-UserInfo', None) + + if encoded_info: + info_json = _base64_decode(encoded_info) + user_info = json.loads(info_json) + else: + user_info = {'id': 'anonymous'} + + return jsonify(user_info) +# [START endpoints_auth_info_backend] + + +@app.route('/auth/info/googlejwt', methods=['GET']) +def auth_info_google_jwt(): + """Auth info with Google signed JWT.""" + return auth_info() + + +@app.route('/auth/info/googleidtoken', methods=['GET']) +def auth_info_google_id_token(): + """Auth info with Google ID token.""" + return auth_info() + + +@app.route('/auth/info/firebase', methods=['GET']) +@cross_origin(send_wildcard=True) +def auth_info_firebase(): + """Auth info with Firebase auth.""" + return auth_info() + + +@app.errorhandler(http_client.INTERNAL_SERVER_ERROR) +def unexpected_error(e): + """Handle exceptions by returning swagger-compliant json.""" + logging.exception('An error occured while processing the request.') + response = jsonify({ + 'code': http_client.INTERNAL_SERVER_ERROR, + 'message': 'Exception: {}'.format(e)}) + response.status_code = http_client.INTERNAL_SERVER_ERROR + return response + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/endpoints/getting-started/main_test.py b/endpoints/getting-started/main_test.py new file mode 100644 index 00000000000..6ca6b5093d3 --- /dev/null +++ b/endpoints/getting-started/main_test.py @@ -0,0 +1,82 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import json +import os + +import pytest + +import main + + +@pytest.fixture +def client(monkeypatch): + monkeypatch.chdir(os.path.dirname(main.__file__)) + main.app.testing = True + client = main.app.test_client() + return client + + +def test_echo(client): + r = client.post( + '/echo', + data='{"message": "Hello"}', + headers={ + 'Content-Type': 'application/json' + }) + + assert r.status_code == 200 + data = json.loads(r.data.decode('utf-8')) + assert data['message'] == 'Hello' + + +def test_auth_info(client): + endpoints = [ + '/auth/info/googlejwt', + '/auth/info/googleidtoken', + '/auth/info/firebase'] + + encoded_info = base64.b64encode(json.dumps({ + 'id': '123' + }).encode('utf-8')) + + for endpoint in endpoints: + r = client.get( + endpoint, + headers={ + 'Content-Type': 'application/json' + }) + + assert r.status_code == 200 + data = json.loads(r.data.decode('utf-8')) + assert data['id'] == 'anonymous' + + r = client.get( + endpoint, + headers={ + 'Content-Type': 'application/json', + 'X-Endpoint-API-UserInfo': encoded_info + }) + + assert r.status_code == 200 + data = json.loads(r.data.decode('utf-8')) + assert data['id'] == '123' + + +def test_cors(client): + r = client.options( + '/auth/info/firebase', headers={'Origin': 'example.com'}) + assert r.status_code == 200 + assert r.headers['Access-Control-Allow-Origin'] == '*' diff --git a/endpoints/getting-started/openapi-appengine.yaml b/endpoints/getting-started/openapi-appengine.yaml new file mode 100644 index 00000000000..ebfc7b7e724 --- /dev/null +++ b/endpoints/getting-started/openapi-appengine.yaml @@ -0,0 +1,157 @@ +# [START swagger] +swagger: "2.0" +info: + description: "A simple Google Cloud Endpoints API example." + title: "Endpoints Example" + version: "1.0.0" +host: "YOUR-PROJECT-ID.appspot.com" +# [END swagger] +consumes: +- "application/json" +produces: +- "application/json" +schemes: +- "https" +paths: + "/echo": + post: + description: "Echo back a given message." + operationId: "echo" + produces: + - "application/json" + responses: + 200: + description: "Echo" + schema: + $ref: "#/definitions/echoMessage" + parameters: + - description: "Message to echo" + in: body + name: message + required: true + schema: + $ref: "#/definitions/echoMessage" + security: + - api_key: [] + "/auth/info/googlejwt": + get: + description: "Returns the requests' authentication information." + operationId: "auth_info_google_jwt" + produces: + - "application/json" + responses: + 200: + description: "Authentication info." + schema: + $ref: "#/definitions/authInfoResponse" + security: + - google_jwt: [] + - gae_default_service_account: [] + - google_service_account: [] + "/auth/info/googleidtoken": + get: + description: "Returns the requests' authentication information." + operationId: "authInfoGoogleIdToken" + produces: + - "application/json" + responses: + 200: + description: "Authentication info." + schema: + $ref: "#/definitions/authInfoResponse" + security: + - google_id_token: [] + "/auth/info/firebase": + get: + description: "Returns the requests' authentication information." + operationId: "authInfoFirebase" + produces: + - "application/json" + responses: + 200: + description: "Authentication info." + schema: + $ref: "#/definitions/authInfoResponse" + security: + - firebase: [] + +definitions: + echoMessage: + type: "object" + properties: + message: + type: "string" + authInfoResponse: + properties: + id: + type: "string" + email: + type: "string" +# [START securityDef] +securityDefinitions: + # This section configures basic authentication with an API key. + api_key: + type: "apiKey" + name: "key" + in: "query" +# [END securityDef] + # This section configures authentication using Google API Service Accounts + # to sign a json web token. This is mostly used for server-to-server + # communication. + google_jwt: + authorizationUrl: "" + flow: "implicit" + type: "oauth2" + # This must match the 'iss' field in the JWT. + x-google-issuer: "jwt-client.endpoints.sample.google.com" + # Update this with your service account's email address. + x-google-jwks_uri: "https://www.googleapis.com/service_accounts/v1/jwk/YOUR-SERVICE-ACCOUNT-EMAIL" + # This must match the "aud" field in the JWT. You can add multiple audiences to accept JWTs from multiple clients. + x-google-audiences: "echo.endpoints.sample.google.com" + # This section configures authentication using Google App Engine default + # service account to sign a json web token. This is mostly used for + # server-to-server communication. + gae_default_service_account: + authorizationUrl: "" + flow: "implicit" + type: "oauth2" + # Replace YOUR-CLIENT-PROJECT-ID with your client project ID. + x-google-issuer: "YOUR-CLIENT-PROJECT-ID@appspot.gserviceaccount.com" + # Replace YOUR-CLIENT-PROJECT-ID with your client project ID. + x-google-jwks_uri: "https://www.googleapis.com/robot/v1/metadata/x509/YOUR-CLIENT-PROJECT-ID@appspot.gserviceaccount.com" + # This must match the "aud" field in the JWT. You can add multiple audiences to accept JWTs from multiple clients. + x-google-audiences: "echo.endpoints.sample.google.com" + # This section configures authentication using a service account + # to sign a json web token. This is mostly used for server-to-server + # communication. + google_service_account: + authorizationUrl: "" + flow: "implicit" + type: "oauth2" + # Replace YOUR-SERVICE-ACCOUNT-EMAIL with your service account email. + x-google-issuer: "YOUR-SERVICE-ACCOUNT-EMAIL" + # Replace YOUR-SERVICE-ACCOUNT-EMAIL with your service account email. + x-google-jwks_uri: "https://www.googleapis.com/robot/v1/metadata/x509/YOUR-SERVICE-ACCOUNT-EMAIL" + # This must match the "aud" field in the JWT. You can add multiple audiences to accept JWTs from multiple clients. + x-google-audiences: "echo.endpoints.sample.google.com" + # This section configures authentication using Google OAuth2 ID Tokens. + # ID Tokens can be obtained using OAuth2 clients, and can be used to access + # your API on behalf of a particular user. + google_id_token: + authorizationUrl: "" + flow: "implicit" + type: "oauth2" + x-google-issuer: "https://accounts.google.com" + x-google-jwks_uri: "https://www.googleapis.com/oauth2/v3/certs" + # Your OAuth2 client's Client ID must be added here. You can add multiple client IDs to accept tokens form multiple clients. + x-google-audiences: "YOUR-CLIENT-ID" + # This section configures authentication using Firebase Auth. + # [START firebaseAuth] + firebase: + authorizationUrl: "" + flow: "implicit" + type: "oauth2" + x-google-issuer: "https://securetoken.google.com/YOUR-PROJECT-ID" + x-google-jwks_uri: "https://www.googleapis.com/service_accounts/v1/metadata/x509/securetoken@system.gserviceaccount.com" + x-google-audiences: "YOUR-PROJECT-ID" + # [END firebaseAuth] diff --git a/endpoints/getting-started/openapi.yaml b/endpoints/getting-started/openapi.yaml new file mode 100644 index 00000000000..4fc5063fdd1 --- /dev/null +++ b/endpoints/getting-started/openapi.yaml @@ -0,0 +1,159 @@ +# [START swagger] +swagger: "2.0" +info: + description: "A simple Google Cloud Endpoints API example." + title: "Endpoints Example" + version: "1.0.0" +host: "echo-api.endpoints.YOUR-PROJECT-ID.cloud.goog" +# [END swagger] +consumes: +- "application/json" +produces: +- "application/json" +schemes: +# Uncomment the next line if you configure SSL for this API. +#- "https" +- "http" +paths: + "/echo": + post: + description: "Echo back a given message." + operationId: "echo" + produces: + - "application/json" + responses: + 200: + description: "Echo" + schema: + $ref: "#/definitions/echoMessage" + parameters: + - description: "Message to echo" + in: body + name: message + required: true + schema: + $ref: "#/definitions/echoMessage" + security: + - api_key: [] + "/auth/info/googlejwt": + get: + description: "Returns the requests' authentication information." + operationId: "auth_info_google_jwt" + produces: + - "application/json" + responses: + 200: + description: "Authentication info." + schema: + $ref: "#/definitions/authInfoResponse" + security: + - google_jwt: [] + - gae_default_service_account: [] + - google_service_account: [] + "/auth/info/googleidtoken": + get: + description: "Returns the requests' authentication information." + operationId: "authInfoGoogleIdToken" + produces: + - "application/json" + responses: + 200: + description: "Authentication info." + schema: + $ref: "#/definitions/authInfoResponse" + security: + - google_id_token: [] + "/auth/info/firebase": + get: + description: "Returns the requests' authentication information." + operationId: "authInfoFirebase" + produces: + - "application/json" + responses: + 200: + description: "Authentication info." + schema: + $ref: "#/definitions/authInfoResponse" + security: + - firebase: [] + +definitions: + echoMessage: + type: "object" + properties: + message: + type: "string" + authInfoResponse: + properties: + id: + type: "string" + email: + type: "string" +# [START securityDef] +securityDefinitions: + # This section configures basic authentication with an API key. + api_key: + type: "apiKey" + name: "key" + in: "query" +# [END securityDef] + # This section configures authentication using Google API Service Accounts + # to sign a json web token. This is mostly used for server-to-server + # communication. + google_jwt: + authorizationUrl: "" + flow: "implicit" + type: "oauth2" + # This must match the 'iss' field in the JWT. + x-google-issuer: "jwt-client.endpoints.sample.google.com" + # Update this with your service account's email address. + x-google-jwks_uri: "https://www.googleapis.com/service_accounts/v1/jwk/YOUR-SERVICE-ACCOUNT-EMAIL" + # This must match the "aud" field in the JWT. You can add multiple audiences to accept JWTs from multiple clients. + x-google-audiences: "echo.endpoints.sample.google.com" + # This section configures authentication using Google App Engine default + # service account to sign a json web token. This is mostly used for + # server-to-server communication. + gae_default_service_account: + authorizationUrl: "" + flow: "implicit" + type: "oauth2" + # Replace YOUR-CLIENT-PROJECT-ID with your client project ID. + x-google-issuer: "YOUR-CLIENT-PROJECT-ID@appspot.gserviceaccount.com" + # Replace YOUR-CLIENT-PROJECT-ID with your client project ID. + x-google-jwks_uri: "https://www.googleapis.com/robot/v1/metadata/x509/YOUR-CLIENT-PROJECT-ID@appspot.gserviceaccount.com" + # This must match the "aud" field in the JWT. You can add multiple audiences to accept JWTs from multiple clients. + x-google-audiences: "echo.endpoints.sample.google.com" + # This section configures authentication using a service account + # to sign a json web token. This is mostly used for server-to-server + # communication. + google_service_account: + authorizationUrl: "" + flow: "implicit" + type: "oauth2" + # Replace YOUR-SERVICE-ACCOUNT-EMAIL with your service account email. + x-google-issuer: "YOUR-SERVICE-ACCOUNT-EMAIL" + # Replace YOUR-SERVICE-ACCOUNT-EMAIL with your service account email. + x-google-jwks_uri: "https://www.googleapis.com/robot/v1/metadata/x509/YOUR-SERVICE-ACCOUNT-EMAIL" + # This must match the "aud" field in the JWT. You can add multiple audiences to accept JWTs from multiple clients. + x-google-audiences: "echo.endpoints.sample.google.com" + # This section configures authentication using Google OAuth2 ID Tokens. + # ID Tokens can be obtained using OAuth2 clients, and can be used to access + # your API on behalf of a particular user. + google_id_token: + authorizationUrl: "" + flow: "implicit" + type: "oauth2" + x-google-issuer: "https://accounts.google.com" + x-google-jwks_uri: "https://www.googleapis.com/oauth2/v3/certs" + # Your OAuth2 client's Client ID must be added here. You can add multiple client IDs to accept tokens form multiple clients. + x-google-audiences: "YOUR-CLIENT-ID" + # This section configures authentication using Firebase Auth. + # [START firebaseAuth] + firebase: + authorizationUrl: "" + flow: "implicit" + type: "oauth2" + x-google-issuer: "https://securetoken.google.com/YOUR-PROJECT-ID" + x-google-jwks_uri: "https://www.googleapis.com/service_accounts/v1/metadata/x509/securetoken@system.gserviceaccount.com" + x-google-audiences: "YOUR-PROJECT-ID" + # [END firebaseAuth] diff --git a/endpoints/getting-started/requirements.txt b/endpoints/getting-started/requirements.txt new file mode 100644 index 00000000000..47b1c80abb1 --- /dev/null +++ b/endpoints/getting-started/requirements.txt @@ -0,0 +1,8 @@ +Flask==1.0.2 +flask-cors==3.0.7 +gunicorn==19.9.0 +six==1.12.0 +pyyaml==3.13 +requests==2.21.0 +google-auth==1.6.2 +google-auth-oauthlib==0.2.0 diff --git a/endpoints/kubernetes/README.md b/endpoints/kubernetes/README.md new file mode 100644 index 00000000000..8b315d41ac3 --- /dev/null +++ b/endpoints/kubernetes/README.md @@ -0,0 +1,7 @@ +# Kubernetes Configuration Example for running Cloud Endpoints with gRPC Bookstore Backend + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=endpoints/kubernetes/README.md + diff --git a/endpoints/kubernetes/grpc-bookstore-server.yaml b/endpoints/kubernetes/grpc-bookstore-server.yaml new file mode 100644 index 00000000000..b426455e3df --- /dev/null +++ b/endpoints/kubernetes/grpc-bookstore-server.yaml @@ -0,0 +1,43 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +apiVersion: v1 +kind: Service +metadata: + name: bookstore +spec: + ports: + - port: 8000 + protocol: TCP + name: grpc + selector: + app: bookstore-app + type: LoadBalancer +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: bookstore +spec: + replicas: 1 + template: + metadata: + labels: + app: bookstore-app + spec: + containers: + - name: bookstore + image: gcr.io/endpointsv2/python-grpc-bookstore-server:1 + ports: + - containerPort: 8000 diff --git a/endpoints/kubernetes/grpc-bookstore.yaml b/endpoints/kubernetes/grpc-bookstore.yaml new file mode 100644 index 00000000000..a61ae987c04 --- /dev/null +++ b/endpoints/kubernetes/grpc-bookstore.yaml @@ -0,0 +1,59 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +# Use this file to deploy the container for the grpc-bookstore sample +# and the container for the Extensible Service Proxy (ESP) to +# Google Kubernetes Engine (GKE). + +apiVersion: v1 +kind: Service +metadata: + name: esp-grpc-bookstore +spec: + ports: + # Port that accepts gRPC and JSON/HTTP2 requests over HTTP. + - port: 80 + targetPort: 9000 + protocol: TCP + name: http2 + selector: + app: esp-grpc-bookstore + type: LoadBalancer +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: esp-grpc-bookstore +spec: + replicas: 1 + template: + metadata: + labels: + app: esp-grpc-bookstore + spec: + containers: + - name: esp + image: gcr.io/endpoints-release/endpoints-runtime:1.16.0 + args: [ + "--http2_port=9000", + "--service=SERVICE_NAME", + "--rollout_strategy=managed", + "--backend=grpc://127.0.0.1:8000" + ] + ports: + - containerPort: 9000 + - name: bookstore + image: gcr.io/endpointsv2/python-grpc-bookstore-server:1 + ports: + - containerPort: 8000 diff --git a/endpoints/kubernetes/k8s-grpc-bookstore.yaml b/endpoints/kubernetes/k8s-grpc-bookstore.yaml new file mode 100644 index 00000000000..54f2b4be619 --- /dev/null +++ b/endpoints/kubernetes/k8s-grpc-bookstore.yaml @@ -0,0 +1,74 @@ +# Copyright 2018 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +# Use this file to deploy the container for the grpc-bookstore sample +# and the container for the Extensible Service Proxy (ESP) to a +# Kubernetes cluster that is not on GCP. + +apiVersion: v1 +kind: Service +metadata: + name: esp-grpc-bookstore +spec: + ports: + # Port that accepts gRPC and JSON/HTTP2 requests over HTTP. + - port: 80 + targetPort: 9000 + protocol: TCP + name: http2 + selector: + app: esp-grpc-bookstore + type: LoadBalancer +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: esp-grpc-bookstore +spec: + replicas: 1 + template: + metadata: + labels: + app: esp-grpc-bookstore + spec: + # [START secret-1] + volumes: + - name: service-account-creds + secret: + secretName: service-account-creds + # [END secret-1] + # [START service] + containers: + - name: esp + image: gcr.io/endpoints-release/endpoints-runtime:1 + args: [ + "--http2_port=9000", + "--service=SERVICE_NAME", + "--rollout_strategy=managed", + "--backend=grpc://127.0.0.1:8000", + "--service_account_key=/etc/nginx/creds/service-account-creds.json" + ] + # [END service] + ports: + - containerPort: 9000 + # [START secret-2] + volumeMounts: + - mountPath: /etc/nginx/creds + name: service-account-creds + readOnly: true + # [END secret-2] + - name: bookstore + image: gcr.io/endpointsv2/python-grpc-bookstore-server:1 + ports: + - containerPort: 8000 diff --git a/error_reporting/api/README.rst b/error_reporting/api/README.rst new file mode 100644 index 00000000000..74bcd6397f2 --- /dev/null +++ b/error_reporting/api/README.rst @@ -0,0 +1,98 @@ +.. This file is automatically generated. Do not edit this file directly. + +Stackdriver Error Reporting Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=error_reporting/api/README.rst + + +This directory contains samples for Stackdriver Error Reporting. `Stackdriver Error Reporting`_ aggregates and displays errors produced in + your running cloud services. + + + + +.. _Stackdriver Error Reporting: https://cloud.google.com/error-reporting/docs/ + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Report Exception ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=error_reporting/api/report_exception.py,error_reporting/api/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python report_exception.py + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/error_reporting/api/README.rst.in b/error_reporting/api/README.rst.in new file mode 100644 index 00000000000..ac60cc80c81 --- /dev/null +++ b/error_reporting/api/README.rst.in @@ -0,0 +1,21 @@ +# This file is used to generate README.rst + +product: + name: Stackdriver Error Reporting + short_name: Error Reporting + url: https://cloud.google.com/error-reporting/docs/ + description: > + `Stackdriver Error Reporting`_ aggregates and displays errors produced in + your running cloud services. + +setup: +- auth +- install_deps + +samples: +- name: Report Exception + file: report_exception.py + +cloud_client_library: true + +folder: error_reporting/api \ No newline at end of file diff --git a/error_reporting/api/report_exception.py b/error_reporting/api/report_exception.py new file mode 100644 index 00000000000..2b7e8f06ba9 --- /dev/null +++ b/error_reporting/api/report_exception.py @@ -0,0 +1,46 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# [START error_reporting] +# [START error_reporting_quickstart] +# [START error_reporting_setup_python] +def simulate_error(): + from google.cloud import error_reporting + + client = error_reporting.Client() + try: + # simulate calling a method that's not defined + raise NameError + except Exception: + client.report_exception() +# [END error_reporting_setup_python] +# [END error_reporting_quickstart] +# [END error_reporting] + + +# [START error_reporting_manual] +# [START error_reporting_setup_python_manual] +def report_manual_error(): + from google.cloud import error_reporting + + client = error_reporting.Client() + client.report("An error has occurred.") +# [START error_reporting_setup_python_manual] +# [END error_reporting_manual] + + +if __name__ == '__main__': + simulate_error() + report_manual_error() diff --git a/error_reporting/api/report_exception_test.py b/error_reporting/api/report_exception_test.py new file mode 100644 index 00000000000..042951e9a48 --- /dev/null +++ b/error_reporting/api/report_exception_test.py @@ -0,0 +1,23 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import report_exception + + +def test_error_sends(): + report_exception.simulate_error() + + +def test_manual_error_sends(): + report_exception.report_manual_error() diff --git a/error_reporting/api/requirements.txt b/error_reporting/api/requirements.txt new file mode 100644 index 00000000000..deb4c891a29 --- /dev/null +++ b/error_reporting/api/requirements.txt @@ -0,0 +1 @@ +google-cloud-error-reporting==0.30.1 diff --git a/error_reporting/fluent_on_compute/README.md b/error_reporting/fluent_on_compute/README.md new file mode 100644 index 00000000000..d3a58c167d8 --- /dev/null +++ b/error_reporting/fluent_on_compute/README.md @@ -0,0 +1,35 @@ +# Google Error Reorting Samples Samples + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=error_reporting/fluent_on_compute/README.md + +This section contains samples for [Google Cloud Error Reporting](https://cloud.google.com/error-reporting). + +A startup script has been provided to demonstrated how to properly provision a GCE +instance with fluentd configured. Note the intallation of fluentd, the addition of the config file, + and the restarting of the fluetnd service. You can start an instance using +it like this: + + gcloud compute instances create example-instance --metadata-from-file startup-script=startup_script.sh + +or simply use it as reference when creating your own instance. + +After fluentd is configured, main.py could be used to simulate an error: + + gcloud compute copy-files main.py example-instance:~/main.py + +Then, + + gcloud compute ssh example-instance + python ~/main.py + +And you will see the message in the Errors Console. + + +These samples are used on the following documentation page: + +> https://cloud.google.com/error-reporting/docs/setting-up-on-compute-engine + + diff --git a/error_reporting/fluent_on_compute/main.py b/error_reporting/fluent_on_compute/main.py new file mode 100644 index 00000000000..45208c913ac --- /dev/null +++ b/error_reporting/fluent_on_compute/main.py @@ -0,0 +1,42 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START error_reporting] +import traceback + +import fluent.event +import fluent.sender + + +def simulate_error(): + fluent.sender.setup('myapp', host='localhost', port=24224) + + def report(ex): + data = {} + data['message'] = '{0}'.format(ex) + data['serviceContext'] = {'service': 'myapp'} + # ... add more metadata + fluent.event.Event('errors', data) + + # report exception data using: + try: + # simulate calling a method that's not defined + raise NameError + except Exception: + report(traceback.format_exc()) +# [END error_reporting] + + +if __name__ == '__main__': + simulate_error() diff --git a/error_reporting/fluent_on_compute/main_test.py b/error_reporting/fluent_on_compute/main_test.py new file mode 100644 index 00000000000..11a24d03543 --- /dev/null +++ b/error_reporting/fluent_on_compute/main_test.py @@ -0,0 +1,23 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +import main + + +@mock.patch("fluent.event") +def test_error_sends(event_mock): + main.simulate_error() + event_mock.Event.assert_called_once_with(mock.ANY, mock.ANY) diff --git a/error_reporting/fluent_on_compute/requirements.txt b/error_reporting/fluent_on_compute/requirements.txt new file mode 100644 index 00000000000..1b1073c1e4c --- /dev/null +++ b/error_reporting/fluent_on_compute/requirements.txt @@ -0,0 +1 @@ +fluent-logger==0.9.3 diff --git a/error_reporting/fluent_on_compute/startup_script.sh b/error_reporting/fluent_on_compute/startup_script.sh new file mode 100644 index 00000000000..f2ef895dcfd --- /dev/null +++ b/error_reporting/fluent_on_compute/startup_script.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -v + +curl -sSO "https://dl.google.com/cloudagents/install-logging-agent.sh" +chmod +x install-logging-agent.sh +./install-logging-agent.sh +mkdir -p /etc/google-fluentd/config.d/ +cat < /etc/google-fluentd/config.d/forward.conf + + type forward + port 24224 + +EOF +service google-fluentd restart + +apt-get update +apt-get install -yq \ + git build-essential supervisor python python-dev python-pip libffi-dev \ + libssl-dev +pip install fluent-logger + diff --git a/firestore/cloud-client/requirements.txt b/firestore/cloud-client/requirements.txt new file mode 100644 index 00000000000..399e498b84c --- /dev/null +++ b/firestore/cloud-client/requirements.txt @@ -0,0 +1 @@ +google-cloud-firestore==0.31.0 diff --git a/firestore/cloud-client/snippets.py b/firestore/cloud-client/snippets.py new file mode 100644 index 00000000000..531f05c1aec --- /dev/null +++ b/firestore/cloud-client/snippets.py @@ -0,0 +1,817 @@ +# Copyright 2017, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +from time import sleep + +from google.cloud import firestore +from google.cloud.firestore_v1beta1 import ArrayRemove, ArrayUnion +import google.cloud.exceptions + + +def quickstart_new_instance(): + # [START quickstart_new_instance] + from google.cloud import firestore + + # Project ID is determined by the GCLOUD_PROJECT environment variable + db = firestore.Client() + # [END quickstart_new_instance] + + return db + + +def quickstart_add_data_one(): + db = firestore.Client() + # [START quickstart_add_data_one] + doc_ref = db.collection(u'users').document(u'alovelace') + doc_ref.set({ + u'first': u'Ada', + u'last': u'Lovelace', + u'born': 1815 + }) + # [END quickstart_add_data_one] + + +def quickstart_add_data_two(): + db = firestore.Client() + # [START quickstart_add_data_two] + doc_ref = db.collection(u'users').document(u'aturing') + doc_ref.set({ + u'first': u'Alan', + u'middle': u'Mathison', + u'last': u'Turing', + u'born': 1912 + }) + # [END quickstart_add_data_two] + + +def quickstart_get_collection(): + db = firestore.Client() + # [START quickstart_get_collection] + users_ref = db.collection(u'users') + docs = users_ref.get() + + for doc in docs: + print(u'{} => {}'.format(doc.id, doc.to_dict())) + # [END quickstart_get_collection] + + +def add_from_dict(): + db = firestore.Client() + # [START add_from_dict] + data = { + u'name': u'Los Angeles', + u'state': u'CA', + u'country': u'USA' + } + + # Add a new doc in collection 'cities' with ID 'LA' + db.collection(u'cities').document(u'LA').set(data) + # [END add_from_dict] + + +def add_data_types(): + db = firestore.Client() + # [START add_data_types] + data = { + u'stringExample': u'Hello, World!', + u'booleanExample': True, + u'numberExample': 3.14159265, + u'dateExample': datetime.datetime.now(), + u'arrayExample': [5, True, u'hello'], + u'nullExample': None, + u'objectExample': { + u'a': 5, + u'b': True + } + } + + db.collection(u'data').document(u'one').set(data) + # [END add_data_types] + + +# [START custom_class_def] +class City(object): + def __init__(self, name, state, country, capital=False, population=0, + regions=[]): + self.name = name + self.state = state + self.country = country + self.capital = capital + self.population = population + self.regions = regions + + @staticmethod + def from_dict(source): + # [START_EXCLUDE] + city = City(source[u'name'], source[u'state'], source[u'country']) + + if u'capital' in source: + city.capital = source[u'capital'] + + if u'population' in source: + city.population = source[u'population'] + + if u'regions' in source: + city.regions = source[u'regions'] + + return city + # [END_EXCLUDE] + + def to_dict(self): + # [START_EXCLUDE] + dest = { + u'name': self.name, + u'state': self.state, + u'country': self.country + } + + if self.capital: + dest[u'capital'] = self.capital + + if self.population: + dest[u'population'] = self.population + + if self.regions: + dest[u'regions'] = self.regions + + return dest + # [END_EXCLUDE] + + def __repr__(self): + return( + u'City(name={}, country={}, population={}, capital={}, regions={})' + .format(self.name, self.country, self.population, self.capital, + self.regions)) +# [END custom_class_def] + + +def add_example_data(): + db = firestore.Client() + # [START add_example_data] + cities_ref = db.collection(u'cities') + cities_ref.document(u'SF').set( + City(u'San Francisco', u'CA', u'USA', False, 860000, + [u'west_coast', u'norcal']).to_dict()) + cities_ref.document(u'LA').set( + City(u'Los Angeles', u'CA', u'USA', False, 3900000, + [u'west_coast', u'socal']).to_dict()) + cities_ref.document(u'DC').set( + City(u'Washington D.C.', None, u'USA', True, 680000, + [u'east_coast']).to_dict()) + cities_ref.document(u'TOK').set( + City(u'Tokyo', None, u'Japan', True, 9000000, + [u'kanto', u'honshu']).to_dict()) + cities_ref.document(u'BJ').set( + City(u'Beijing', None, u'China', True, 21500000, [u'hebei']).to_dict()) + # [END add_example_data] + + +def add_custom_class_with_id(): + db = firestore.Client() + # [START add_custom_class_with_id] + city = City(name=u'Los Angeles', state=u'CA', country=u'USA') + db.collection(u'cities').document(u'LA').set(city.to_dict()) + # [END add_custom_class_with_id] + + +def add_data_with_id(): + db = firestore.Client() + data = {} + # [START add_data_with_id] + db.collection(u'cities').document(u'new-city-id').set(data) + # [END add_data_with_id] + + +def add_custom_class_generated_id(): + db = firestore.Client() + # [START add_custom_class_generated_id] + city = City(name=u'Tokyo', state=None, country=u'Japan') + db.collection(u'cities').add(city.to_dict()) + # [END add_custom_class_generated_id] + + +def add_new_doc(): + db = firestore.Client() + # [START add_new_doc] + new_city_ref = db.collection(u'cities').document() + + # later... + new_city_ref.set({ + # ... + }) + # [END add_new_doc] + + +def get_check_exists(): + db = firestore.Client() + # [START get_check_exists] + doc_ref = db.collection(u'cities').document(u'SF') + + try: + doc = doc_ref.get() + print(u'Document data: {}'.format(doc.to_dict())) + except google.cloud.exceptions.NotFound: + print(u'No such document!') + # [END get_check_exists] + + +def get_custom_class(): + db = firestore.Client() + # [START get_custom_class] + doc_ref = db.collection(u'cities').document(u'BJ') + + doc = doc_ref.get() + city = City.from_dict(doc.to_dict()) + print(city) + # [END get_custom_class] + + +def get_simple_query(): + db = firestore.Client() + # [START get_simple_query] + docs = db.collection(u'cities').where(u'capital', u'==', True).get() + + for doc in docs: + print(u'{} => {}'.format(doc.id, doc.to_dict())) + # [END get_simple_query] + + +def array_contains_filter(): + db = firestore.Client() + # [START fs_array_contains_filter] + cities_ref = db.collection(u'cities') + + query = cities_ref.where(u'regions', u'array_contains', u'west_coast') + # [END fs_array_contains_filter] + docs = query.get() + for doc in docs: + print(u'{} => {}'.format(doc.id, doc.to_dict())) + + +def get_full_collection(): + db = firestore.Client() + # [START get_full_collection] + docs = db.collection(u'cities').get() + + for doc in docs: + print(u'{} => {}'.format(doc.id, doc.to_dict())) + # [END get_full_collection] + + +def structure_doc_ref(): + db = firestore.Client() + # [START structure_doc_ref] + a_lovelace_ref = db.collection(u'users').document(u'alovelace') + # [END structure_doc_ref] + print(a_lovelace_ref) + + +def structure_collection_ref(): + db = firestore.Client() + # [START structure_collection_ref] + users_ref = db.collection(u'users') + # [END structure_collection_ref] + print(users_ref) + + +def structure_doc_ref_alternate(): + db = firestore.Client() + # [START structure_doc_ref_alternate] + a_lovelace_ref = db.document(u'users/alovelace') + # [END structure_doc_ref_alternate] + + return a_lovelace_ref + + +def structure_subcollection_ref(): + db = firestore.Client() + # [START structure_subcollection_ref] + room_a_ref = db.collection(u'rooms').document(u'roomA') + message_ref = room_a_ref.collection(u'messages').document(u'message1') + # [END structure_subcollection_ref] + print(message_ref) + + +def update_doc(): + db = firestore.Client() + # [START update_doc] + city_ref = db.collection(u'cities').document(u'DC') + + # Set the capital field + city_ref.update({u'capital': True}) + # [END update_doc] + + +def update_doc_array(): + db = firestore.Client() + # [START fs_update_doc_array] + city_ref = db.collection(u'cities').document(u'DC') + + # Atomically add a new region to the 'regions' array field. + city_ref.update({u'regions': ArrayUnion([u'greater_virginia'])}) + + # // Atomically remove a region from the 'regions' array field. + city_ref.update({u'regions': ArrayRemove([u'east_coast'])}) + # [END fs_update_doc_array] + city = city_ref.get() + print(u'Updated the regions field of the DC. {}'.format(city.to_dict())) + + +def update_multiple(): + db = firestore.Client() + # [START update_multiple] + doc_ref = db.collection(u'cities').document(u'DC') + + doc_ref.update({ + u'name': u'Washington D.C.', + u'country': u'USA', + u'capital': True + }) + # [END update_multiple] + + +def update_create_if_missing(): + db = firestore.Client() + # [START update_create_if_missing] + city_ref = db.collection(u'cities').document(u'BJ') + + city_ref.set({ + u'capital': True + }, merge=True) + # [END update_create_if_missing] + + +def update_nested(): + db = firestore.Client() + # [START update_nested] + # Create an initial document to update + frank_ref = db.collection(u'users').document(u'frank') + frank_ref.set({ + u'name': u'Frank', + u'favorites': { + u'food': u'Pizza', + u'color': u'Blue', + u'subject': u'Recess' + }, + u'age': 12 + }) + + # Update age and favorite color + frank_ref.update({ + u'age': 13, + u'favorites.color': u'Red' + }) + # [END update_nested] + + +def update_server_timestamp(): + db = firestore.Client() + # [START update_server_timestamp] + city_ref = db.collection(u'objects').document(u'some-id') + city_ref.update({ + u'timestamp': firestore.SERVER_TIMESTAMP + }) + # [END update_server_timestamp] + + +def update_data_transaction(): + db = firestore.Client() + # [START update_data_transaction] + transaction = db.transaction() + city_ref = db.collection(u'cities').document(u'SF') + + @firestore.transactional + def update_in_transaction(transaction, city_ref): + snapshot = city_ref.get(transaction=transaction) + transaction.update(city_ref, { + u'population': snapshot.get(u'population') + 1 + }) + + update_in_transaction(transaction, city_ref) + # [END update_data_transaction] + + +def update_data_transaction_result(): + db = firestore.Client() + # [START update_data_transaction_result] + transaction = db.transaction() + city_ref = db.collection(u'cities').document(u'SF') + + @firestore.transactional + def update_in_transaction(transaction, city_ref): + snapshot = city_ref.get(transaction=transaction) + new_population = snapshot.get(u'population') + 1 + + if new_population < 1000000: + transaction.update(city_ref, { + u'population': new_population + }) + return True + else: + return False + + result = update_in_transaction(transaction, city_ref) + if result: + print(u'Population updated') + else: + print(u'Sorry! Population is too big.') + # [END update_data_transaction_result] + + +def update_data_batch(): + db = firestore.Client() + # [START update_data_batch] + batch = db.batch() + + # Set the data for NYC + nyc_ref = db.collection(u'cities').document(u'NYC') + batch.set(nyc_ref, {u'name': u'New York City'}) + + # Update the population for SF + sf_ref = db.collection(u'cities').document(u'SF') + batch.update(sf_ref, {u'population': 1000000}) + + # Delete LA + la_ref = db.collection(u'cities').document(u'LA') + batch.delete(la_ref) + + # Commit the batch + batch.commit() + # [END update_data_batch] + + +def compound_query_example(): + db = firestore.Client() + # [START compound_query_example] + # Create a reference to the cities collection + cities_ref = db.collection(u'cities') + + # Create a query against the collection + query_ref = cities_ref.where(u'state', u'==', u'CA') + # [END compound_query_example] + + return query_ref + + +def compound_query_simple(): + db = firestore.Client() + # [START compound_query_simple] + cities_ref = db.collection(u'cities') + + query = cities_ref.where(u'capital', u'==', True) + # [END compound_query_simple] + + print(query) + + +def compound_query_single_clause(): + db = firestore.Client() + # [START compound_query_single_clause] + cities_ref = db.collection(u'cities') + + cities_ref.where(u'state', u'==', u'CA') + cities_ref.where(u'population', u'<', 1000000) + cities_ref.where(u'name', u'>=', u'San Francisco') + # [END compound_query_single_clause] + + +def compound_query_valid_multi_clause(): + db = firestore.Client() + # [START compound_query_valid_multi_clause] + cities_ref = db.collection(u'cities') + + sydney_query = cities_ref.where( + u'state', u'==', u'CO').where(u'name', u'==', u'Denver') + large_us_cities_query = cities_ref.where( + u'state', u'==', u'CA').where(u'population', u'>', 1000000) + # [END compound_query_valid_multi_clause] + print(sydney_query) + print(large_us_cities_query) + + +def compound_query_valid_single_field(): + db = firestore.Client() + # [START compound_query_valid_single_field] + cities_ref = db.collection(u'cities') + cities_ref.where(u'state', u'>=', u'CA').where(u'state', u'<=', u'IN') + # [END compound_query_valid_single_field] + + +def compound_query_invalid_multi_field(): + db = firestore.Client() + # [START compound_query_invalid_multi_field] + cities_ref = db.collection(u'cities') + cities_ref.where( + u'state', u'>=', u'CA').where(u'population', u'>=', 1000000) + # [END compound_query_invalid_multi_field] + + +def order_simple_limit(): + db = firestore.Client() + # [START order_simple_limit] + db.collection(u'cities').order_by(u'name').limit(3).get() + # [END order_simple_limit] + + +def order_simple_limit_desc(): + db = firestore.Client() + # [START order_simple_limit_desc] + cities_ref = db.collection(u'cities') + query = cities_ref.order_by( + u'name', direction=firestore.Query.DESCENDING).limit(3) + results = query.get() + # [END order_simple_limit_desc] + print(results) + + +def order_multiple(): + db = firestore.Client() + # [START order_multiple] + cities_ref = db.collection(u'cities') + cities_ref.order_by(u'state').order_by( + u'population', direction=firestore.Query.DESCENDING) + # [END order_multiple] + + +def order_where_limit(): + db = firestore.Client() + # [START order_where_limit] + cities_ref = db.collection(u'cities') + query = cities_ref.where( + u'population', u'>', 2500000).order_by(u'population').limit(2) + results = query.get() + # [END order_where_limit] + print(results) + + +def order_where_valid(): + db = firestore.Client() + # [START order_where_valid] + cities_ref = db.collection(u'cities') + query = cities_ref.where( + u'population', u'>', 2500000).order_by(u'population') + results = query.get() + # [END order_where_valid] + print(results) + + +def order_where_invalid(): + db = firestore.Client() + # [START order_where_invalid] + cities_ref = db.collection(u'cities') + query = cities_ref.where(u'population', u'>', 2500000).order_by(u'country') + results = query.get() + # [END order_where_invalid] + print(results) + + +def cursor_simple_start_at(): + db = firestore.Client() + # [START cursor_simple_start_at] + cities_ref = db.collection(u'cities') + query_start_at = cities_ref.order_by(u'population').start_at({ + u'population': 1000000 + }) + # [END cursor_simple_start_at] + + return query_start_at + + +def cursor_simple_end_at(): + db = firestore.Client() + # [START cursor_simple_end_at] + cities_ref = db.collection(u'cities') + query_end_at = cities_ref.order_by(u'population').end_at({ + u'population': 1000000 + }) + # [END cursor_simple_end_at] + + return query_end_at + + +def snapshot_cursors(): + db = firestore.Client() + # [START fs_start_at_snapshot_query_cursor] + doc_ref = db.collection(u'cities').document(u'SF') + + snapshot = doc_ref.get() + start_at_snapshot = db.collection( + u'cities').order_by(u'population').start_at(snapshot) + # [END fs_start_at_snapshot_query_cursor] + results = start_at_snapshot.limit(10).get() + for doc in results: + print(u'{}'.format(doc.id)) + + return results + + +def cursor_paginate(): + db = firestore.Client() + # [START cursor_paginate] + cities_ref = db.collection(u'cities') + first_query = cities_ref.order_by(u'population').limit(3) + + # Get the last document from the results + docs = first_query.get() + last_doc = list(docs)[-1] + + # Construct a new query starting at this document + # Note: this will not have the desired effect if + # multiple cities have the exact same population value + last_pop = last_doc.to_dict()[u'population'] + + next_query = ( + cities_ref + .order_by(u'population') + .start_after({ + u'population': last_pop + }) + .limit(3) + ) + # Use the query for pagination + # ... + # [END cursor_paginate] + + return next_query + + +def listen_document(): + db = firestore.Client() + # [START listen_document] + + # Create a callback on_snapshot function to capture changes + def on_snapshot(doc_snapshot, changes, read_time): + for doc in doc_snapshot: + print(u'Received document snapshot: {}'.format(doc.id)) + + doc_ref = db.collection(u'cities').document(u'SF') + + # Watch the document + doc_watch = doc_ref.on_snapshot(on_snapshot) + # [END listen_document] + + # Creating document + data = { + u'name': u'San Francisco', + u'state': u'CA', + u'country': u'USA', + u'capital': False, + u'population': 860000 + } + doc_ref.set(data) + sleep(3) + # [START detach_listener] + # Terminate watch on a document + doc_watch.unsubscribe() + # [END detach_listener] + + +def listen_multiple(): + db = firestore.Client() + # [START listen_multiple] + + # Create a callback on_snapshot function to capture changes + def on_snapshot(col_snapshot, changes, read_time): + print(u'Callback received query snapshot.') + print(u'Current cities in California:') + for doc in col_snapshot: + print(u'{}'.format(doc.id)) + + col_query = db.collection(u'cities').where(u'state', u'==', u'CA') + + # Watch the collection query + query_watch = col_query.on_snapshot(on_snapshot) + + # [END listen_multiple] + # Creating document + data = { + u'name': u'San Francisco', + u'state': u'CA', + u'country': u'USA', + u'capital': False, + u'population': 860000 + } + db.collection(u'cities').document(u'SF').set(data) + sleep(1) + query_watch.unsubscribe() + + +def listen_for_changes(): + db = firestore.Client() + # [START listen_for_changes] + + # Create a callback on_snapshot function to capture changes + def on_snapshot(col_snapshot, changes, read_time): + print(u'Callback received query snapshot.') + print(u'Current cities in California: ') + for change in changes: + if change.type.name == 'ADDED': + print(u'New city: {}'.format(change.document.id)) + elif change.type.name == 'MODIFIED': + print(u'Modified city: {}'.format(change.document.id)) + elif change.type.name == 'REMOVED': + print(u'Removed city: {}'.format(change.document.id)) + + col_query = db.collection(u'cities').where(u'state', u'==', u'CA') + + # Watch the collection query + query_watch = col_query.on_snapshot(on_snapshot) + + # [END listen_for_changes] + mtv_document = db.collection(u'cities').document(u'MTV') + # Creating document + mtv_document.set({ + u'name': u'Mountain View', + u'state': u'CA', + u'country': u'USA', + u'capital': False, + u'population': 80000 + }) + + # Modifying document + mtv_document.update({ + u'name': u'Mountain View', + u'state': u'CA', + u'country': u'USA', + u'capital': False, + u'population': 90000 + }) + + # Delete document + mtv_document.delete() + sleep(1) + query_watch.unsubscribe() + + +def cursor_multiple_conditions(): + db = firestore.Client() + # [START cursor_multiple_conditions] + start_at_name = ( + db.collection(u'cities') + .order_by(u'name') + .order_by(u'state') + .start_at({ + u'name': u'Springfield' + }) + ) + + start_at_name_and_state = ( + db.collection(u'cities') + .order_by(u'name') + .order_by(u'state') + .start_at({ + u'name': u'Springfield', + u'state': u'Missouri' + }) + ) + # [END cursor_multiple_conditions] + + return start_at_name, start_at_name_and_state + + +def delete_single_doc(): + db = firestore.Client() + # [START delete_single_doc] + db.collection(u'cities').document(u'DC').delete() + # [END delete_single_doc] + + +def delete_field(): + db = firestore.Client() + # [START delete_field] + city_ref = db.collection(u'cities').document(u'BJ') + city_ref.update({ + u'capital': firestore.DELETE_FIELD + }) + # [END delete_field] + + +def delete_full_collection(): + db = firestore.Client() + + # [START delete_full_collection] + def delete_collection(coll_ref, batch_size): + docs = coll_ref.limit(10).get() + deleted = 0 + + for doc in docs: + print(u'Deleting doc {} => {}'.format(doc.id, doc.to_dict())) + doc.reference.delete() + deleted = deleted + 1 + + if deleted >= batch_size: + return delete_collection(coll_ref, batch_size) + # [END delete_full_collection] + + delete_collection(db.collection(u'cities'), 10) diff --git a/firestore/cloud-client/snippets_test.py b/firestore/cloud-client/snippets_test.py new file mode 100644 index 00000000000..346ec548d6a --- /dev/null +++ b/firestore/cloud-client/snippets_test.py @@ -0,0 +1,257 @@ +# Copyright 2017, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from google.cloud import firestore +import pytest + +import snippets + +os.environ['GOOGLE_CLOUD_PROJECT'] = os.environ['FIRESTORE_PROJECT'] + + +@pytest.fixture +def db(): + yield firestore.Client() + + +def test_quickstart_new_instance(): + snippets.quickstart_new_instance() + + +def test_quickstart_add_data_two(): + snippets.quickstart_add_data_two() + + +def test_quickstart_get_collection(): + snippets.quickstart_get_collection() + + +def test_quickstart_add_data_one(): + snippets.quickstart_add_data_one() + + +def test_add_from_dict(): + snippets.add_from_dict() + + +def test_add_data_types(): + snippets.add_data_types() + + +def test_add_example_data(): + snippets.add_example_data() + + +def test_add_custom_class_with_id(): + snippets.add_custom_class_with_id() + + +def test_add_data_with_id(): + snippets.add_data_with_id() + + +def test_add_custom_class_generated_id(): + snippets.add_custom_class_generated_id() + + +def test_add_new_doc(): + snippets.add_new_doc() + + +def test_get_simple_query(): + snippets.get_simple_query() + + +def test_array_contains_filter(capsys): + snippets.array_contains_filter() + out, _ = capsys.readouterr() + assert 'SF' in out + + +def test_get_full_collection(): + snippets.get_full_collection() + + +def test_get_custom_class(): + snippets.get_custom_class() + + +def test_get_check_exists(): + snippets.get_check_exists() + + +def test_structure_subcollection_ref(): + snippets.structure_subcollection_ref() + + +def test_structure_collection_ref(): + snippets.structure_collection_ref() + + +def test_structure_doc_ref_alternate(): + snippets.structure_doc_ref_alternate() + + +def test_structure_doc_ref(): + snippets.structure_doc_ref() + + +def test_update_create_if_missing(): + snippets.update_create_if_missing() + + +def test_update_doc(): + snippets.update_doc() + + +def test_update_doc_array(capsys): + snippets.update_doc_array() + out, _ = capsys.readouterr() + assert 'greater_virginia' in out + + +def test_update_multiple(): + snippets.update_multiple() + + +def test_update_server_timestamp(db): + db.collection(u'objects').document(u'some-id').set({'timestamp': 0}) + snippets.update_server_timestamp() + + +def test_update_data_transaction(db): + db.collection('cities').document('SF').set({'population': 1}) + snippets.update_data_transaction() + + +def test_update_data_transaction_result(db): + db.collection('cities').document('SF').set({'population': 1}) + snippets.update_data_transaction_result() + + +def test_update_data_batch(db): + db.collection('cities').document('SF').set({}) + db.collection('cities').document('LA').set({}) + snippets.update_data_batch() + + +def test_update_nested(): + snippets.update_nested() + + +def test_compound_query_example(): + snippets.compound_query_example() + + +def test_compound_query_valid_multi_clause(): + snippets.compound_query_valid_multi_clause() + + +def test_compound_query_simple(): + snippets.compound_query_simple() + + +def test_compound_query_invalid_multi_field(): + snippets.compound_query_invalid_multi_field() + + +def test_compound_query_single_clause(): + snippets.compound_query_single_clause() + + +def test_compound_query_valid_single_field(): + snippets.compound_query_valid_single_field() + + +def test_order_simple_limit(): + snippets.order_simple_limit() + + +def test_order_simple_limit_desc(): + snippets.order_simple_limit_desc() + + +def test_order_multiple(): + snippets.order_multiple() + + +def test_order_where_limit(): + snippets.order_where_limit() + + +def test_order_where_invalid(): + snippets.order_where_invalid() + + +def test_order_where_valid(): + snippets.order_where_valid() + + +def test_cursor_simple_start_at(): + snippets.cursor_simple_start_at() + + +def test_cursor_simple_end_at(): + snippets.cursor_simple_end_at() + + +def test_snapshot_cursors(capsys): + snippets.snapshot_cursors() + out, _ = capsys.readouterr() + assert 'SF' in out + assert 'TOK' in out + assert 'BJ' in out + + +def test_cursor_paginate(): + snippets.cursor_paginate() + + +def test_cursor_multiple_conditions(): + snippets.cursor_multiple_conditions() + + +def test_delete_single_doc(): + snippets.delete_single_doc() + + +def test_delete_field(db): + db.collection('cities').document('Beijing').set({'capital': True}) + snippets.delete_field() + + +def test_listen_document(capsys): + snippets.listen_document() + out, _ = capsys.readouterr() + assert 'Received document snapshot: SF' in out + + +def test_listen_multiple(capsys): + snippets.listen_multiple() + out, _ = capsys.readouterr() + assert 'Current cities in California:' in out + assert 'SF' in out + + +def test_listen_for_changes(capsys): + snippets.listen_for_changes() + out, _ = capsys.readouterr() + assert 'New city: MTV' in out + assert 'Modified city: MTV' in out + assert 'Removed city: MTV' in out + + +def test_delete_full_collection(): + snippets.delete_full_collection() diff --git a/functions/README.rst b/functions/README.rst new file mode 100644 index 00000000000..f81a7a6611a --- /dev/null +++ b/functions/README.rst @@ -0,0 +1,84 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Functions Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=/README.rst + + +This directory contains samples for Google Cloud Functions. `Cloud Functions`_ is a lightweight, event-based, asynchronous compute solution that allows you to create small, single-purpose functions that respond to Cloud events without the need to manage a server or a runtime environment. + + + + +.. _Cloud Functions: https://cloud.google.com/functions/docs/ + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +- `Hello World`_ +- Concepts_ +- `Logging & Monitoring`_ +- Tips_ + + +.. _Hello World: helloworld/ +.. _Concepts: concepts/ +.. _Logging & Monitoring: log/ +.. _Tips: tips/ + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ diff --git a/functions/README.rst.in b/functions/README.rst.in new file mode 100644 index 00000000000..f86d5bd823b --- /dev/null +++ b/functions/README.rst.in @@ -0,0 +1,18 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Functions + short_name: GCF + url: https://cloud.google.com/functions/docs/ + description: > + Cloud Functions is a lightweight, event-based, asynchronous compute solution that allows you to create small, single-purpose functions that respond to Cloud events without the need to manage a server or a runtime environment. + +setup: +- auth +- install_deps + +samples: +- name: Hello World + file: helloworld/main.py + +cloud_client_library: true diff --git a/functions/billing/main.py b/functions/billing/main.py new file mode 100644 index 00000000000..ceb2e6f9297 --- /dev/null +++ b/functions/billing/main.py @@ -0,0 +1,168 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START functions_billing_limit] +# [START functions_billing_stop] +import base64 +import json +# [END functions_billing_stop] +import os +# [END functions_billing_limit] + +# [START functions_billing_limit] +# [START functions_billing_stop] +from googleapiclient import discovery +from oauth2client.client import GoogleCredentials + +# [END functions_billing_stop] +# [END functions_billing_limit] + +# [START functions_billing_slack] +from slackclient import SlackClient +# [END functions_billing_slack] + +# [START functions_billing_limit] +# [START functions_billing_stop] +PROJECT_ID = os.getenv('GCP_PROJECT') +PROJECT_NAME = f'projects/{PROJECT_ID}' +# [END functions_billing_stop] +# [END functions_billing_limit] + +# [START functions_billing_slack] + +# See https://api.slack.com/docs/token-types#bot for more info +BOT_ACCESS_TOKEN = 'xxxx-111111111111-abcdefghidklmnopq' + +CHANNEL_ID = 'C0XXXXXX' + +slack_client = SlackClient(BOT_ACCESS_TOKEN) + + +def notify_slack(data, context): + pubsub_message = data + + notification_attrs = json.dumps(pubsub_message['attributes']) + notification_data = base64.b64decode(data['data']).decode('utf-8') + budget_notification_text = f'{notification_attrs}, {notification_data}' + + slack_client.api_call( + 'chat.postMessage', + channel=CHANNEL_ID, + text=budget_notification_text) +# [END functions_billing_slack] + + +# [START functions_billing_limit] +def stop_billing(data, context): + pubsub_data = base64.b64decode(data['data']).decode('utf-8') + pubsub_json = json.loads(pubsub_data) + cost_amount = pubsub_json['costAmount'] + budget_amount = pubsub_json['budgetAmount'] + if cost_amount <= budget_amount: + print(f'No action necessary. (Current cost: {cost_amount})') + return + + billing = discovery.build( + 'cloudbilling', + 'v1', + cache_discovery=False, + credentials=GoogleCredentials.get_application_default() + ) + + projects = billing.projects() + + if __is_billing_enabled(PROJECT_NAME, projects): + print(__disable_billing_for_project(PROJECT_NAME, projects)) + else: + print('Billing already disabled') + + +def __is_billing_enabled(project_name, projects): + """ + Determine whether billing is enabled for a project + @param {string} project_name Name of project to check if billing is enabled + @return {bool} Whether project has billing enabled or not + """ + res = projects.getBillingInfo(name=project_name).execute() + return res['billingEnabled'] + + +def __disable_billing_for_project(project_name, projects): + """ + Disable billing for a project by removing its billing account + @param {string} project_name Name of project disable billing on + @return {string} Text containing response from disabling billing + """ + body = {'billingAccountName': ''} # Disable billing + res = projects.updateBillingInfo(name=project_name, body=body).execute() + print(f'Billing disabled: {json.dumps(res)}') +# [END functions_billing_stop] + + +# [START functions_billing_limit] +ZONE = 'us-west1-b' + + +def limit_use(data, context): + pubsub_data = base64.b64decode(data['data']).decode('utf-8') + pubsub_json = json.loads(pubsub_data) + cost_amount = pubsub_json['costAmount'] + budget_amount = pubsub_json['budgetAmount'] + if cost_amount <= budget_amount: + print(f'No action necessary. (Current cost: {cost_amount})') + return + + compute = discovery.build( + 'compute', + 'v1', + cache_discovery=False, + credentials=GoogleCredentials.get_application_default() + ) + instances = compute.instances() + + instance_names = __list_running_instances(PROJECT_ID, ZONE, instances) + __stop_instances(PROJECT_ID, ZONE, instance_names, instances) + + +def __list_running_instances(project_id, zone, instances): + """ + @param {string} project_id ID of project that contains instances to stop + @param {string} zone Zone that contains instances to stop + @return {Promise} Array of names of running instances + """ + res = instances.list(project=project_id, zone=zone).execute() + + items = res['items'] + running_names = [i['name'] for i in items if i['status'] == 'RUNNING'] + return running_names + + +def __stop_instances(project_id, zone, instance_names, instances): + """ + @param {string} project_id ID of project that contains instances to stop + @param {string} zone Zone that contains instances to stop + @param {Array} instance_names Names of instance to stop + @return {Promise} Response from stopping instances + """ + if not len(instance_names): + print('No running instances were found.') + return + + for name in instance_names: + instances.stop( + project=project_id, + zone=zone, + instance=name).execute() + print(f'Instance stopped successfully: {name}') +# [END functions_billing_limit] diff --git a/functions/billing/main_test.py b/functions/billing/main_test.py new file mode 100644 index 00000000000..13524debabf --- /dev/null +++ b/functions/billing/main_test.py @@ -0,0 +1,116 @@ +# Copyright 2018, Google, LLC. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import json + +from mock import MagicMock, patch + +import main + + +@patch('main.slack_client') +def test_notify_slack(slack_client): + slack_client.api_call = MagicMock() + + data = {"budgetAmount": 400, "costAmount": 500} + attrs = {"foo": "bar"} + + pubsub_message = { + "data": base64.b64encode(bytes(json.dumps(data), 'utf-8')), + "attributes": attrs + } + + main.notify_slack(pubsub_message, None) + + assert slack_client.api_call.called + + +@patch('main.PROJECT_ID') +@patch('main.discovery') +def test_disable_billing(discovery_mock, PROJECT_ID): + PROJECT_ID = 'my-project' + PROJECT_NAME = f'projects/{PROJECT_ID}' + + data = {"budgetAmount": 400, "costAmount": 500} + + pubsub_message = { + "data": base64.b64encode(bytes(json.dumps(data), 'utf-8')), + "attributes": {} + } + + projects_mock = MagicMock() + projects_mock.projects = MagicMock(return_value=projects_mock) + projects_mock.getBillingInfo = MagicMock(return_value=projects_mock) + projects_mock.updateBillingInfo = MagicMock(return_value=projects_mock) + projects_mock.execute = MagicMock(return_value={'billingEnabled': True}) + + discovery_mock.build = MagicMock(return_value=projects_mock) + + main.stop_billing(pubsub_message, None) + + assert projects_mock.getBillingInfo.called_with(name=PROJECT_NAME) + assert projects_mock.updateBillingInfo.called_with( + name=PROJECT_NAME, + body={'billingAccountName': ''} + ) + assert projects_mock.execute.call_count == 2 + + +@patch('main.PROJECT_ID') +@patch('main.ZONE') +@patch('main.discovery') +def test_limit_use(discovery_mock, ZONE, PROJECT_ID): + PROJECT_ID = 'my-project' + PROJECT_NAME = f'projects/{PROJECT_ID}' + ZONE = 'my-zone' + + data = {"budgetAmount": 400, "costAmount": 500} + + pubsub_message = { + "data": base64.b64encode(bytes(json.dumps(data), 'utf-8')), + "attributes": {} + } + + instances_list = { + "items": [ + {"name": "instance-1", "status": "RUNNING"}, + {"name": "instance-2", "status": "TERMINATED"} + ] + } + + instances_mock = MagicMock() + instances_mock.instances = MagicMock(return_value=instances_mock) + instances_mock.list = MagicMock(return_value=instances_mock) + instances_mock.stop = MagicMock(return_value=instances_mock) + instances_mock.execute = MagicMock(return_value=instances_list) + + projects_mock = MagicMock() + projects_mock.projects = MagicMock(return_value=projects_mock) + projects_mock.getBillingInfo = MagicMock(return_value=projects_mock) + projects_mock.execute = MagicMock(return_value={'billingEnabled': True}) + + def discovery_mocker(x, *args, **kwargs): + if x == 'compute': + return instances_mock + else: + return projects_mock + + discovery_mock.build = MagicMock(side_effect=discovery_mocker) + + main.limit_use(pubsub_message, None) + + assert projects_mock.getBillingInfo.called_with(name=PROJECT_NAME) + assert instances_mock.list.calledWith(project=PROJECT_ID, zone=ZONE) + assert instances_mock.stop.call_count == 1 + assert instances_mock.execute.call_count == 2 diff --git a/functions/billing/requirements.txt b/functions/billing/requirements.txt new file mode 100644 index 00000000000..4c2c13edc0c --- /dev/null +++ b/functions/billing/requirements.txt @@ -0,0 +1,3 @@ +slackclient==1.3.0 +oauth2client==4.1.3 +google-api-python-client==1.7.8 diff --git a/functions/concepts/README.md b/functions/concepts/README.md new file mode 100644 index 00000000000..a889048b17a --- /dev/null +++ b/functions/concepts/README.md @@ -0,0 +1,11 @@ +Google Cloud Platform logo + +# Google Cloud Functions - Concepts sample + +See: + +* [Cloud Functions Concepts tutorial][tutorial] +* [Cloud Functions Concepts sample source code][code] + +[tutorial]: https://cloud.google.com/functions/docs/concepts/exec +[code]: main.py diff --git a/functions/concepts/main.py b/functions/concepts/main.py new file mode 100644 index 00000000000..f6cf5a08053 --- /dev/null +++ b/functions/concepts/main.py @@ -0,0 +1,123 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + + +# [START functions_concepts_stateless] +# Global variable, modified within the function by using the global keyword. +count = 0 + + +def statelessness(request): + """ + HTTP Cloud Function that counts how many times it is executed + within a specific instance. + Args: + request (flask.Request): The request object. + + Returns: + The response text, or any set of values that can be turned into a + Response object using `make_response` + . + """ + global count + count += 1 + + # Note: the total function invocation count across + # all instances may not be equal to this value! + return 'Instance execution count: {}'.format(count) +# [END functions_concepts_stateless] + + +# Placeholder +def heavy_computation(): + return time.time() + + +# Placeholder +def light_computation(): + return time.time() + + +# [START functions_tips_scopes] +# Global (instance-wide) scope +# This computation runs at instance cold-start +instance_var = heavy_computation() + + +def scope_demo(request): + """ + HTTP Cloud Function that declares a variable. + Args: + request (flask.Request): The request object. + + Returns: + The response text, or any set of values that can be turned into a + Response object using `make_response` + . + """ + + # Per-function scope + # This computation runs every time this function is called + function_var = light_computation() + return 'Instance: {}; function: {}'.format(instance_var, function_var) +# [END functions_tips_scopes] + + +# [START functions_concepts_requests] +def make_request(request): + """ + HTTP Cloud Function that makes another HTTP request. + Args: + request (flask.Request): The request object. + + Returns: + The response text, or any set of values that can be turned into a + Response object using `make_response` + . + """ + import requests + + # The URL to send the request to + url = 'http://example.com' + + # Process the request + response = requests.get(url) + response.raise_for_status() + return 'Success!' +# [END functions_concepts_requests] + + +# [START functions_concepts_after_timeout] +def timeout(request): + print('Function running...') + time.sleep(120) + + # May not execute if function's timeout is <2 minutes + print('Function completed!') + return 'Function completed!' +# [END functions_concepts_after_timeout] + + +# [START functions_concepts_filesystem] +def list_files(request): + import os + from os import path + + root = path.dirname(path.abspath(__file__)) + children = os.listdir(root) + files = [c for c in children if path.isfile(path.join(root, c))] + return 'Files: {}'.format(files) +# [END functions_concepts_filesystem] diff --git a/functions/concepts/main_test.py b/functions/concepts/main_test.py new file mode 100644 index 00000000000..71e07b50587 --- /dev/null +++ b/functions/concepts/main_test.py @@ -0,0 +1,63 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import flask +import pytest +import requests +import responses + +import main + + +# Create a fake "app" for generating test request contexts. +@pytest.fixture(scope="module") +def app(): + return flask.Flask(__name__) + + +def test_statelessness(app): + with app.test_request_context(): + res = main.statelessness(flask.request) + assert res == 'Instance execution count: 1' + res = main.statelessness(flask.request) + assert res == 'Instance execution count: 2' + + +def test_scope_demo(app): + with app.test_request_context(): + main.scope_demo(flask.request) + + +@responses.activate +def test_make_request_200(app): + responses.add(responses.GET, 'http://example.com', + json={'status': 'OK'}, status=200) + with app.test_request_context(): + main.make_request(flask.request) + + +@responses.activate +def test_make_request_404(app): + responses.add(responses.GET, 'http://example.com', + json={'error': 'not found'}, status=404) + with app.test_request_context(): + with pytest.raises(requests.exceptions.HTTPError): + main.make_request(flask.request) + + +def test_list_files(app): + with app.test_request_context(): + res = main.list_files(flask.request) + assert 'main.py' in res diff --git a/functions/env_vars/README.md b/functions/env_vars/README.md new file mode 100644 index 00000000000..75a4cc1c2ba --- /dev/null +++ b/functions/env_vars/README.md @@ -0,0 +1,9 @@ +Google Cloud Platform logo + +# Google Cloud Functions - Using Environment Variables sample + +See: +* [Cloud Functions Using Environment Variables tutorial][tutorial] +* [Cloud Functions Using Environment Variables sample source code][code] + [tutorial]: https://cloud.google.com/functions/docs/env-var#functions_env_var-python +[code]: main.py \ No newline at end of file diff --git a/functions/env_vars/main.py b/functions/env_vars/main.py new file mode 100644 index 00000000000..740fd2166ea --- /dev/null +++ b/functions/env_vars/main.py @@ -0,0 +1,22 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# [START functions_env_vars] +import os + + +def env_vars(request): + return os.environ.get('FOO', 'Specified environment variable is not set.') +# [END functions_env_vars] diff --git a/functions/env_vars/main_test.py b/functions/env_vars/main_test.py new file mode 100644 index 00000000000..40b8f645c81 --- /dev/null +++ b/functions/env_vars/main_test.py @@ -0,0 +1,33 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import flask +import pytest + +import main + + +# Create a fake "app" for generating test request contexts. +@pytest.fixture(scope="module") +def app(): + return flask.Flask(__name__) + + +def test_env_vars(app): + with app.test_request_context(): + os.environ['FOO'] = 'bar' + res = main.env_vars(flask.request) + assert res == 'bar' diff --git a/functions/firebase/main.py b/functions/firebase/main.py new file mode 100644 index 00000000000..e7ad09b875b --- /dev/null +++ b/functions/firebase/main.py @@ -0,0 +1,138 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START functions_firebase_analytics] +from datetime import datetime +# [END functions_firebase_analytics] + +# [START functions_firebase_rtdb] +# [START functions_firebase_firestore] +# [START functions_firebase_auth] +import json +# [END functions_firebase_rtdb] +# [END functions_firebase_firestore] +# [END functions_firebase_auth] + +# [START functions_firebase_reactive] +from google.cloud import firestore +# [END functions_firebase_reactive] + + +# [START functions_firebase_rtdb] +def hello_rtdb(data, context): + """ Triggered by a change to a Firebase RTDB reference. + Args: + data (dict): The event payload. + context (google.cloud.functions.Context): Metadata for the event. + """ + trigger_resource = context.resource + + print('Function triggered by change to: %s' % trigger_resource) + print('Admin?: %s' % data.get("admin", False)) + print('Delta:') + print(json.dumps(data["delta"])) +# [END functions_firebase_rtdb] + + +# [START functions_firebase_firestore] +def hello_firestore(data, context): + """ Triggered by a change to a Firestore document. + Args: + data (dict): The event payload. + context (google.cloud.functions.Context): Metadata for the event. + """ + trigger_resource = context.resource + + print('Function triggered by change to: %s' % trigger_resource) + + print('\nOld value:') + print(json.dumps(data["oldValue"])) + + print('\nNew value:') + print(json.dumps(data["value"])) +# [END functions_firebase_firestore] + + +# [START functions_firebase_auth] +def hello_auth(data, context): + """ Triggered by creation or deletion of a Firebase Auth user object. + Args: + data (dict): The event payload. + context (google.cloud.functions.Context): Metadata for the event. + """ + print('Function triggered by creation/deletion of user: %s' % data["uid"]) + print('Created at: %s' % data["metadata"]["createdAt"]) + + if 'email' in data: + print('Email: %s' % data["email"]) +# [END functions_firebase_auth] + + +# [START functions_firebase_reactive] +client = firestore.Client() + + +# Converts strings added to /messages/{pushId}/original to uppercase +def make_upper_case(data, context): + path_parts = context.resource.split('/documents/')[1].split('/') + collection_path = path_parts[0] + document_path = '/'.join(path_parts[1:]) + + affected_doc = client.collection(collection_path).document(document_path) + + cur_value = data["value"]["fields"]["original"]["stringValue"] + new_value = cur_value.upper() + print(f'Replacing value: {cur_value} --> {new_value}') + + affected_doc.set({ + u'original': new_value + }) +# [END functions_firebase_reactive] + + +# [START functions_firebase_analytics] +def hello_analytics(data, context): + """ Triggered by a Google Analytics for Firebase log event. + Args: + data (dict): The event payload. + context (google.cloud.functions.Context): Metadata for the event. + """ + trigger_resource = context.resource + print(f'Function triggered by the following event: {trigger_resource}') + + event = data["eventDim"][0] + print(f'Name: {event["name"]}') + + event_timestamp = int(event["timestampMicros"][:-6]) + print(f'Timestamp: {datetime.utcfromtimestamp(event_timestamp)}') + + user_obj = data["userDim"] + print(f'Device Model: {user_obj["deviceInfo"]["deviceModel"]}') + + geo_info = user_obj["geoInfo"] + print(f'Location: {geo_info["city"]}, {geo_info["country"]}') +# [END functions_firebase_analytics] + + +# [START functions_firebase_remote_config] +def hello_remote_config(data, context): + """ Triggered by a change to a Firebase Remote Config value. + Args: + data (dict): The event payload. + context (google.cloud.functions.Context): Metadata for the event. + """ + print(f'Update type: {data["updateType"]}') + print(f'Origin: {data["updateOrigin"]}') + print(f'Version: {data["versionNumber"]}') +# [END functions_firebase_remote_config] diff --git a/functions/firebase/main_test.py b/functions/firebase/main_test.py new file mode 100644 index 00000000000..dac0cc025f7 --- /dev/null +++ b/functions/firebase/main_test.py @@ -0,0 +1,167 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import UserDict +from datetime import datetime +import json +import uuid + +from mock import MagicMock, patch + +import main + + +class Context(object): + pass + + +def test_rtdb(capsys): + data = { + 'admin': True, + 'delta': {'id': 'my-data'} + } + + context = Context() + context.resource = 'my-resource' + + main.hello_rtdb(data, context) + + out, _ = capsys.readouterr() + + assert 'Function triggered by change to: my-resource' in out + assert 'Admin?: True' in out + assert 'my-data' in out + + +def test_firestore(capsys): + context = Context() + context.resource = 'my-resource' + + data = { + 'oldValue': {'a': 1}, + 'value': {'b': 2} + } + + main.hello_firestore(data, context) + + out, _ = capsys.readouterr() + + assert 'Function triggered by change to: my-resource' in out + assert json.dumps(data['oldValue']) in out + assert json.dumps(data['value']) in out + + +def test_auth(capsys): + date_string = datetime.now().isoformat() + + data = { + 'uid': 'my-user', + 'metadata': {'createdAt': date_string}, + 'email': 'me@example.com' + } + + main.hello_auth(data, None) + + out, _ = capsys.readouterr() + + assert 'Function triggered by creation/deletion of user: my-user' in out + assert date_string in out + assert 'Email: me@example.com' in out + + +@patch('main.client') +def test_make_upper_case(firestore_mock, capsys): + + firestore_mock.collection = MagicMock(return_value=firestore_mock) + firestore_mock.document = MagicMock(return_value=firestore_mock) + firestore_mock.set = MagicMock(return_value=firestore_mock) + + user_id = str(uuid.uuid4()) + date_string = datetime.now().isoformat() + email_string = '%s@%s.com' % (uuid.uuid4(), uuid.uuid4()) + + data = { + 'uid': user_id, + 'metadata': {'createdAt': date_string}, + 'email': email_string, + 'value': { + 'fields': { + 'original': { + 'stringValue': 'foobar' + } + } + } + } + + context = UserDict() + context.resource = '/documents/some_collection/path/some/path' + + main.make_upper_case(data, context) + + out, _ = capsys.readouterr() + + assert 'Replacing value: foobar --> FOOBAR' in out + firestore_mock.collection.assert_called_with('some_collection') + firestore_mock.document.assert_called_with('path/some/path') + firestore_mock.set.assert_called_with({'original': 'FOOBAR'}) + + +def test_analytics(capsys): + timestamp = int(datetime.utcnow().timestamp()) + + data = { + 'eventDim': [{ + 'name': 'my-event', + 'timestampMicros': f'{str(timestamp)}000000' + }], + 'userDim': { + 'deviceInfo': { + 'deviceModel': 'Pixel' + }, + 'geoInfo': { + 'city': 'London', + 'country': 'UK' + } + } + } + + context = Context() + context.resource = 'my-resource' + + main.hello_analytics(data, context) + + out, _ = capsys.readouterr() + + assert 'Function triggered by the following event: my-resource' in out + assert f'Timestamp: {datetime.utcfromtimestamp(timestamp)}' in out + assert 'Name: my-event' in out + assert 'Device Model: Pixel' in out + assert 'Location: London, UK' in out + + +def test_remote_config(capsys): + data = { + 'updateOrigin': 'CONSOLE', + 'updateType': 'INCREMENTAL_UPDATE', + 'versionNumber': '1' + } + context = Context() + + main.hello_remote_config(data, context) + + out, _ = capsys.readouterr() + + assert 'Update type: INCREMENTAL_UPDATE' in out + assert 'Origin: CONSOLE' in out + assert 'Version: 1' in out diff --git a/functions/firebase/requirements.txt b/functions/firebase/requirements.txt new file mode 100644 index 00000000000..399e498b84c --- /dev/null +++ b/functions/firebase/requirements.txt @@ -0,0 +1 @@ +google-cloud-firestore==0.31.0 diff --git a/functions/gcs/README.md b/functions/gcs/README.md new file mode 100644 index 00000000000..8d067c5c00b --- /dev/null +++ b/functions/gcs/README.md @@ -0,0 +1,11 @@ +Google Cloud Platform logo + +# Google Cloud Functions - Cloud Storage sample + +See: + +* [Cloud Functions Cloud Storage tutorial][tutorial] +* [Cloud Functions Cloud Storage source code][code] + +[tutorial]: https://cloud.google.com/functions/docs/tutorials/storage +[code]: main.py diff --git a/functions/gcs/main.py b/functions/gcs/main.py new file mode 100644 index 00000000000..6e50d6a124e --- /dev/null +++ b/functions/gcs/main.py @@ -0,0 +1,34 @@ +# Copyright 2018, Google, LLC. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# [START functions_helloworld_storage_generic] +def hello_gcs_generic(data, context): + """Background Cloud Function to be triggered by Cloud Storage. + This generic function logs relevant data when a file is changed. + + Args: + data (dict): The Cloud Functions event payload. + context (google.cloud.functions.Context): Metadata of triggering event. + Returns: + None; the output is written to Stackdriver Logging + """ + + print('Event ID: {}'.format(context.event_id)) + print('Event type: {}'.format(context.event_type)) + print('Bucket: {}'.format(data['bucket'])) + print('File: {}'.format(data['name'])) + print('Metageneration: {}'.format(data['metageneration'])) + print('Created: {}'.format(data['timeCreated'])) + print('Updated: {}'.format(data['updated'])) +# [END functions_helloworld_storage_generic] diff --git a/functions/gcs/main_test.py b/functions/gcs/main_test.py new file mode 100644 index 00000000000..015a7ba8670 --- /dev/null +++ b/functions/gcs/main_test.py @@ -0,0 +1,37 @@ +# Copyright 2018, Google, LLC. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +import main + + +class TestGCFPyGCSSample(object): + def test_hello_gcs_generic(self, capsys): + event = { + 'bucket': 'some-bucket', + 'name': 'some-filename', + 'metageneration': 'some-metageneration', + 'timeCreated': '0', + 'updated': '0' + } + context = mock.MagicMock() + context.event_id = 'some-id' + context.event_type = 'gcs-event' + + main.hello_gcs_generic(event, context) + + out, _ = capsys.readouterr() + + assert 'some-bucket' in out + assert 'some-id' in out diff --git a/functions/helloworld/.gcloudignore b/functions/helloworld/.gcloudignore new file mode 100644 index 00000000000..0ba261093e3 --- /dev/null +++ b/functions/helloworld/.gcloudignore @@ -0,0 +1 @@ +*test.py diff --git a/functions/helloworld/README.md b/functions/helloworld/README.md new file mode 100644 index 00000000000..11d68f93063 --- /dev/null +++ b/functions/helloworld/README.md @@ -0,0 +1,11 @@ +Google Cloud Platform logo + +# Google Cloud Functions - Hello World sample + +See: + +* [Cloud Functions Hello World tutorial][tutorial] +* [Cloud Functions Hello World sample source code][code] + +[tutorial]: https://cloud.google.com/functions/docs/quickstart +[code]: main.py diff --git a/functions/helloworld/main.py b/functions/helloworld/main.py new file mode 100644 index 00000000000..da9b593d475 --- /dev/null +++ b/functions/helloworld/main.py @@ -0,0 +1,198 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START functions_helloworld_http] +# [START functions_http_content] +from flask import escape + +# [END functions_helloworld_http] +# [END functions_http_content] + + +# [START functions_tips_terminate] +# [START functions_helloworld_get] +def hello_get(request): + """HTTP Cloud Function. + Args: + request (flask.Request): The request object. + + Returns: + The response text, or any set of values that can be turned into a + Response object using `make_response` + . + """ + return 'Hello World!' +# [END functions_helloworld_get] + + +# [START functions_helloworld_background] +def hello_background(data, context): + """Background Cloud Function. + Args: + data (dict): The dictionary with data specific to the given event. + context (google.cloud.functions.Context): The Cloud Functions event + metadata. + """ + if data and 'name' in data: + name = data['name'] + else: + name = 'World' + return 'Hello {}!'.format(name) +# [END functions_helloworld_background] +# [END functions_tips_terminate] + + +# [START functions_helloworld_http] +def hello_http(request): + """HTTP Cloud Function. + Args: + request (flask.Request): The request object. + + Returns: + The response text, or any set of values that can be turned into a + Response object using `make_response` + . + """ + request_json = request.get_json(silent=True) + request_args = request.args + + if request_json and 'name' in request_json: + name = request_json['name'] + elif request_args and 'name' in request_args: + name = request_args['name'] + else: + name = 'World' + return 'Hello {}!'.format(escape(name)) +# [END functions_helloworld_http] + + +# [START functions_helloworld_pubsub] +def hello_pubsub(data, context): + """Background Cloud Function to be triggered by Pub/Sub. + Args: + data (dict): The dictionary with data specific to this type of event. + context (google.cloud.functions.Context): The Cloud Functions event + metadata. + """ + import base64 + + if 'data' in data: + name = base64.b64decode(data['data']).decode('utf-8') + else: + name = 'World' + print('Hello {}!'.format(name)) +# [END functions_helloworld_pubsub] + + +# [START functions_helloworld_storage] +def hello_gcs(data, context): + """Background Cloud Function to be triggered by Cloud Storage. + Args: + data (dict): The dictionary with data specific to this type of event. + context (google.cloud.functions.Context): The Cloud Functions + event metadata. + """ + print("File: {}.".format(data['objectId'])) +# [END functions_helloworld_storage] + + +# [START functions_http_content] +def hello_content(request): + """ Responds to an HTTP request using data from the request body parsed + according to the "content-type" header. + Args: + request (flask.Request): The request object. + + Returns: + The response text, or any set of values that can be turned into a + Response object using `make_response` + . + """ + content_type = request.headers['content-type'] + if content_type == 'application/json': + request_json = request.get_json(silent=True) + if request_json and 'name' in request_json: + name = request_json['name'] + else: + raise ValueError("JSON is invalid, or missing a 'name' property") + elif content_type == 'application/octet-stream': + name = request.data + elif content_type == 'text/plain': + name = request.data + elif content_type == 'application/x-www-form-urlencoded': + name = request.form.get('name') + else: + raise ValueError("Unknown content type: {}".format(content_type)) + return 'Hello {}!'.format(escape(name)) +# [END functions_http_content] + + +# [START functions_http_methods] +def hello_method(request): + """ Responds to a GET request with "Hello world!". Forbids a PUT request. + Args: + request (flask.Request): The request object. + + Returns: + The response text, or any set of values that can be turned into a + Response object using `make_response` + . + """ + from flask import abort + + if request.method == 'GET': + return 'Hello World!' + elif request.method == 'PUT': + return abort(403) + else: + return abort(405) +# [END functions_http_methods] + + +def hello_error_1(request): + # [START functions_helloworld_error] + # This WILL be reported to Stackdriver Error + # Reporting, and WILL NOT show up in logs or + # terminate the function. + from google.cloud import error_reporting + client = error_reporting.Client() + + try: + raise RuntimeError('I failed you') + except RuntimeError: + client.report_exception() + + # This WILL be reported to Stackdriver Error Reporting, + # and WILL terminate the function + raise RuntimeError('I failed you') + + # [END functions_helloworld_error] + + +def hello_error_2(request): + # [START functions_helloworld_error] + # WILL NOT be reported to Stackdriver Error Reporting, but will show up + # in logs + import logging + print(RuntimeError('I failed you (print to stdout)')) + logging.warn(RuntimeError('I failed you (logging.warn)')) + logging.error(RuntimeError('I failed you (logging.error)')) + sys.stderr.write('I failed you (sys.stderr.write)\n') + + # This WILL be reported to Stackdriver Error Reporting + from flask import abort + return abort(500) + # [END functions_helloworld_error] diff --git a/functions/helloworld/main_test.py b/functions/helloworld/main_test.py new file mode 100644 index 00000000000..e033276749c --- /dev/null +++ b/functions/helloworld/main_test.py @@ -0,0 +1,94 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import flask +import pytest + +import main + + +# Create a fake "app" for generating test request contexts. +@pytest.fixture(scope="module") +def app(): + return flask.Flask(__name__) + + +def test_hello_get(app): + with app.test_request_context(): + res = main.hello_get(flask.request) + assert 'Hello World!' in res + + +def test_hello_http_no_args(app): + with app.test_request_context(): + res = main.hello_http(flask.request) + assert 'Hello World!' in res + + +def test_hello_http_get(app): + with app.test_request_context(query_string={'name': 'test'}): + res = main.hello_http(flask.request) + assert 'Hello test!' in res + + +def test_hello_http_args(app): + with app.test_request_context(json={'name': 'test'}): + res = main.hello_http(flask.request) + assert 'Hello test!' in res + + +def test_hello_http_empty_json(app): + with app.test_request_context(json=''): + res = main.hello_http(flask.request) + assert 'Hello World!' in res + + +def test_hello_http_xss(app): + with app.test_request_context(json={'name': ''}): + res = main.hello_http(flask.request) + assert ''}): + res = main.hello_content(flask.request) + assert ' - - - -{# [END index] #} diff --git a/memorystore/redis/README.md b/memorystore/redis/README.md new file mode 100644 index 00000000000..6e6be596917 --- /dev/null +++ b/memorystore/redis/README.md @@ -0,0 +1,15 @@ +# Getting started with Googe Cloud Memorystore +Simple HTTP server example to demonstrate connecting to [Google Cloud Memorystore](https://cloud.google.com/memorystore/docs/redis). +This sample uses the [redis-py client](https://github.com/andymccurdy/redis-py). + +## Running on GCE + +Follow the instructions in [this guide](https://cloud.google.com/memorystore/docs/redis/connect-redis-instance-gce) to deploy the sample application on a GCE VM. + +## Running on GKE + +Follow the instructions in [this guide](https://cloud.google.com/memorystore/docs/redis/connect-redis-instance-gke) to deploy the sample application on GKE. + +## Running on Google App Engine Flex + +Follow the instructions in [this guide](https://cloud.google.com/memorystore/docs/redis/connect-redis-instance-flex) to deploy the sample application on GAE Flex. diff --git a/memorystore/redis/app.yaml b/memorystore/redis/app.yaml new file mode 100644 index 00000000000..249eeabc08b --- /dev/null +++ b/memorystore/redis/app.yaml @@ -0,0 +1,31 @@ +# Copyright 2018 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START memorystore_app_yaml] +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 + +# update with Redis instance host IP, port +env_variables: + REDISHOST: redis-ip + REDISPORT: 6379 + +# update with Redis instance network name +network: + name: default + +#[END memorystore_app_yaml] diff --git a/memorystore/redis/gce_deployment/deploy.sh b/memorystore/redis/gce_deployment/deploy.sh new file mode 100755 index 00000000000..5c7bf7ecdb6 --- /dev/null +++ b/memorystore/redis/gce_deployment/deploy.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +# Copyright 2018 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# [START memorystore_deploy_sh] +if [ -z "$REDISHOST" ]; then + echo "Must set \$REDISHOST. For example: REDISHOST=127.0.0.1" + exit 1 +fi + +if [ -z "$REDISPORT" ]; then + echo "Must set \$REDISPORT. For example: REDISPORT=6379" + exit 1 +fi + +if [ -z "$GCS_BUCKET_NAME" ]; then + echo "Must set \$GCS_BUCKET_NAME. For example: GCS_BUCKET_NAME=my-bucket" + exit 1 +fi + +if [ -z "$ZONE" ]; then + ZONE=$(gcloud config get-value compute/zone -q) + echo $ZONE +fi + +#Upload the tar to GCS +tar -cvf app.tar -C .. requirements.txt main.py +# Copy to GCS bucket +gsutil cp app.tar gs://"$GCS_BUCKET_NAME"/gce/ + +# Create an instance +gcloud compute instances create my-instance \ + --image-family=debian-9 \ + --image-project=debian-cloud \ + --machine-type=g1-small \ + --scopes cloud-platform \ + --metadata-from-file startup-script=startup-script.sh \ + --metadata gcs-bucket=$GCS_BUCKET_NAME,redis-host=$REDISHOST,redis-port=$REDISPORT \ + --zone $ZONE \ + --tags http-server + +gcloud compute firewall-rules create allow-http-server-8080 \ + --allow tcp:8080 \ + --source-ranges 0.0.0.0/0 \ + --target-tags http-server \ + --description "Allow port 8080 access to http-server" +# [END memorystore_deploy_sh] diff --git a/memorystore/redis/gce_deployment/startup-script.sh b/memorystore/redis/gce_deployment/startup-script.sh new file mode 100644 index 00000000000..7a925f8244d --- /dev/null +++ b/memorystore/redis/gce_deployment/startup-script.sh @@ -0,0 +1,65 @@ +#! /bin/bash + +# Copyright 2018 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# [START memorystore_startup_script_sh] +set -v + +# Talk to the metadata server to get the project id and location of application binary. +PROJECTID=$(curl -s "http://metadata.google.internal/computeMetadata/v1/project/project-id" -H "Metadata-Flavor: Google") +GCS_BUCKET_NAME=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/gcs-bucket" -H "Metadata-Flavor: Google") +REDISHOST=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/redis-host" -H "Metadata-Flavor: Google") +REDISPORT=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/redis-port" -H "Metadata-Flavor: Google") + +# Install dependencies from apt +apt-get update +apt-get install -yq \ + git build-essential supervisor python python-dev python-pip libffi-dev \ + libssl-dev + +# Install logging monitor. The monitor will automatically pickup logs send to +# syslog. +curl -s "https://storage.googleapis.com/signals-agents/logging/google-fluentd-install.sh" | bash +service google-fluentd restart & + +gsutil cp gs://"$GCS_BUCKET_NAME"/gce/app.tar /app.tar +mkdir -p /app +tar -x -f /app.tar -C /app +cd /app + +# Install the app dependencies +pip install --upgrade pip virtualenv +virtualenv /app/env +/app/env/bin/pip install -r /app/requirements.txt + +# Create a pythonapp user. The application will run as this user. +getent passwd pythonapp || useradd -m -d /home/pythonapp pythonapp +chown -R pythonapp:pythonapp /app + +# Configure supervisor to run the Go app. +cat >/etc/supervisor/conf.d/pythonapp.conf << EOF +[program:pythonapp] +directory=/app +environment=HOME="/home/pythonapp",USER="pythonapp",REDISHOST=$REDISHOST,REDISPORT=$REDISPORT +command=/app/env/bin/gunicorn main:app --bind 0.0.0:8080 +autostart=true +autorestart=true +user=pythonapp +stdout_logfile=syslog +stderr_logfile=syslog +EOF + +supervisorctl reread +supervisorctl update +# [END memorystore_startup_script_sh] diff --git a/memorystore/redis/gce_deployment/teardown.sh b/memorystore/redis/gce_deployment/teardown.sh new file mode 100755 index 00000000000..b54a91fb210 --- /dev/null +++ b/memorystore/redis/gce_deployment/teardown.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2018 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# [START memorystore_teardown_sh] +gcloud compute instances delete my-instance + +gcloud compute firewall-rules delete allow-http-server-8080 +# [END memorystore_teardown_sh] diff --git a/memorystore/redis/gke_deployment/Dockerfile b/memorystore/redis/gke_deployment/Dockerfile new file mode 100644 index 00000000000..4b0a3259c1a --- /dev/null +++ b/memorystore/redis/gke_deployment/Dockerfile @@ -0,0 +1,30 @@ +# The Google App Engine python runtime is Debian Jessie with Python installed +# and various os-level packages to allow installation of popular Python +# libraries. The source is on github at: +# https://github.com/GoogleCloudPlatform/python-docker +FROM gcr.io/google_appengine/python + +# Create a virtualenv for the application dependencies. +# If you want to use Python 2, add the -p python2.7 flag. +RUN virtualenv -p python3.4 /env + +# Set virtualenv environment variables. This is equivalent to running +# source /env/bin/activate. This ensures the application is executed within +# the context of the virtualenv and will have access to its dependencies. +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH + +# Note: REDISHOST value here is only used for local testing +# See README.md on how to inject environment variable as ConfigMap on GKE +ENV REDISHOST 127.0.0.1 +ENV REDISPORT 6379 + +# Install dependencies. +ADD requirements.txt /app/requirements.txt +RUN pip install -r /app/requirements.txt + +# Add application code. +ADD . /app + +CMD ["gunicorn", "-b", "0.0.0.0:8080", "main:app"] + diff --git a/memorystore/redis/gke_deployment/visit-counter.yaml b/memorystore/redis/gke_deployment/visit-counter.yaml new file mode 100644 index 00000000000..4607d0dcde0 --- /dev/null +++ b/memorystore/redis/gke_deployment/visit-counter.yaml @@ -0,0 +1,39 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: visit-counter + labels: + app: visit-counter +spec: + replicas: 1 + template: + metadata: + labels: + app: visit-counter + spec: + containers: + - name: visit-counter + image: "gcr.io//visit-counter:v1" + env: + - name: REDISHOST + valueFrom: + configMapKeyRef: + name: redishost + key: REDISHOST + ports: + - name: http + containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: visit-counter +spec: + type: LoadBalancer + selector: + app: visit-counter + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + diff --git a/memorystore/redis/main.py b/memorystore/redis/main.py new file mode 100644 index 00000000000..111f9a98f9c --- /dev/null +++ b/memorystore/redis/main.py @@ -0,0 +1,46 @@ +# Copyright 2018 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# [START memorystore_main_py] +import logging +import os + +from flask import Flask +import redis + +app = Flask(__name__) + +redis_host = os.environ.get('REDISHOST', 'localhost') +redis_port = int(os.environ.get('REDISPORT', 6379)) +redis_client = redis.StrictRedis(host=redis_host, port=redis_port) + + +@app.route('/') +def index(): + value = redis_client.incr('counter', 1) + return 'Visitor number: {}'.format(value) + + +@app.errorhandler(500) +def server_error(e): + logging.exception('An error occurred during a request.') + return """ + An internal error occurred: 
          {}
          + See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) +# [END memorystore_main_py] diff --git a/memorystore/redis/requirements.txt b/memorystore/redis/requirements.txt new file mode 100644 index 00000000000..fc5d58cd762 --- /dev/null +++ b/memorystore/redis/requirements.txt @@ -0,0 +1,17 @@ +# Copyright 2018 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# [START memorystore_requirements] +Flask==1.0.2 +gunicorn==19.9.0 +redis==3.1.0 +# [END memorystore_requirements] diff --git a/ml_engine/custom-prediction-routines/README.md b/ml_engine/custom-prediction-routines/README.md new file mode 100644 index 00000000000..86e66e8e2cb --- /dev/null +++ b/ml_engine/custom-prediction-routines/README.md @@ -0,0 +1,28 @@ +# Custom prediction routines (beta) + +Read the AI Platform documentation about custom prediction routines to learn how +to use these samples: + +* [Custom prediction routines (with a TensorFlow Keras + example)](https://cloud.google.com/ml-engine/docs/tensorflow/custom-prediction-routines) +* [Custom prediction routines (with a scikit-learn + example)](https://cloud.google.com/ml-engine/docs/scikit/custom-prediction-routines) + +If you want to package a predictor directly from this directory, make sure to +edit `setup.py`: replace the reference to `predictor.py` with either +`tensorflow-predictor.py` or `scikit-predictor.py`. + +## What's next + +For a more complete example of how to train and deploy a custom prediction +routine, check out one of the following tutorials: + +* [Creating a custom prediction routine with + Keras](https://cloud.google.com/ml-engine/docs/tensorflow/custom-prediction-routine-keras) + (also available as [a Jupyter + notebook](https://colab.research.google.com/github/GoogleCloudPlatform/cloudml-samples/blob/master/notebooks/tensorflow/custom-prediction-routine-keras.ipynb)) + +* [Creating a custom prediction routine with + scikit-learn](https://cloud.google.com/ml-engine/docs/scikit/custom-prediction-routine-scikit-learn) + (also available as [a Jupyter + notebook](https://colab.research.google.com/github/GoogleCloudPlatform/cloudml-samples/blob/master/notebooks/scikit-learn/custom-prediction-routine-scikit-learn.ipynb)) \ No newline at end of file diff --git a/ml_engine/custom-prediction-routines/predictor-interface.py b/ml_engine/custom-prediction-routines/predictor-interface.py new file mode 100644 index 00000000000..a45ea763f80 --- /dev/null +++ b/ml_engine/custom-prediction-routines/predictor-interface.py @@ -0,0 +1,50 @@ +# Copyright 2019 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# https://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class Predictor(object): + """Interface for constructing custom predictors.""" + + def predict(self, instances, **kwargs): + """Performs custom prediction. + + Instances are the decoded values from the request. They have already + been deserialized from JSON. + + Args: + instances: A list of prediction input instances. + **kwargs: A dictionary of keyword args provided as additional + fields on the predict request body. + + Returns: + A list of outputs containing the prediction results. This list must + be JSON serializable. + """ + raise NotImplementedError() + + @classmethod + def from_path(cls, model_dir): + """Creates an instance of Predictor using the given path. + + Loading of the predictor should be done in this method. + + Args: + model_dir: The local directory that contains the exported model + file along with any additional files uploaded when creating the + version resource. + + Returns: + An instance implementing this Predictor class. + """ + raise NotImplementedError() diff --git a/ml_engine/custom-prediction-routines/preprocess.py b/ml_engine/custom-prediction-routines/preprocess.py new file mode 100644 index 00000000000..e28aaf357df --- /dev/null +++ b/ml_engine/custom-prediction-routines/preprocess.py @@ -0,0 +1,43 @@ +# Copyright 2019 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# https://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np + + +class ZeroCenterer(object): + """Stores means of each column of a matrix and uses them for preprocessing. + """ + + def __init__(self): + """On initialization, is not tied to any distribution.""" + self._means = None + + def preprocess(self, data): + """Transforms a matrix. + + The first time this is called, it stores the means of each column of + the input. Then it transforms the input so each column has mean 0. For + subsequent calls, it subtracts the stored means from each column. This + lets you 'center' data at prediction time based on the distribution of + the original training data. + + Args: + data: A NumPy matrix of numerical data. + + Returns: + A transformed matrix with the same dimensions as the input. + """ + if self._means is None: # during training only + self._means = np.mean(data, axis=0) + return data - self._means diff --git a/ml_engine/custom-prediction-routines/scikit-predictor.py b/ml_engine/custom-prediction-routines/scikit-predictor.py new file mode 100644 index 00000000000..ca2998bc68f --- /dev/null +++ b/ml_engine/custom-prediction-routines/scikit-predictor.py @@ -0,0 +1,73 @@ +# Copyright 2019 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# https://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pickle + +import numpy as np +from sklearn.externals import joblib + + +class MyPredictor(object): + """An example Predictor for an AI Platform custom prediction routine.""" + + def __init__(self, model, preprocessor): + """Stores artifacts for prediction. Only initialized via `from_path`. + """ + self._model = model + self._preprocessor = preprocessor + + def predict(self, instances, **kwargs): + """Performs custom prediction. + + Preprocesses inputs, then performs prediction using the trained + scikit-learn model. + + Args: + instances: A list of prediction input instances. + **kwargs: A dictionary of keyword args provided as additional + fields on the predict request body. + + Returns: + A list of outputs containing the prediction results. + """ + inputs = np.asarray(instances) + preprocessed_inputs = self._preprocessor.preprocess(inputs) + outputs = self._model.predict(preprocessed_inputs) + return outputs.tolist() + + @classmethod + def from_path(cls, model_dir): + """Creates an instance of MyPredictor using the given path. + + This loads artifacts that have been copied from your model directory in + Cloud Storage. MyPredictor uses them during prediction. + + Args: + model_dir: The local directory that contains the trained + scikit-learn model and the pickled preprocessor instance. These + are copied from the Cloud Storage model directory you provide + when you deploy a version resource. + + Returns: + An instance of `MyPredictor`. + """ + model_path = os.path.join(model_dir, 'model.joblib') + model = joblib.load(model_path) + + preprocessor_path = os.path.join(model_dir, 'preprocessor.pkl') + with open(preprocessor_path, 'rb') as f: + preprocessor = pickle.load(f) + + return cls(model, preprocessor) diff --git a/ml_engine/custom-prediction-routines/setup.py b/ml_engine/custom-prediction-routines/setup.py new file mode 100644 index 00000000000..f313c0f1404 --- /dev/null +++ b/ml_engine/custom-prediction-routines/setup.py @@ -0,0 +1,20 @@ +# Copyright 2019 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# https://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from setuptools import setup + +setup( + name='my_custom_code', + version='0.1', + scripts=['predictor.py', 'preprocess.py']) diff --git a/ml_engine/custom-prediction-routines/tensorflow-predictor.py b/ml_engine/custom-prediction-routines/tensorflow-predictor.py new file mode 100644 index 00000000000..3d8ed8422f8 --- /dev/null +++ b/ml_engine/custom-prediction-routines/tensorflow-predictor.py @@ -0,0 +1,73 @@ +# Copyright 2019 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# https://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pickle + +import numpy as np +from tensorflow import keras + + +class MyPredictor(object): + """An example Predictor for an AI Platform custom prediction routine.""" + + def __init__(self, model, preprocessor): + """Stores artifacts for prediction. Only initialized via `from_path`. + """ + self._model = model + self._preprocessor = preprocessor + + def predict(self, instances, **kwargs): + """Performs custom prediction. + + Preprocesses inputs, then performs prediction using the trained Keras + model. + + Args: + instances: A list of prediction input instances. + **kwargs: A dictionary of keyword args provided as additional + fields on the predict request body. + + Returns: + A list of outputs containing the prediction results. + """ + inputs = np.asarray(instances) + preprocessed_inputs = self._preprocessor.preprocess(inputs) + outputs = self._model.predict(preprocessed_inputs) + return outputs.tolist() + + @classmethod + def from_path(cls, model_dir): + """Creates an instance of MyPredictor using the given path. + + This loads artifacts that have been copied from your model directory in + Cloud Storage. MyPredictor uses them during prediction. + + Args: + model_dir: The local directory that contains the trained Keras + model and the pickled preprocessor instance. These are copied + from the Cloud Storage model directory you provide when you + deploy a version resource. + + Returns: + An instance of `MyPredictor`. + """ + model_path = os.path.join(model_dir, 'model.h5') + model = keras.models.load_model(model_path) + + preprocessor_path = os.path.join(model_dir, 'preprocessor.pkl') + with open(preprocessor_path, 'rb') as f: + preprocessor = pickle.load(f) + + return cls(model, preprocessor) diff --git a/ml_engine/online_prediction/README.md b/ml_engine/online_prediction/README.md new file mode 100644 index 00000000000..c0a3909a3aa --- /dev/null +++ b/ml_engine/online_prediction/README.md @@ -0,0 +1,6 @@ +https://cloud.google.com/ml-engine/docs/concepts/prediction-overview + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=ml_engine/online_prediction/README.md diff --git a/ml_engine/online_prediction/predict.py b/ml_engine/online_prediction/predict.py new file mode 100644 index 00000000000..6275ea7f453 --- /dev/null +++ b/ml_engine/online_prediction/predict.py @@ -0,0 +1,198 @@ +#!/bin/python +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Examples of using AI Platform's online prediction service.""" +import argparse +import base64 +import json + +# [START import_libraries] +import googleapiclient.discovery +# [END import_libraries] +import six + + +# [START predict_json] +def predict_json(project, model, instances, version=None): + """Send json data to a deployed model for prediction. + + Args: + project (str): project where the AI Platform Model is deployed. + model (str): model name. + instances ([Mapping[str: Any]]): Keys should be the names of Tensors + your deployed model expects as inputs. Values should be datatypes + convertible to Tensors, or (potentially nested) lists of datatypes + convertible to tensors. + version: str, version of the model to target. + Returns: + Mapping[str: any]: dictionary of prediction results defined by the + model. + """ + # Create the AI Platform service object. + # To authenticate set the environment variable + # GOOGLE_APPLICATION_CREDENTIALS= + service = googleapiclient.discovery.build('ml', 'v1') + name = 'projects/{}/models/{}'.format(project, model) + + if version is not None: + name += '/versions/{}'.format(version) + + response = service.projects().predict( + name=name, + body={'instances': instances} + ).execute() + + if 'error' in response: + raise RuntimeError(response['error']) + + return response['predictions'] +# [END predict_json] + + +# [START predict_tf_records] +def predict_examples(project, + model, + example_bytes_list, + version=None): + """Send protocol buffer data to a deployed model for prediction. + + Args: + project (str): project where the AI Platform Model is deployed. + model (str): model name. + example_bytes_list ([str]): A list of bytestrings representing + serialized tf.train.Example protocol buffers. The contents of this + protocol buffer will change depending on the signature of your + deployed model. + version: str, version of the model to target. + Returns: + Mapping[str: any]: dictionary of prediction results defined by the + model. + """ + service = googleapiclient.discovery.build('ml', 'v1') + name = 'projects/{}/models/{}'.format(project, model) + + if version is not None: + name += '/versions/{}'.format(version) + + response = service.projects().predict( + name=name, + body={'instances': [ + {'b64': base64.b64encode(example_bytes).decode('utf-8')} + for example_bytes in example_bytes_list + ]} + ).execute() + + if 'error' in response: + raise RuntimeError(response['error']) + + return response['predictions'] +# [END predict_tf_records] + + +# [START census_to_example_bytes] +def census_to_example_bytes(json_instance): + """Serialize a JSON example to the bytes of a tf.train.Example. + This method is specific to the signature of the Census example. + See: https://cloud.google.com/ml-engine/docs/concepts/prediction-overview + for details. + + Args: + json_instance (Mapping[str: Any]): Keys should be the names of Tensors + your deployed model expects to parse using it's tf.FeatureSpec. + Values should be datatypes convertible to Tensors, or (potentially + nested) lists of datatypes convertible to tensors. + Returns: + str: A string as a container for the serialized bytes of + tf.train.Example protocol buffer. + """ + import tensorflow as tf + feature_dict = {} + for key, data in six.iteritems(json_instance): + if isinstance(data, six.string_types): + feature_dict[key] = tf.train.Feature( + bytes_list=tf.train.BytesList(value=[data.encode('utf-8')])) + elif isinstance(data, float): + feature_dict[key] = tf.train.Feature( + float_list=tf.train.FloatList(value=[data])) + elif isinstance(data, int): + feature_dict[key] = tf.train.Feature( + int64_list=tf.train.Int64List(value=[data])) + return tf.train.Example( + features=tf.train.Features( + feature=feature_dict + ) + ).SerializeToString() +# [END census_to_example_bytes] + + +def main(project, model, version=None, force_tfrecord=False): + """Send user input to the prediction service.""" + while True: + try: + user_input = json.loads(input("Valid JSON >>>")) + except KeyboardInterrupt: + return + + if not isinstance(user_input, list): + user_input = [user_input] + try: + if force_tfrecord: + example_bytes_list = [ + census_to_example_bytes(e) + for e in user_input + ] + result = predict_examples( + project, model, example_bytes_list, version=version) + else: + result = predict_json( + project, model, user_input, version=version) + except RuntimeError as err: + print(str(err)) + else: + print(result) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + '--project', + help='Project in which the model is deployed', + type=str, + required=True + ) + parser.add_argument( + '--model', + help='Model name', + type=str, + required=True + ) + parser.add_argument( + '--version', + help='Name of the version.', + type=str + ) + parser.add_argument( + '--force-tfrecord', + help='Send predictions as TFRecords rather than raw JSON', + action='store_true', + default=False + ) + args = parser.parse_args() + main( + args.project, + args.model, + version=args.version, + force_tfrecord=args.force_tfrecord + ) diff --git a/ml_engine/online_prediction/predict_test.py b/ml_engine/online_prediction/predict_test.py new file mode 100644 index 00000000000..0fd17ff9ccd --- /dev/null +++ b/ml_engine/online_prediction/predict_test.py @@ -0,0 +1,71 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import socket + +from gcp_devrel.testing.flaky import flaky +import pytest + +import predict + +MODEL = 'census' +JSON_VERSION = 'v1json' +EXAMPLES_VERSION = 'v1example' +PROJECT = 'python-docs-samples-tests' +EXPECTED_OUTPUT = { + u'confidence': 0.7760371565818787, + u'predictions': u' <=50K' +} + +# Raise the socket timeout. The requests involved in the sample can take +# a long time to complete. +socket.setdefaulttimeout(60) + + +with open('resources/census_test_data.json') as f: + JSON = json.load(f) + + +with open('resources/census_example_bytes.pb', 'rb') as f: + BYTESTRING = f.read() + + +@flaky +def test_predict_json(): + result = predict.predict_json( + PROJECT, MODEL, [JSON, JSON], version=JSON_VERSION) + assert [EXPECTED_OUTPUT, EXPECTED_OUTPUT] == result + + +@flaky +def test_predict_json_error(): + with pytest.raises(RuntimeError): + predict.predict_json( + PROJECT, MODEL, [{"foo": "bar"}], version=JSON_VERSION) + + +@flaky +def test_census_example_to_bytes(): + import tensorflow as tf + b = predict.census_to_example_bytes(JSON) + assert tf.train.Example.FromString(b) == tf.train.Example.FromString( + BYTESTRING) + + +@flaky(max_runs=6) +def test_predict_examples(): + result = predict.predict_examples( + PROJECT, MODEL, [BYTESTRING, BYTESTRING], version=EXAMPLES_VERSION) + assert [EXPECTED_OUTPUT, EXPECTED_OUTPUT] == result diff --git a/ml_engine/online_prediction/requirements.txt b/ml_engine/online_prediction/requirements.txt new file mode 100644 index 00000000000..481aed6392b --- /dev/null +++ b/ml_engine/online_prediction/requirements.txt @@ -0,0 +1,4 @@ +tensorflow==1.12.0 +google-api-python-client==1.7.8 +google-auth==1.6.2 +google-auth-httplib2==0.0.3 diff --git a/ml_engine/online_prediction/resources/census_example_bytes.pb b/ml_engine/online_prediction/resources/census_example_bytes.pb new file mode 100644 index 00000000000..8cd9013d5b6 Binary files /dev/null and b/ml_engine/online_prediction/resources/census_example_bytes.pb differ diff --git a/ml_engine/online_prediction/resources/census_test_data.json b/ml_engine/online_prediction/resources/census_test_data.json new file mode 100644 index 00000000000..18fa3802a0b --- /dev/null +++ b/ml_engine/online_prediction/resources/census_test_data.json @@ -0,0 +1 @@ +{"hours_per_week": 40, "native_country": " United-States", "relationship": " Own-child", "capital_loss": 0, "education": " 11th", "capital_gain": 0, "occupation": " Machine-op-inspct", "workclass": " Private", "gender": " Male", "age": 25, "marital_status": " Never-married", "race": " Black", "education_num": 7} \ No newline at end of file diff --git a/ml_engine/online_prediction/scikit-xg-predict.py b/ml_engine/online_prediction/scikit-xg-predict.py new file mode 100644 index 00000000000..73d9f152267 --- /dev/null +++ b/ml_engine/online_prediction/scikit-xg-predict.py @@ -0,0 +1,53 @@ + +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Examples of using AI Platform's online prediction service, + modified for scikit-learn and XGBoost.""" + +import googleapiclient.discovery + + +# [START predict_json] +def predict_json(project, model, instances, version=None): + """Send json data to a deployed model for prediction. + Args: + project (str): project where the AI Platform Model is deployed. + model (str): model name. + instances ([[float]]): List of input instances, where each input + instance is a list of floats. + version: str, version of the model to target. + Returns: + Mapping[str: any]: dictionary of prediction results defined by the + model. + """ + # Create the AI Platform service object. + # To authenticate set the environment variable + # GOOGLE_APPLICATION_CREDENTIALS= + service = googleapiclient.discovery.build('ml', 'v1') + name = 'projects/{}/models/{}'.format(project, model) + + if version is not None: + name += '/versions/{}'.format(version) + + response = service.projects().predict( + name=name, + body={'instances': instances} + ).execute() + + if 'error' in response: + raise RuntimeError(response['error']) + + return response['predictions'] +# [END predict_json] diff --git a/monitoring/README.md b/monitoring/README.md deleted file mode 100644 index ad73848bcc7..00000000000 --- a/monitoring/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# Google Cloud Monitoring Samples - -This section contains samples for [Google Cloud Monitoring](https://cloud.google.com/monitoring). - -## Running the samples - -1. Your environment must be setup with [authentication -information](https://developers.google.com/identity/protocols/application-default-credentials#howtheywork). *Note* that Cloud Monitoring does not currently work -with `gcloud auth`. You will need to use a *service account* when running -locally and set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable. - - $ export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service_account.json - -2. Install dependencies from `requirements.txt`: - - $ pip install -r requirements.txt - -3. Depending on the sample, you may also need to create resources on the [Google Developers Console](https://console.developers.google.com). Refer to the sample description and associated documentation page. - -## Additional resources - -For more information on Cloud Monitoring you can visit: - -> https://cloud.google.com/monitoring - -For more information on the Cloud Monitoring API Python library surface you -can visit: - -> https://developers.google.com/resources/api-libraries/documentation/storage/v2beta2/python/latest/ - -For information on the Python Client Library visit: - -> https://developers.google.com/api-client-library/python diff --git a/monitoring/api/v2/README.md b/monitoring/api/v2/README.md deleted file mode 100644 index 953e3545ab9..00000000000 --- a/monitoring/api/v2/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# Cloud Monitoring API Samples - - -These samples are used on the following documentation pages: - -> -* https://cloud.google.com/monitoring/demos/ -* https://cloud.google.com/monitoring/api/authentication - - diff --git a/monitoring/api/v2/auth.py b/monitoring/api/v2/auth.py deleted file mode 100644 index fb349d74ec9..00000000000 --- a/monitoring/api/v2/auth.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env pyhton - -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Sample command-line program for retrieving Google Cloud Monitoring API data. - -Prerequisites: To run locally, download a Service Account JSON file from -your project and point GOOGLE_APPLICATION_CREDENTIALS to the file. - - -This sample is used on this page: - - https://cloud.google.com/monitoring/api/authentication - -For more information, see the README.md under /monitoring. -""" - -# [START all] -import argparse -import json - -from googleapiclient.discovery import build -from oauth2client.client import GoogleCredentials - -METRIC = 'compute.googleapis.com/instance/disk/read_ops_count' -YOUNGEST = '2015-01-01T00:00:00Z' - - -def list_timeseries(monitoring, project_name): - """Query the Timeseries.list API method. - - Args: - monitoring: the CloudMonitoring service object. - project_name: the name of the project you'd like to monitor. - """ - timeseries = monitoring.timeseries() - - response = timeseries.list( - project=project_name, metric=METRIC, youngest=YOUNGEST).execute() - - print('Timeseries.list raw response:') - print(json.dumps(response, - sort_keys=True, - indent=4, - separators=(',', ': '))) - - -def main(project_name): - credentials = GoogleCredentials.get_application_default() - monitoring = build('cloudmonitoring', 'v2beta2', credentials=credentials) - - list_timeseries(monitoring, project_name) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('project_id', help='Your Google Cloud project ID.') - - args = parser.parse_args() - - main(args.project_id) -# [END all] diff --git a/monitoring/api/v2/auth_test.py b/monitoring/api/v2/auth_test.py deleted file mode 100644 index 917e33e870d..00000000000 --- a/monitoring/api/v2/auth_test.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -import auth - - -def test_main(cloud_config, capsys): - auth.main(cloud_config.project) - output, _ = capsys.readouterr() - - assert re.search( - re.compile( - r'Timeseries.list raw response:\s*' - r'{\s*"kind": "[^"]+",' - r'\s*"oldest": *"[0-9]+', re.S), - output) diff --git a/monitoring/api/v2/labeled_custom_metric.py b/monitoring/api/v2/labeled_custom_metric.py deleted file mode 100644 index fec346d852f..00000000000 --- a/monitoring/api/v2/labeled_custom_metric.py +++ /dev/null @@ -1,196 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -"""Creates, writes, and reads a labeled custom metric. - -This is an example of how to use the Google Cloud Monitoring API to create, -write, and read a labeled custom metric. -The metric has two labels: color and size, and the data points represent -the number of shirts of the given color and size in inventory. - -Prerequisites: To run locally, download a Service Account JSON file from -your project and point GOOGLE_APPLICATION_CREDENTIALS to the file. - -From App Engine or a GCE instance with the correct scope, the Service -Account step is not required. - -Typical usage: Run the following shell commands on the instance: - python labeled_custom_metric.py --project_id / - --color yellow --size large --count 10 -""" - -import argparse -import datetime -import time - -from googleapiclient import discovery -from oauth2client.client import GoogleCredentials - -CUSTOM_METRIC_DOMAIN = "custom.cloudmonitoring.googleapis.com" -CUSTOM_METRIC_NAME = "{}/shirt_inventory".format(CUSTOM_METRIC_DOMAIN) - - -def format_rfc3339(datetime_instance=None): - """Formats a datetime per RFC 3339. - :param datetime_instance: Datetime instance to format, defaults to utcnow - """ - return datetime_instance.isoformat("T") + "Z" - - -def get_now_rfc3339(): - return format_rfc3339(datetime.datetime.utcnow()) - - -def create_custom_metric(client, project_id): - """Create metric descriptor for the custom metric and send it to the - API.""" - # You need to execute this operation only once. The operation is - # idempotent, so, for simplicity, this sample code calls it each time - - # Create a label descriptor for each of the metric labels. The - # "description" field should be more meaningful for your metrics. - label_descriptors = [] - for label in ["color", "size", ]: - label_descriptors.append({"key": "/{}".format(label), - "description": "The {}.".format(label)}) - - # Create the metric descriptor for the custom metric. - metric_descriptor = { - "name": CUSTOM_METRIC_NAME, - "project": project_id, - "typeDescriptor": { - "metricType": "gauge", - "valueType": "int64", - }, - "labels": label_descriptors, - "description": "The size of my shirt inventory.", - } - # Submit the custom metric creation request. - try: - request = client.metricDescriptors().create( - project=project_id, body=metric_descriptor) - request.execute() # ignore the response - except Exception as e: - print("Failed to create custom metric: exception={})".format(e)) - raise - - -def write_custom_metric(client, project_id, now_rfc3339, color, size, count): - """Write a data point to a single time series of the custom metric.""" - # Identify the particular time series to which to write the data by - # specifying the metric and values for each label. - timeseries_descriptor = { - "project": project_id, - "metric": CUSTOM_METRIC_NAME, - "labels": { - "{}/color".format(CUSTOM_METRIC_DOMAIN): color, - "{}/size".format(CUSTOM_METRIC_DOMAIN): size, - } - } - - # Specify a new data point for the time series. - timeseries_data = { - "timeseriesDesc": timeseries_descriptor, - "point": { - "start": now_rfc3339, - "end": now_rfc3339, - "int64Value": count, - } - } - - # Submit the write request. - request = client.timeseries().write( - project=project_id, body={"timeseries": [timeseries_data, ]}) - try: - request.execute() # ignore the response - except Exception as e: - print("Failed to write data to custom metric: exception={}".format(e)) - raise - - -def read_custom_metric(client, project_id, now_rfc3339, color, size): - """Read all the timeseries data points for a given set of label values.""" - # To identify a time series, specify values for in label as a list. - labels = ["{}/color=={}".format(CUSTOM_METRIC_DOMAIN, color), - "{}/size=={}".format(CUSTOM_METRIC_DOMAIN, size), ] - - # Submit the read request. - request = client.timeseries().list( - project=project_id, - metric=CUSTOM_METRIC_NAME, - youngest=now_rfc3339, - labels=labels) - - # When a custom metric is created, it may take a few seconds - # to propagate throughout the system. Retry a few times. - start = time.time() - while True: - try: - response = request.execute() - for point in response["timeseries"][0]["points"]: - print("{}: {}".format(point["end"], point["int64Value"])) - break - except Exception as e: - if time.time() < start + 20: - print("Failed to read custom metric data, retrying...") - time.sleep(3) - else: - print("Failed to read custom metric data, aborting: " - "exception={}".format(e)) - raise - - -def get_client(): - """Builds an http client authenticated with the application default - credentials.""" - credentials = GoogleCredentials.get_application_default() - client = discovery.build( - 'cloudmonitoring', 'v2beta2', - credentials=credentials) - return client - - -def main(project_id, color, size, count): - now_rfc3339 = get_now_rfc3339() - - client = get_client() - - print ("Labels: color: {}, size: {}.".format(color, size)) - print ("Creating custom metric...") - create_custom_metric(client, project_id) - time.sleep(2) - print ("Writing new data to custom metric timeseries...") - write_custom_metric(client, project_id, now_rfc3339, - color, size, count) - print ("Reading data from custom metric timeseries...") - read_custom_metric(client, project_id, now_rfc3339, color, size) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - - parser.add_argument( - '--project_id', help='Project ID you want to access.', required=True) - parser.add_argument("--color", required=True) - parser.add_argument("--size", required=True) - parser.add_argument("--count", required=True) - - args = parser.parse_args() - main(args.project_id, args.color, args.size, args.count) diff --git a/monitoring/api/v2/labeled_custom_metric_test.py b/monitoring/api/v2/labeled_custom_metric_test.py deleted file mode 100644 index f564eef0fd5..00000000000 --- a/monitoring/api/v2/labeled_custom_metric_test.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -import labeled_custom_metric - - -def test_main(cloud_config, capsys): - labeled_custom_metric.main(cloud_config.project, "yellow", "large", "10") - output, _ = capsys.readouterr() - - assert re.search( - re.compile( - r'.*Creating.*' - r'Writing.*' - r'Reading.*', flags=re.DOTALL), - output) diff --git a/monitoring/api/v2/lightweight_custom_metric.py b/monitoring/api/v2/lightweight_custom_metric.py deleted file mode 100644 index b2fc550f781..00000000000 --- a/monitoring/api/v2/lightweight_custom_metric.py +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -"""Writes and reads a lightweight custom metric. - -This is an example of how to use the Google Cloud Monitoring API to write -and read a lightweight custom metric. Lightweight custom metrics have no -labels and you do not need to create a metric descriptor for them. - -Prerequisites: To run locally, download a Service Account JSON file from -your project and point GOOGLE_APPLICATION_CREDENTIALS to the file. - -From App Engine or a GCE instance with the correct scope, the Service -Account step is not required. - -Typical usage: Run the following shell commands on the instance: - - python lightweight_custom_metric.py --project_id= -""" - -import argparse -import datetime -import os -import time - -from googleapiclient import discovery -from oauth2client.client import GoogleCredentials - -CUSTOM_METRIC_NAME = "custom.cloudmonitoring.googleapis.com/pid" - - -def format_rfc3339(datetime_instance=None): - """Formats a datetime per RFC 3339. - :param datetime_instance: Datetime instanec to format, defaults to utcnow - """ - return datetime_instance.isoformat("T") + "Z" - - -def get_now_rfc3339(): - return format_rfc3339(datetime.datetime.utcnow()) - - -def get_client(): - """Builds an http client authenticated with the service account - credentials.""" - credentials = GoogleCredentials.get_application_default() - client = discovery.build( - 'cloudmonitoring', 'v2beta2', - credentials=credentials) - return client - - -def main(project_id): - # Set up the write request. - client = get_client() - now = get_now_rfc3339() - desc = {"project": project_id, - "metric": CUSTOM_METRIC_NAME} - point = {"start": now, - "end": now, - "doubleValue": os.getpid()} - print("Writing {} at {}".format(point["doubleValue"], now)) - - # Write a new data point. - try: - write_request = client.timeseries().write( - project=project_id, - body={"timeseries": [{"timeseriesDesc": desc, "point": point}]}) - write_request.execute() # Ignore the response. - except Exception as e: - print("Failed to write custom metric data: exception={}".format(e)) - raise - - # Read all data points from the time series. - # When a custom metric is created, it may take a few seconds - # to propagate throughout the system. Retry a few times. - print("Reading data from custom metric timeseries...") - read_request = client.timeseries().list( - project=project_id, - metric=CUSTOM_METRIC_NAME, - youngest=now) - start = time.time() - while True: - try: - read_response = read_request.execute() - for point in read_response["timeseries"][0]["points"]: - print("Point: {}: {}".format( - point["end"], point["doubleValue"])) - break - except Exception as e: - if time.time() < start + 20: - print("Failed to read custom metric data, retrying...") - time.sleep(3) - else: - print("Failed to read custom metric data, aborting: " - "exception={}".format(e)) - raise - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter - ) - - parser.add_argument( - '--project_id', help='Project ID you want to access.', required=True) - - args = parser.parse_args() - main(args.project_id) diff --git a/monitoring/api/v2/lightweight_custom_metric_test.py b/monitoring/api/v2/lightweight_custom_metric_test.py deleted file mode 100644 index f8c0598a61b..00000000000 --- a/monitoring/api/v2/lightweight_custom_metric_test.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -import lightweight_custom_metric - - -def test_main(cloud_config, capsys): - lightweight_custom_metric.main(cloud_config.project) - output, _ = capsys.readouterr() - - assert re.search( - re.compile( - r'Point:\s*'), - output) diff --git a/monitoring/api/v2/requirements.txt b/monitoring/api/v2/requirements.txt deleted file mode 100644 index c3b2784ce87..00000000000 --- a/monitoring/api/v2/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -google-api-python-client==1.5.0 diff --git a/monitoring/api/v3/README.md b/monitoring/api/v3/README.md deleted file mode 100644 index e5a9ea37e0f..00000000000 --- a/monitoring/api/v3/README.md +++ /dev/null @@ -1,71 +0,0 @@ -# Cloud Monitoring v3 Sample - -Sample command-line programs for retrieving Google Monitoring API V3 data. - -`list_resources.py` is a simple command-line program to demonstrate connecting to the Google -Monitoring API to retrieve API data and print out some of the resources. - -`custom_metric.py` demonstrates how to create a custom metric and write a TimeSeries -value to it. - -## Prerequisites to run locally: - -* [pip](https://pypi.python.org/pypi/pip) - -Go to the [Google Cloud Console](https://console.cloud.google.com). - - -# Set Up Your Local Dev Environment -To install, run the following commands. If you want to use [virtualenv](https://virtualenv.readthedocs.org/en/latest/) -(recommended), run the commands within a virtualenv. - - * pip install -r requirements.txt - -Create local credentials by running the following command and following the oauth2 flow: - - gcloud beta auth application-default login - -To run: - - python list_resources.py --project_id= - python custom_metric.py --project_id= Credentials -* Click 'New Credentials', and create a Service Account or [click here](https://console.cloud.google -.com/project/_/apiui/credential/serviceaccount) - Download the JSON for this service account, and set the `GOOGLE_APPLICATION_CREDENTIALS` - environment variable to point to the file containing the JSON credentials. - - - export GOOGLE_APPLICATION_CREDENTIALS=~/Downloads/-0123456789abcdef.json - - -## Contributing changes - -* See [CONTRIBUTING.md](CONTRIBUTING.md) - -## Licensing - -* See [LICENSE](LICENSE) - - diff --git a/monitoring/api/v3/alerts-client/.gitignore b/monitoring/api/v3/alerts-client/.gitignore new file mode 100644 index 00000000000..de0a466d79c --- /dev/null +++ b/monitoring/api/v3/alerts-client/.gitignore @@ -0,0 +1 @@ +backup.json diff --git a/monitoring/api/v3/alerts-client/README.rst b/monitoring/api/v3/alerts-client/README.rst new file mode 100644 index 00000000000..68eba2344eb --- /dev/null +++ b/monitoring/api/v3/alerts-client/README.rst @@ -0,0 +1,117 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Stackdriver Alerting API Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=monitoring/api/v3/alerts-client/README.rst + + +This directory contains samples for Google Stackdriver Alerting API. Stackdriver Monitoring collects metrics, events, and metadata from Google Cloud Platform, Amazon Web Services (AWS), hosted uptime probes, application instrumentation, and a variety of common application components including Cassandra, Nginx, Apache Web Server, Elasticsearch and many others. Stackdriver's Alerting API allows you to create, delete, and make back up copies of your alert policies. + + + + +.. _Google Stackdriver Alerting API: https://cloud.google.com/monitoring/alerts/ + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Snippets ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=monitoring/api/v3/alerts-client/snippets.py,monitoring/api/v3/alerts-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python snippets.py + + usage: snippets.py [-h] + {list-alert-policies,list-notification-channels,enable-alert-policies,disable-alert-policies,replace-notification-channels,backup,restore} + ... + + Demonstrates AlertPolicy API operations. + + positional arguments: + {list-alert-policies,list-notification-channels,enable-alert-policies,disable-alert-policies,replace-notification-channels,backup,restore} + list-alert-policies + list-notification-channels + enable-alert-policies + disable-alert-policies + replace-notification-channels + backup + restore + + optional arguments: + -h, --help show this help message and exit + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/monitoring/api/v3/alerts-client/README.rst.in b/monitoring/api/v3/alerts-client/README.rst.in new file mode 100644 index 00000000000..ed7f6a3bcf1 --- /dev/null +++ b/monitoring/api/v3/alerts-client/README.rst.in @@ -0,0 +1,26 @@ +# This file is used to generate README.rst + +product: + name: Google Stackdriver Alerting API + short_name: Stackdriver Alerting API + url: https://cloud.google.com/monitoring/alerts/ + description: > + Stackdriver Monitoring collects metrics, events, and metadata from Google + Cloud Platform, Amazon Web Services (AWS), hosted uptime probes, + application instrumentation, and a variety of common application + components including Cassandra, Nginx, Apache Web Server, Elasticsearch + and many others. Stackdriver's Alerting API allows you to create, + delete, and make back up copies of your alert policies. + +setup: +- auth +- install_deps + +samples: +- name: Snippets + file: snippets.py + show_help: true + +cloud_client_library: true + +folder: monitoring/api/v3/alerts-client \ No newline at end of file diff --git a/monitoring/api/v3/alerts-client/requirements.txt b/monitoring/api/v3/alerts-client/requirements.txt new file mode 100644 index 00000000000..807c0443c4a --- /dev/null +++ b/monitoring/api/v3/alerts-client/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-monitoring==0.31.1 +tabulate==0.8.3 diff --git a/monitoring/api/v3/alerts-client/snippets.py b/monitoring/api/v3/alerts-client/snippets.py new file mode 100644 index 00000000000..62c84bf8d80 --- /dev/null +++ b/monitoring/api/v3/alerts-client/snippets.py @@ -0,0 +1,331 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import argparse +import json +import os + +from google.cloud import monitoring_v3 +import google.protobuf.json_format +import tabulate + + +# [START monitoring_alert_list_policies] +def list_alert_policies(project_name): + client = monitoring_v3.AlertPolicyServiceClient() + policies = client.list_alert_policies(project_name) + print(tabulate.tabulate( + [(policy.name, policy.display_name) for policy in policies], + ('name', 'display_name'))) +# [END monitoring_alert_list_policies] + + +# [START monitoring_alert_list_channels] +def list_notification_channels(project_name): + client = monitoring_v3.NotificationChannelServiceClient() + channels = client.list_notification_channels(project_name) + print(tabulate.tabulate( + [(channel.name, channel.display_name) for channel in channels], + ('name', 'display_name'))) +# [END monitoring_alert_list_channels] + + +# [START monitoring_alert_enable_policies] +def enable_alert_policies(project_name, enable, filter_=None): + """Enable or disable alert policies in a project. + + Arguments: + project_name (str) + enable (bool): Enable or disable the policies. + filter_ (str, optional): Only enable/disable alert policies that match + this filter_. See + https://cloud.google.com/monitoring/api/v3/sorting-and-filtering + """ + + client = monitoring_v3.AlertPolicyServiceClient() + policies = client.list_alert_policies(project_name, filter_=filter_) + + for policy in policies: + if bool(enable) == policy.enabled.value: + print('Policy', policy.name, 'is already', + 'enabled' if policy.enabled.value else 'disabled') + else: + policy.enabled.value = bool(enable) + mask = monitoring_v3.types.field_mask_pb2.FieldMask() + mask.paths.append('enabled') + client.update_alert_policy(policy, mask) + print('Enabled' if enable else 'Disabled', policy.name) +# [END monitoring_alert_enable_policies] + + +# [START monitoring_alert_replace_channels] +def replace_notification_channels(project_name, alert_policy_id, channel_ids): + _, project_id = project_name.split('/') + alert_client = monitoring_v3.AlertPolicyServiceClient() + channel_client = monitoring_v3.NotificationChannelServiceClient() + policy = monitoring_v3.types.alert_pb2.AlertPolicy() + policy.name = alert_client.alert_policy_path(project_id, alert_policy_id) + + for channel_id in channel_ids: + policy.notification_channels.append( + channel_client.notification_channel_path(project_id, channel_id)) + + mask = monitoring_v3.types.field_mask_pb2.FieldMask() + mask.paths.append('notification_channels') + updated_policy = alert_client.update_alert_policy(policy, mask) + print('Updated', updated_policy.name) +# [END monitoring_alert_replace_channels] + + +# [START monitoring_alert_delete_channel] +def delete_notification_channels(project_name, channel_ids, force=None): + channel_client = monitoring_v3.NotificationChannelServiceClient() + for channel_id in channel_ids: + channel_name = '{}/notificationChannels/{}'.format( + project_name, channel_id) + try: + channel_client.delete_notification_channel( + channel_name, force=force) + print('Channel {} deleted'.format(channel_name)) + except ValueError: + print('The parameters are invalid') + except Exception as e: + print('API call failed: {}'.format(e)) +# [END monitoring_alert_delete_channel] + + +# [START monitoring_alert_backup_policies] +def backup(project_name): + alert_client = monitoring_v3.AlertPolicyServiceClient() + channel_client = monitoring_v3.NotificationChannelServiceClient() + record = {'project_name': project_name, + 'policies': list(alert_client.list_alert_policies(project_name)), + 'channels': list(channel_client.list_notification_channels( + project_name))} + json.dump(record, open('backup.json', 'wt'), cls=ProtoEncoder, indent=2) + print('Backed up alert policies and notification channels to backup.json.') + + +class ProtoEncoder(json.JSONEncoder): + """Uses google.protobuf.json_format to encode protobufs as json.""" + def default(self, obj): + if type(obj) in (monitoring_v3.types.alert_pb2.AlertPolicy, + monitoring_v3.types.notification_pb2. + NotificationChannel): + text = google.protobuf.json_format.MessageToJson(obj) + return json.loads(text) + return super(ProtoEncoder, self).default(obj) +# [END monitoring_alert_backup_policies] + + +# [START monitoring_alert_restore_policies] +# [START monitoring_alert_create_policy] +# [START monitoring_alert_create_channel] +# [START monitoring_alert_update_channel] +# [START monitoring_alert_enable_channel] +def restore(project_name): + print('Loading alert policies and notification channels from backup.json.') + record = json.load(open('backup.json', 'rt')) + is_same_project = project_name == record['project_name'] + # Convert dicts to AlertPolicies. + policies_json = [json.dumps(policy) for policy in record['policies']] + policies = [google.protobuf.json_format.Parse( + policy_json, monitoring_v3.types.alert_pb2.AlertPolicy()) + for policy_json in policies_json] + # Convert dicts to NotificationChannels + channels_json = [json.dumps(channel) for channel in record['channels']] + channels = [google.protobuf.json_format.Parse( + channel_json, monitoring_v3.types.notification_pb2. + NotificationChannel()) for channel_json in channels_json] + + # Restore the channels. + channel_client = monitoring_v3.NotificationChannelServiceClient() + channel_name_map = {} + + for channel in channels: + updated = False + print('Updating channel', channel.display_name) + # This field is immutable and it is illegal to specify a + # non-default value (UNVERIFIED or VERIFIED) in the + # Create() or Update() operations. + channel.verification_status = monitoring_v3.enums.NotificationChannel.\ + VerificationStatus.VERIFICATION_STATUS_UNSPECIFIED + + if is_same_project: + try: + channel_client.update_notification_channel(channel) + updated = True + except google.api_core.exceptions.NotFound: + pass # The channel was deleted. Create it below. + + if not updated: + # The channel no longer exists. Recreate it. + old_name = channel.name + channel.ClearField("name") + new_channel = channel_client.create_notification_channel( + project_name, channel) + channel_name_map[old_name] = new_channel.name + + # Restore the alerts + alert_client = monitoring_v3.AlertPolicyServiceClient() + + for policy in policies: + print('Updating policy', policy.display_name) + # These two fields cannot be set directly, so clear them. + policy.ClearField('creation_record') + policy.ClearField('mutation_record') + + # Update old channel names with new channel names. + for i, channel in enumerate(policy.notification_channels): + new_channel = channel_name_map.get(channel) + if new_channel: + policy.notification_channels[i] = new_channel + + updated = False + + if is_same_project: + try: + alert_client.update_alert_policy(policy) + updated = True + except google.api_core.exceptions.NotFound: + pass # The policy was deleted. Create it below. + except google.api_core.exceptions.InvalidArgument: + # Annoying that API throws InvalidArgument when the policy + # does not exist. Seems like it should throw NotFound. + pass # The policy was deleted. Create it below. + + if not updated: + # The policy no longer exists. Recreate it. + old_name = policy.name + policy.ClearField("name") + for condition in policy.conditions: + condition.ClearField("name") + policy = alert_client.create_alert_policy(project_name, policy) + print('Updated', policy.name) +# [END monitoring_alert_enable_channel] +# [END monitoring_alert_restore_policies] +# [END monitoring_alert_create_policy] +# [END monitoring_alert_create_channel] +# [END monitoring_alert_update_channel] + + +class MissingProjectIdError(Exception): + pass + + +def project_id(): + """Retreieves the project id from the environment variable. + + Raises: + MissingProjectIdError -- When not set. + + Returns: + str -- the project name + """ + project_id = os.environ['GCLOUD_PROJECT'] + + if not project_id: + raise MissingProjectIdError( + 'Set the environment variable ' + + 'GCLOUD_PROJECT to your Google Cloud Project Id.') + return project_id + + +def project_name(): + return 'projects/' + project_id() + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser( + description='Demonstrates AlertPolicy API operations.') + + subparsers = parser.add_subparsers(dest='command') + + list_alert_policies_parser = subparsers.add_parser( + 'list-alert-policies', + help=list_alert_policies.__doc__ + ) + + list_notification_channels_parser = subparsers.add_parser( + 'list-notification-channels', + help=list_alert_policies.__doc__ + ) + + enable_alert_policies_parser = subparsers.add_parser( + 'enable-alert-policies', + help=enable_alert_policies.__doc__ + ) + enable_alert_policies_parser.add_argument( + '--filter', + ) + + disable_alert_policies_parser = subparsers.add_parser( + 'disable-alert-policies', + help=enable_alert_policies.__doc__ + ) + disable_alert_policies_parser.add_argument( + '--filter', + ) + + replace_notification_channels_parser = subparsers.add_parser( + 'replace-notification-channels', + help=replace_notification_channels.__doc__ + ) + replace_notification_channels_parser.add_argument( + '-p', '--alert_policy_id', + required=True + ) + replace_notification_channels_parser.add_argument( + '-c', '--notification_channel_id', + required=True, + action='append' + ) + + backup_parser = subparsers.add_parser( + 'backup', + help=backup.__doc__ + ) + + restore_parser = subparsers.add_parser( + 'restore', + help=restore.__doc__ + ) + + args = parser.parse_args() + + if args.command == 'list-alert-policies': + list_alert_policies(project_name()) + + elif args.command == 'list-notification-channels': + list_notification_channels(project_name()) + + elif args.command == 'enable-alert-policies': + enable_alert_policies(project_name(), enable=True, filter_=args.filter) + + elif args.command == 'disable-alert-policies': + enable_alert_policies(project_name(), enable=False, + filter_=args.filter) + + elif args.command == 'replace-notification-channels': + replace_notification_channels(project_name(), args.alert_policy_id, + args.notification_channel_id) + + elif args.command == 'backup': + backup(project_name()) + + elif args.command == 'restore': + restore(project_name()) diff --git a/monitoring/api/v3/alerts-client/snippets_test.py b/monitoring/api/v3/alerts-client/snippets_test.py new file mode 100644 index 00000000000..8aba3fc5943 --- /dev/null +++ b/monitoring/api/v3/alerts-client/snippets_test.py @@ -0,0 +1,126 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import random +import string + +from google.cloud import monitoring_v3 +import google.protobuf.json_format +import pytest + +import snippets + + +def random_name(length): + return ''.join( + [random.choice(string.ascii_lowercase) for i in range(length)]) + + +class PochanFixture: + """A test fixture that creates an alert POlicy and a notification CHANnel, + hence the name, pochan. + """ + + def __init__(self): + self.project_id = snippets.project_id() + self.project_name = snippets.project_name() + self.alert_policy_client = monitoring_v3.AlertPolicyServiceClient() + self.notification_channel_client = ( + monitoring_v3.NotificationChannelServiceClient()) + + def __enter__(self): + # Create a policy. + policy = monitoring_v3.types.alert_pb2.AlertPolicy() + json = open('test_alert_policy.json').read() + google.protobuf.json_format.Parse(json, policy) + policy.display_name = 'snippets-test-' + random_name(10) + self.alert_policy = self.alert_policy_client.create_alert_policy( + self.project_name, policy) + # Create a notification channel. + notification_channel = ( + monitoring_v3.types.notification_pb2.NotificationChannel()) + json = open('test_notification_channel.json').read() + google.protobuf.json_format.Parse(json, notification_channel) + notification_channel.display_name = 'snippets-test-' + random_name(10) + self.notification_channel = ( + self.notification_channel_client.create_notification_channel( + self.project_name, notification_channel)) + return self + + def __exit__(self, type, value, traceback): + # Delete the policy and channel we created. + self.alert_policy_client.delete_alert_policy(self.alert_policy.name) + if self.notification_channel.name: + self.notification_channel_client.delete_notification_channel( + self.notification_channel.name) + + +@pytest.fixture(scope='session') +def pochan(): + with PochanFixture() as pochan: + yield pochan + + +def test_list_alert_policies(capsys, pochan): + snippets.list_alert_policies(pochan.project_name) + out, _ = capsys.readouterr() + assert pochan.alert_policy.display_name in out + + +def test_enable_alert_policies(capsys, pochan): + snippets.enable_alert_policies(pochan.project_name, False) + out, _ = capsys.readouterr() + + snippets.enable_alert_policies(pochan.project_name, False) + out, _ = capsys.readouterr() + assert "already disabled" in out + + snippets.enable_alert_policies(pochan.project_name, True) + out, _ = capsys.readouterr() + assert "Enabled {0}".format(pochan.project_name) in out + + snippets.enable_alert_policies(pochan.project_name, True) + out, _ = capsys.readouterr() + assert "already enabled" in out + + +def test_replace_channels(capsys, pochan): + alert_policy_id = pochan.alert_policy.name.split('/')[-1] + notification_channel_id = pochan.notification_channel.name.split('/')[-1] + snippets.replace_notification_channels( + pochan.project_name, alert_policy_id, [notification_channel_id]) + out, _ = capsys.readouterr() + assert "Updated {0}".format(pochan.alert_policy.name) in out + + +def test_backup_and_restore(capsys, pochan): + snippets.backup(pochan.project_name) + out, _ = capsys.readouterr() + + snippets.restore(pochan.project_name) + out, _ = capsys.readouterr() + assert "Updated {0}".format(pochan.alert_policy.name) in out + assert "Updating channel {0}".format( + pochan.notification_channel.display_name) in out + + +def test_delete_channels(capsys, pochan): + notification_channel_id = pochan.notification_channel.name.split('/')[-1] + snippets.delete_notification_channels( + pochan.project_name, [notification_channel_id], force=True) + out, _ = capsys.readouterr() + assert "{0} deleted".format(notification_channel_id) in out + pochan.notification_channel.name = '' # So teardown is not tried diff --git a/monitoring/api/v3/alerts-client/test_alert_policy.json b/monitoring/api/v3/alerts-client/test_alert_policy.json new file mode 100644 index 00000000000..d728949f9bb --- /dev/null +++ b/monitoring/api/v3/alerts-client/test_alert_policy.json @@ -0,0 +1,31 @@ +{ + "displayName": "test_alert_policy.json", + "combiner": "OR", + "conditions": [ + { + "conditionThreshold": { + "filter": "metric.label.state=\"blocked\" AND metric.type=\"agent.googleapis.com/processes/count_by_state\" AND resource.type=\"gce_instance\"", + "comparison": "COMPARISON_GT", + "thresholdValue": 100, + "duration": "900s", + "trigger": { + "percent": 0 + }, + "aggregations": [ + { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_MEAN", + "crossSeriesReducer": "REDUCE_MEAN", + "groupByFields": [ + "project", + "resource.label.instance_id", + "resource.label.zone" + ] + } + ] + }, + "displayName": "test_alert_policy.json" + } + ], + "enabled": false +} \ No newline at end of file diff --git a/monitoring/api/v3/alerts-client/test_notification_channel.json b/monitoring/api/v3/alerts-client/test_notification_channel.json new file mode 100644 index 00000000000..6a0d53c00cd --- /dev/null +++ b/monitoring/api/v3/alerts-client/test_notification_channel.json @@ -0,0 +1,15 @@ +{ + "type": "email", + "displayName": "Email joe.", + "description": "test_notification_channel.json", + "labels": { + "email_address": "joe@example.com" + }, + "userLabels": { + "office": "california_westcoast_usa", + "division": "fulfillment", + "role": "operations", + "level": "5" + }, + "enabled": true +} \ No newline at end of file diff --git a/monitoring/api/v3/api-client/README.rst b/monitoring/api/v3/api-client/README.rst new file mode 100644 index 00000000000..b905644a9e8 --- /dev/null +++ b/monitoring/api/v3/api-client/README.rst @@ -0,0 +1,137 @@ +.. This file is automatically generated. Do not edit this file directly. + +Stackdriver Monitoring Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=monitoring/api/v3/api-client/README.rst + + +This directory contains samples for Stackdriver Monitoring. `Stackdriver Monitoring `_ collects metrics, events, and metadata from Google Cloud Platform, Amazon Web Services (AWS), hosted uptime probes, application instrumentation, and a variety of common application components including Cassandra, Nginx, Apache Web Server, Elasticsearch and many others. Stackdriver ingests that data and generates insights via dashboards, charts, and alerts. + + + + +.. _Stackdriver Monitoring: https://cloud.google.com/monitoring/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +List resources ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=monitoring/api/v3/api-client/list_resources.py,monitoring/api/v3/api-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python list_resources.py + + usage: list_resources.py [-h] --project_id PROJECT_ID + + Sample command-line program for retrieving Stackdriver Monitoring API V3 + data. + + See README.md for instructions on setting up your development environment. + + To run locally: + + python list_resources.py --project_id= + + optional arguments: + -h, --help show this help message and exit + --project_id PROJECT_ID + Project ID you want to access. + + + +Custom metrics ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=monitoring/api/v3/api-client/custom_metric.py,monitoring/api/v3/api-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python custom_metric.py + + usage: custom_metric.py [-h] --project_id PROJECT_ID + + Sample command-line program for writing and reading Stackdriver Monitoring + API V3 custom metrics. + + Simple command-line program to demonstrate connecting to the Google + Monitoring API to write custom metrics and read them back. + + See README.md for instructions on setting up your development environment. + + This example creates a custom metric based on a hypothetical GAUGE measurement. + + To run locally: + + python custom_metric.py --project_id= + + optional arguments: + -h, --help show this help message and exit + --project_id PROJECT_ID + Project ID you want to access. + + + + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/monitoring/api/v3/api-client/README.rst.in b/monitoring/api/v3/api-client/README.rst.in new file mode 100644 index 00000000000..e71bd6de87c --- /dev/null +++ b/monitoring/api/v3/api-client/README.rst.in @@ -0,0 +1,27 @@ +# This file is used to generate README.rst + +product: + name: Stackdriver Monitoring + short_name: Stackdriver Monitoring + url: https://cloud.google.com/monitoring/docs + description: > + `Stackdriver Monitoring `_ collects metrics, events, and metadata from + Google Cloud Platform, Amazon Web Services (AWS), hosted uptime probes, + application instrumentation, and a variety of common application components + including Cassandra, Nginx, Apache Web Server, Elasticsearch and many + others. Stackdriver ingests that data and generates insights via + dashboards, charts, and alerts. + +setup: +- auth +- install_deps + +samples: +- name: List resources + file: list_resources.py + show_help: true +- name: Custom metrics + file: custom_metric.py + show_help: true + +folder: monitoring/api/v3/api-client \ No newline at end of file diff --git a/monitoring/api/v3/custom_metric.py b/monitoring/api/v3/api-client/custom_metric.py similarity index 91% rename from monitoring/api/v3/custom_metric.py rename to monitoring/api/v3/api-client/custom_metric.py index 9f420b1aafd..6cf83980357 100644 --- a/monitoring/api/v3/custom_metric.py +++ b/monitoring/api/v3/api-client/custom_metric.py @@ -11,8 +11,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" Sample command-line program for writing and reading Google Monitoring API -V3 custom metrics. +""" Sample command-line program for writing and reading Stackdriver Monitoring +API V3 custom metrics. Simple command-line program to demonstrate connecting to the Google Monitoring API to write custom metrics and read them back. @@ -34,7 +34,7 @@ import random import time -import list_resources +import googleapiclient.discovery def format_rfc3339(datetime_instance=None): @@ -59,14 +59,12 @@ def create_custom_metric(client, project_id, custom_metric_type, metric_kind): """Create custom metric descriptor""" metrics_descriptor = { - "name": "projects/{}/metricDescriptors/{}".format( - project_id, custom_metric_type), "type": custom_metric_type, "labels": [ { "key": "environment", "valueType": "STRING", - "description": "An abritrary measurement" + "description": "An arbitrary measurement" } ], "metricKind": metric_kind, @@ -76,10 +74,17 @@ def create_custom_metric(client, project_id, "displayName": "Custom Metric" } - client.projects().metricDescriptors().create( + return client.projects().metricDescriptors().create( name=project_id, body=metrics_descriptor).execute() +def delete_metric_descriptor( + client, custom_metric_name): + """Delete a custom metric descriptor.""" + client.projects().metricDescriptors().delete( + name=custom_metric_name).execute() + + def get_custom_metric(client, project_id, custom_metric_type): """Retrieve the custom metric we created""" request = client.projects().metricDescriptors().list( @@ -102,6 +107,7 @@ def get_custom_data_point(): return length +# [START write_timeseries] def write_timeseries_value(client, project_resource, custom_metric_type, instance_id, metric_kind): """Write the custom metric obtained by get_custom_data_point at a point in @@ -122,8 +128,6 @@ def write_timeseries_value(client, project_resource, 'zone': 'us-central1-f' } }, - "metricKind": metric_kind, - "valueType": "INT64", "points": [ { "interval": { @@ -140,6 +144,7 @@ def write_timeseries_value(client, project_resource, request = client.projects().timeSeries().create( name=project_resource, body={"timeSeries": [timeseries_data]}) request.execute() +# [END write_timeseries] def read_timeseries(client, project_resource, custom_metric_type): @@ -168,7 +173,7 @@ def main(project_id): METRIC_KIND = "GAUGE" project_resource = "projects/{0}".format(project_id) - client = list_resources.get_client() + client = googleapiclient.discovery.build('monitoring', 'v3') create_custom_metric(client, project_resource, CUSTOM_METRIC_TYPE, METRIC_KIND) custom_metric = None diff --git a/monitoring/api/v3/api-client/custom_metric_test.py b/monitoring/api/v3/api-client/custom_metric_test.py new file mode 100644 index 00000000000..e2b0d8f366c --- /dev/null +++ b/monitoring/api/v3/api-client/custom_metric_test.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Integration test for custom_metric.py + +GOOGLE_APPLICATION_CREDENTIALS must be set to a Service Account for a project +that has enabled the Monitoring API. + +Currently the TEST_PROJECT_ID is hard-coded to run using the project created +for this test, but it could be changed to a different project. +""" + +import os +import random +import time + +from gcp_devrel.testing import eventually_consistent +from gcp_devrel.testing.flaky import flaky +import googleapiclient.discovery +import pytest + +from custom_metric import create_custom_metric +from custom_metric import delete_metric_descriptor +from custom_metric import get_custom_metric +from custom_metric import read_timeseries +from custom_metric import write_timeseries_value + +PROJECT = os.environ['GCLOUD_PROJECT'] + +""" Custom metric domain for all custom metrics""" +CUSTOM_METRIC_DOMAIN = "custom.googleapis.com" + +METRIC = 'compute.googleapis.com/instance/cpu/usage_time' +METRIC_NAME = ''.join( + random.choice('0123456789ABCDEF') for i in range(16)) +METRIC_RESOURCE = "{}/{}".format( + CUSTOM_METRIC_DOMAIN, METRIC_NAME) + + +@pytest.fixture(scope='module') +def client(): + return googleapiclient.discovery.build('monitoring', 'v3') + + +@flaky +def test_custom_metric(client): + PROJECT_RESOURCE = "projects/{}".format(PROJECT) + # Use a constant seed so psuedo random number is known ahead of time + random.seed(1) + pseudo_random_value = random.randint(0, 10) + # Reseed it + random.seed(1) + + INSTANCE_ID = "test_instance" + METRIC_KIND = "GAUGE" + + custom_metric_descriptor = create_custom_metric( + client, PROJECT_RESOURCE, METRIC_RESOURCE, METRIC_KIND) + + # wait until metric has been created, use the get call to wait until + # a response comes back with the new metric + custom_metric = None + while not custom_metric: + time.sleep(1) + custom_metric = get_custom_metric( + client, PROJECT_RESOURCE, METRIC_RESOURCE) + + write_timeseries_value(client, PROJECT_RESOURCE, + METRIC_RESOURCE, INSTANCE_ID, + METRIC_KIND) + + # Sometimes on new metric descriptors, writes have a delay in being + # read back. Use eventually_consistent to account for this. + @eventually_consistent.call + def _(): + response = read_timeseries(client, PROJECT_RESOURCE, METRIC_RESOURCE) + value = int( + response['timeSeries'][0]['points'][0]['value']['int64Value']) + # using seed of 1 will create a value of 1 + assert value == pseudo_random_value + + delete_metric_descriptor(client, custom_metric_descriptor['name']) diff --git a/monitoring/api/v3/list_resources.py b/monitoring/api/v3/api-client/list_resources.py similarity index 89% rename from monitoring/api/v3/list_resources.py rename to monitoring/api/v3/api-client/list_resources.py index 2d088f414ab..0c4e27a8259 100644 --- a/monitoring/api/v3/list_resources.py +++ b/monitoring/api/v3/api-client/list_resources.py @@ -11,7 +11,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" Sample command-line program for retrieving Google Monitoring API V3 data. +""" Sample command-line program for retrieving Stackdriver Monitoring API V3 +data. See README.md for instructions on setting up your development environment. @@ -26,8 +27,7 @@ import datetime import pprint -from apiclient import discovery -from oauth2client.client import GoogleCredentials +import googleapiclient.discovery def format_rfc3339(datetime_instance): @@ -95,17 +95,10 @@ def list_timeseries(client, project_resource, metric): print('list_timeseries response:\n{}'.format(pprint.pformat(response))) -def get_client(): - """Builds an http client authenticated with the service account - credentials.""" - credentials = GoogleCredentials.get_application_default() - client = discovery.build('monitoring', 'v3', credentials=credentials) - return client - - def main(project_id): + client = googleapiclient.discovery.build('monitoring', 'v3') + project_resource = "projects/{}".format(project_id) - client = get_client() list_monitored_resource_descriptors(client, project_resource) # Metric to list metric = 'compute.googleapis.com/instance/cpu/usage_time' diff --git a/monitoring/api/v3/api-client/list_resources_test.py b/monitoring/api/v3/api-client/list_resources_test.py new file mode 100644 index 00000000000..f2a9bbf035d --- /dev/null +++ b/monitoring/api/v3/api-client/list_resources_test.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Integration test for list_env.py + +GOOGLE_APPLICATION_CREDENTIALS must be set to a Service Account for a project +that has enabled the Monitoring API. + +Currently the TEST_PROJECT_ID is hard-coded to run using the project created +for this test, but it could be changed to a different project. +""" + +import os +import re + +from gcp_devrel.testing.flaky import flaky +import googleapiclient.discovery +import pytest + +import list_resources + +PROJECT = os.environ['GCLOUD_PROJECT'] +METRIC = 'compute.googleapis.com/instance/cpu/usage_time' + + +@pytest.fixture(scope='module') +def client(): + return googleapiclient.discovery.build('monitoring', 'v3') + + +@flaky +def test_list_monitored_resources(client, capsys): + PROJECT_RESOURCE = "projects/{}".format(PROJECT) + list_resources.list_monitored_resource_descriptors( + client, PROJECT_RESOURCE) + stdout, _ = capsys.readouterr() + regex = re.compile( + 'An application running', re.I) + assert regex.search(stdout) is not None + + +@flaky +def test_list_metrics(client, capsys): + PROJECT_RESOURCE = "projects/{}".format(PROJECT) + list_resources.list_metric_descriptors( + client, PROJECT_RESOURCE, METRIC) + stdout, _ = capsys.readouterr() + regex = re.compile( + u'Delta CPU', re.I) + assert regex.search(stdout) is not None + + +@flaky +def test_list_timeseries(client, capsys): + PROJECT_RESOURCE = "projects/{}".format(PROJECT) + list_resources.list_timeseries( + client, PROJECT_RESOURCE, METRIC) + stdout, _ = capsys.readouterr() + regex = re.compile(u'list_timeseries response:\n', re.I) + assert regex.search(stdout) is not None diff --git a/monitoring/api/v3/api-client/requirements.txt b/monitoring/api/v3/api-client/requirements.txt new file mode 100644 index 00000000000..7e4359ce08d --- /dev/null +++ b/monitoring/api/v3/api-client/requirements.txt @@ -0,0 +1,3 @@ +google-api-python-client==1.7.8 +google-auth==1.6.2 +google-auth-httplib2==0.0.3 diff --git a/monitoring/api/v3/cloud-client/README.rst b/monitoring/api/v3/cloud-client/README.rst new file mode 100644 index 00000000000..18d2acccf7d --- /dev/null +++ b/monitoring/api/v3/cloud-client/README.rst @@ -0,0 +1,139 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Stackdriver Monitoring API Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=monitoring/api/v3/cloud-client/README.rst + + +This directory contains samples for Google Stackdriver Monitoring API. Stackdriver Monitoring collects metrics, events, and metadata from Google Cloud Platform, Amazon Web Services (AWS), hosted uptime probes, application instrumentation, and a variety of common application components including Cassandra, Nginx, Apache Web Server, Elasticsearch + and many others. Stackdriver ingests that data and generates insights + via dashboards, charts, and alerts. + + + + +.. _Google Stackdriver Monitoring API: https://cloud.google.com/monitoring/docs/ + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Quickstart ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=monitoring/api/v3/cloud-client/quickstart.py,monitoring/api/v3/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python quickstart.py + + +Snippets ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=monitoring/api/v3/cloud-client/snippets.py,monitoring/api/v3/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python snippets.py + + usage: snippets.py [-h] + {create-metric-descriptor,list-metric-descriptors,get-metric-descriptor,delete-metric-descriptor,list-resources,get-resource,write-time-series,list-time-series,list-time-series-header,list-time-series-reduce,list-time-series-aggregate} + ... + + Demonstrates Monitoring API operations. + + positional arguments: + {create-metric-descriptor,list-metric-descriptors,get-metric-descriptor,delete-metric-descriptor,list-resources,get-resource,write-time-series,list-time-series,list-time-series-header,list-time-series-reduce,list-time-series-aggregate} + create-metric-descriptor + list-metric-descriptors + get-metric-descriptor + delete-metric-descriptor + list-resources + get-resource + write-time-series + list-time-series + list-time-series-header + list-time-series-reduce + list-time-series-aggregate + + optional arguments: + -h, --help show this help message and exit + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/monitoring/api/v3/cloud-client/README.rst.in b/monitoring/api/v3/cloud-client/README.rst.in new file mode 100644 index 00000000000..d6a0dd46099 --- /dev/null +++ b/monitoring/api/v3/cloud-client/README.rst.in @@ -0,0 +1,28 @@ +# This file is used to generate README.rst + +product: + name: Google Stackdriver Monitoring API + short_name: Stackdriver Monitoring API + url: https://cloud.google.com/monitoring/docs/ + description: > + Stackdriver Monitoring collects metrics, events, and metadata from Google + Cloud Platform, Amazon Web Services (AWS), hosted uptime probes, + application instrumentation, and a variety of common application + components including Cassandra, Nginx, Apache Web Server, Elasticsearch + and many others. Stackdriver ingests that data and generates insights + via dashboards, charts, and alerts. + +setup: +- auth +- install_deps + +samples: +- name: Quickstart + file: quickstart.py +- name: Snippets + file: snippets.py + show_help: true + +cloud_client_library: true + +folder: monitoring/api/v3/cloud-client \ No newline at end of file diff --git a/monitoring/api/v3/cloud-client/quickstart.py b/monitoring/api/v3/cloud-client/quickstart.py new file mode 100644 index 00000000000..0527acae545 --- /dev/null +++ b/monitoring/api/v3/cloud-client/quickstart.py @@ -0,0 +1,43 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def run_quickstart(): + # [START monitoring_quickstart] + from google.cloud import monitoring_v3 + + import time + + client = monitoring_v3.MetricServiceClient() + project = 'my-project' # TODO: Update to your project ID. + project_name = client.project_path(project) + + series = monitoring_v3.types.TimeSeries() + series.metric.type = 'custom.googleapis.com/my_metric' + series.resource.type = 'gce_instance' + series.resource.labels['instance_id'] = '1234567890123456789' + series.resource.labels['zone'] = 'us-central1-f' + point = series.points.add() + point.value.double_value = 3.14 + now = time.time() + point.interval.end_time.seconds = int(now) + point.interval.end_time.nanos = int( + (now - point.interval.end_time.seconds) * 10**9) + client.create_time_series(project_name, [series]) + print('Successfully wrote time series.') + # [END monitoring_quickstart] + + +if __name__ == '__main__': + run_quickstart() diff --git a/monitoring/api/v3/cloud-client/quickstart_test.py b/monitoring/api/v3/cloud-client/quickstart_test.py new file mode 100644 index 00000000000..d9b54b62c48 --- /dev/null +++ b/monitoring/api/v3/cloud-client/quickstart_test.py @@ -0,0 +1,41 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import mock +import pytest + +import quickstart + + +PROJECT = os.environ['GCLOUD_PROJECT'] + + +@pytest.fixture +def mock_project_path(): + """Mock out project and replace with project from environment.""" + project_patch = mock.patch( + 'google.cloud.monitoring_v3.MetricServiceClient.' + 'project_path') + + with project_patch as project_mock: + project_mock.return_value = 'projects/{}'.format(PROJECT) + yield project_mock + + +def test_quickstart(capsys, mock_project_path): + quickstart.run_quickstart() + out, _ = capsys.readouterr() + assert 'wrote' in out diff --git a/monitoring/api/v3/cloud-client/requirements.txt b/monitoring/api/v3/cloud-client/requirements.txt new file mode 100644 index 00000000000..ae711707fd7 --- /dev/null +++ b/monitoring/api/v3/cloud-client/requirements.txt @@ -0,0 +1 @@ +google-cloud-monitoring==0.31.1 diff --git a/monitoring/api/v3/cloud-client/snippets.py b/monitoring/api/v3/cloud-client/snippets.py new file mode 100644 index 00000000000..7bfd829d7bc --- /dev/null +++ b/monitoring/api/v3/cloud-client/snippets.py @@ -0,0 +1,334 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import pprint +import random +import time + +from google.cloud import monitoring_v3 + + +# Avoid collisions with other runs +RANDOM_SUFFIX = str(random.randint(1000, 9999)) + + +def create_metric_descriptor(project_id): + # [START monitoring_create_metric] + client = monitoring_v3.MetricServiceClient() + project_name = client.project_path(project_id) + descriptor = monitoring_v3.types.MetricDescriptor() + descriptor.type = 'custom.googleapis.com/my_metric' + RANDOM_SUFFIX + descriptor.metric_kind = ( + monitoring_v3.enums.MetricDescriptor.MetricKind.GAUGE) + descriptor.value_type = ( + monitoring_v3.enums.MetricDescriptor.ValueType.DOUBLE) + descriptor.description = 'This is a simple example of a custom metric.' + descriptor = client.create_metric_descriptor(project_name, descriptor) + print('Created {}.'.format(descriptor.name)) + # [END monitoring_create_metric] + + +def delete_metric_descriptor(descriptor_name): + # [START monitoring_delete_metric] + client = monitoring_v3.MetricServiceClient() + client.delete_metric_descriptor(descriptor_name) + print('Deleted metric descriptor {}.'.format(descriptor_name)) + # [END monitoring_delete_metric] + + +def write_time_series(project_id): + # [START monitoring_write_timeseries] + client = monitoring_v3.MetricServiceClient() + project_name = client.project_path(project_id) + + series = monitoring_v3.types.TimeSeries() + series.metric.type = 'custom.googleapis.com/my_metric' + RANDOM_SUFFIX + series.resource.type = 'gce_instance' + series.resource.labels['instance_id'] = '1234567890123456789' + series.resource.labels['zone'] = 'us-central1-f' + point = series.points.add() + point.value.double_value = 3.14 + now = time.time() + point.interval.end_time.seconds = int(now) + point.interval.end_time.nanos = int( + (now - point.interval.end_time.seconds) * 10**9) + client.create_time_series(project_name, [series]) + # [END monitoring_write_timeseries] + + +def list_time_series(project_id): + # [START monitoring_read_timeseries_simple] + client = monitoring_v3.MetricServiceClient() + project_name = client.project_path(project_id) + interval = monitoring_v3.types.TimeInterval() + now = time.time() + interval.end_time.seconds = int(now) + interval.end_time.nanos = int( + (now - interval.end_time.seconds) * 10**9) + interval.start_time.seconds = int(now - 300) + interval.start_time.nanos = interval.end_time.nanos + results = client.list_time_series( + project_name, + 'metric.type = "compute.googleapis.com/instance/cpu/utilization"', + interval, + monitoring_v3.enums.ListTimeSeriesRequest.TimeSeriesView.FULL) + for result in results: + print(result) + # [END monitoring_read_timeseries_simple] + + +def list_time_series_header(project_id): + # [START monitoring_read_timeseries_fields] + client = monitoring_v3.MetricServiceClient() + project_name = client.project_path(project_id) + interval = monitoring_v3.types.TimeInterval() + now = time.time() + interval.end_time.seconds = int(now) + interval.end_time.nanos = int( + (now - interval.end_time.seconds) * 10**9) + interval.start_time.seconds = int(now - 300) + interval.start_time.nanos = interval.end_time.nanos + results = client.list_time_series( + project_name, + 'metric.type = "compute.googleapis.com/instance/cpu/utilization"', + interval, + monitoring_v3.enums.ListTimeSeriesRequest.TimeSeriesView.HEADERS) + for result in results: + print(result) + # [END monitoring_read_timeseries_fields] + + +def list_time_series_aggregate(project_id): + # [START monitoring_read_timeseries_align] + client = monitoring_v3.MetricServiceClient() + project_name = client.project_path(project_id) + interval = monitoring_v3.types.TimeInterval() + now = time.time() + interval.end_time.seconds = int(now) + interval.end_time.nanos = int( + (now - interval.end_time.seconds) * 10**9) + interval.start_time.seconds = int(now - 3600) + interval.start_time.nanos = interval.end_time.nanos + aggregation = monitoring_v3.types.Aggregation() + aggregation.alignment_period.seconds = 300 # 5 minutes + aggregation.per_series_aligner = ( + monitoring_v3.enums.Aggregation.Aligner.ALIGN_MEAN) + + results = client.list_time_series( + project_name, + 'metric.type = "compute.googleapis.com/instance/cpu/utilization"', + interval, + monitoring_v3.enums.ListTimeSeriesRequest.TimeSeriesView.FULL, + aggregation) + for result in results: + print(result) + # [END monitoring_read_timeseries_align] + + +def list_time_series_reduce(project_id): + # [START monitoring_read_timeseries_reduce] + client = monitoring_v3.MetricServiceClient() + project_name = client.project_path(project_id) + interval = monitoring_v3.types.TimeInterval() + now = time.time() + interval.end_time.seconds = int(now) + interval.end_time.nanos = int( + (now - interval.end_time.seconds) * 10**9) + interval.start_time.seconds = int(now - 3600) + interval.start_time.nanos = interval.end_time.nanos + aggregation = monitoring_v3.types.Aggregation() + aggregation.alignment_period.seconds = 300 # 5 minutes + aggregation.per_series_aligner = ( + monitoring_v3.enums.Aggregation.Aligner.ALIGN_MEAN) + aggregation.cross_series_reducer = ( + monitoring_v3.enums.Aggregation.Reducer.REDUCE_MEAN) + aggregation.group_by_fields.append('resource.zone') + + results = client.list_time_series( + project_name, + 'metric.type = "compute.googleapis.com/instance/cpu/utilization"', + interval, + monitoring_v3.enums.ListTimeSeriesRequest.TimeSeriesView.FULL, + aggregation) + for result in results: + print(result) + # [END monitoring_read_timeseries_reduce] + + +def list_metric_descriptors(project_id): + # [START monitoring_list_descriptors] + client = monitoring_v3.MetricServiceClient() + project_name = client.project_path(project_id) + for descriptor in client.list_metric_descriptors(project_name): + print(descriptor.type) + # [END monitoring_list_descriptors] + + +def list_monitored_resources(project_id): + # [START monitoring_list_resources] + client = monitoring_v3.MetricServiceClient() + project_name = client.project_path(project_id) + resource_descriptors = ( + client.list_monitored_resource_descriptors(project_name)) + for descriptor in resource_descriptors: + print(descriptor.type) + # [END monitoring_list_resources] + + +def get_monitored_resource_descriptor(project_id, resource_type_name): + # [START monitoring_get_resource] + client = monitoring_v3.MetricServiceClient() + resource_path = client.monitored_resource_descriptor_path( + project_id, resource_type_name) + pprint.pprint(client.get_monitored_resource_descriptor(resource_path)) + # [END monitoring_get_resource] + + +def get_metric_descriptor(metric_name): + # [START monitoring_get_descriptor] + client = monitoring_v3.MetricServiceClient() + descriptor = client.get_metric_descriptor(metric_name) + pprint.pprint(descriptor) + # [END monitoring_get_descriptor] + + +class MissingProjectIdError(Exception): + pass + + +def project_id(): + """Retreives the project id from the environment variable. + + Raises: + MissingProjectIdError -- When not set. + + Returns: + str -- the project name + """ + project_id = (os.environ['GOOGLE_CLOUD_PROJECT'] or + os.environ['GCLOUD_PROJECT']) + + if not project_id: + raise MissingProjectIdError( + 'Set the environment variable ' + + 'GCLOUD_PROJECT to your Google Cloud Project Id.') + return project_id + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Demonstrates Monitoring API operations.') + + subparsers = parser.add_subparsers(dest='command') + + create_metric_descriptor_parser = subparsers.add_parser( + 'create-metric-descriptor', + help=create_metric_descriptor.__doc__ + ) + + list_metric_descriptor_parser = subparsers.add_parser( + 'list-metric-descriptors', + help=list_metric_descriptors.__doc__ + ) + + get_metric_descriptor_parser = subparsers.add_parser( + 'get-metric-descriptor', + help=get_metric_descriptor.__doc__ + ) + + get_metric_descriptor_parser.add_argument( + '--metric-type-name', + help='The metric type of the metric descriptor to see details about.', + required=True + ) + + delete_metric_descriptor_parser = subparsers.add_parser( + 'delete-metric-descriptor', + help=list_metric_descriptors.__doc__ + ) + + delete_metric_descriptor_parser.add_argument( + '--metric-descriptor-name', + help='Metric descriptor to delete', + required=True + ) + + list_resources_parser = subparsers.add_parser( + 'list-resources', + help=list_monitored_resources.__doc__ + ) + + get_resource_parser = subparsers.add_parser( + 'get-resource', + help=get_monitored_resource_descriptor.__doc__ + ) + + get_resource_parser.add_argument( + '--resource-type-name', + help='Monitored resource to view more information about.', + required=True + ) + + write_time_series_parser = subparsers.add_parser( + 'write-time-series', + help=write_time_series.__doc__ + ) + + list_time_series_parser = subparsers.add_parser( + 'list-time-series', + help=list_time_series.__doc__ + ) + + list_time_series_header_parser = subparsers.add_parser( + 'list-time-series-header', + help=list_time_series_header.__doc__ + ) + + read_time_series_reduce = subparsers.add_parser( + 'list-time-series-reduce', + help=list_time_series_reduce.__doc__ + ) + + read_time_series_aggregate = subparsers.add_parser( + 'list-time-series-aggregate', + help=list_time_series_aggregate.__doc__ + ) + + args = parser.parse_args() + + if args.command == 'create-metric-descriptor': + create_metric_descriptor(project_id()) + if args.command == 'list-metric-descriptors': + list_metric_descriptors(project_id()) + if args.command == 'get-metric-descriptor': + get_metric_descriptor(args.metric_type_name) + if args.command == 'delete-metric-descriptor': + delete_metric_descriptor(args.metric_descriptor_name) + if args.command == 'list-resources': + list_monitored_resources(project_id()) + if args.command == 'get-resource': + get_monitored_resource_descriptor( + project_id(), args.resource_type_name) + if args.command == 'write-time-series': + write_time_series(project_id()) + if args.command == 'list-time-series': + list_time_series(project_id()) + if args.command == 'list-time-series-header': + list_time_series_header(project_id()) + if args.command == 'list-time-series-reduce': + list_time_series_reduce(project_id()) + if args.command == 'list-time-series-aggregate': + list_time_series_aggregate(project_id()) diff --git a/monitoring/api/v3/cloud-client/snippets_test.py b/monitoring/api/v3/cloud-client/snippets_test.py new file mode 100644 index 00000000000..bff93575df3 --- /dev/null +++ b/monitoring/api/v3/cloud-client/snippets_test.py @@ -0,0 +1,82 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +from gcp_devrel.testing import eventually_consistent + +import snippets + + +def test_create_get_delete_metric_descriptor(capsys): + snippets.create_metric_descriptor(snippets.project_id()) + out, _ = capsys.readouterr() + match = re.search(r'Created (.*)\.', out) + metric_name = match.group(1) + try: + @eventually_consistent.call + def __(): + snippets.get_metric_descriptor(metric_name) + + out, _ = capsys.readouterr() + assert 'DOUBLE' in out + finally: + snippets.delete_metric_descriptor(metric_name) + out, _ = capsys.readouterr() + assert 'Deleted metric' in out + + +def test_list_metric_descriptors(capsys): + snippets.list_metric_descriptors(snippets.project_id()) + out, _ = capsys.readouterr() + assert 'logging.googleapis.com/byte_count' in out + + +def test_list_resources(capsys): + snippets.list_monitored_resources(snippets.project_id()) + out, _ = capsys.readouterr() + assert 'pubsub_topic' in out + + +def test_get_resources(capsys): + snippets.get_monitored_resource_descriptor( + snippets.project_id(), 'pubsub_topic') + out, _ = capsys.readouterr() + assert 'A topic in Google Cloud Pub/Sub' in out + + +def test_time_series(capsys): + snippets.write_time_series(snippets.project_id()) + + snippets.list_time_series(snippets.project_id()) + out, _ = capsys.readouterr() + assert 'gce_instance' in out + + snippets.list_time_series_header(snippets.project_id()) + out, _ = capsys.readouterr() + assert 'gce_instance' in out + + snippets.list_time_series_aggregate(snippets.project_id()) + out, _ = capsys.readouterr() + assert 'points' in out + assert 'interval' in out + assert 'start_time' in out + assert 'end_time' in out + + snippets.list_time_series_reduce(snippets.project_id()) + out, _ = capsys.readouterr() + assert 'points' in out + assert 'interval' in out + assert 'start_time' in out + assert 'end_time' in out diff --git a/monitoring/api/v3/custom_metric_test.py b/monitoring/api/v3/custom_metric_test.py deleted file mode 100644 index 3d457de4d39..00000000000 --- a/monitoring/api/v3/custom_metric_test.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env python -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" Integration test for custom_metric.py - -GOOGLE_APPLICATION_CREDENTIALS must be set to a Service Account for a project -that has enabled the Monitoring API. - -Currently the TEST_PROJECT_ID is hard-coded to run using the project created -for this test, but it could be changed to a different project. -""" - -import random -import time - -from custom_metric import create_custom_metric, get_custom_metric -from custom_metric import read_timeseries, write_timeseries_value -import list_resources - -""" Custom metric domain for all custom metrics""" -CUSTOM_METRIC_DOMAIN = "custom.googleapis.com" - -METRIC = 'compute.googleapis.com/instance/cpu/usage_time' -METRIC_NAME = ''.join( - random.choice('0123456789ABCDEF') for i in range(16)) -METRIC_RESOURCE = "{}/{}".format( - CUSTOM_METRIC_DOMAIN, METRIC_NAME) - - -def test_custom_metric(cloud_config): - PROJECT_RESOURCE = "projects/{}".format(cloud_config.project) - client = list_resources.get_client() - # Use a constant seed so psuedo random number is known ahead of time - random.seed(1) - pseudo_random_value = random.randint(0, 10) - # Reseed it - random.seed(1) - - INSTANCE_ID = "test_instance" - METRIC_KIND = "GAUGE" - - create_custom_metric( - client, PROJECT_RESOURCE, METRIC_RESOURCE, METRIC_KIND) - custom_metric = None - # wait until metric has been created, use the get call to wait until - # a response comes back with the new metric - while not custom_metric: - time.sleep(1) - custom_metric = get_custom_metric( - client, PROJECT_RESOURCE, METRIC_RESOURCE) - - write_timeseries_value(client, PROJECT_RESOURCE, - METRIC_RESOURCE, INSTANCE_ID, - METRIC_KIND) - # Sometimes on new metric descriptors, writes have a delay in being - # read back. 3 seconds should be enough to make sure our read call - # picks up the write - time.sleep(3) - response = read_timeseries(client, PROJECT_RESOURCE, METRIC_RESOURCE) - value = int( - response['timeSeries'][0]['points'][0]['value']['int64Value']) - # using seed of 1 will create a value of 1 - assert value == pseudo_random_value diff --git a/monitoring/api/v3/list_resources_test.py b/monitoring/api/v3/list_resources_test.py deleted file mode 100644 index 3669aa9f2c2..00000000000 --- a/monitoring/api/v3/list_resources_test.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" Integration test for list_env.py - -GOOGLE_APPLICATION_CREDENTIALS must be set to a Service Account for a project -that has enabled the Monitoring API. - -Currently the TEST_PROJECT_ID is hard-coded to run using the project created -for this test, but it could be changed to a different project. -""" - -import re - -import list_resources - -METRIC = 'compute.googleapis.com/instance/cpu/usage_time' - - -def test_list_monitored_resources(cloud_config, capsys): - PROJECT_RESOURCE = "projects/{}".format(cloud_config.project) - client = list_resources.get_client() - list_resources.list_monitored_resource_descriptors( - client, PROJECT_RESOURCE) - stdout, _ = capsys.readouterr() - regex = re.compile( - 'An application running') - assert regex.search(stdout) is not None - - -def test_list_metrics(cloud_config, capsys): - PROJECT_RESOURCE = "projects/{}".format(cloud_config.project) - client = list_resources.get_client() - list_resources.list_metric_descriptors( - client, PROJECT_RESOURCE, METRIC) - stdout, _ = capsys.readouterr() - regex = re.compile( - u'Delta CPU usage time') - assert regex.search(stdout) is not None - - -def test_list_timeseries(cloud_config, capsys): - PROJECT_RESOURCE = "projects/{}".format(cloud_config.project) - client = list_resources.get_client() - list_resources.list_timeseries( - client, PROJECT_RESOURCE, METRIC) - stdout, _ = capsys.readouterr() - regex = re.compile(u'list_timeseries response:\n') - assert regex.search(stdout) is not None diff --git a/monitoring/api/v3/requirements.txt b/monitoring/api/v3/requirements.txt deleted file mode 100644 index 1cf239c12f3..00000000000 --- a/monitoring/api/v3/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -google-api-python-client==1.5.0 -httplib2==0.9.2 -oauth2client==2.0.1 -pyasn1==0.1.9 -pyasn1-modules==0.0.8 -rsa==3.3 -simplejson==3.8.2 -six==1.10.0 -uritemplate==0.6 -wheel==0.24.0 diff --git a/monitoring/api/v3/uptime-check-client/README.rst b/monitoring/api/v3/uptime-check-client/README.rst new file mode 100644 index 00000000000..30046bdef9d --- /dev/null +++ b/monitoring/api/v3/uptime-check-client/README.rst @@ -0,0 +1,115 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Stackdriver Uptime Checks API Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=monitoring/api/v3/uptime-check-client/README.rst + + +This directory contains samples for Google Stackdriver Uptime Checks API. Stackdriver Monitoring collects metrics, events, and metadata from Google Cloud Platform, Amazon Web Services (AWS), hosted uptime probes, application instrumentation, and a variety of common application components including Cassandra, Nginx, Apache Web Server, Elasticsearch and many others. Stackdriver's Uptime Checks API allows you to create, delete, and list your project's Uptime Checks. + + + + +.. _Google Stackdriver Uptime Checks API: https://cloud.google.com/monitoring/uptime-checks/management + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Snippets ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=monitoring/api/v3/uptime-check-client/snippets.py,monitoring/api/v3/uptime-check-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python snippets.py + + usage: snippets.py [-h] + {list-uptime-check-configs,list-uptime-check-ips,create-uptime-check,get-uptime-check-config,delete-uptime-check-config} + ... + + Demonstrates Uptime Check API operations. + + positional arguments: + {list-uptime-check-configs,list-uptime-check-ips,create-uptime-check,get-uptime-check-config,delete-uptime-check-config} + list-uptime-check-configs + list-uptime-check-ips + create-uptime-check + get-uptime-check-config + delete-uptime-check-config + + optional arguments: + -h, --help show this help message and exit + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/monitoring/api/v3/uptime-check-client/README.rst.in b/monitoring/api/v3/uptime-check-client/README.rst.in new file mode 100644 index 00000000000..1174962e48d --- /dev/null +++ b/monitoring/api/v3/uptime-check-client/README.rst.in @@ -0,0 +1,26 @@ +# This file is used to generate README.rst + +product: + name: Google Stackdriver Uptime Checks API + short_name: Stackdriver Uptime Checks API + url: https://cloud.google.com/monitoring/uptime-checks/management + description: > + Stackdriver Monitoring collects metrics, events, and metadata from Google + Cloud Platform, Amazon Web Services (AWS), hosted uptime probes, + application instrumentation, and a variety of common application + components including Cassandra, Nginx, Apache Web Server, Elasticsearch + and many others. Stackdriver's Uptime Checks API allows you to create, + delete, and list your project's Uptime Checks. + +setup: +- auth +- install_deps + +samples: +- name: Snippets + file: snippets.py + show_help: true + +cloud_client_library: true + +folder: monitoring/api/v3/uptime-check-client \ No newline at end of file diff --git a/monitoring/api/v3/uptime-check-client/requirements.txt b/monitoring/api/v3/uptime-check-client/requirements.txt new file mode 100644 index 00000000000..807c0443c4a --- /dev/null +++ b/monitoring/api/v3/uptime-check-client/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-monitoring==0.31.1 +tabulate==0.8.3 diff --git a/monitoring/api/v3/uptime-check-client/snippets.py b/monitoring/api/v3/uptime-check-client/snippets.py new file mode 100644 index 00000000000..78c4a5f6394 --- /dev/null +++ b/monitoring/api/v3/uptime-check-client/snippets.py @@ -0,0 +1,213 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import argparse +import os +import pprint + +from google.cloud import monitoring_v3 +import tabulate + + +# [START monitoring_uptime_check_create] +def create_uptime_check_config(project_name, host_name=None, + display_name=None): + config = monitoring_v3.types.uptime_pb2.UptimeCheckConfig() + config.display_name = display_name or 'New uptime check' + config.monitored_resource.type = 'uptime_url' + config.monitored_resource.labels.update( + {'host': host_name or 'example.com'}) + config.http_check.path = '/' + config.http_check.port = 80 + config.timeout.seconds = 10 + config.period.seconds = 300 + + client = monitoring_v3.UptimeCheckServiceClient() + new_config = client.create_uptime_check_config(project_name, config) + pprint.pprint(new_config) + return new_config +# [END monitoring_uptime_check_create] + + +# [START monitoring_uptime_check_update] +def update_uptime_check_config(config_name, new_display_name=None, + new_http_check_path=None): + client = monitoring_v3.UptimeCheckServiceClient() + config = client.get_uptime_check_config(config_name) + field_mask = monitoring_v3.types.FieldMask() + if new_display_name: + field_mask.paths.append('display_name') + config.display_name = new_display_name + if new_http_check_path: + field_mask.paths.append('http_check.path') + config.http_check.path = new_http_check_path + client.update_uptime_check_config(config, field_mask) +# [END monitoring_uptime_check_update] + + +# [START monitoring_uptime_check_list_configs] +def list_uptime_check_configs(project_name): + client = monitoring_v3.UptimeCheckServiceClient() + configs = client.list_uptime_check_configs(project_name) + + for config in configs: + pprint.pprint(config) +# [END monitoring_uptime_check_list_configs] + + +# [START monitoring_uptime_check_list_ips] +def list_uptime_check_ips(): + client = monitoring_v3.UptimeCheckServiceClient() + ips = client.list_uptime_check_ips() + print(tabulate.tabulate( + [(ip.region, ip.location, ip.ip_address) for ip in ips], + ('region', 'location', 'ip_address') + )) +# [END monitoring_uptime_check_list_ips] + + +# [START monitoring_uptime_check_get] +def get_uptime_check_config(config_name): + client = monitoring_v3.UptimeCheckServiceClient() + config = client.get_uptime_check_config(config_name) + pprint.pprint(config) +# [END monitoring_uptime_check_get] + + +# [START monitoring_uptime_check_delete] +def delete_uptime_check_config(config_name): + client = monitoring_v3.UptimeCheckServiceClient() + client.delete_uptime_check_config(config_name) + print('Deleted ', config_name) +# [END monitoring_uptime_check_delete] + + +class MissingProjectIdError(Exception): + pass + + +def project_id(): + """Retreieves the project id from the environment variable. + + Raises: + MissingProjectIdError -- When not set. + + Returns: + str -- the project name + """ + project_id = os.environ['GCLOUD_PROJECT'] + + if not project_id: + raise MissingProjectIdError( + 'Set the environment variable ' + + 'GCLOUD_PROJECT to your Google Cloud Project Id.') + return project_id + + +def project_name(): + return 'projects/' + project_id() + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser( + description='Demonstrates Uptime Check API operations.') + + subparsers = parser.add_subparsers(dest='command') + + list_uptime_check_configs_parser = subparsers.add_parser( + 'list-uptime-check-configs', + help=list_uptime_check_configs.__doc__ + ) + + list_uptime_check_ips_parser = subparsers.add_parser( + 'list-uptime-check-ips', + help=list_uptime_check_ips.__doc__ + ) + + create_uptime_check_config_parser = subparsers.add_parser( + 'create-uptime-check', + help=create_uptime_check_config.__doc__ + ) + create_uptime_check_config_parser.add_argument( + '-d', '--display_name', + required=False, + ) + create_uptime_check_config_parser.add_argument( + '-o', '--host_name', + required=False, + ) + + get_uptime_check_config_parser = subparsers.add_parser( + 'get-uptime-check-config', + help=get_uptime_check_config.__doc__ + ) + get_uptime_check_config_parser.add_argument( + '-m', '--name', + required=True, + ) + + delete_uptime_check_config_parser = subparsers.add_parser( + 'delete-uptime-check-config', + help=delete_uptime_check_config.__doc__ + ) + delete_uptime_check_config_parser.add_argument( + '-m', '--name', + required=True, + ) + + update_uptime_check_config_parser = subparsers.add_parser( + 'update-uptime-check-config', + help=update_uptime_check_config.__doc__ + ) + update_uptime_check_config_parser.add_argument( + '-m', '--name', + required=True, + ) + update_uptime_check_config_parser.add_argument( + '-d', '--display_name', + required=False, + ) + update_uptime_check_config_parser.add_argument( + '-p', '--uptime_check_path', + required=False, + ) + + args = parser.parse_args() + + if args.command == 'list-uptime-check-configs': + list_uptime_check_configs(project_name()) + + elif args.command == 'list-uptime-check-ips': + list_uptime_check_ips() + + elif args.command == 'create-uptime-check': + create_uptime_check_config(project_name(), args.host_name, + args.display_name) + + elif args.command == 'get-uptime-check-config': + get_uptime_check_config(args.name) + + elif args.command == 'delete-uptime-check-config': + delete_uptime_check_config(args.name) + + elif args.command == 'update-uptime-check-config': + if not args.display_name and not args.uptime_check_path: + print('Nothing to update. Pass --display_name or ' + '--uptime_check_path.') + else: + update_uptime_check_config(args.name, args.display_name, + args.uptime_check_path) diff --git a/monitoring/api/v3/uptime-check-client/snippets_test.py b/monitoring/api/v3/uptime-check-client/snippets_test.py new file mode 100644 index 00000000000..1411607c37c --- /dev/null +++ b/monitoring/api/v3/uptime-check-client/snippets_test.py @@ -0,0 +1,90 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import random +import string + +import pytest + +import snippets + + +def random_name(length): + return ''.join( + [random.choice(string.ascii_lowercase) for i in range(length)]) + + +class UptimeFixture: + """A test fixture that creates uptime check config. + """ + + def __init__(self): + self.project_id = snippets.project_id() + self.project_name = snippets.project_name() + + def __enter__(self): + # Create an uptime check config. + self.config = snippets.create_uptime_check_config( + self.project_name, display_name=random_name(10)) + return self + + def __exit__(self, type, value, traceback): + # Delete the config. + snippets.delete_uptime_check_config(self.config.name) + + +@pytest.fixture(scope='session') +def uptime(): + with UptimeFixture() as uptime: + yield uptime + + +def test_create_and_delete(capsys): + # create and delete happen in uptime fixture. + with UptimeFixture(): + pass + + +def test_update_uptime_config(capsys): + # create and delete happen in uptime fixture. + new_display_name = random_name(10) + new_uptime_check_path = '/' + random_name(10) + with UptimeFixture() as fixture: + snippets.update_uptime_check_config( + fixture.config.name, new_display_name, new_uptime_check_path) + out, _ = capsys.readouterr() + snippets.get_uptime_check_config(fixture.config.name) + out, _ = capsys.readouterr() + assert new_display_name in out + assert new_uptime_check_path in out + + +def test_get_uptime_check_config(capsys, uptime): + snippets.get_uptime_check_config(uptime.config.name) + out, _ = capsys.readouterr() + assert uptime.config.display_name in out + + +def test_list_uptime_check_configs(capsys, uptime): + snippets.list_uptime_check_configs(uptime.project_name) + out, _ = capsys.readouterr() + assert uptime.config.display_name in out + + +def test_list_uptime_check_ips(capsys): + snippets.list_uptime_check_ips() + out, _ = capsys.readouterr() + assert 'Singapore' in out diff --git a/notebooks/README.md b/notebooks/README.md new file mode 100644 index 00000000000..4ae67189085 --- /dev/null +++ b/notebooks/README.md @@ -0,0 +1,50 @@ +# Notebook Tutorials + +This directory contains Jupyter notebook tutorials for Google Cloud Platform. +The tutorials assume you have performed the following steps: + +1. Install Jupyter notebooks ([instructions](https://jupyter.org/install)) +1. Install the dependencies in the [requirements.txt](./requirements.txt) file ([instructions below](#install-the-dependencies)) +1. Registered the `google-cloud-bigquery` magic commands ([instructions below](#register-magics-and-configure-matplotlib)) +1. Set `matplotlib` to render inline ([instructions below](#register-magics-and-configure-matplotlib)) + +## Install the dependencies + +Install the dependencies with the following command: + + pip install --upgrade -r requirements.txt + +## Register magics and configure matplotlib + +You can either perform these set up steps in a single notebook, or add the +steps to your IPython configuration file to apply to all notebooks. + +### Perform set up steps within a notebook + +To perform the set up steps for a single notebook, run the following commands +in your notebook to register the BigQuery magic commands and set `matplotlib` +to render inline: +```python +%load_ext google.cloud.bigquery +%matplotlib inline +``` + +### Perform set up steps in your IPython configuration file + +To perform the set up steps implicitly for all of your notebooks, add the +following code to your `ipython_config.py` file to register the BigQuery magic +commands and set `matplotlib` to render inline: +```python +c = get_config() + +# Register magic commands +c.InteractiveShellApp.extensions = [ + 'google.cloud.bigquery', +] + +# Enable matplotlib renderings to render inline in the notebook. +c.InteractiveShellApp.matplotlib = 'inline' +``` +See +[IPython documentation](https://ipython.readthedocs.io/en/stable/config/intro.html) +for more information about IPython configuration. diff --git a/notebooks/rendered/bigquery-basics.md b/notebooks/rendered/bigquery-basics.md new file mode 100644 index 00000000000..c72b37f6fc0 --- /dev/null +++ b/notebooks/rendered/bigquery-basics.md @@ -0,0 +1,232 @@ + +# BigQuery basics + +[BigQuery](https://cloud.google.com/bigquery/docs/) is a petabyte-scale analytics data warehouse that you can use to run SQL queries over vast amounts of data in near realtime. This page shows you how to get started with the Google BigQuery API using the Python client library. + +## Import the libraries used in this tutorial + + +```python +from google.cloud import bigquery +import pandas +``` + +## Initialize a client + +To use the BigQuery Python client library, start by initializing a client. The BigQuery client is used to send and receive messages from the BigQuery API. + +### Client project +The `bigquery.Client` object uses your default project. Alternatively, you can specify a project in the `Client` constructor. For more information about how the default project is determined, see the [google-auth documentation](https://google-auth.readthedocs.io/en/latest/reference/google.auth.html). + + +### Client location +Locations are required for certain BigQuery operations such as creating a dataset. If a location is provided to the client when it is initialized, it will be the default location for jobs, datasets, and tables. + +Run the following to create a client with your default project: + + +```python +client = bigquery.Client(location="US") +print("Client creating using default project: {}".format(client.project)) +``` + +To explicitly specify a project when constructing the client, set the `project` parameter: + + +```python +# client = bigquery.Client(location="US", project="your-project-id") +``` + +## Run a query on a public dataset + +The following example queries the BigQuery `usa_names` public dataset to find the 10 most popular names. `usa_names` is a Social Security Administration dataset that contains all names from Social Security card applications for births that occurred in the United States after 1879. + +Use the [Client.query](https://googleapis.github.io/google-cloud-python/latest/bigquery/generated/google.cloud.bigquery.client.Client.html#google.cloud.bigquery.client.Client.query) method to run the query, and the [QueryJob.to_dataframe](https://googleapis.github.io/google-cloud-python/latest/bigquery/generated/google.cloud.bigquery.job.QueryJob.html#google.cloud.bigquery.job.QueryJob.to_dataframe) method to return the results as a pandas [`DataFrame`](http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html). + + +```python +query = """ + SELECT name, SUM(number) as total + FROM `bigquery-public-data.usa_names.usa_1910_current` + GROUP BY name + ORDER BY total DESC + LIMIT 10 +""" +query_job = client.query( + query, + # Location must match that of the dataset(s) referenced in the query. + location="US", +) # API request - starts the query + +df = query_job.to_dataframe() +df +``` + +## Run a parameterized query + +BigQuery supports query parameters to help prevent [SQL injection](https://en.wikipedia.org/wiki/SQL_injection) when you construct a query with user input. Query parameters are only available with [standard SQL syntax](https://cloud.google.com/bigquery/docs/reference/standard-sql/). Query parameters can be used as substitutes for arbitrary expressions. Parameters cannot be used as substitutes for identifiers, column names, table names, or other parts of the query. + +To specify a parameter, use the `@` character followed by an [identifier](https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#identifiers), such as `@param_name`. For example, the following query finds all the words in a specific Shakespeare corpus with counts that are at least the specified value. + +For more information, see [Running parameterized queries](https://cloud.google.com/bigquery/docs/parameterized-queries) in the BigQuery documentation. + + +```python +# Define the query +sql = """ + SELECT word, word_count + FROM `bigquery-public-data.samples.shakespeare` + WHERE corpus = @corpus + AND word_count >= @min_word_count + ORDER BY word_count DESC; +""" + +# Define the parameter values in a query job configuration +job_config = bigquery.QueryJobConfig( + query_parameters=[ + bigquery.ScalarQueryParameter("corpus", "STRING", "romeoandjuliet"), + bigquery.ScalarQueryParameter("min_word_count", "INT64", 250), + ] +) + +# Start the query job +query_job = client.query(sql, location="US", job_config=job_config) + +# Return the results as a pandas DataFrame +query_job.to_dataframe() +``` + +## Create a new dataset + +A dataset is contained within a specific [project](https://cloud.google.com/bigquery/docs/projects). Datasets are top-level containers that are used to organize and control access to your [tables](https://cloud.google.com/bigquery/docs/tables) and [views](https://cloud.google.com/bigquery/docs/views). A table or view must belong to a dataset. You need to create at least one dataset before [loading data into BigQuery](https://cloud.google.com/bigquery/loading-data-into-bigquery). + + +```python +# Define a name for the new dataset. +dataset_id = 'your_new_dataset' + +# The project defaults to the Client's project if not specified. +dataset = client.create_dataset(dataset_id) # API request +``` + +## Write query results to a destination table + +For more information, see [Writing query results](https://cloud.google.com/bigquery/docs/writing-results) in the BigQuery documentation. + + +```python +sql = """ + SELECT corpus + FROM `bigquery-public-data.samples.shakespeare` + GROUP BY corpus; +""" +table_ref = dataset.table("your_new_table_id") +job_config = bigquery.QueryJobConfig( + destination=table_ref +) + +# Start the query, passing in the extra configuration. +query_job = client.query(sql, location="US", job_config=job_config) + +query_job.result() # Waits for the query to finish +print("Query results loaded to table {}".format(table_ref.path)) +``` + +## Load data from a pandas DataFrame to a new table + + +```python +records = [ + {"title": "The Meaning of Life", "release_year": 1983}, + {"title": "Monty Python and the Holy Grail", "release_year": 1975}, + {"title": "Life of Brian", "release_year": 1979}, + {"title": "And Now for Something Completely Different", "release_year": 1971}, +] + +# Optionally set explicit indices. +# If indices are not specified, a column will be created for the default +# indices created by pandas. +index = ["Q24980", "Q25043", "Q24953", "Q16403"] +df = pandas.DataFrame(records, index=pandas.Index(index, name="wikidata_id")) + +table_ref = dataset.table("monty_python") +job = client.load_table_from_dataframe(df, table_ref, location="US") + +job.result() # Waits for table load to complete. +print("Loaded dataframe to {}".format(table_ref.path)) +``` + +## Load data from a local file to a table + +The following example demonstrates how to load a local CSV file into a new table. See [SourceFormat](https://googleapis.github.io/google-cloud-python/latest/bigquery/generated/google.cloud.bigquery.job.SourceFormat.html#google.cloud.bigquery.job.SourceFormat) in the Python client library documentation for a list of available source formats. For more information, see [Loading Data into BigQuery from a local data source](https://cloud.google.com/bigquery/docs/loading-data-local) in the BigQuery documentation. + + +```python +source_filename = 'resources/us-states.csv' + +table_ref = dataset.table('us_states_from_local_file') +job_config = bigquery.LoadJobConfig( + source_format=bigquery.SourceFormat.CSV, + skip_leading_rows=1, + autodetect=True +) + +with open(source_filename, 'rb') as source_file: + job = client.load_table_from_file( + source_file, + table_ref, + location='US', # Must match the destination dataset location. + job_config=job_config) # API request + +job.result() # Waits for table load to complete. + +print('Loaded {} rows into {}:{}.'.format( + job.output_rows, dataset_id, table_ref.path)) +``` + +## Load data from Cloud Storage to a table + +The following example demonstrates how to load a local CSV file into a new table. See [SourceFormat](https://googleapis.github.io/google-cloud-python/latest/bigquery/generated/google.cloud.bigquery.job.SourceFormat.html#google.cloud.bigquery.job.SourceFormat) in the Python client library documentation for a list of available source formats. For more information, see [Introduction to loading data from Cloud Storage](https://cloud.google.com/bigquery/docs/loading-data-cloud-storage) in the BigQuery documentation. + + +```python +# Configure the load job +job_config = bigquery.LoadJobConfig( + schema=[ + bigquery.SchemaField('name', 'STRING'), + bigquery.SchemaField('post_abbr', 'STRING') + ], + skip_leading_rows=1, + # The source format defaults to CSV. The line below is optional. + source_format=bigquery.SourceFormat.CSV +) +uri = 'gs://cloud-samples-data/bigquery/us-states/us-states.csv' +destination_table_ref = dataset.table('us_states_from_gcs') + +# Start the load job +load_job = client.load_table_from_uri( + uri, destination_table_ref, job_config=job_config) +print('Starting job {}'.format(load_job.job_id)) + +load_job.result() # Waits for table load to complete. +print('Job finished.') + +# Retreive the destination table +destination_table = client.get_table(table_ref) +print('Loaded {} rows.'.format(destination_table.num_rows)) +``` + +## Cleaning Up + +The following code deletes the dataset created for this tutorial, including all tables in the dataset. + + +```python +# Retrieve the dataset from the API +dataset = client.get_dataset(client.dataset(dataset_id)) + +# Delete the dataset and its contents +client.delete_dataset(dataset, delete_contents=True) + +print('Deleted dataset: {}'.format(dataset.path)) +``` diff --git a/notebooks/rendered/bigquery-command-line-tool.md b/notebooks/rendered/bigquery-command-line-tool.md new file mode 100644 index 00000000000..9d824a09065 --- /dev/null +++ b/notebooks/rendered/bigquery-command-line-tool.md @@ -0,0 +1,101 @@ + +# BigQuery command-line tool + +The BigQuery command-line tool is installed as part of the [Cloud SDK](https://cloud-dot-devsite.googleplex.com/sdk/docs/) and can be used to interact with BigQuery. When you use CLI commands in a notebook, the command must be prepended with a `!`. + +## View available commands + +To view the available commands for the BigQuery command-line tool, use the `help` command. + + +```python +!bq help +``` + +## Create a new dataset + +A dataset is contained within a specific [project](https://cloud.google.com/bigquery/docs/projects). Datasets are top-level containers that are used to organize and control access to your [tables](https://cloud.google.com/bigquery/docs/tables) and [views](https://cloud.google.com/bigquery/docs/views). A table or view must belong to a dataset. You need to create at least one dataset before [loading data into BigQuery](https://cloud.google.com/bigquery/loading-data-into-bigquery). + +First, name your new dataset: + + +```python +dataset_id = "your_new_dataset" +``` + +The following command creates a new dataset in the US using the ID defined above. + +NOTE: In the examples in this notebook, the `dataset_id` variable is referenced in the commands using both `{}` and `$`. To avoid creating and using variables, replace these interpolated variables with literal values and remove the `{}` and `$` characters. + + +```python +!bq --location=US mk --dataset $dataset_id +``` + +The response should look like the following: + +``` +Dataset 'your-project-id:your_new_dataset' successfully created. +``` + +## List datasets + +The following command lists all datasets in your default project. + + +```python +!bq ls +``` + +The response should look like the following: + +``` + datasetId + ------------------------------ + your_new_dataset +``` + +## Load data from a local file to a table + +The following example demonstrates how to load a local CSV file into a new or existing table. See [SourceFormat](https://googleapis.github.io/google-cloud-python/latest/bigquery/generated/google.cloud.bigquery.job.SourceFormat.html#google.cloud.bigquery.job.SourceFormat) in the Python client library documentation for a list of available source formats. For more information, see [Loading Data into BigQuery from a local data source](https://cloud.google.com/bigquery/docs/loading-data-local) in the BigQuery documentation. + + +```python +!bq \ + --location=US \ + load \ + --autodetect \ + --skip_leading_rows=1 \ + --source_format=CSV \ + {dataset_id}.us_states_local_file \ + 'resources/us-states.csv' +``` + +## Load data from Cloud Storage to a table + +The following example demonstrates how to load a local CSV file into a new table. See [SourceFormat](https://googleapis.github.io/google-cloud-python/latest/bigquery/generated/google.cloud.bigquery.job.SourceFormat.html#google.cloud.bigquery.job.SourceFormat) in the Python client library documentation for a list of available source formats. For more information, see [Introduction to loading data from Cloud Storage](https://cloud.google.com/bigquery/docs/loading-data-cloud-storage) in the BigQuery documentation. + + +```python +!bq \ + --location=US \ + load \ + --autodetect \ + --skip_leading_rows=1 \ + --source_format=CSV \ + {dataset_id}.us_states_gcs \ + 'gs://cloud-samples-data/bigquery/us-states/us-states.csv' +``` + +## Run a query + +The BigQuery command-line tool has a `query` command for running queries, but it is recommended to use the [magic command](./BigQuery%20Query%20Magic.ipynb) for this purpose. + +## Cleaning Up + +The following code deletes the dataset created for this tutorial, including all tables in the dataset. + + +```python +!bq rm -r -f --dataset $dataset_id +``` diff --git a/notebooks/rendered/bigquery-query-magic.md b/notebooks/rendered/bigquery-query-magic.md new file mode 100644 index 00000000000..6200ac53084 --- /dev/null +++ b/notebooks/rendered/bigquery-query-magic.md @@ -0,0 +1,91 @@ + +# BigQuery query magic + +Jupyter magics are notebook-specific shortcuts that allow you to run commands with minimal syntax. Jupyter notebooks come with many [built-in commands](https://ipython.readthedocs.io/en/stable/interactive/magics.html). The BigQuery client library, `google-cloud-bigquery`, provides a cell magic, `%%bigquery`. The `%%bigquery` magic runs a SQL query and returns the results as a pandas `DataFrame`. + +## Run a query on a public dataset + +The following example queries the BigQuery `usa_names` public dataset. `usa_names` is a Social Security Administration dataset that contains all names from Social Security card applications for births that occurred in the United States after 1879. + +The following example shows how to invoke the magic (`%%bigquery`), and how to pass in a standard SQL query in the body of the code cell. The results are displayed below the input cell as a pandas [`DataFrame`](http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html). + + +```python +%%bigquery +SELECT name, SUM(number) as count +FROM `bigquery-public-data.usa_names.usa_1910_current` +GROUP BY name +ORDER BY count DESC +LIMIT 10 +``` + +## Display verbose output + +As the query job is running, status messages below the cell update with the query job ID and the amount of time the query has been running. By default, this output is erased and replaced with the results of the query. If you pass the `--verbose` flag, the output will remain below the cell after query completion. + + +```python +%%bigquery --verbose +SELECT name, SUM(number) as count +FROM `bigquery-public-data.usa_names.usa_1910_current` +GROUP BY name +ORDER BY count DESC +LIMIT 10 +``` + +## Explicitly specify a project + +By default, the `%%bigquery` magic command uses your default project to run the query. You may also explicitly provide a project ID using the `--project` flag. Note that your credentials must have permissions to create query jobs in the project you specify. + + +```python +project_id = 'your-project-id' +``` + + +```python +%%bigquery --project $project_id +SELECT name, SUM(number) as count +FROM `bigquery-public-data.usa_names.usa_1910_current` +GROUP BY name +ORDER BY count DESC +LIMIT 10 +``` + +## Assign the query results to a variable + +To save the results of your query to a variable, provide a variable name as a parameter to `%%bigquery`. The following example saves the results of the query to a variable named `df`. Note that when a variable is provided, the results are not displayed below the cell that invokes the magic command. + + +```python +%%bigquery df +SELECT name, SUM(number) as count +FROM `bigquery-public-data.usa_names.usa_1910_current` +GROUP BY name +ORDER BY count DESC +LIMIT 10 +``` + + +```python +df +``` + +## Run a parameterized query + +Parameterized queries are useful if you need to run a query with certain parameters that are calculated at run time. Note that the value types must be JSON serializable. The following example defines a parameters dictionary and passes it to the `--params` flag. The key of the dictionary is the name of the parameter, and the value of the dictionary is the value of the parameter. + + +```python +params = {"limit": 10} +``` + + +```python +%%bigquery --params $params +SELECT name, SUM(number) as count +FROM `bigquery-public-data.usa_names.usa_1910_current` +GROUP BY name +ORDER BY count DESC +LIMIT @limit +``` diff --git a/notebooks/rendered/cloud-storage-client-library.md b/notebooks/rendered/cloud-storage-client-library.md new file mode 100644 index 00000000000..7fde6e40117 --- /dev/null +++ b/notebooks/rendered/cloud-storage-client-library.md @@ -0,0 +1,158 @@ + +# Cloud Storage client library + +This tutorial shows how to get started with the [Cloud Storage Python client library](https://googleapis.github.io/google-cloud-python/latest/storage/index.html). + +## Create a storage bucket + +Buckets are the basic containers that hold your data. Everything that you store in Cloud Storage must be contained in a bucket. You can use buckets to organize your data and control access to your data. + +Start by importing the library: + + +```python +from google.cloud import storage +``` + +The `storage.Client` object uses your default project. Alternatively, you can specify a project in the `Client` constructor. For more information about how the default project is determined, see the [google-auth documentation](https://google-auth.readthedocs.io/en/latest/reference/google.auth.html). + +Run the following to create a client with your default project: + + +```python +client = storage.Client() +print("Client created using default project: {}".format(client.project)) +``` + +To explicitly specify a project when constructing the client, set the `project` parameter: + + +```python +# client = storage.Client(project='your-project-id') +``` + +Finally, create a bucket with a globally unique name. + +For more information about naming buckets, see [Bucket name requirements](https://cloud.google.com/storage/docs/naming#requirements). + + +```python +# Replace the string below with a unique name for the new bucket +bucket_name = "your-new-bucket" + +# Creates the new bucket +bucket = client.create_bucket(bucket_name) + +print("Bucket {} created.".format(bucket.name)) +``` + +## List buckets in a project + + +```python +buckets = client.list_buckets() + +print("Buckets in {}:".format(client.project)) +for item in buckets: + print("\t" + item.name) +``` + +## Get bucket metadata + +The next cell shows how to get information on metadata of your Cloud Storage buckets. + +To learn more about specific bucket properties, see [Bucket locations](https://cloud.google.com/storage/docs/locations) and [Storage classes](https://cloud.google.com/storage/docs/storage-classes). + + +```python +bucket = client.get_bucket(bucket_name) + +print("Bucket name: {}".format(bucket.name)) +print("Bucket location: {}".format(bucket.location)) +print("Bucket storage class: {}".format(bucket.storage_class)) +``` + +## Upload a local file to a bucket + +Objects are the individual pieces of data that you store in Cloud Storage. Objects are referred to as "blobs" in the Python client library. There is no limit on the number of objects that you can create in a bucket. + +An object's name is treated as a piece of object metadata in Cloud Storage. Object names can contain any combination of Unicode characters (UTF-8 encoded) and must be less than 1024 bytes in length. + +For more information, including how to rename an object, see the [Object name requirements](https://cloud.google.com/storage/docs/naming#objectnames). + + +```python +blob_name = "us-states.txt" +blob = bucket.blob(blob_name) + +source_file_name = "resources/us-states.txt" +blob.upload_from_filename(source_file_name) + +print("File uploaded to {}.".format(bucket.name)) +``` + +## List blobs in a bucket + + +```python +blobs = bucket.list_blobs() + +print("Blobs in {}:".format(bucket.name)) +for item in blobs: + print("\t" + item.name) +``` + +## Get a blob and display metadata + +See [documentation](https://cloud.google.com/storage/docs/viewing-editing-metadata) for more information about object metadata. + + +```python +blob = bucket.get_blob(blob_name) + +print("Name: {}".format(blob.id)) +print("Size: {} bytes".format(blob.size)) +print("Content type: {}".format(blob.content_type)) +print("Public URL: {}".format(blob.public_url)) +``` + +## Download a blob to a local directory + + +```python +output_file_name = "resources/downloaded-us-states.txt" +blob.download_to_filename(output_file_name) + +print("Downloaded blob {} to {}.".format(blob.name, output_file_name)) +``` + +## Cleaning up + +### Delete a blob + + +```python +blob = client.get_bucket(bucket_name).get_blob(blob_name) +blob.delete() + +print("Blob {} deleted.".format(blob.name)) +``` + +### Delete a bucket + +Note that the bucket must be empty before it can be deleted. + + +```python +bucket = client.get_bucket(bucket_name) +bucket.delete() + +print("Bucket {} deleted.".format(bucket.name)) +``` + +## Next Steps + +Read more about Cloud Storage in the documentation: ++ [Storage key terms](https://cloud.google.com/storage/docs/key-terms) ++ [How-to guides](https://cloud.google.com/storage/docs/how-to) ++ [Pricing](https://cloud.google.com/storage/pricing) diff --git a/notebooks/rendered/getting-started-with-bigquery-ml.md b/notebooks/rendered/getting-started-with-bigquery-ml.md new file mode 100644 index 00000000000..be92acf0b8a --- /dev/null +++ b/notebooks/rendered/getting-started-with-bigquery-ml.md @@ -0,0 +1,239 @@ + +# Getting started with BigQuery ML + +BigQuery ML enables users to create and execute machine learning models in BigQuery using SQL queries. The goal is to democratize machine learning by enabling SQL practitioners to build models using their existing tools and to increase development speed by eliminating the need for data movement. + +In this tutorial, you use the sample [Google Analytics sample dataset for BigQuery](https://support.google.com/analytics/answer/7586738?hl=en&ref_topic=3416089) to create a model that predicts whether a website visitor will make a transaction. For information on the schema of the Analytics dataset, see [BigQuery export schema](https://support.google.com/analytics/answer/3437719) in the Google Analytics Help Center. + + +## Objectives +In this tutorial, you use: + ++ BigQuery ML to create a binary logistic regression model using the `CREATE MODEL` statement ++ The `ML.EVALUATE` function to evaluate the ML model ++ The `ML.PREDICT` function to make predictions using the ML model + +## Create your dataset + +Enter the following code to import the BigQuery Python client library and initialize a client. The BigQuery client is used to send and receive messages from the BigQuery API. + + +```python +from google.cloud import bigquery + +client = bigquery.Client(location="US") +``` + +Next, you create a BigQuery dataset to store your ML model. Run the following to create your dataset: + + +```python +dataset = client.create_dataset("bqml_tutorial") +``` + +## Create your model + +Next, you create a logistic regression model using the Google Analytics sample +dataset for BigQuery. The model is used to predict whether a +website visitor will make a transaction. The standard SQL query uses a +`CREATE MODEL` statement to create and train the model. Standard SQL is the +default query syntax for the BigQuery python client library. + +The BigQuery python client library provides a cell magic, +`%%bigquery`, which runs a SQL query and returns the results as a Pandas +`DataFrame`. + +To run the `CREATE MODEL` query to create and train your model: + + +```python +%%bigquery +CREATE OR REPLACE MODEL `bqml_tutorial.sample_model` +OPTIONS(model_type='logistic_reg') AS +SELECT + IF(totals.transactions IS NULL, 0, 1) AS label, + IFNULL(device.operatingSystem, "") AS os, + device.isMobile AS is_mobile, + IFNULL(geoNetwork.country, "") AS country, + IFNULL(totals.pageviews, 0) AS pageviews +FROM + `bigquery-public-data.google_analytics_sample.ga_sessions_*` +WHERE + _TABLE_SUFFIX BETWEEN '20160801' AND '20170630' +``` + +The query takes several minutes to complete. After the first iteration is +complete, your model (`sample_model`) appears in the navigation panel of the +BigQuery web UI. Because the query uses a `CREATE MODEL` statement to create a +table, you do not see query results. The output is an empty `DataFrame`. + +## Get training statistics + +To see the results of the model training, you can use the +[`ML.TRAINING_INFO`](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-train) +function, or you can view the statistics in the BigQuery web UI. This functionality +is not currently available in the BigQuery Classic web UI. +In this tutorial, you use the `ML.TRAINING_INFO` function. + +A machine learning algorithm builds a model by examining many examples and +attempting to find a model that minimizes loss. This process is called empirical +risk minimization. + +Loss is the penalty for a bad prediction — a number indicating +how bad the model's prediction was on a single example. If the model's +prediction is perfect, the loss is zero; otherwise, the loss is greater. The +goal of training a model is to find a set of weights that have low +loss, on average, across all examples. + +To see the model training statistics that were generated when you ran the +`CREATE MODEL` query: + + +```python +%%bigquery +SELECT + * +FROM + ML.TRAINING_INFO(MODEL `bqml_tutorial.sample_model`) +``` + +Note: Typically, it is not a best practice to use a `SELECT *` query. Because the model output is a small table, this query does not process a large amount of data. As a result, the cost is minimal. + +When the query is complete, the results appear below the query. The results should look like the following: + +![Training statistics table](../tutorials/bigquery/resources/training-statistics.png) + +The `loss` column represents the loss metric calculated after the given iteration +on the training dataset. Since you performed a logistic regression, this column +is the [log loss](https://en.wikipedia.org/wiki/Cross_entropy#Cross-entropy_error_function_and_logistic_regression). +The `eval_loss` column is the same loss metric calculated on +the holdout dataset (data that is held back from training to validate the model). + +For more details on the `ML.TRAINING_INFO` function, see the +[BigQuery ML syntax reference](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-train). + +## Evaluate your model + +After creating your model, you evaluate the performance of the classifier using +the [`ML.EVALUATE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-evaluate) +function. You can also use the [`ML.ROC_CURVE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-roc) +function for logistic regression specific metrics. + +A classifier is one of a set of enumerated target values for a label. For +example, in this tutorial you are using a binary classification model that +detects transactions. The two classes are the values in the `label` column: +`0` (no transactions) and not `1` (transaction made). + +To run the `ML.EVALUATE` query that evaluates the model: + + +```python +%%bigquery +SELECT + * +FROM ML.EVALUATE(MODEL `bqml_tutorial.sample_model`, ( + SELECT + IF(totals.transactions IS NULL, 0, 1) AS label, + IFNULL(device.operatingSystem, "") AS os, + device.isMobile AS is_mobile, + IFNULL(geoNetwork.country, "") AS country, + IFNULL(totals.pageviews, 0) AS pageviews + FROM + `bigquery-public-data.google_analytics_sample.ga_sessions_*` + WHERE + _TABLE_SUFFIX BETWEEN '20170701' AND '20170801')) +``` + +When the query is complete, the results appear below the query. The +results should look like the following: + +![Model evaluation results table](../tutorials/bigquery/resources/model-evaluation.png) + +Because you performed a logistic regression, the results include the following +columns: + ++ [`precision`](https://developers.google.com/machine-learning/glossary/#precision) ++ [`recall`](https://developers.google.com/machine-learning/glossary/#recall) ++ [`accuracy`](https://developers.google.com/machine-learning/glossary/#accuracy) ++ [`f1_score`](https://en.wikipedia.org/wiki/F1_score) ++ [`log_loss`](https://developers.google.com/machine-learning/glossary/#Log_Loss) ++ [`roc_auc`](https://developers.google.com/machine-learning/glossary/#AUC) + + +## Use your model to predict outcomes + +Now that you have evaluated your model, the next step is to use it to predict +outcomes. You use your model to predict the number of transactions made by +website visitors from each country. And you use it to predict purchases per user. + +To run the query that uses the model to predict the number of transactions: + + +```python +%%bigquery +SELECT + country, + SUM(predicted_label) as total_predicted_purchases +FROM ML.PREDICT(MODEL `bqml_tutorial.sample_model`, ( + SELECT + IFNULL(device.operatingSystem, "") AS os, + device.isMobile AS is_mobile, + IFNULL(totals.pageviews, 0) AS pageviews, + IFNULL(geoNetwork.country, "") AS country + FROM + `bigquery-public-data.google_analytics_sample.ga_sessions_*` + WHERE + _TABLE_SUFFIX BETWEEN '20170701' AND '20170801')) + GROUP BY country + ORDER BY total_predicted_purchases DESC + LIMIT 10 +``` + +When the query is complete, the results appear below the query. The +results should look like the following. Because model training is not +deterministic, your results may differ. + +![Model predictions table](../tutorials/bigquery/resources/transaction-predictions.png) + +In the next example, you try to predict the number of transactions each website +visitor will make. This query is identical to the previous query except for the +`GROUP BY` clause. Here the `GROUP BY` clause — `GROUP BY fullVisitorId` +— is used to group the results by visitor ID. + +To run the query that predicts purchases per user: + + +```python +%%bigquery +SELECT + fullVisitorId, + SUM(predicted_label) as total_predicted_purchases +FROM ML.PREDICT(MODEL `bqml_tutorial.sample_model`, ( + SELECT + IFNULL(device.operatingSystem, "") AS os, + device.isMobile AS is_mobile, + IFNULL(totals.pageviews, 0) AS pageviews, + IFNULL(geoNetwork.country, "") AS country, + fullVisitorId + FROM + `bigquery-public-data.google_analytics_sample.ga_sessions_*` + WHERE + _TABLE_SUFFIX BETWEEN '20170701' AND '20170801')) + GROUP BY fullVisitorId + ORDER BY total_predicted_purchases DESC + LIMIT 10 +``` + +When the query is complete, the results appear below the query. The +results should look like the following: + +![Purchase predictions table](../tutorials/bigquery/resources/purchase-predictions.png) + +## Cleaning up + +To delete the resources created by this tutorial, execute the following code to delete the dataset and its contents: + + +```python +client.delete_dataset(dataset, delete_contents=True) +``` diff --git a/notebooks/rendered/storage-command-line-tool.md b/notebooks/rendered/storage-command-line-tool.md new file mode 100644 index 00000000000..d6fffa759de --- /dev/null +++ b/notebooks/rendered/storage-command-line-tool.md @@ -0,0 +1,155 @@ + +# Storage command-line tool + +The [Google Cloud SDK](https://cloud-dot-devsite.googleplex.com/sdk/docs/) provides a set of commands for working with data stored in Cloud Storage. This notebook introduces several `gsutil` commands for interacting with Cloud Storage. Note that shell commands in a notebook must be prepended with a `!`. + +## List available commands + +The `gsutil` command can be used to perform a wide array of tasks. Run the `help` command to view a list of available commands: + + +```python +!gsutil help +``` + +## Create a storage bucket + +Buckets are the basic containers that hold your data. Everything that you store in Cloud Storage must be contained in a bucket. You can use buckets to organize your data and control access to your data. + +Start by defining a globally unique name. + +For more information about naming buckets, see [Bucket name requirements](https://cloud.google.com/storage/docs/naming#requirements). + + +```python +# Replace the string below with a unique name for the new bucket +bucket_name = "your-new-bucket" +``` + +NOTE: In the examples below, the `bucket_name` and `project_id` variables are referenced in the commands using `{}` and `$`. If you want to avoid creating and using variables, replace these interpolated variables with literal values and remove the `{}` and `$` characters. + +Next, create the new bucket with the `gsutil mb` command: + + +```python +!gsutil mb gs://{bucket_name}/ +``` + +## List buckets in a project + +Replace 'your-project-id' in the cell below with your project ID and run the cell to list the storage buckets in your project. + + +```python +# Replace the string below with your project ID +project_id = "your-project-id" +``` + + +```python +!gsutil ls -p $project_id +``` + +The response should look like the following: + +``` +gs://your-new-bucket/ +``` + +## Get bucket metadata + +The next cell shows how to get information on metadata of your Cloud Storage buckets. + +To learn more about specific bucket properties, see [Bucket locations](https://cloud.google.com/storage/docs/locations) and [Storage classes](https://cloud.google.com/storage/docs/storage-classes). + + +```python +!gsutil ls -L -b gs://{bucket_name}/ +``` + +The response should look like the following: +``` +gs://your-new-bucket/ : + Storage class: MULTI_REGIONAL + Location constraint: US + ... +``` + +## Upload a local file to a bucket + +Objects are the individual pieces of data that you store in Cloud Storage. Objects are referred to as "blobs" in the Python client library. There is no limit on the number of objects that you can create in a bucket. + +An object's name is treated as a piece of object metadata in Cloud Storage. Object names can contain any combination of Unicode characters (UTF-8 encoded) and must be less than 1024 bytes in length. + +For more information, including how to rename an object, see the [Object name requirements](https://cloud.google.com/storage/docs/naming#objectnames). + + +```python +!gsutil cp resources/us-states.txt gs://{bucket_name}/ +``` + +## List blobs in a bucket + + +```python +!gsutil ls -r gs://{bucket_name}/** +``` + +The response should look like the following: +``` +gs://your-new-bucket/us-states.txt +``` + +## Get a blob and display metadata + +See [Viewing and editing object metadata](https://cloud.google.com/storage/docs/viewing-editing-metadata) for more information about object metadata. + + +```python +!gsutil ls -L gs://{bucket_name}/us-states.txt +``` + +The response should look like the following: + +``` +gs://your-new-bucket/us-states.txt: + Creation time: Fri, 08 Feb 2019 05:23:28 GMT + Update time: Fri, 08 Feb 2019 05:23:28 GMT + Storage class: STANDARD + Content-Language: en + Content-Length: 637 + Content-Type: text/plain +... +``` + +## Download a blob to a local directory + + +```python +!gsutil cp gs://{bucket_name}/us-states.txt resources/downloaded-us-states.txt +``` + +## Cleaning up + +### Delete a blob + + +```python +!gsutil rm gs://{bucket_name}/us-states.txt +``` + +### Delete a bucket + +The following command deletes all objects in the bucket before deleting the bucket itself. + + +```python +!gsutil rm -r gs://{bucket_name}/ +``` + +## Next Steps + +Read more about Cloud Storage in the documentation: ++ [Storage key terms](https://cloud.google.com/storage/docs/key-terms) ++ [How-to guides](https://cloud.google.com/storage/docs/how-to) ++ [Pricing](https://cloud.google.com/storage/pricing) diff --git a/notebooks/rendered/visualizing-bigquery-public-data.md b/notebooks/rendered/visualizing-bigquery-public-data.md new file mode 100644 index 00000000000..bbd4bb34830 --- /dev/null +++ b/notebooks/rendered/visualizing-bigquery-public-data.md @@ -0,0 +1,146 @@ + +# Vizualizing BigQuery data in a Jupyter notebook + +[BigQuery](https://cloud.google.com/bigquery/docs/) is a petabyte-scale analytics data warehouse that you can use to run SQL queries over vast amounts of data in near realtime. + +Data visualization tools can help you make sense of your BigQuery data and help you analyze the data interactively. You can use visualization tools to help you identify trends, respond to them, and make predictions using your data. In this tutorial, you use the BigQuery Python client library and pandas in a Jupyter notebook to visualize data in the BigQuery natality sample table. + +## Using Jupyter magics to query BigQuery data + +The BigQuery Python client library provides a magic command that allows you to run queries with minimal code. + +The BigQuery client library provides a cell magic, `%%bigquery`. The `%%bigquery` magic runs a SQL query and returns the results as a pandas `DataFrame`. The following cell executes a query of the BigQuery natality public dataset and returns the total births by year. + + +```python +%%bigquery +SELECT + source_year AS year, + COUNT(is_male) AS birth_count +FROM `bigquery-public-data.samples.natality` +GROUP BY year +ORDER BY year DESC +LIMIT 15 +``` + +The following command to runs the same query, but this time the results are saved to a variable. The variable name, `total_births`, is given as an argument to the `%%bigquery`. The results can then be used for further analysis and visualization. + + +```python +%%bigquery total_births +SELECT + source_year AS year, + COUNT(is_male) AS birth_count +FROM `bigquery-public-data.samples.natality` +GROUP BY year +ORDER BY year DESC +LIMIT 15 +``` + +The next cell uses the pandas `DataFrame.plot` method to visualize the query results as a bar chart. See the [pandas documentation](https://pandas.pydata.org/pandas-docs/stable/visualization.html) to learn more about data visualization with pandas. + + +```python +total_births.plot(kind='bar', x='year', y='birth_count'); +``` + +Run the following query to retrieve the number of births by weekday. Because the `wday` (weekday) field allows null values, the query excludes records where wday is null. + + +```python +%%bigquery births_by_weekday +SELECT + wday, + SUM(CASE WHEN is_male THEN 1 ELSE 0 END) AS male_births, + SUM(CASE WHEN is_male THEN 0 ELSE 1 END) AS female_births +FROM `bigquery-public-data.samples.natality` +WHERE wday IS NOT NULL +GROUP BY wday +ORDER BY wday ASC +``` + +Visualize the query results using a line chart. + + +```python +births_by_weekday.plot(x='wday'); +``` + +## Using Python to query BigQuery data + +Magic commands allow you to use minimal syntax to interact with BigQuery. Behind the scenes, `%%bigquery` uses the BigQuery Python client library to run the given query, convert the results to a pandas `Dataframe`, optionally save the results to a variable, and finally display the results. Using the BigQuery Python client library directly instead of through magic commands gives you more control over your queries and allows for more complex configurations. The library's integrations with pandas enable you to combine the power of declarative SQL with imperative code (Python) to perform interesting data analysis, visualization, and transformation tasks. + +To use the BigQuery Python client library, start by importing the library and initializing a client. The BigQuery client is used to send and receive messages from the BigQuery API. + + +```python +from google.cloud import bigquery + +client = bigquery.Client() +``` + +Use the [`Client.query`](https://googleapis.github.io/google-cloud-python/latest/bigquery/generated/google.cloud.bigquery.client.Client.html#google.cloud.bigquery.client.Client.query) method to run a query. Execute the following cell to run a query to retrieve the annual count of plural births by plurality (2 for twins, 3 for triplets, etc.). + + +```python +sql = """ +SELECT + plurality, + COUNT(1) AS count, + year +FROM + `bigquery-public-data.samples.natality` +WHERE + NOT IS_NAN(plurality) AND plurality > 1 +GROUP BY + plurality, year +ORDER BY + count DESC +""" +df = client.query(sql).to_dataframe() +df.head() +``` + +To chart the query results in your `DataFrame`, run the following cell to pivot the data and create a stacked bar chart of the count of plural births over time. + + +```python +pivot_table = df.pivot(index='year', columns='plurality', values='count') +pivot_table.plot(kind='bar', stacked=True, figsize=(15, 7)); +``` + +Run the following query to retrieve the count of births by the number of gestation weeks. + + +```python +sql = """ +SELECT + gestation_weeks, + COUNT(1) AS count +FROM + `bigquery-public-data.samples.natality` +WHERE + NOT IS_NAN(gestation_weeks) AND gestation_weeks <> 99 +GROUP BY + gestation_weeks +ORDER BY + gestation_weeks +""" +df = client.query(sql).to_dataframe() +``` + +Finally, chart the query results in your `DataFrame`. + + +```python +ax = df.plot(kind='bar', x='gestation_weeks', y='count', figsize=(15,7)) +ax.set_title('Count of Births by Gestation Weeks') +ax.set_xlabel('Gestation Weeks') +ax.set_ylabel('Count'); +``` + +## What's Next + ++ __Learn more about writing queries for BigQuery__ — [Querying Data](https://cloud.google.com/bigquery/querying-data) in the BigQuery documentation explains how to run queries, create user-defined functions (UDFs), and more. + ++ __Explore BigQuery syntax__ — The preferred dialect for SQL queries in BigQuery is standard SQL. Standard SQL syntax is described in the [SQL Reference](https://cloud.google.com/bigquery/docs/reference/standard-sql/). BigQuery's legacy SQL-like syntax is described in the [Query Reference (legacy SQL)](https://cloud.google.com/bigquery/query-reference). diff --git a/notebooks/requirements.txt b/notebooks/requirements.txt new file mode 100644 index 00000000000..d13f3e9f733 --- /dev/null +++ b/notebooks/requirements.txt @@ -0,0 +1,3 @@ +google-cloud-storage==1.14.0 +google-cloud-bigquery[pandas,pyarrow]==1.9.0 +matplotlib diff --git a/notebooks/tutorials/bigquery/BigQuery basics.ipynb b/notebooks/tutorials/bigquery/BigQuery basics.ipynb new file mode 100644 index 00000000000..2ade591fbfb --- /dev/null +++ b/notebooks/tutorials/bigquery/BigQuery basics.ipynb @@ -0,0 +1,361 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# BigQuery basics\n", + "\n", + "[BigQuery](https://cloud.google.com/bigquery/docs/) is a petabyte-scale analytics data warehouse that you can use to run SQL queries over vast amounts of data in near realtime. This page shows you how to get started with the Google BigQuery API using the Python client library." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Import the libraries used in this tutorial" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from google.cloud import bigquery\n", + "import pandas" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Initialize a client\n", + "\n", + "To use the BigQuery Python client library, start by initializing a client. The BigQuery client is used to send and receive messages from the BigQuery API.\n", + "\n", + "### Client project\n", + "The `bigquery.Client` object uses your default project. Alternatively, you can specify a project in the `Client` constructor. For more information about how the default project is determined, see the [google-auth documentation](https://google-auth.readthedocs.io/en/latest/reference/google.auth.html).\n", + "\n", + "\n", + "### Client location\n", + "Locations are required for certain BigQuery operations such as creating a dataset. If a location is provided to the client when it is initialized, it will be the default location for jobs, datasets, and tables.\n", + "\n", + "Run the following to create a client with your default project:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "client = bigquery.Client(location=\"US\")\n", + "print(\"Client creating using default project: {}\".format(client.project))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To explicitly specify a project when constructing the client, set the `project` parameter:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# client = bigquery.Client(location=\"US\", project=\"your-project-id\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run a query on a public dataset\n", + "\n", + "The following example queries the BigQuery `usa_names` public dataset to find the 10 most popular names. `usa_names` is a Social Security Administration dataset that contains all names from Social Security card applications for births that occurred in the United States after 1879.\n", + "\n", + "Use the [Client.query](https://googleapis.github.io/google-cloud-python/latest/bigquery/generated/google.cloud.bigquery.client.Client.html#google.cloud.bigquery.client.Client.query) method to run the query, and the [QueryJob.to_dataframe](https://googleapis.github.io/google-cloud-python/latest/bigquery/generated/google.cloud.bigquery.job.QueryJob.html#google.cloud.bigquery.job.QueryJob.to_dataframe) method to return the results as a pandas [`DataFrame`](http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "query = \"\"\"\n", + " SELECT name, SUM(number) as total\n", + " FROM `bigquery-public-data.usa_names.usa_1910_current`\n", + " GROUP BY name\n", + " ORDER BY total DESC\n", + " LIMIT 10\n", + "\"\"\"\n", + "query_job = client.query(\n", + " query,\n", + " # Location must match that of the dataset(s) referenced in the query.\n", + " location=\"US\",\n", + ") # API request - starts the query\n", + "\n", + "df = query_job.to_dataframe()\n", + "df" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run a parameterized query\n", + "\n", + "BigQuery supports query parameters to help prevent [SQL injection](https://en.wikipedia.org/wiki/SQL_injection) when you construct a query with user input. Query parameters are only available with [standard SQL syntax](https://cloud.google.com/bigquery/docs/reference/standard-sql/). Query parameters can be used as substitutes for arbitrary expressions. Parameters cannot be used as substitutes for identifiers, column names, table names, or other parts of the query.\n", + "\n", + "To specify a parameter, use the `@` character followed by an [identifier](https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#identifiers), such as `@param_name`. For example, the following query finds all the words in a specific Shakespeare corpus with counts that are at least the specified value.\n", + "\n", + "For more information, see [Running parameterized queries](https://cloud.google.com/bigquery/docs/parameterized-queries) in the BigQuery documentation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Define the query\n", + "sql = \"\"\"\n", + " SELECT word, word_count\n", + " FROM `bigquery-public-data.samples.shakespeare`\n", + " WHERE corpus = @corpus\n", + " AND word_count >= @min_word_count\n", + " ORDER BY word_count DESC;\n", + "\"\"\"\n", + "\n", + "# Define the parameter values in a query job configuration\n", + "job_config = bigquery.QueryJobConfig(\n", + " query_parameters=[\n", + " bigquery.ScalarQueryParameter(\"corpus\", \"STRING\", \"romeoandjuliet\"),\n", + " bigquery.ScalarQueryParameter(\"min_word_count\", \"INT64\", 250),\n", + " ]\n", + ")\n", + "\n", + "# Start the query job\n", + "query_job = client.query(sql, location=\"US\", job_config=job_config)\n", + "\n", + "# Return the results as a pandas DataFrame\n", + "query_job.to_dataframe()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create a new dataset\n", + "\n", + "A dataset is contained within a specific [project](https://cloud.google.com/bigquery/docs/projects). Datasets are top-level containers that are used to organize and control access to your [tables](https://cloud.google.com/bigquery/docs/tables) and [views](https://cloud.google.com/bigquery/docs/views). A table or view must belong to a dataset. You need to create at least one dataset before [loading data into BigQuery](https://cloud.google.com/bigquery/loading-data-into-bigquery)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Define a name for the new dataset.\n", + "dataset_id = 'your_new_dataset'\n", + "\n", + "# The project defaults to the Client's project if not specified.\n", + "dataset = client.create_dataset(dataset_id) # API request" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Write query results to a destination table\n", + "\n", + "For more information, see [Writing query results](https://cloud.google.com/bigquery/docs/writing-results) in the BigQuery documentation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sql = \"\"\"\n", + " SELECT corpus\n", + " FROM `bigquery-public-data.samples.shakespeare`\n", + " GROUP BY corpus;\n", + "\"\"\"\n", + "table_ref = dataset.table(\"your_new_table_id\")\n", + "job_config = bigquery.QueryJobConfig(\n", + " destination=table_ref\n", + ")\n", + "\n", + "# Start the query, passing in the extra configuration.\n", + "query_job = client.query(sql, location=\"US\", job_config=job_config)\n", + "\n", + "query_job.result() # Waits for the query to finish\n", + "print(\"Query results loaded to table {}\".format(table_ref.path))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load data from a pandas DataFrame to a new table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "records = [\n", + " {\"title\": \"The Meaning of Life\", \"release_year\": 1983},\n", + " {\"title\": \"Monty Python and the Holy Grail\", \"release_year\": 1975},\n", + " {\"title\": \"Life of Brian\", \"release_year\": 1979},\n", + " {\"title\": \"And Now for Something Completely Different\", \"release_year\": 1971},\n", + "]\n", + "\n", + "# Optionally set explicit indices.\n", + "# If indices are not specified, a column will be created for the default\n", + "# indices created by pandas.\n", + "index = [\"Q24980\", \"Q25043\", \"Q24953\", \"Q16403\"]\n", + "df = pandas.DataFrame(records, index=pandas.Index(index, name=\"wikidata_id\"))\n", + "\n", + "table_ref = dataset.table(\"monty_python\")\n", + "job = client.load_table_from_dataframe(df, table_ref, location=\"US\")\n", + "\n", + "job.result() # Waits for table load to complete.\n", + "print(\"Loaded dataframe to {}\".format(table_ref.path))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load data from a local file to a table\n", + "\n", + "The following example demonstrates how to load a local CSV file into a new table. See [SourceFormat](https://googleapis.github.io/google-cloud-python/latest/bigquery/generated/google.cloud.bigquery.job.SourceFormat.html#google.cloud.bigquery.job.SourceFormat) in the Python client library documentation for a list of available source formats. For more information, see [Loading Data into BigQuery from a local data source](https://cloud.google.com/bigquery/docs/loading-data-local) in the BigQuery documentation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "source_filename = 'resources/us-states.csv'\n", + "\n", + "table_ref = dataset.table('us_states_from_local_file')\n", + "job_config = bigquery.LoadJobConfig(\n", + " source_format=bigquery.SourceFormat.CSV,\n", + " skip_leading_rows=1,\n", + " autodetect=True\n", + ")\n", + "\n", + "with open(source_filename, 'rb') as source_file:\n", + " job = client.load_table_from_file(\n", + " source_file,\n", + " table_ref,\n", + " location='US', # Must match the destination dataset location.\n", + " job_config=job_config) # API request\n", + "\n", + "job.result() # Waits for table load to complete.\n", + "\n", + "print('Loaded {} rows into {}:{}.'.format(\n", + " job.output_rows, dataset_id, table_ref.path))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load data from Cloud Storage to a table\n", + "\n", + "The following example demonstrates how to load a local CSV file into a new table. See [SourceFormat](https://googleapis.github.io/google-cloud-python/latest/bigquery/generated/google.cloud.bigquery.job.SourceFormat.html#google.cloud.bigquery.job.SourceFormat) in the Python client library documentation for a list of available source formats. For more information, see [Introduction to loading data from Cloud Storage](https://cloud.google.com/bigquery/docs/loading-data-cloud-storage) in the BigQuery documentation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Configure the load job\n", + "job_config = bigquery.LoadJobConfig(\n", + " schema=[\n", + " bigquery.SchemaField('name', 'STRING'),\n", + " bigquery.SchemaField('post_abbr', 'STRING')\n", + " ],\n", + " skip_leading_rows=1,\n", + " # The source format defaults to CSV. The line below is optional.\n", + " source_format=bigquery.SourceFormat.CSV\n", + ")\n", + "uri = 'gs://cloud-samples-data/bigquery/us-states/us-states.csv'\n", + "destination_table_ref = dataset.table('us_states_from_gcs')\n", + "\n", + "# Start the load job\n", + "load_job = client.load_table_from_uri(\n", + " uri, destination_table_ref, job_config=job_config)\n", + "print('Starting job {}'.format(load_job.job_id))\n", + "\n", + "load_job.result() # Waits for table load to complete.\n", + "print('Job finished.')\n", + "\n", + "# Retreive the destination table\n", + "destination_table = client.get_table(table_ref)\n", + "print('Loaded {} rows.'.format(destination_table.num_rows))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Cleaning Up\n", + "\n", + "The following code deletes the dataset created for this tutorial, including all tables in the dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Retrieve the dataset from the API\n", + "dataset = client.get_dataset(client.dataset(dataset_id))\n", + "\n", + "# Delete the dataset and its contents\n", + "client.delete_dataset(dataset, delete_contents=True)\n", + "\n", + "print('Deleted dataset: {}'.format(dataset.path))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/tutorials/bigquery/BigQuery command-line tool.ipynb b/notebooks/tutorials/bigquery/BigQuery command-line tool.ipynb new file mode 100644 index 00000000000..f9c709e4533 --- /dev/null +++ b/notebooks/tutorials/bigquery/BigQuery command-line tool.ipynb @@ -0,0 +1,209 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# BigQuery command-line tool\n", + "\n", + "The BigQuery command-line tool is installed as part of the [Cloud SDK](https://cloud-dot-devsite.googleplex.com/sdk/docs/) and can be used to interact with BigQuery. When you use CLI commands in a notebook, the command must be prepended with a `!`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## View available commands\n", + "\n", + "To view the available commands for the BigQuery command-line tool, use the `help` command." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!bq help" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create a new dataset\n", + "\n", + "A dataset is contained within a specific [project](https://cloud.google.com/bigquery/docs/projects). Datasets are top-level containers that are used to organize and control access to your [tables](https://cloud.google.com/bigquery/docs/tables) and [views](https://cloud.google.com/bigquery/docs/views). A table or view must belong to a dataset. You need to create at least one dataset before [loading data into BigQuery](https://cloud.google.com/bigquery/loading-data-into-bigquery).\n", + "\n", + "First, name your new dataset:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dataset_id = \"your_new_dataset\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following command creates a new dataset in the US using the ID defined above.\n", + "\n", + "NOTE: In the examples in this notebook, the `dataset_id` variable is referenced in the commands using both `{}` and `$`. To avoid creating and using variables, replace these interpolated variables with literal values and remove the `{}` and `$` characters." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!bq --location=US mk --dataset $dataset_id" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The response should look like the following:\n", + "\n", + "```\n", + "Dataset 'your-project-id:your_new_dataset' successfully created.\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## List datasets\n", + "\n", + "The following command lists all datasets in your default project." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!bq ls" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The response should look like the following:\n", + "\n", + "```\n", + " datasetId \n", + " ------------------------------ \n", + " your_new_dataset \n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load data from a local file to a table\n", + "\n", + "The following example demonstrates how to load a local CSV file into a new or existing table. See [SourceFormat](https://googleapis.github.io/google-cloud-python/latest/bigquery/generated/google.cloud.bigquery.job.SourceFormat.html#google.cloud.bigquery.job.SourceFormat) in the Python client library documentation for a list of available source formats. For more information, see [Loading Data into BigQuery from a local data source](https://cloud.google.com/bigquery/docs/loading-data-local) in the BigQuery documentation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!bq \\\n", + " --location=US \\\n", + " load \\\n", + " --autodetect \\\n", + " --skip_leading_rows=1 \\\n", + " --source_format=CSV \\\n", + " {dataset_id}.us_states_local_file \\\n", + " 'resources/us-states.csv'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load data from Cloud Storage to a table\n", + "\n", + "The following example demonstrates how to load a local CSV file into a new table. See [SourceFormat](https://googleapis.github.io/google-cloud-python/latest/bigquery/generated/google.cloud.bigquery.job.SourceFormat.html#google.cloud.bigquery.job.SourceFormat) in the Python client library documentation for a list of available source formats. For more information, see [Introduction to loading data from Cloud Storage](https://cloud.google.com/bigquery/docs/loading-data-cloud-storage) in the BigQuery documentation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!bq \\\n", + " --location=US \\\n", + " load \\\n", + " --autodetect \\\n", + " --skip_leading_rows=1 \\\n", + " --source_format=CSV \\\n", + " {dataset_id}.us_states_gcs \\\n", + " 'gs://cloud-samples-data/bigquery/us-states/us-states.csv'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run a query\n", + "\n", + "The BigQuery command-line tool has a `query` command for running queries, but it is recommended to use the [magic command](./BigQuery%20Query%20Magic.ipynb) for this purpose." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Cleaning Up\n", + "\n", + "The following code deletes the dataset created for this tutorial, including all tables in the dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!bq rm -r -f --dataset $dataset_id" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/tutorials/bigquery/BigQuery query magic.ipynb b/notebooks/tutorials/bigquery/BigQuery query magic.ipynb new file mode 100644 index 00000000000..9c948679a7b --- /dev/null +++ b/notebooks/tutorials/bigquery/BigQuery query magic.ipynb @@ -0,0 +1,180 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# BigQuery query magic\n", + "\n", + "Jupyter magics are notebook-specific shortcuts that allow you to run commands with minimal syntax. Jupyter notebooks come with many [built-in commands](https://ipython.readthedocs.io/en/stable/interactive/magics.html). The BigQuery client library, `google-cloud-bigquery`, provides a cell magic, `%%bigquery`. The `%%bigquery` magic runs a SQL query and returns the results as a pandas `DataFrame`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run a query on a public dataset\n", + "\n", + "The following example queries the BigQuery `usa_names` public dataset. `usa_names` is a Social Security Administration dataset that contains all names from Social Security card applications for births that occurred in the United States after 1879.\n", + "\n", + "The following example shows how to invoke the magic (`%%bigquery`), and how to pass in a standard SQL query in the body of the code cell. The results are displayed below the input cell as a pandas [`DataFrame`](http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "%%bigquery\n", + "SELECT name, SUM(number) as count\n", + "FROM `bigquery-public-data.usa_names.usa_1910_current`\n", + "GROUP BY name\n", + "ORDER BY count DESC\n", + "LIMIT 10" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Display verbose output\n", + "\n", + "As the query job is running, status messages below the cell update with the query job ID and the amount of time the query has been running. By default, this output is erased and replaced with the results of the query. If you pass the `--verbose` flag, the output will remain below the cell after query completion." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%bigquery --verbose\n", + "SELECT name, SUM(number) as count\n", + "FROM `bigquery-public-data.usa_names.usa_1910_current`\n", + "GROUP BY name\n", + "ORDER BY count DESC\n", + "LIMIT 10" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Explicitly specify a project\n", + "\n", + "By default, the `%%bigquery` magic command uses your default project to run the query. You may also explicitly provide a project ID using the `--project` flag. Note that your credentials must have permissions to create query jobs in the project you specify." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "project_id = 'your-project-id'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%bigquery --project $project_id\n", + "SELECT name, SUM(number) as count\n", + "FROM `bigquery-public-data.usa_names.usa_1910_current`\n", + "GROUP BY name\n", + "ORDER BY count DESC\n", + "LIMIT 10" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Assign the query results to a variable\n", + "\n", + "To save the results of your query to a variable, provide a variable name as a parameter to `%%bigquery`. The following example saves the results of the query to a variable named `df`. Note that when a variable is provided, the results are not displayed below the cell that invokes the magic command." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%bigquery df\n", + "SELECT name, SUM(number) as count\n", + "FROM `bigquery-public-data.usa_names.usa_1910_current`\n", + "GROUP BY name\n", + "ORDER BY count DESC\n", + "LIMIT 10" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run a parameterized query\n", + "\n", + "Parameterized queries are useful if you need to run a query with certain parameters that are calculated at run time. Note that the value types must be JSON serializable. The following example defines a parameters dictionary and passes it to the `--params` flag. The key of the dictionary is the name of the parameter, and the value of the dictionary is the value of the parameter." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "params = {\"limit\": 10}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%bigquery --params $params\n", + "SELECT name, SUM(number) as count\n", + "FROM `bigquery-public-data.usa_names.usa_1910_current`\n", + "GROUP BY name\n", + "ORDER BY count DESC\n", + "LIMIT @limit" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/tutorials/bigquery/Getting started with BigQuery ML.ipynb b/notebooks/tutorials/bigquery/Getting started with BigQuery ML.ipynb new file mode 100644 index 00000000000..e3f8625f10e --- /dev/null +++ b/notebooks/tutorials/bigquery/Getting started with BigQuery ML.ipynb @@ -0,0 +1,384 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Getting started with BigQuery ML\n", + "\n", + "BigQuery ML enables users to create and execute machine learning models in BigQuery using SQL queries. The goal is to democratize machine learning by enabling SQL practitioners to build models using their existing tools and to increase development speed by eliminating the need for data movement.\n", + "\n", + "In this tutorial, you use the sample [Google Analytics sample dataset for BigQuery](https://support.google.com/analytics/answer/7586738?hl=en&ref_topic=3416089) to create a model that predicts whether a website visitor will make a transaction. For information on the schema of the Analytics dataset, see [BigQuery export schema](https://support.google.com/analytics/answer/3437719) in the Google Analytics Help Center.\n", + "\n", + "\n", + "## Objectives\n", + "In this tutorial, you use:\n", + "\n", + "+ BigQuery ML to create a binary logistic regression model using the `CREATE MODEL` statement\n", + "+ The `ML.EVALUATE` function to evaluate the ML model\n", + "+ The `ML.PREDICT` function to make predictions using the ML model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create your dataset\n", + "\n", + "Enter the following code to import the BigQuery Python client library and initialize a client. The BigQuery client is used to send and receive messages from the BigQuery API." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from google.cloud import bigquery\n", + "\n", + "client = bigquery.Client(location=\"US\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, you create a BigQuery dataset to store your ML model. Run the following to create your dataset:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dataset = client.create_dataset(\"bqml_tutorial\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create your model\n", + "\n", + "Next, you create a logistic regression model using the Google Analytics sample\n", + "dataset for BigQuery. The model is used to predict whether a\n", + "website visitor will make a transaction. The standard SQL query uses a\n", + "`CREATE MODEL` statement to create and train the model. Standard SQL is the\n", + "default query syntax for the BigQuery python client library.\n", + "\n", + "The BigQuery python client library provides a cell magic,\n", + "`%%bigquery`, which runs a SQL query and returns the results as a Pandas\n", + "`DataFrame`.\n", + "\n", + "To run the `CREATE MODEL` query to create and train your model:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%bigquery\n", + "CREATE OR REPLACE MODEL `bqml_tutorial.sample_model`\n", + "OPTIONS(model_type='logistic_reg') AS\n", + "SELECT\n", + " IF(totals.transactions IS NULL, 0, 1) AS label,\n", + " IFNULL(device.operatingSystem, \"\") AS os,\n", + " device.isMobile AS is_mobile,\n", + " IFNULL(geoNetwork.country, \"\") AS country,\n", + " IFNULL(totals.pageviews, 0) AS pageviews\n", + "FROM\n", + " `bigquery-public-data.google_analytics_sample.ga_sessions_*`\n", + "WHERE\n", + " _TABLE_SUFFIX BETWEEN '20160801' AND '20170630'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The query takes several minutes to complete. After the first iteration is\n", + "complete, your model (`sample_model`) appears in the navigation panel of the\n", + "BigQuery web UI. Because the query uses a `CREATE MODEL` statement to create a\n", + "table, you do not see query results. The output is an empty `DataFrame`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Get training statistics\n", + "\n", + "To see the results of the model training, you can use the\n", + "[`ML.TRAINING_INFO`](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-train)\n", + "function, or you can view the statistics in the BigQuery web UI. This functionality\n", + "is not currently available in the BigQuery Classic web UI.\n", + "In this tutorial, you use the `ML.TRAINING_INFO` function.\n", + "\n", + "A machine learning algorithm builds a model by examining many examples and\n", + "attempting to find a model that minimizes loss. This process is called empirical\n", + "risk minimization.\n", + "\n", + "Loss is the penalty for a bad prediction — a number indicating\n", + "how bad the model's prediction was on a single example. If the model's\n", + "prediction is perfect, the loss is zero; otherwise, the loss is greater. The\n", + "goal of training a model is to find a set of weights that have low\n", + "loss, on average, across all examples.\n", + "\n", + "To see the model training statistics that were generated when you ran the\n", + "`CREATE MODEL` query:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%bigquery\n", + "SELECT\n", + " *\n", + "FROM\n", + " ML.TRAINING_INFO(MODEL `bqml_tutorial.sample_model`)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note: Typically, it is not a best practice to use a `SELECT *` query. Because the model output is a small table, this query does not process a large amount of data. As a result, the cost is minimal." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When the query is complete, the results appear below the query. The results should look like the following:\n", + "\n", + "![Training statistics table](./resources/training-statistics.png)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `loss` column represents the loss metric calculated after the given iteration\n", + "on the training dataset. Since you performed a logistic regression, this column\n", + "is the [log loss](https://en.wikipedia.org/wiki/Cross_entropy#Cross-entropy_error_function_and_logistic_regression).\n", + "The `eval_loss` column is the same loss metric calculated on\n", + "the holdout dataset (data that is held back from training to validate the model).\n", + "\n", + "For more details on the `ML.TRAINING_INFO` function, see the\n", + "[BigQuery ML syntax reference](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-train)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Evaluate your model\n", + "\n", + "After creating your model, you evaluate the performance of the classifier using\n", + "the [`ML.EVALUATE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-evaluate)\n", + "function. You can also use the [`ML.ROC_CURVE`](https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-roc)\n", + "function for logistic regression specific metrics.\n", + "\n", + "A classifier is one of a set of enumerated target values for a label. For\n", + "example, in this tutorial you are using a binary classification model that\n", + "detects transactions. The two classes are the values in the `label` column:\n", + "`0` (no transactions) and not `1` (transaction made).\n", + "\n", + "To run the `ML.EVALUATE` query that evaluates the model:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%bigquery\n", + "SELECT\n", + " *\n", + "FROM ML.EVALUATE(MODEL `bqml_tutorial.sample_model`, (\n", + " SELECT\n", + " IF(totals.transactions IS NULL, 0, 1) AS label,\n", + " IFNULL(device.operatingSystem, \"\") AS os,\n", + " device.isMobile AS is_mobile,\n", + " IFNULL(geoNetwork.country, \"\") AS country,\n", + " IFNULL(totals.pageviews, 0) AS pageviews\n", + " FROM\n", + " `bigquery-public-data.google_analytics_sample.ga_sessions_*`\n", + " WHERE\n", + " _TABLE_SUFFIX BETWEEN '20170701' AND '20170801'))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When the query is complete, the results appear below the query. The\n", + "results should look like the following:\n", + "\n", + "![Model evaluation results table](./resources/model-evaluation.png)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Because you performed a logistic regression, the results include the following\n", + "columns:\n", + "\n", + "+ [`precision`](https://developers.google.com/machine-learning/glossary/#precision)\n", + "+ [`recall`](https://developers.google.com/machine-learning/glossary/#recall)\n", + "+ [`accuracy`](https://developers.google.com/machine-learning/glossary/#accuracy)\n", + "+ [`f1_score`](https://en.wikipedia.org/wiki/F1_score)\n", + "+ [`log_loss`](https://developers.google.com/machine-learning/glossary/#Log_Loss)\n", + "+ [`roc_auc`](https://developers.google.com/machine-learning/glossary/#AUC)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Use your model to predict outcomes\n", + "\n", + "Now that you have evaluated your model, the next step is to use it to predict\n", + "outcomes. You use your model to predict the number of transactions made by\n", + "website visitors from each country. And you use it to predict purchases per user.\n", + "\n", + "To run the query that uses the model to predict the number of transactions:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%bigquery\n", + "SELECT\n", + " country,\n", + " SUM(predicted_label) as total_predicted_purchases\n", + "FROM ML.PREDICT(MODEL `bqml_tutorial.sample_model`, (\n", + " SELECT\n", + " IFNULL(device.operatingSystem, \"\") AS os,\n", + " device.isMobile AS is_mobile,\n", + " IFNULL(totals.pageviews, 0) AS pageviews,\n", + " IFNULL(geoNetwork.country, \"\") AS country\n", + " FROM\n", + " `bigquery-public-data.google_analytics_sample.ga_sessions_*`\n", + " WHERE\n", + " _TABLE_SUFFIX BETWEEN '20170701' AND '20170801'))\n", + " GROUP BY country\n", + " ORDER BY total_predicted_purchases DESC\n", + " LIMIT 10" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When the query is complete, the results appear below the query. The\n", + "results should look like the following. Because model training is not\n", + "deterministic, your results may differ.\n", + "\n", + "![Model predictions table](./resources/transaction-predictions.png)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the next example, you try to predict the number of transactions each website\n", + "visitor will make. This query is identical to the previous query except for the\n", + "`GROUP BY` clause. Here the `GROUP BY` clause — `GROUP BY fullVisitorId`\n", + "— is used to group the results by visitor ID.\n", + "\n", + "To run the query that predicts purchases per user:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%bigquery\n", + "SELECT\n", + " fullVisitorId,\n", + " SUM(predicted_label) as total_predicted_purchases\n", + "FROM ML.PREDICT(MODEL `bqml_tutorial.sample_model`, (\n", + " SELECT\n", + " IFNULL(device.operatingSystem, \"\") AS os,\n", + " device.isMobile AS is_mobile,\n", + " IFNULL(totals.pageviews, 0) AS pageviews,\n", + " IFNULL(geoNetwork.country, \"\") AS country,\n", + " fullVisitorId\n", + " FROM\n", + " `bigquery-public-data.google_analytics_sample.ga_sessions_*`\n", + " WHERE\n", + " _TABLE_SUFFIX BETWEEN '20170701' AND '20170801'))\n", + " GROUP BY fullVisitorId\n", + " ORDER BY total_predicted_purchases DESC\n", + " LIMIT 10" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When the query is complete, the results appear below the query. The\n", + "results should look like the following:\n", + "\n", + "![Purchase predictions table](./resources/purchase-predictions.png)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Cleaning up\n", + "\n", + "To delete the resources created by this tutorial, execute the following code to delete the dataset and its contents:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "client.delete_dataset(dataset, delete_contents=True)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/tutorials/bigquery/Visualizing BigQuery public data.ipynb b/notebooks/tutorials/bigquery/Visualizing BigQuery public data.ipynb new file mode 100644 index 00000000000..607938d6fbd --- /dev/null +++ b/notebooks/tutorials/bigquery/Visualizing BigQuery public data.ipynb @@ -0,0 +1,271 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Vizualizing BigQuery data in a Jupyter notebook\n", + "\n", + "[BigQuery](https://cloud.google.com/bigquery/docs/) is a petabyte-scale analytics data warehouse that you can use to run SQL queries over vast amounts of data in near realtime.\n", + "\n", + "Data visualization tools can help you make sense of your BigQuery data and help you analyze the data interactively. You can use visualization tools to help you identify trends, respond to them, and make predictions using your data. In this tutorial, you use the BigQuery Python client library and pandas in a Jupyter notebook to visualize data in the BigQuery natality sample table." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using Jupyter magics to query BigQuery data\n", + "\n", + "The BigQuery Python client library provides a magic command that allows you to run queries with minimal code.\n", + "\n", + "The BigQuery client library provides a cell magic, `%%bigquery`. The `%%bigquery` magic runs a SQL query and returns the results as a pandas `DataFrame`. The following cell executes a query of the BigQuery natality public dataset and returns the total births by year." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%bigquery\n", + "SELECT\n", + " source_year AS year,\n", + " COUNT(is_male) AS birth_count\n", + "FROM `bigquery-public-data.samples.natality`\n", + "GROUP BY year\n", + "ORDER BY year DESC\n", + "LIMIT 15" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following command to runs the same query, but this time the results are saved to a variable. The variable name, `total_births`, is given as an argument to the `%%bigquery`. The results can then be used for further analysis and visualization." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%bigquery total_births\n", + "SELECT\n", + " source_year AS year,\n", + " COUNT(is_male) AS birth_count\n", + "FROM `bigquery-public-data.samples.natality`\n", + "GROUP BY year\n", + "ORDER BY year DESC\n", + "LIMIT 15" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The next cell uses the pandas `DataFrame.plot` method to visualize the query results as a bar chart. See the [pandas documentation](https://pandas.pydata.org/pandas-docs/stable/visualization.html) to learn more about data visualization with pandas." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "total_births.plot(kind='bar', x='year', y='birth_count');" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Run the following query to retrieve the number of births by weekday. Because the `wday` (weekday) field allows null values, the query excludes records where wday is null." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%bigquery births_by_weekday\n", + "SELECT\n", + " wday,\n", + " SUM(CASE WHEN is_male THEN 1 ELSE 0 END) AS male_births,\n", + " SUM(CASE WHEN is_male THEN 0 ELSE 1 END) AS female_births\n", + "FROM `bigquery-public-data.samples.natality`\n", + "WHERE wday IS NOT NULL\n", + "GROUP BY wday\n", + "ORDER BY wday ASC" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Visualize the query results using a line chart." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "births_by_weekday.plot(x='wday');" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using Python to query BigQuery data\n", + "\n", + "Magic commands allow you to use minimal syntax to interact with BigQuery. Behind the scenes, `%%bigquery` uses the BigQuery Python client library to run the given query, convert the results to a pandas `Dataframe`, optionally save the results to a variable, and finally display the results. Using the BigQuery Python client library directly instead of through magic commands gives you more control over your queries and allows for more complex configurations. The library's integrations with pandas enable you to combine the power of declarative SQL with imperative code (Python) to perform interesting data analysis, visualization, and transformation tasks.\n", + "\n", + "To use the BigQuery Python client library, start by importing the library and initializing a client. The BigQuery client is used to send and receive messages from the BigQuery API." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from google.cloud import bigquery\n", + "\n", + "client = bigquery.Client()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use the [`Client.query`](https://googleapis.github.io/google-cloud-python/latest/bigquery/generated/google.cloud.bigquery.client.Client.html#google.cloud.bigquery.client.Client.query) method to run a query. Execute the following cell to run a query to retrieve the annual count of plural births by plurality (2 for twins, 3 for triplets, etc.)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sql = \"\"\"\n", + "SELECT\n", + " plurality,\n", + " COUNT(1) AS count,\n", + " year\n", + "FROM\n", + " `bigquery-public-data.samples.natality`\n", + "WHERE\n", + " NOT IS_NAN(plurality) AND plurality > 1\n", + "GROUP BY\n", + " plurality, year\n", + "ORDER BY\n", + " count DESC\n", + "\"\"\"\n", + "df = client.query(sql).to_dataframe()\n", + "df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To chart the query results in your `DataFrame`, run the following cell to pivot the data and create a stacked bar chart of the count of plural births over time." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pivot_table = df.pivot(index='year', columns='plurality', values='count')\n", + "pivot_table.plot(kind='bar', stacked=True, figsize=(15, 7));" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Run the following query to retrieve the count of births by the number of gestation weeks." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sql = \"\"\"\n", + "SELECT\n", + " gestation_weeks,\n", + " COUNT(1) AS count\n", + "FROM\n", + " `bigquery-public-data.samples.natality`\n", + "WHERE\n", + " NOT IS_NAN(gestation_weeks) AND gestation_weeks <> 99\n", + "GROUP BY\n", + " gestation_weeks\n", + "ORDER BY\n", + " gestation_weeks\n", + "\"\"\"\n", + "df = client.query(sql).to_dataframe()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, chart the query results in your `DataFrame`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ax = df.plot(kind='bar', x='gestation_weeks', y='count', figsize=(15,7))\n", + "ax.set_title('Count of Births by Gestation Weeks')\n", + "ax.set_xlabel('Gestation Weeks')\n", + "ax.set_ylabel('Count');" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## What's Next\n", + "\n", + "+ __Learn more about writing queries for BigQuery__ — [Querying Data](https://cloud.google.com/bigquery/querying-data) in the BigQuery documentation explains how to run queries, create user-defined functions (UDFs), and more.\n", + "\n", + "+ __Explore BigQuery syntax__ — The preferred dialect for SQL queries in BigQuery is standard SQL. Standard SQL syntax is described in the [SQL Reference](https://cloud.google.com/bigquery/docs/reference/standard-sql/). BigQuery's legacy SQL-like syntax is described in the [Query Reference (legacy SQL)](https://cloud.google.com/bigquery/query-reference)." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/tutorials/bigquery/resources/model-evaluation.png b/notebooks/tutorials/bigquery/resources/model-evaluation.png new file mode 100644 index 00000000000..814e8175f3d Binary files /dev/null and b/notebooks/tutorials/bigquery/resources/model-evaluation.png differ diff --git a/notebooks/tutorials/bigquery/resources/purchase-predictions.png b/notebooks/tutorials/bigquery/resources/purchase-predictions.png new file mode 100644 index 00000000000..b48aae714c8 Binary files /dev/null and b/notebooks/tutorials/bigquery/resources/purchase-predictions.png differ diff --git a/notebooks/tutorials/bigquery/resources/training-statistics.png b/notebooks/tutorials/bigquery/resources/training-statistics.png new file mode 100644 index 00000000000..6bb8176446b Binary files /dev/null and b/notebooks/tutorials/bigquery/resources/training-statistics.png differ diff --git a/notebooks/tutorials/bigquery/resources/transaction-predictions.png b/notebooks/tutorials/bigquery/resources/transaction-predictions.png new file mode 100644 index 00000000000..877500ed830 Binary files /dev/null and b/notebooks/tutorials/bigquery/resources/transaction-predictions.png differ diff --git a/notebooks/tutorials/bigquery/resources/us-states.csv b/notebooks/tutorials/bigquery/resources/us-states.csv new file mode 100644 index 00000000000..54a60e29de9 --- /dev/null +++ b/notebooks/tutorials/bigquery/resources/us-states.csv @@ -0,0 +1,51 @@ +name,post_abbr +Alabama,AL +Alaska,AK +Arizona,AZ +Arkansas,AR +California,CA +Colorado,CO +Connecticut,CT +Delaware,DE +Florida,FL +Georgia,GA +Hawaii,HI +Idaho,ID +Illinois,IL +Indiana,IN +Iowa,IA +Kansas,KS +Kentucky,KY +Louisiana,LA +Maine,ME +Maryland,MD +Massachusetts,MA +Michigan,MI +Minnesota,MN +Mississippi,MS +Missouri,MO +Montana,MT +Nebraska,NE +Nevada,NV +New Hampshire,NH +New Jersey,NJ +New Mexico,NM +New York,NY +North Carolina,NC +North Dakota,ND +Ohio,OH +Oklahoma,OK +Oregon,OR +Pennsylvania,PA +Rhode Island,RI +South Carolina,SC +South Dakota,SD +Tennessee,TN +Texas,TX +Utah,UT +Vermont,VT +Virginia,VA +Washington,WA +West Virginia,WV +Wisconsin,WI +Wyoming,WY diff --git a/notebooks/tutorials/cloud-ml-engine/Training and prediction with scikit-learn.ipynb b/notebooks/tutorials/cloud-ml-engine/Training and prediction with scikit-learn.ipynb new file mode 100644 index 00000000000..c9d641c610e --- /dev/null +++ b/notebooks/tutorials/cloud-ml-engine/Training and prediction with scikit-learn.ipynb @@ -0,0 +1,570 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Training and prediction with scikit-learn\n", + "\n", + "This notebook demonstrates how to use AI Platform to train a simple classification model using `scikit-learn`, and then deploy the model to get predictions.\n", + "\n", + "You train the model to predict a person's income level based on the [Census Income data set](https://archive.ics.uci.edu/ml/datasets/Census+Income).\n", + "\n", + "Before you jump in, let’s cover some of the different tools you’ll be using:\n", + "\n", + "+ [AI Platform](https://cloud.google.com/ml-engine/) is a managed service that enables you to easily build machine learning models that work on any type of data, of any size.\n", + "\n", + "+ [Cloud Storage](https://cloud.google.com/storage/) is a unified object storage for developers and enterprises, from live data serving to data analytics/ML to data archiving.\n", + "\n", + "+ [Cloud SDK](https://cloud.google.com/sdk/) is a command line tool which allows you to interact with Google Cloud products. This notebook introduces several `gcloud` and `gsutil` commands, which are part of the Cloud SDK. Note that shell commands in a notebook must be prepended with a `!`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Set up your environment\n", + "\n", + "### Enable the required APIs\n", + "\n", + "In order to use AI Platform, confirm that the required APIs are enabled:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!gcloud services enable ml.googleapis.com\n", + "!gcloud services enable compute.googleapis.com" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create a storage bucket\n", + "Buckets are the basic containers that hold your data. Everything that you store in Cloud Storage must be contained in a bucket. You can use buckets to organize your data and control access to your data.\n", + "\n", + "Start by defining a globally unique name.\n", + "\n", + "For more information about naming buckets, see [Bucket name requirements](https://cloud.google.com/storage/docs/naming#requirements)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "BUCKET_NAME = 'your-new-bucket'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the examples below, the `BUCKET_NAME` variable is referenced in the commands using `$`.\n", + "\n", + "Create the new bucket with the `gsutil mb` command:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!gsutil mb gs://$BUCKET_NAME/" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### About the data\n", + "\n", + "The [Census Income Data Set](https://archive.ics.uci.edu/ml/datasets/Census+Income) that this sample\n", + "uses for training is provided by the [UC Irvine Machine Learning\n", + "Repository](https://archive.ics.uci.edu/ml/datasets/).\n", + "\n", + "Census data courtesy of: Lichman, M. (2013). UCI Machine Learning Repository http://archive.ics.uci.edu/ml. Irvine, CA: University of California, School of Information and Computer Science. This dataset is publicly available for anyone to use under the following terms provided by the Dataset Source - http://archive.ics.uci.edu/ml - and is provided \"AS IS\" without any warranty, express or implied, from Google. Google disclaims all liability for any damages, direct or indirect, resulting from the use of the dataset.\n", + "\n", + "The data used in this tutorial is located in a public Cloud Storage bucket:\n", + "\n", + " gs://cloud-samples-data/ml-engine/sklearn/census_data/ \n", + "\n", + "The training file is `adult.data` ([download](https://storage.googleapis.com/cloud-samples-data/ml-engine/sklearn/census_data/adult.data)) and the evaluation file is `adult.test` ([download](https://storage.googleapis.com/cloud-samples-data/ml-engine/sklearn/census_data/adult.test)). The evaluation file is not used in this tutorial." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Create training application package\n", + "\n", + "The easiest (and recommended) way to create a training application package is to use `gcloud` to package and upload the application when you submit your training job. This method allows you to create a very simple file structure with only two files. For this tutorial, the file structure of your training application package should appear similar to the following:\n", + "\n", + " census_training/\n", + " __init__.py\n", + " train.py" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create a directory locally:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!mkdir census_training" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create a blank file named `__init__.py`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!touch ./census_training/__init__.py" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Save training code in one Python file in the `census_training` directory. The following cell writes a training file to the `census_training` directory. The training file performs the following operations:\n", + "+ Loads the data into a pandas `DataFrame` that can be used by `scikit-learn`\n", + "+ Fits the model is against the training data\n", + "+ Exports the model with the [Python `pickle` library](https://docs.python.org/3/library/pickle.html)\n", + "\n", + "The following model training code is not executed within this notebook. Instead, it is saved to a Python file and packaged as a Python module that runs on AI Platform after you submit the training job." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%writefile ./census_training/train.py\n", + "import argparse\n", + "import pickle\n", + "import pandas as pd\n", + "\n", + "from google.cloud import storage\n", + "\n", + "from sklearn.ensemble import RandomForestClassifier\n", + "from sklearn.feature_selection import SelectKBest\n", + "from sklearn.pipeline import FeatureUnion\n", + "from sklearn.pipeline import Pipeline\n", + "from sklearn.preprocessing import LabelBinarizer\n", + "\n", + "parser = argparse.ArgumentParser()\n", + "parser.add_argument(\"--bucket-name\", help=\"The bucket name\", required=True)\n", + "\n", + "arguments, unknown = parser.parse_known_args()\n", + "bucket_name = arguments.bucket_name\n", + "\n", + "# Define the format of your input data, including unused columns.\n", + "# These are the columns from the census data files.\n", + "COLUMNS = (\n", + " 'age',\n", + " 'workclass',\n", + " 'fnlwgt',\n", + " 'education',\n", + " 'education-num',\n", + " 'marital-status',\n", + " 'occupation',\n", + " 'relationship',\n", + " 'race',\n", + " 'sex',\n", + " 'capital-gain',\n", + " 'capital-loss',\n", + " 'hours-per-week',\n", + " 'native-country',\n", + " 'income-level'\n", + ")\n", + "\n", + "# Categorical columns are columns that need to be turned into a numerical value\n", + "# to be used by scikit-learn\n", + "CATEGORICAL_COLUMNS = (\n", + " 'workclass',\n", + " 'education',\n", + " 'marital-status',\n", + " 'occupation',\n", + " 'relationship',\n", + " 'race',\n", + " 'sex',\n", + " 'native-country'\n", + ")\n", + "\n", + "# Create a Cloud Storage client to download the census data\n", + "storage_client = storage.Client()\n", + "\n", + "# Download the data\n", + "public_bucket = storage_client.bucket('cloud-samples-data')\n", + "blob = public_bucket.blob('ml-engine/sklearn/census_data/adult.data')\n", + "blob.download_to_filename('adult.data')\n", + "\n", + "# Load the training census dataset\n", + "with open(\"./adult.data\", \"r\") as train_data:\n", + " raw_training_data = pd.read_csv(train_data, header=None, names=COLUMNS)\n", + " # Removing the whitespaces in categorical features\n", + " for col in CATEGORICAL_COLUMNS:\n", + " raw_training_data[col] = raw_training_data[col].apply(lambda x: str(x).strip())\n", + "\n", + "# Remove the column we are trying to predict ('income-level') from our features\n", + "# list and convert the DataFrame to a lists of lists\n", + "train_features = raw_training_data.drop(\"income-level\", axis=1).values.tolist()\n", + "# Create our training labels list, convert the DataFrame to a lists of lists\n", + "train_labels = (raw_training_data[\"income-level\"] == \" >50K\").values.tolist()\n", + "\n", + "# Since the census data set has categorical features, we need to convert\n", + "# them to numerical values. We'll use a list of pipelines to convert each\n", + "# categorical column and then use FeatureUnion to combine them before calling\n", + "# the RandomForestClassifier.\n", + "categorical_pipelines = []\n", + "\n", + "# Each categorical column needs to be extracted individually and converted to a\n", + "# numerical value. To do this, each categorical column will use a pipeline that\n", + "# extracts one feature column via SelectKBest(k=1) and a LabelBinarizer() to\n", + "# convert the categorical value to a numerical one. A scores array (created\n", + "# below) will select and extract the feature column. The scores array is\n", + "# created by iterating over the columns and checking if it is a\n", + "# categorical column.\n", + "for i, col in enumerate(COLUMNS[:-1]):\n", + " if col in CATEGORICAL_COLUMNS:\n", + " # Create a scores array to get the individual categorical column.\n", + " # Example:\n", + " # data = [\n", + " # 39, 'State-gov', 77516, 'Bachelors', 13, 'Never-married',\n", + " # 'Adm-clerical', 'Not-in-family', 'White', 'Male', 2174, 0,\n", + " # 40, 'United-States'\n", + " # ]\n", + " # scores = [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", + " #\n", + " # Returns: [['State-gov']]\n", + " # Build the scores array\n", + " scores = [0] * len(COLUMNS[:-1])\n", + " # This column is the categorical column we want to extract.\n", + " scores[i] = 1\n", + " skb = SelectKBest(k=1)\n", + " skb.scores_ = scores\n", + " # Convert the categorical column to a numerical value\n", + " lbn = LabelBinarizer()\n", + " r = skb.transform(train_features)\n", + " lbn.fit(r)\n", + " # Create the pipeline to extract the categorical feature\n", + " categorical_pipelines.append(\n", + " (\n", + " 'categorical-{}'.format(i), \n", + " Pipeline([\n", + " ('SKB-{}'.format(i), skb),\n", + " ('LBN-{}'.format(i), lbn)])\n", + " )\n", + " )\n", + "\n", + "# Create pipeline to extract the numerical features\n", + "skb = SelectKBest(k=6)\n", + "# From COLUMNS use the features that are numerical\n", + "skb.scores_ = [1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0]\n", + "categorical_pipelines.append((\"numerical\", skb))\n", + "\n", + "# Combine all the features using FeatureUnion\n", + "preprocess = FeatureUnion(categorical_pipelines)\n", + "\n", + "# Create the classifier\n", + "classifier = RandomForestClassifier()\n", + "\n", + "# Transform the features and fit them to the classifier\n", + "classifier.fit(preprocess.transform(train_features), train_labels)\n", + "\n", + "# Create the overall model as a single pipeline\n", + "pipeline = Pipeline([(\"union\", preprocess), (\"classifier\", classifier)])\n", + "\n", + "# Create the model file\n", + "# It is required to name the model file \"model.pkl\" if you are using pickle\n", + "model_filename = \"model.pkl\"\n", + "with open(model_filename, \"wb\") as model_file:\n", + " pickle.dump(pipeline, model_file)\n", + "\n", + "# Upload the model to Cloud Storage\n", + "bucket = storage_client.bucket(bucket_name)\n", + "blob = bucket.blob(model_filename)\n", + "blob.upload_from_filename(model_filename)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Submit the training job\n", + "\n", + "In this section, you use [`gcloud ml-engine jobs submit training`](https://cloud.google.com/sdk/gcloud/reference/ml-engine/jobs/submit/training) to submit your training job. The `--` argument passed to the command is a separator; anything after the separator will be passed to the Python code as input arguments.\n", + "\n", + "For more information about the arguments preceeding the separator, run the following:\n", + "\n", + " gcloud ml-engine jobs submit training --help\n", + "\n", + "The argument given to the python script is `--bucket-name`. The `--bucket-name` argument is used to specify the name of the bucket to save the model file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "import time\n", + "\n", + "# Define a timestamped job name\n", + "JOB_NAME = \"census_training_{}\".format(int(time.time()))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# Submit the training job:\n", + "!gcloud ml-engine jobs submit training $JOB_NAME \\\n", + " --job-dir gs://$BUCKET_NAME/census_job_dir \\\n", + " --package-path ./census_training \\\n", + " --module-name census_training.train \\\n", + " --region us-central1 \\\n", + " --runtime-version=1.12 \\\n", + " --python-version=3.5 \\\n", + " --scale-tier BASIC \\\n", + " --stream-logs \\\n", + " -- \\\n", + " --bucket-name $BUCKET_NAME" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Verify model file in Cloud Storage\n", + "\n", + "View the contents of the destination model directory to verify that your model file has been uploaded to Cloud Storage.\n", + "\n", + "Note: The model can take a few minutes to train and show up in Cloud Storage." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!gsutil ls gs://$BUCKET_NAME/" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Serve the model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Once the model is successfully created and trained, you can serve it. A model can have different versions. In order to serve the model, create a model and version in AI Platform.\n", + "\n", + "Define the model and version names:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "MODEL_NAME = \"CensusPredictor\"\n", + "VERSION_NAME = \"census_predictor_{}\".format(int(time.time()))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create the model in AI Platform:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!gcloud ml-engine models create $MODEL_NAME --regions us-central1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create a version that points to your model file in Cloud Storage:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!gcloud ml-engine versions create $VERSION_NAME \\\n", + " --model=$MODEL_NAME \\\n", + " --framework=scikit-learn \\\n", + " --origin=gs://$BUCKET_NAME/ \\\n", + " --python-version=3.5 \\\n", + " --runtime-version=1.12" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Make predictions\n", + "\n", + "### Format data for prediction\n", + "\n", + "Before you send an online prediction request, you must format your test data to prepare it for use by the AI Platform prediction service. Make sure that the format of your input instances matches what your model expects.\n", + "\n", + "Create an `input.json` file with each input instance on a separate line. The following example uses ten data instances. Note that the format of input instances needs to match what your model expects. In this example, the Census model requires 14 features, so your input must be a matrix of shape (`num_instances, 14`)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Define a name for the input file\n", + "INPUT_FILE = \"./census_training/input.json\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "%%writefile $INPUT_FILE\n", + "[25, \"Private\", 226802, \"11th\", 7, \"Never-married\", \"Machine-op-inspct\", \"Own-child\", \"Black\", \"Male\", 0, 0, 40, \"United-States\"]\n", + "[38, \"Private\", 89814, \"HS-grad\", 9, \"Married-civ-spouse\", \"Farming-fishing\", \"Husband\", \"White\", \"Male\", 0, 0, 50, \"United-States\"]\n", + "[28, \"Local-gov\", 336951, \"Assoc-acdm\", 12, \"Married-civ-spouse\", \"Protective-serv\", \"Husband\", \"White\", \"Male\", 0, 0, 40, \"United-States\"]\n", + "[44, \"Private\", 160323, \"Some-college\", 10, \"Married-civ-spouse\", \"Machine-op-inspct\", \"Husband\", \"Black\", \"Male\", 7688, 0, 40, \"United-States\"]\n", + "[18, \"?\", 103497, \"Some-college\", 10, \"Never-married\", \"?\", \"Own-child\", \"White\", \"Female\", 0, 0, 30, \"United-States\"]\n", + "[34, \"Private\", 198693, \"10th\", 6, \"Never-married\", \"Other-service\", \"Not-in-family\", \"White\", \"Male\", 0, 0, 30, \"United-States\"]\n", + "[29, \"?\", 227026, \"HS-grad\", 9, \"Never-married\", \"?\", \"Unmarried\", \"Black\", \"Male\", 0, 0, 40, \"United-States\"]\n", + "[63, \"Self-emp-not-inc\", 104626, \"Prof-school\", 15, \"Married-civ-spouse\", \"Prof-specialty\", \"Husband\", \"White\", \"Male\", 3103, 0, 32, \"United-States\"]\n", + "[24, \"Private\", 369667, \"Some-college\", 10, \"Never-married\", \"Other-service\", \"Unmarried\", \"White\", \"Female\", 0, 0, 40, \"United-States\"]\n", + "[55, \"Private\", 104996, \"7th-8th\", 4, \"Married-civ-spouse\", \"Craft-repair\", \"Husband\", \"White\", \"Male\", 0, 0, 10, \"United-States\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Send the online prediction request\n", + "\n", + "The prediction results return `True` if the person's income is predicted to be greater than $50,000 per year, and `False` otherwise. The output of the command below may appear similar to the following:\n", + "\n", + " [False, False, False, True, False, False, False, False, False, False]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "!gcloud ml-engine predict --model $MODEL_NAME --version \\\n", + " $VERSION_NAME --json-instances $INPUT_FILE" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Clean up" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To delete all resources you created in this tutorial, run the following commands:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# Delete the model version\n", + "!gcloud ml-engine versions delete $VERSION_NAME --model=$MODEL_NAME --quiet\n", + "\n", + "# Delete the model\n", + "!gcloud ml-engine models delete $MODEL_NAME --quiet\n", + "\n", + "# Delete the bucket and contents\n", + "!gsutil rm -r gs://$BUCKET_NAME\n", + " \n", + "# Delete the local files created by the tutorial\n", + "!rm -rf census_training" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/tutorials/storage/Cloud Storage client library.ipynb b/notebooks/tutorials/storage/Cloud Storage client library.ipynb new file mode 100644 index 00000000000..34f747b5786 --- /dev/null +++ b/notebooks/tutorials/storage/Cloud Storage client library.ipynb @@ -0,0 +1,312 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Cloud Storage client library\n", + "\n", + "This tutorial shows how to get started with the [Cloud Storage Python client library](https://googleapis.github.io/google-cloud-python/latest/storage/index.html)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create a storage bucket\n", + "\n", + "Buckets are the basic containers that hold your data. Everything that you store in Cloud Storage must be contained in a bucket. You can use buckets to organize your data and control access to your data.\n", + "\n", + "Start by importing the library:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from google.cloud import storage" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `storage.Client` object uses your default project. Alternatively, you can specify a project in the `Client` constructor. For more information about how the default project is determined, see the [google-auth documentation](https://google-auth.readthedocs.io/en/latest/reference/google.auth.html).\n", + "\n", + "Run the following to create a client with your default project:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "client = storage.Client()\n", + "print(\"Client created using default project: {}\".format(client.project))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To explicitly specify a project when constructing the client, set the `project` parameter:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# client = storage.Client(project='your-project-id')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, create a bucket with a globally unique name.\n", + "\n", + "For more information about naming buckets, see [Bucket name requirements](https://cloud.google.com/storage/docs/naming#requirements)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Replace the string below with a unique name for the new bucket\n", + "bucket_name = \"your-new-bucket\"\n", + "\n", + "# Creates the new bucket\n", + "bucket = client.create_bucket(bucket_name)\n", + "\n", + "print(\"Bucket {} created.\".format(bucket.name))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## List buckets in a project" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "buckets = client.list_buckets()\n", + "\n", + "print(\"Buckets in {}:\".format(client.project))\n", + "for item in buckets:\n", + " print(\"\\t\" + item.name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Get bucket metadata\n", + "\n", + "The next cell shows how to get information on metadata of your Cloud Storage buckets.\n", + "\n", + "To learn more about specific bucket properties, see [Bucket locations](https://cloud.google.com/storage/docs/locations) and [Storage classes](https://cloud.google.com/storage/docs/storage-classes)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "bucket = client.get_bucket(bucket_name)\n", + "\n", + "print(\"Bucket name: {}\".format(bucket.name))\n", + "print(\"Bucket location: {}\".format(bucket.location))\n", + "print(\"Bucket storage class: {}\".format(bucket.storage_class))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Upload a local file to a bucket\n", + "\n", + "Objects are the individual pieces of data that you store in Cloud Storage. Objects are referred to as \"blobs\" in the Python client library. There is no limit on the number of objects that you can create in a bucket.\n", + "\n", + "An object's name is treated as a piece of object metadata in Cloud Storage. Object names can contain any combination of Unicode characters (UTF-8 encoded) and must be less than 1024 bytes in length.\n", + "\n", + "For more information, including how to rename an object, see the [Object name requirements](https://cloud.google.com/storage/docs/naming#objectnames)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "blob_name = \"us-states.txt\"\n", + "blob = bucket.blob(blob_name)\n", + "\n", + "source_file_name = \"resources/us-states.txt\"\n", + "blob.upload_from_filename(source_file_name)\n", + "\n", + "print(\"File uploaded to {}.\".format(bucket.name))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## List blobs in a bucket" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "blobs = bucket.list_blobs()\n", + "\n", + "print(\"Blobs in {}:\".format(bucket.name))\n", + "for item in blobs:\n", + " print(\"\\t\" + item.name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Get a blob and display metadata\n", + "\n", + "See [documentation](https://cloud.google.com/storage/docs/viewing-editing-metadata) for more information about object metadata." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "blob = bucket.get_blob(blob_name)\n", + "\n", + "print(\"Name: {}\".format(blob.id))\n", + "print(\"Size: {} bytes\".format(blob.size))\n", + "print(\"Content type: {}\".format(blob.content_type))\n", + "print(\"Public URL: {}\".format(blob.public_url))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Download a blob to a local directory" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "output_file_name = \"resources/downloaded-us-states.txt\"\n", + "blob.download_to_filename(output_file_name)\n", + "\n", + "print(\"Downloaded blob {} to {}.\".format(blob.name, output_file_name))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Cleaning up" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Delete a blob" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "blob = client.get_bucket(bucket_name).get_blob(blob_name)\n", + "blob.delete()\n", + "\n", + "print(\"Blob {} deleted.\".format(blob.name))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Delete a bucket\n", + "\n", + "Note that the bucket must be empty before it can be deleted." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "bucket = client.get_bucket(bucket_name)\n", + "bucket.delete()\n", + "\n", + "print(\"Bucket {} deleted.\".format(bucket.name))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Next Steps\n", + "\n", + "Read more about Cloud Storage in the documentation:\n", + "+ [Storage key terms](https://cloud.google.com/storage/docs/key-terms)\n", + "+ [How-to guides](https://cloud.google.com/storage/docs/how-to)\n", + "+ [Pricing](https://cloud.google.com/storage/pricing)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/notebooks/tutorials/storage/Storage command-line tool.ipynb b/notebooks/tutorials/storage/Storage command-line tool.ipynb new file mode 100644 index 00000000000..21e62ae8236 --- /dev/null +++ b/notebooks/tutorials/storage/Storage command-line tool.ipynb @@ -0,0 +1,328 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Storage command-line tool\n", + "\n", + "The [Google Cloud SDK](https://cloud-dot-devsite.googleplex.com/sdk/docs/) provides a set of commands for working with data stored in Cloud Storage. This notebook introduces several `gsutil` commands for interacting with Cloud Storage. Note that shell commands in a notebook must be prepended with a `!`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## List available commands\n", + "\n", + "The `gsutil` command can be used to perform a wide array of tasks. Run the `help` command to view a list of available commands:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "!gsutil help" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create a storage bucket\n", + "\n", + "Buckets are the basic containers that hold your data. Everything that you store in Cloud Storage must be contained in a bucket. You can use buckets to organize your data and control access to your data.\n", + "\n", + "Start by defining a globally unique name.\n", + "\n", + "For more information about naming buckets, see [Bucket name requirements](https://cloud.google.com/storage/docs/naming#requirements)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Replace the string below with a unique name for the new bucket\n", + "bucket_name = \"your-new-bucket\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "NOTE: In the examples below, the `bucket_name` and `project_id` variables are referenced in the commands using `{}` and `$`. If you want to avoid creating and using variables, replace these interpolated variables with literal values and remove the `{}` and `$` characters.\n", + "\n", + "Next, create the new bucket with the `gsutil mb` command:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!gsutil mb gs://{bucket_name}/" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## List buckets in a project\n", + "\n", + "Replace 'your-project-id' in the cell below with your project ID and run the cell to list the storage buckets in your project." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Replace the string below with your project ID\n", + "project_id = \"your-project-id\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!gsutil ls -p $project_id" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The response should look like the following:\n", + "\n", + "```\n", + "gs://your-new-bucket/\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Get bucket metadata\n", + "\n", + "The next cell shows how to get information on metadata of your Cloud Storage buckets.\n", + "\n", + "To learn more about specific bucket properties, see [Bucket locations](https://cloud.google.com/storage/docs/locations) and [Storage classes](https://cloud.google.com/storage/docs/storage-classes)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "!gsutil ls -L -b gs://{bucket_name}/" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The response should look like the following:\n", + "```\n", + "gs://your-new-bucket/ :\n", + " Storage class: MULTI_REGIONAL\n", + " Location constraint: US\n", + " ...\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Upload a local file to a bucket\n", + "\n", + "Objects are the individual pieces of data that you store in Cloud Storage. Objects are referred to as \"blobs\" in the Python client library. There is no limit on the number of objects that you can create in a bucket.\n", + "\n", + "An object's name is treated as a piece of object metadata in Cloud Storage. Object names can contain any combination of Unicode characters (UTF-8 encoded) and must be less than 1024 bytes in length.\n", + "\n", + "For more information, including how to rename an object, see the [Object name requirements](https://cloud.google.com/storage/docs/naming#objectnames)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!gsutil cp resources/us-states.txt gs://{bucket_name}/" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## List blobs in a bucket" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "!gsutil ls -r gs://{bucket_name}/**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The response should look like the following:\n", + "```\n", + "gs://your-new-bucket/us-states.txt\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Get a blob and display metadata\n", + "\n", + "See [Viewing and editing object metadata](https://cloud.google.com/storage/docs/viewing-editing-metadata) for more information about object metadata." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!gsutil ls -L gs://{bucket_name}/us-states.txt" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The response should look like the following:\n", + "\n", + "```\n", + "gs://your-new-bucket/us-states.txt:\n", + " Creation time: Fri, 08 Feb 2019 05:23:28 GMT\n", + " Update time: Fri, 08 Feb 2019 05:23:28 GMT\n", + " Storage class: STANDARD\n", + " Content-Language: en\n", + " Content-Length: 637\n", + " Content-Type: text/plain\n", + "...\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Download a blob to a local directory" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "!gsutil cp gs://{bucket_name}/us-states.txt resources/downloaded-us-states.txt" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Cleaning up" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Delete a blob" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "!gsutil rm gs://{bucket_name}/us-states.txt" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Delete a bucket\n", + "\n", + "The following command deletes all objects in the bucket before deleting the bucket itself." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!gsutil rm -r gs://{bucket_name}/" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Next Steps\n", + "\n", + "Read more about Cloud Storage in the documentation:\n", + "+ [Storage key terms](https://cloud.google.com/storage/docs/key-terms)\n", + "+ [How-to guides](https://cloud.google.com/storage/docs/how-to)\n", + "+ [Pricing](https://cloud.google.com/storage/pricing)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/notebooks/tutorials/storage/resources/downloaded-us-states.txt b/notebooks/tutorials/storage/resources/downloaded-us-states.txt new file mode 100644 index 00000000000..54a60e29de9 --- /dev/null +++ b/notebooks/tutorials/storage/resources/downloaded-us-states.txt @@ -0,0 +1,51 @@ +name,post_abbr +Alabama,AL +Alaska,AK +Arizona,AZ +Arkansas,AR +California,CA +Colorado,CO +Connecticut,CT +Delaware,DE +Florida,FL +Georgia,GA +Hawaii,HI +Idaho,ID +Illinois,IL +Indiana,IN +Iowa,IA +Kansas,KS +Kentucky,KY +Louisiana,LA +Maine,ME +Maryland,MD +Massachusetts,MA +Michigan,MI +Minnesota,MN +Mississippi,MS +Missouri,MO +Montana,MT +Nebraska,NE +Nevada,NV +New Hampshire,NH +New Jersey,NJ +New Mexico,NM +New York,NY +North Carolina,NC +North Dakota,ND +Ohio,OH +Oklahoma,OK +Oregon,OR +Pennsylvania,PA +Rhode Island,RI +South Carolina,SC +South Dakota,SD +Tennessee,TN +Texas,TX +Utah,UT +Vermont,VT +Virginia,VA +Washington,WA +West Virginia,WV +Wisconsin,WI +Wyoming,WY diff --git a/notebooks/tutorials/storage/resources/us-states.txt b/notebooks/tutorials/storage/resources/us-states.txt new file mode 100644 index 00000000000..54a60e29de9 --- /dev/null +++ b/notebooks/tutorials/storage/resources/us-states.txt @@ -0,0 +1,51 @@ +name,post_abbr +Alabama,AL +Alaska,AK +Arizona,AZ +Arkansas,AR +California,CA +Colorado,CO +Connecticut,CT +Delaware,DE +Florida,FL +Georgia,GA +Hawaii,HI +Idaho,ID +Illinois,IL +Indiana,IN +Iowa,IA +Kansas,KS +Kentucky,KY +Louisiana,LA +Maine,ME +Maryland,MD +Massachusetts,MA +Michigan,MI +Minnesota,MN +Mississippi,MS +Missouri,MO +Montana,MT +Nebraska,NE +Nevada,NV +New Hampshire,NH +New Jersey,NJ +New Mexico,NM +New York,NY +North Carolina,NC +North Dakota,ND +Ohio,OH +Oklahoma,OK +Oregon,OR +Pennsylvania,PA +Rhode Island,RI +South Carolina,SC +South Dakota,SD +Tennessee,TN +Texas,TX +Utah,UT +Vermont,VT +Virginia,VA +Washington,WA +West Virginia,WV +Wisconsin,WI +Wyoming,WY diff --git a/nox.py b/nox.py deleted file mode 100644 index 6772a97d4bf..00000000000 --- a/nox.py +++ /dev/null @@ -1,124 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import fnmatch -import os -import tempfile - -import nox - -REPO_TOOLS_REQ =\ - 'git+https://github.com/GoogleCloudPlatform/python-repo-tools.git' - -COMMON_PYTEST_ARGS = [ - '-x', '--no-success-flaky-report', '--cov', '--cov-config', - '.coveragerc', '--cov-append', '--cov-report='] - -SESSION_TESTS_BLACKLIST = set(('appengine', 'testing')) - - -def session_lint(session): - session.install('flake8', 'flake8-import-order') - session.run( - 'flake8', '--builtin=gettext', '--max-complexity=10', - '--import-order-style=google', - '--exclude', - 'container_engine/django_tutorial/polls/migrations/*,.nox,.cache,env', - *(session.posargs or ['.'])) - - -def list_files(folder, pattern): - for root, folders, files in os.walk(folder): - for filename in files: - if fnmatch.fnmatch(filename, pattern): - yield os.path.join(root, filename) - - -def session_reqcheck(session): - session.install(REPO_TOOLS_REQ) - - if 'update' in session.posargs: - command = 'update-requirements' - else: - command = 'check-requirements' - - for reqfile in list_files('.', 'requirements*.txt'): - session.run('gcprepotools', command, reqfile) - - -def collect_sample_dirs(start_dir, blacklist=set()): - """Recursively collects a list of dirs that contain tests.""" - # Collect all the directories that have tests in them. - for parent, subdirs, files in os.walk(start_dir): - if any(f for f in files if f[-8:] == '_test.py'): - # Don't recurse further, since py.test will do that. - del subdirs[:] - # This dir has tests in it. yield it. - yield parent - else: - # Filter out dirs we don't want to recurse into - subdirs[:] = [s for s in subdirs - if s[0].isalpha() and s not in blacklist] - - -@nox.parametrize('interpreter', ['python2.7', 'python3.4']) -def session_tests(session, interpreter, extra_pytest_args=None): - session.interpreter = interpreter - session.install(REPO_TOOLS_REQ) - session.install('-r', 'requirements-{}-dev.txt'.format(interpreter)) - - # extra_pytest_args can be send by another session calling this session, - # see session_travis. - pytest_args = COMMON_PYTEST_ARGS + (extra_pytest_args or []) - - # session.posargs is any leftover arguments from the command line, which - # allows users to run a particular test instead of all of them. - for sample in (session.posargs or - collect_sample_dirs('.', SESSION_TESTS_BLACKLIST)): - session.run( - 'py.test', sample, - *pytest_args, - success_codes=[0, 5]) # Treat no test collected as success. - - -def session_gae(session, extra_pytest_args=None): - session.interpreter = 'python2.7' - session.install(REPO_TOOLS_REQ) - session.install('-r', 'requirements-python2.7-dev.txt') - - # Install the app engine sdk and setup import paths. - gae_root = os.environ.get('GAE_ROOT', tempfile.gettempdir()) - session.env['PYTHONPATH'] = os.path.join(gae_root, 'google_appengine') - session.run('gcprepotools', 'download-appengine-sdk', gae_root) - - # Create a lib directory to prevent the GAE vendor library from - # complaining. - if not os.path.exists('lib'): - os.makedirs('lib') - - pytest_args = COMMON_PYTEST_ARGS + (extra_pytest_args or []) - - for sample in (session.posargs or collect_sample_dirs('appengine')): - session.run( - 'py.test', sample, - *pytest_args, - success_codes=[0, 5]) # Treat no test collected as success. - - -def session_travis(session): - """On travis, just run with python3.4 and don't run slow or flaky tests.""" - session_tests( - session, 'python3.4', extra_pytest_args=['-m not slow and not flaky']) - session_gae( - session, extra_pytest_args=['-m not slow and not flaky']) diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 00000000000..d89cddcbb6d --- /dev/null +++ b/noxfile.py @@ -0,0 +1,307 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import fnmatch +import os +import tempfile + +import nox + +try: + import ci_diff_helper +except ImportError: + ci_diff_helper = None + + +# +# Helpers and utility functions +# + +def _list_files(folder, pattern): + """Lists all files below the given folder that match the pattern.""" + for root, folders, files in os.walk(folder): + for filename in files: + if fnmatch.fnmatch(filename, pattern): + yield os.path.join(root, filename) + + +def _collect_dirs( + start_dir, + blacklist=set(['conftest.py', 'noxfile.py', 'lib', 'third_party']), + suffix='_test.py', + recurse_further=False): + """Recursively collects a list of dirs that contain a file matching the + given suffix. + + This works by listing the contents of directories and finding + directories that have `*_test.py` files. + """ + # Collect all the directories that have tests in them. + for parent, subdirs, files in os.walk(start_dir): + if './.' in parent: + continue # Skip top-level dotfiles + elif any( + f for f in files if f.endswith(suffix) and f not in blacklist + ): + # Don't recurse further for tests, since py.test will do that. + if not recurse_further: + del subdirs[:] + # This dir has desired files in it. yield it. + yield parent + else: + # Filter out dirs we don't want to recurse into + subdirs[:] = [ + s for s in subdirs + if s[0].isalpha() and + s not in blacklist] + + +def _get_changed_files(): + """Returns a list of files changed for this pull request / push. + + If running on a public CI like Travis or Circle this is used to only + run tests/lint for changed files. + """ + if not ci_diff_helper: + return None + + try: + config = ci_diff_helper.get_config() + except OSError: # Not on CI. + return None + + changed_files = ci_diff_helper.get_changed_files('HEAD', config.base) + + changed_files = set([ + './{}'.format(filename) for filename in changed_files]) + + return changed_files + + +def _filter_samples(sample_dirs, changed_files): + """Filers the list of sample directories to only include directories that + contain files in the list of changed files.""" + result = [] + for sample_dir in sample_dirs: + for changed_file in changed_files: + if changed_file.startswith(sample_dir): + result.append(sample_dir) + + return list(set(result)) + + +def _determine_local_import_names(start_dir): + """Determines all import names that should be considered "local". + + This is used when running the linter to insure that import order is + properly checked. + """ + file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)] + return [ + basename + for basename, extension + in file_ext_pairs + if extension == '.py' or os.path.isdir( + os.path.join(start_dir, basename)) + and basename not in ('__pycache__')] + +# +# App Engine specific helpers +# + + +_GAE_ROOT = os.environ.get('GAE_ROOT') +if _GAE_ROOT is None: + _GAE_ROOT = tempfile.mkdtemp() + + +def _setup_appengine_sdk(session): + """Installs the App Engine SDK, if needed.""" + session.env['GAE_SDK_PATH'] = os.path.join(_GAE_ROOT, 'google_appengine') + session.run('gcp-devrel-py-tools', 'download-appengine-sdk', _GAE_ROOT) + + +# +# Test sessions +# + + +PYTEST_COMMON_ARGS = ['--junitxml=sponge_log.xml'] + +# Ignore I202 "Additional newline in a section of imports." to accommodate +# region tags in import blocks. Since we specify an explicit ignore, we also +# have to explicitly ignore the list of default ignores: +# `E121,E123,E126,E226,E24,E704,W503,W504` as shown by `flake8 --help`. +FLAKE8_COMMON_ARGS = [ + '--show-source', '--builtin', 'gettext', '--max-complexity', '20', + '--import-order-style', 'google', + '--exclude', '.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py', + '--ignore=E121,E123,E126,E226,E24,E704,W503,W504,I100,I201,I202', +] + + +# Collect sample directories. +ALL_TESTED_SAMPLES = sorted(list(_collect_dirs('.'))) +ALL_SAMPLE_DIRECTORIES = sorted(list(_collect_dirs( + '.', + suffix='.py', + recurse_further=True +))) +GAE_STANDARD_SAMPLES = [ + sample for sample in ALL_TESTED_SAMPLES + if sample.startswith('./appengine/standard/')] +PY2_ONLY_SAMPLES = GAE_STANDARD_SAMPLES +PY3_ONLY_SAMPLES = [ + sample for sample in ALL_TESTED_SAMPLES + if (sample.startswith('./appengine/standard_python37') + or sample.startswith('./functions/'))] +NON_GAE_STANDARD_SAMPLES_PY2 = sorted(list(( + set(ALL_TESTED_SAMPLES) - + set(GAE_STANDARD_SAMPLES)) - + set(PY3_ONLY_SAMPLES) +)) +NON_GAE_STANDARD_SAMPLES_PY3 = sorted( + list(set(ALL_TESTED_SAMPLES) - set(PY2_ONLY_SAMPLES))) + + +# Filter sample directories if on a CI like Travis or Circle to only run tests +# for changed samples. +CHANGED_FILES = _get_changed_files() + +if CHANGED_FILES is not None: + print('Filtering based on changed files.') + ALL_TESTED_SAMPLES = _filter_samples( + ALL_TESTED_SAMPLES, CHANGED_FILES) + ALL_SAMPLE_DIRECTORIES = _filter_samples( + ALL_SAMPLE_DIRECTORIES, CHANGED_FILES) + GAE_STANDARD_SAMPLES = _filter_samples( + GAE_STANDARD_SAMPLES, CHANGED_FILES) + NON_GAE_STANDARD_SAMPLES_PY2 = _filter_samples( + NON_GAE_STANDARD_SAMPLES_PY2, CHANGED_FILES) + NON_GAE_STANDARD_SAMPLES_PY3 = _filter_samples( + NON_GAE_STANDARD_SAMPLES_PY3, CHANGED_FILES) + + +def _session_tests(session, sample, post_install=None): + """Runs py.test for a particular sample.""" + session.install('-r', 'testing/requirements.txt') + + session.chdir(sample) + + if os.path.exists('requirements.txt'): + session.install('-r', 'requirements.txt') + + if post_install: + post_install(session) + + session.run( + 'pytest', + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5]) + + +@nox.session(python='2.7') +@nox.parametrize('sample', GAE_STANDARD_SAMPLES) +def gae(session, sample): + """Runs py.test for an App Engine standard sample.""" + + # Create a lib directory if needed, otherwise the App Engine vendor library + # will complain. + if not os.path.isdir(os.path.join(sample, 'lib')): + os.mkdir(os.path.join(sample, 'lib')) + + _session_tests(session, sample, _setup_appengine_sdk) + + +@nox.session(python='2.7') +@nox.parametrize('sample', NON_GAE_STANDARD_SAMPLES_PY2) +def py27(session, sample): + """Runs py.test for a sample using Python 2.7""" + _session_tests(session, sample) + + +@nox.session(python='3.6') +@nox.parametrize('sample', NON_GAE_STANDARD_SAMPLES_PY3) +def py36(session, sample): + """Runs py.test for a sample using Python 3.6""" + _session_tests(session, sample) + + +@nox.session +@nox.parametrize('sample', ALL_SAMPLE_DIRECTORIES) +def lint(session, sample): + """Runs flake8 on the sample.""" + session.install('flake8', 'flake8-import-order') + + local_names = _determine_local_import_names(sample) + args = FLAKE8_COMMON_ARGS + [ + '--application-import-names', ','.join(local_names), + '.'] + + session.chdir(sample) + session.run('flake8', *args) + + +# +# Utility sessions +# + +@nox.session +def missing_tests(session): + """Lists all sample directories that do not have tests.""" + print('The following samples do not have tests:') + for sample in set(ALL_SAMPLE_DIRECTORIES) - set(ALL_TESTED_SAMPLES): + print('* {}'.format(sample)) + + +SAMPLES_WITH_GENERATED_READMES = sorted( + list(_collect_dirs('.', suffix='.rst.in'))) + + +@nox.session +@nox.parametrize('sample', SAMPLES_WITH_GENERATED_READMES) +def readmegen(session, sample): + """(Re-)generates the readme for a sample.""" + session.install('jinja2', 'pyyaml') + + if os.path.exists(os.path.join(sample, 'requirements.txt')): + session.install('-r', os.path.join(sample, 'requirements.txt')) + + in_file = os.path.join(sample, 'README.rst.in') + session.run('python', 'scripts/readme-gen/readme_gen.py', in_file) + + +@nox.session +def check_requirements(session): + """Checks for out of date requirements and optionally updates them. + + This is intentionally not parametric, as it's desired to never have two + samples with differing versions of dependencies. + """ + session.install('-r', 'testing/requirements.txt') + + if 'update' in session.posargs: + command = 'update-requirements' + else: + command = 'check-requirements' + + reqfiles = list(_list_files('.', 'requirements*.txt')) + + for reqfile in reqfiles: + session.run('gcp-devrel-py-tools', command, reqfile) diff --git a/opencensus/README.md b/opencensus/README.md new file mode 100644 index 00000000000..4ffe0a5f510 --- /dev/null +++ b/opencensus/README.md @@ -0,0 +1,35 @@ +OpenCensus logo + +# OpenCensus Stackdriver Metrics Sample + +[OpenCensus](https://opencensus.io) is a toolkit for collecting application +performance and behavior data. OpenCensus includes utilities for distributed +tracing, metrics collection, and context propagation within and between +services. + +This example demonstrates using the OpenCensus client to send metrics data to +the [Stackdriver Monitoring](https://cloud.google.com/monitoring/docs/) +backend. + +## Prerequisites + +Install the OpenCensus core and Stackdriver exporter libraries: + +```sh +pip install -r opencensus/requirements.txt +``` + +Make sure that your environment is configured to [authenticate with +GCP](https://cloud.google.com/docs/authentication/getting-started). + +## Running the example + +```sh +python opencensus/metrics_quickstart.py +``` + +The example generates a histogram of simulated latencies, which is exported to +Stackdriver after 60 seconds. After it's exported, the histogram will be +visible on the [Stackdriver Metrics +Explorer](https://app.google.stackdriver.com/metrics-explorer) page as +`OpenCensus/task_latency_view`. diff --git a/opencensus/metrics_quickstart.py b/opencensus/metrics_quickstart.py new file mode 100755 index 00000000000..eefa20c5fd5 --- /dev/null +++ b/opencensus/metrics_quickstart.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START monitoring_opencensus_metrics_quickstart] + +from random import random +import time + +from opencensus.ext.stackdriver import stats_exporter +from opencensus.stats import aggregation +from opencensus.stats import measure +from opencensus.stats import stats +from opencensus.stats import view + + +# A measure that represents task latency in ms. +LATENCY_MS = measure.MeasureFloat( + "task_latency", + "The task latency in milliseconds", + "ms") + +# A view of the task latency measure that aggregates measurements according to +# a histogram with predefined bucket boundaries. This aggregate is periodically +# exported to Stackdriver Monitoring. +LATENCY_VIEW = view.View( + "task_latency_distribution", + "The distribution of the task latencies", + [], + LATENCY_MS, + # Latency in buckets: [>=0ms, >=100ms, >=200ms, >=400ms, >=1s, >=2s, >=4s] + aggregation.DistributionAggregation( + [100.0, 200.0, 400.0, 1000.0, 2000.0, 4000.0])) + + +def main(): + # Register the view. Measurements are only aggregated and exported if + # they're associated with a registered view. + stats.stats.view_manager.register_view(LATENCY_VIEW) + + # Create the Stackdriver stats exporter and start exporting metrics in the + # background, once every 60 seconds by default. + exporter = stats_exporter.new_stats_exporter() + print('Exporting stats to project "{}"' + .format(exporter.options.project_id)) + + # Record 100 fake latency values between 0 and 5 seconds. + for num in range(100): + ms = random() * 5 * 1000 + print("Latency {}: {}".format(num, ms)) + + mmap = stats.stats.stats_recorder.new_measurement_map() + mmap.measure_float_put(LATENCY_MS, ms) + mmap.record() + + # Keep the thread alive long enough for the exporter to export at least + # once. + time.sleep(65) + + +if __name__ == '__main__': + main() + +# [END monitoring_opencensus_metrics_quickstart] diff --git a/opencensus/metrics_quickstart_test.py b/opencensus/metrics_quickstart_test.py new file mode 100644 index 00000000000..bd9f745edc3 --- /dev/null +++ b/opencensus/metrics_quickstart_test.py @@ -0,0 +1,80 @@ +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import mock +import unittest + +from opencensus.ext.stackdriver import stats_exporter +from opencensus.stats import aggregation +from opencensus.stats import measure +from opencensus.stats import stats +from opencensus.stats import view + + +class TestMetricsQuickstart(unittest.TestCase): + """Sanity checks for the OpenCensus metrics quickstart examples. + + These tests check that a few basic features of the library work as expected + in order for the demo to run. See the opencensus and + opencensus-ext-stackdriver source for actual library tests. + """ + def test_measure_creation(self): + measure.MeasureFloat( + "task_latency", + "The task latency in milliseconds", + "ms") + + def test_view_creation(self): + test_view = view.View( + "task_latency_distribution", + "The distribution of the task latencies", + [], + mock.Mock(), + aggregation.DistributionAggregation([1.0, 2.0, 3.0])) + # Check that metric descriptor conversion doesn't crash + test_view.get_metric_descriptor() + + # Don't modify global stats + @mock.patch('opencensus.ext.stackdriver.stats_exporter.stats.stats', + stats._Stats()) + def test_measurement_map_record(self): + mock_measure = mock.Mock() + mock_measure_name = mock.Mock() + mock_measure.name = mock_measure_name + mock_view = mock.Mock() + mock_view.columns = [] + mock_view.measure = mock_measure + + stats.stats.view_manager.register_view(mock_view) + + mmap = stats.stats.stats_recorder.new_measurement_map() + mmap.measure_float_put(mock_measure, 1.0) + mmap.record() + + # Reaching into the stats internals here to check that recording the + # measurement map affects view data. + m2vd = (stats.stats.view_manager.measure_to_view_map + ._measure_to_view_data_list_map) + self.assertIn(mock_measure_name, m2vd) + [view_data] = m2vd[mock_measure_name] + agg_data = view_data.tag_value_aggregation_data_map[tuple()] + agg_data.add_sample.assert_called_once() + + @mock.patch('opencensus.ext.stackdriver.stats_exporter' + '.monitoring_v3.MetricServiceClient') + def test_new_stats_exporter(self, mock_client): + transport = stats_exporter.new_stats_exporter() + self.assertIsNotNone(transport) + self.assertIsNotNone(transport.options) + self.assertIsNotNone(transport.options.project_id) diff --git a/opencensus/requirements.txt b/opencensus/requirements.txt new file mode 100644 index 00000000000..0b9d34aa12f --- /dev/null +++ b/opencensus/requirements.txt @@ -0,0 +1,3 @@ +grpcio +opencensus-ext-stackdriver==0.2.1 +opencensus==0.4.1 diff --git a/profiler/appengine/flexible/app.yaml b/profiler/appengine/flexible/app.yaml new file mode 100644 index 00000000000..55439a72cf3 --- /dev/null +++ b/profiler/appengine/flexible/app.yaml @@ -0,0 +1,19 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +# App Engine flexible Python 3 runtime uses Python 3.7.x, which supports both +# CPU and Wall profiling. +runtime_config: + python_version: 3 + +# This sample incurs costs to run on the App Engine flexible environment. +# The settings below are to reduce costs during testing and are not appropriate +# for production use. For more information, see: +# https://cloud.google.com/appengine/docs/flexible/python/configuring-your-app-with-app-yaml +manual_scaling: + instances: 1 +resources: + cpu: 1 + memory_gb: 0.5 + disk_size_gb: 10 diff --git a/profiler/appengine/flexible/main.py b/profiler/appengine/flexible/main.py new file mode 100644 index 00000000000..03feabad277 --- /dev/null +++ b/profiler/appengine/flexible/main.py @@ -0,0 +1,45 @@ +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""An example of using https://cloud.google.com/profiler on GAE flex.""" + +from flask import Flask +# [START profiler_python_appengine_flex] +import googlecloudprofiler + +# Profiler initialization. It starts a daemon thread which continuously +# collects and uploads profiles. Best done as early as possible. +try: + # service and service_version can be automatically inferred when + # running on App Engine. project_id must be set if not running + # on GCP. + googlecloudprofiler.start(verbose=3) +except (ValueError, NotImplementedError) as exc: + print(exc) # Handle errors here + +# [END profiler_python_appengine_flex] + + +app = Flask(__name__) + + +@app.route('/') +def hello(): + """Return a friendly HTTP greeting.""" + return 'Hello World!' + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/profiler/appengine/flexible/main_test.py b/profiler/appengine/flexible/main_test.py new file mode 100644 index 00000000000..6df63eacbc3 --- /dev/null +++ b/profiler/appengine/flexible/main_test.py @@ -0,0 +1,24 @@ +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import main + + +def test_index(): + main.app.testing = True + client = main.app.test_client() + + r = client.get('/') + assert r.status_code == 200 + assert 'Hello World' in r.data.decode('utf-8') diff --git a/profiler/appengine/flexible/requirements.txt b/profiler/appengine/flexible/requirements.txt new file mode 100644 index 00000000000..306c4ffd814 --- /dev/null +++ b/profiler/appengine/flexible/requirements.txt @@ -0,0 +1,3 @@ +Flask==1.0.2 +gunicorn==19.9.0 +google-cloud-profiler diff --git a/profiler/appengine/standard_python37/app.yaml b/profiler/appengine/standard_python37/app.yaml new file mode 100644 index 00000000000..a0b719d6dd4 --- /dev/null +++ b/profiler/appengine/standard_python37/app.yaml @@ -0,0 +1 @@ +runtime: python37 diff --git a/profiler/appengine/standard_python37/main.py b/profiler/appengine/standard_python37/main.py new file mode 100644 index 00000000000..ce41cc9e404 --- /dev/null +++ b/profiler/appengine/standard_python37/main.py @@ -0,0 +1,48 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""An example of using https://cloud.google.com/profiler on GAE standard.""" + +from flask import Flask +# [START profiler_python_appengine_standard_python37] +import googlecloudprofiler + +# Profiler initialization. It starts a daemon thread which continuously +# collects and uploads profiles. Best done as early as possible. +try: + # service and service_version can be automatically inferred when + # running on App Engine. project_id must be set if not running + # on GCP. + googlecloudprofiler.start(verbose=3) +except (ValueError, NotImplementedError) as exc: + print(exc) # Handle errors here + +# [END profiler_python_appengine_standard_python37] + + +# If `entrypoint` is not defined in app.yaml, App Engine will look for an app +# called `app` in `main.py`. +app = Flask(__name__) + + +@app.route('/') +def hello(): + """Return a friendly HTTP greeting.""" + return 'Hello World!' + + +if __name__ == '__main__': + # This is used when running locally only. When deploying to Google App + # Engine, a webserver process such as Gunicorn will serve the app. This + # can be configured by adding an `entrypoint` to app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/profiler/appengine/standard_python37/main_test.py b/profiler/appengine/standard_python37/main_test.py new file mode 100644 index 00000000000..6df63eacbc3 --- /dev/null +++ b/profiler/appengine/standard_python37/main_test.py @@ -0,0 +1,24 @@ +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import main + + +def test_index(): + main.app.testing = True + client = main.app.test_client() + + r = client.get('/') + assert r.status_code == 200 + assert 'Hello World' in r.data.decode('utf-8') diff --git a/profiler/appengine/standard_python37/requirements.txt b/profiler/appengine/standard_python37/requirements.txt new file mode 100644 index 00000000000..94e54c91bf2 --- /dev/null +++ b/profiler/appengine/standard_python37/requirements.txt @@ -0,0 +1,2 @@ +Flask==1.0.2 +google-cloud-profiler diff --git a/profiler/quickstart/main.py b/profiler/quickstart/main.py new file mode 100644 index 00000000000..98206a39674 --- /dev/null +++ b/profiler/quickstart/main.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""An example of starting https://cloud.google.com/profiler.""" + +# [START profiler_python_quickstart] +import googlecloudprofiler + + +def main(): + # Profiler initialization. It starts a daemon thread which continuously + # collects and uploads profiles. Best done as early as possible. + try: + googlecloudprofiler.start( + service='hello-profiler', + service_version='1.0.1', + # verbose is the logging level. 0-error, 1-warning, 2-info, + # 3-debug. It defaults to 0 (error) if not set. + verbose=3, + # project_id must be set if not running on GCP. + # project_id='my-project-id', + ) + except (ValueError, NotImplementedError) as exc: + print(exc) # Handle errors here +# [END profiler_python_quickstart] + busyloop() + + +# A loop function which spends 30% CPU time on loop3() and 70% CPU time +# on loop7(). +def busyloop(): + while True: + loop3() + loop7() + + +def loop3(): + for _ in range(3): + loop() + + +def loop7(): + for _ in range(7): + loop() + + +def loop(): + for _ in range(10000): + pass + + +if __name__ == '__main__': + main() diff --git a/profiler/quickstart/requirements.txt b/profiler/quickstart/requirements.txt new file mode 100644 index 00000000000..ffbe653f12a --- /dev/null +++ b/profiler/quickstart/requirements.txt @@ -0,0 +1 @@ +google-cloud-profiler \ No newline at end of file diff --git a/pubsub/cloud-client/README.rst b/pubsub/cloud-client/README.rst new file mode 100644 index 00000000000..e0e265f8d42 --- /dev/null +++ b/pubsub/cloud-client/README.rst @@ -0,0 +1,247 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Pub/Sub Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/README.rst + + +This directory contains samples for Google Cloud Pub/Sub. `Google Cloud Pub/Sub`_ is a fully-managed real-time messaging service that allows you to send and receive messages between independent applications. + + + + +.. _Google Cloud Pub/Sub: https://cloud.google.com/pubsub/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Quickstart ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/quickstart.py,pubsub/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python quickstart.py + + +Publisher ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/publisher.py,pubsub/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python publisher.py + + usage: publisher.py [-h] + project + {list,create,delete,publish,publish-with-custom-attributes,publish-with-futures,publish-with-batch-settings} + ... + + This application demonstrates how to perform basic operations on topics + with the Cloud Pub/Sub API. + + For more information, see the README.md under /pubsub and the documentation + at https://cloud.google.com/pubsub/docs. + + positional arguments: + project Your Google Cloud project ID + {list,create,delete,publish,publish-with-custom-attributes,publish-with-futures,publish-with-batch-settings} + list Lists all Pub/Sub topics in the given project. + create Create a new Pub/Sub topic. + delete Deletes an existing Pub/Sub topic. + publish Publishes multiple messages to a Pub/Sub topic. + publish-with-custom-attributes + Publishes multiple messages with custom attributes to + a Pub/Sub topic. + publish-with-futures + Publishes multiple messages to a Pub/Sub topic and + prints their message IDs. + publish-with-batch-settings + Publishes multiple messages to a Pub/Sub topic with + batch settings. + + optional arguments: + -h, --help show this help message and exit + + + +Subscribers ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/subscriber.py,pubsub/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python subscriber.py + + usage: subscriber.py [-h] + project + {list_in_topic,list_in_project,create,create-push,delete,update,receive,receive-custom-attributes,receive-flow-control,listen_for_errors} + ... + + This application demonstrates how to perform basic operations on + subscriptions with the Cloud Pub/Sub API. + + For more information, see the README.md under /pubsub and the documentation + at https://cloud.google.com/pubsub/docs. + + positional arguments: + project Your Google Cloud project ID + {list_in_topic,list_in_project,create,create-push,delete,update,receive,receive-custom-attributes,receive-flow-control,listen_for_errors} + list_in_topic Lists all subscriptions for a given topic. + list_in_project Lists all subscriptions in the current project. + create Create a new pull subscription on the given topic. + create-push Create a new push subscription on the given topic. For + example, endpoint is "https://my-test- + project.appspot.com/push". + delete Deletes an existing Pub/Sub topic. + update Updates an existing Pub/Sub subscription's push + endpoint URL. Note that certain properties of a + subscription, such as its topic, are not modifiable. + For example, endpoint is "https://my-test- + project.appspot.com/push". + receive Receives messages from a pull subscription. + receive-custom-attributes + Receives messages from a pull subscription. + receive-flow-control + Receives messages from a pull subscription with flow + control. + listen_for_errors Receives messages and catches errors from a pull + subscription. + + optional arguments: + -h, --help show this help message and exit + + + +Identity and Access Management ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/iam.py,pubsub/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python iam.py + + usage: iam.py [-h] + project + {get-topic-policy,get-subscription-policy,set-topic-policy,set-subscription-policy,check-topic-permissions,check-subscription-permissions} + ... + + This application demonstrates how to perform basic operations on IAM + policies with the Cloud Pub/Sub API. + + For more information, see the README.md under /pubsub and the documentation + at https://cloud.google.com/pubsub/docs. + + positional arguments: + project Your Google Cloud project ID + {get-topic-policy,get-subscription-policy,set-topic-policy,set-subscription-policy,check-topic-permissions,check-subscription-permissions} + get-topic-policy Prints the IAM policy for the given topic. + get-subscription-policy + Prints the IAM policy for the given subscription. + set-topic-policy Sets the IAM policy for a topic. + set-subscription-policy + Sets the IAM policy for a topic. + check-topic-permissions + Checks to which permissions are available on the given + topic. + check-subscription-permissions + Checks to which permissions are available on the given + subscription. + + optional arguments: + -h, --help show this help message and exit + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/pubsub/cloud-client/README.rst.in b/pubsub/cloud-client/README.rst.in new file mode 100644 index 00000000000..ddbc647121b --- /dev/null +++ b/pubsub/cloud-client/README.rst.in @@ -0,0 +1,30 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Pub/Sub + short_name: Cloud Pub/Sub + url: https://cloud.google.com/pubsub/docs + description: > + `Google Cloud Pub/Sub`_ is a fully-managed real-time messaging service that + allows you to send and receive messages between independent applications. + +setup: +- auth +- install_deps + +samples: +- name: Quickstart + file: quickstart.py +- name: Publisher + file: publisher.py + show_help: true +- name: Subscribers + file: subscriber.py + show_help: true +- name: Identity and Access Management + file: iam.py + show_help: true + +cloud_client_library: true + +folder: pubsub/cloud-client \ No newline at end of file diff --git a/pubsub/cloud-client/iam.py b/pubsub/cloud-client/iam.py new file mode 100644 index 00000000000..bd44f1ab6e0 --- /dev/null +++ b/pubsub/cloud-client/iam.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to perform basic operations on IAM +policies with the Cloud Pub/Sub API. + +For more information, see the README.md under /pubsub and the documentation +at https://cloud.google.com/pubsub/docs. +""" + +import argparse + +from google.cloud import pubsub_v1 + + +def get_topic_policy(project, topic_name): + """Prints the IAM policy for the given topic.""" + # [START pubsub_get_topic_policy] + client = pubsub_v1.PublisherClient() + topic_path = client.topic_path(project, topic_name) + + policy = client.get_iam_policy(topic_path) + + print('Policy for topic {}:'.format(topic_path)) + for binding in policy.bindings: + print('Role: {}, Members: {}'.format(binding.role, binding.members)) + # [END pubsub_get_topic_policy] + + +def get_subscription_policy(project, subscription_name): + """Prints the IAM policy for the given subscription.""" + # [START pubsub_get_subscription_policy] + client = pubsub_v1.SubscriberClient() + subscription_path = client.subscription_path(project, subscription_name) + + policy = client.get_iam_policy(subscription_path) + + print('Policy for subscription {}:'.format(subscription_path)) + for binding in policy.bindings: + print('Role: {}, Members: {}'.format(binding.role, binding.members)) + # [END pubsub_get_subscription_policy] + + +def set_topic_policy(project, topic_name): + """Sets the IAM policy for a topic.""" + # [START pubsub_set_topic_policy] + client = pubsub_v1.PublisherClient() + topic_path = client.topic_path(project, topic_name) + + policy = client.get_iam_policy(topic_path) + + # Add all users as viewers. + policy.bindings.add( + role='roles/pubsub.viewer', + members=['allUsers']) + + # Add a group as a publisher. + policy.bindings.add( + role='roles/pubsub.publisher', + members=['group:cloud-logs@google.com']) + + # Set the policy + policy = client.set_iam_policy(topic_path, policy) + + print('IAM policy for topic {} set: {}'.format( + topic_name, policy)) + # [END pubsub_set_topic_policy] + + +def set_subscription_policy(project, subscription_name): + """Sets the IAM policy for a topic.""" + # [START pubsub_set_subscription_policy] + client = pubsub_v1.SubscriberClient() + subscription_path = client.subscription_path(project, subscription_name) + + policy = client.get_iam_policy(subscription_path) + + # Add all users as viewers. + policy.bindings.add( + role='roles/pubsub.viewer', + members=['allUsers']) + + # Add a group as an editor. + policy.bindings.add( + role='roles/editor', + members=['group:cloud-logs@google.com']) + + # Set the policy + policy = client.set_iam_policy(subscription_path, policy) + + print('IAM policy for subscription {} set: {}'.format( + subscription_name, policy)) + # [END pubsub_set_subscription_policy] + + +def check_topic_permissions(project, topic_name): + """Checks to which permissions are available on the given topic.""" + # [START pubsub_test_topic_permissions] + client = pubsub_v1.PublisherClient() + topic_path = client.topic_path(project, topic_name) + + permissions_to_check = [ + 'pubsub.topics.publish', + 'pubsub.topics.update' + ] + + allowed_permissions = client.test_iam_permissions( + topic_path, permissions_to_check) + + print('Allowed permissions for topic {}: {}'.format( + topic_path, allowed_permissions)) + # [END pubsub_test_topic_permissions] + + +def check_subscription_permissions(project, subscription_name): + """Checks to which permissions are available on the given subscription.""" + # [START pubsub_test_subscription_permissions] + client = pubsub_v1.SubscriberClient() + subscription_path = client.subscription_path(project, subscription_name) + + permissions_to_check = [ + 'pubsub.subscriptions.consume', + 'pubsub.subscriptions.update' + ] + + allowed_permissions = client.test_iam_permissions( + subscription_path, permissions_to_check) + + print('Allowed permissions for subscription {}: {}'.format( + subscription_path, allowed_permissions)) + # [END pubsub_test_subscription_permissions] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument('project', help='Your Google Cloud project ID') + + subparsers = parser.add_subparsers(dest='command') + + get_topic_policy_parser = subparsers.add_parser( + 'get-topic-policy', help=get_topic_policy.__doc__) + get_topic_policy_parser.add_argument('topic_name') + + get_subscription_policy_parser = subparsers.add_parser( + 'get-subscription-policy', help=get_subscription_policy.__doc__) + get_subscription_policy_parser.add_argument('subscription_name') + + set_topic_policy_parser = subparsers.add_parser( + 'set-topic-policy', help=set_topic_policy.__doc__) + set_topic_policy_parser.add_argument('topic_name') + + set_subscription_policy_parser = subparsers.add_parser( + 'set-subscription-policy', help=set_subscription_policy.__doc__) + set_subscription_policy_parser.add_argument('subscription_name') + + check_topic_permissions_parser = subparsers.add_parser( + 'check-topic-permissions', help=check_topic_permissions.__doc__) + check_topic_permissions_parser.add_argument('topic_name') + + check_subscription_permissions_parser = subparsers.add_parser( + 'check-subscription-permissions', + help=check_subscription_permissions.__doc__) + check_subscription_permissions_parser.add_argument('subscription_name') + + args = parser.parse_args() + + if args.command == 'get-topic-policy': + get_topic_policy(args.project, args.topic_name) + elif args.command == 'get-subscription-policy': + get_subscription_policy(args.project, args.subscription_name) + elif args.command == 'set-topic-policy': + set_topic_policy(args.project, args.topic_name) + elif args.command == 'set-subscription-policy': + set_subscription_policy(args.project, args.subscription_name) + elif args.command == 'check-topic-permissions': + check_topic_permissions(args.project, args.topic_name) + elif args.command == 'check-subscription-permissions': + check_subscription_permissions(args.project, args.subscription_name) diff --git a/pubsub/cloud-client/iam_test.py b/pubsub/cloud-client/iam_test.py new file mode 100644 index 00000000000..8a524c35a06 --- /dev/null +++ b/pubsub/cloud-client/iam_test.py @@ -0,0 +1,111 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from google.cloud import pubsub_v1 +import pytest + +import iam + +PROJECT = os.environ['GCLOUD_PROJECT'] +TOPIC = 'iam-test-topic' +SUBSCRIPTION = 'iam-test-subscription' + + +@pytest.fixture(scope='module') +def publisher_client(): + yield pubsub_v1.PublisherClient() + + +@pytest.fixture(scope='module') +def topic(publisher_client): + topic_path = publisher_client.topic_path(PROJECT, TOPIC) + + try: + publisher_client.delete_topic(topic_path) + except Exception: + pass + + publisher_client.create_topic(topic_path) + + yield topic_path + + +@pytest.fixture(scope='module') +def subscriber_client(): + yield pubsub_v1.SubscriberClient() + + +@pytest.fixture +def subscription(subscriber_client, topic): + subscription_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION) + + try: + subscriber_client.delete_subscription(subscription_path) + except Exception: + pass + + subscriber_client.create_subscription(subscription_path, topic=topic) + + yield subscription_path + + +def test_get_topic_policy(topic, capsys): + iam.get_topic_policy(PROJECT, TOPIC) + + out, _ = capsys.readouterr() + assert topic in out + + +def test_get_subscription_policy(subscription, capsys): + iam.get_subscription_policy(PROJECT, SUBSCRIPTION) + + out, _ = capsys.readouterr() + assert subscription in out + + +def test_set_topic_policy(publisher_client, topic): + iam.set_topic_policy(PROJECT, TOPIC) + + policy = publisher_client.get_iam_policy(topic) + assert 'roles/pubsub.publisher' in str(policy) + assert 'allUsers' in str(policy) + + +def test_set_subscription_policy(subscriber_client, subscription): + iam.set_subscription_policy(PROJECT, SUBSCRIPTION) + + policy = subscriber_client.get_iam_policy(subscription) + assert 'roles/pubsub.viewer' in str(policy) + assert 'allUsers' in str(policy) + + +def test_check_topic_permissions(topic, capsys): + iam.check_topic_permissions(PROJECT, TOPIC) + + out, _ = capsys.readouterr() + + assert topic in out + assert 'pubsub.topics.publish' in out + + +def test_check_subscription_permissions(subscription, capsys): + iam.check_subscription_permissions(PROJECT, SUBSCRIPTION) + + out, _ = capsys.readouterr() + + assert subscription in out + assert 'pubsub.subscriptions.consume' in out diff --git a/pubsub/cloud-client/publisher.py b/pubsub/cloud-client/publisher.py new file mode 100644 index 00000000000..fcb0d9b0f2e --- /dev/null +++ b/pubsub/cloud-client/publisher.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to perform basic operations on topics +with the Cloud Pub/Sub API. + +For more information, see the README.md under /pubsub and the documentation +at https://cloud.google.com/pubsub/docs. +""" + +import argparse + + +def list_topics(project_id): + """Lists all Pub/Sub topics in the given project.""" + # [START pubsub_list_topics] + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + + publisher = pubsub_v1.PublisherClient() + project_path = publisher.project_path(project_id) + + for topic in publisher.list_topics(project_path): + print(topic) + # [END pubsub_list_topics] + + +def create_topic(project_id, topic_name): + """Create a new Pub/Sub topic.""" + # [START pubsub_quickstart_create_topic] + # [START pubsub_create_topic] + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO topic_name = "Your Pub/Sub topic name" + + publisher = pubsub_v1.PublisherClient() + topic_path = publisher.topic_path(project_id, topic_name) + + topic = publisher.create_topic(topic_path) + + print('Topic created: {}'.format(topic)) + # [END pubsub_quickstart_create_topic] + # [END pubsub_create_topic] + + +def delete_topic(project_id, topic_name): + """Deletes an existing Pub/Sub topic.""" + # [START pubsub_delete_topic] + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO topic_name = "Your Pub/Sub topic name" + + publisher = pubsub_v1.PublisherClient() + topic_path = publisher.topic_path(project_id, topic_name) + + publisher.delete_topic(topic_path) + + print('Topic deleted: {}'.format(topic_path)) + # [END pubsub_delete_topic] + + +def publish_messages(project_id, topic_name): + """Publishes multiple messages to a Pub/Sub topic.""" + # [START pubsub_quickstart_publisher] + # [START pubsub_publish] + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO topic_name = "Your Pub/Sub topic name" + + publisher = pubsub_v1.PublisherClient() + # The `topic_path` method creates a fully qualified identifier + # in the form `projects/{project_id}/topics/{topic_name}` + topic_path = publisher.topic_path(project_id, topic_name) + + for n in range(1, 10): + data = u'Message number {}'.format(n) + # Data must be a bytestring + data = data.encode('utf-8') + # When you publish a message, the client returns a future. + future = publisher.publish(topic_path, data=data) + print('Published {} of message ID {}.'.format(data, future.result())) + + print('Published messages.') + # [END pubsub_quickstart_publisher] + # [END pubsub_publish] + + +def publish_messages_with_custom_attributes(project_id, topic_name): + """Publishes multiple messages with custom attributes + to a Pub/Sub topic.""" + # [START pubsub_publish_custom_attributes] + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO topic_name = "Your Pub/Sub topic name" + + publisher = pubsub_v1.PublisherClient() + topic_path = publisher.topic_path(project_id, topic_name) + + for n in range(1, 10): + data = u'Message number {}'.format(n) + # Data must be a bytestring + data = data.encode('utf-8') + # Add two attributes, origin and username, to the message + publisher.publish( + topic_path, data, origin='python-sample', username='gcp') + + print('Published messages with custom attributes.') + # [END pubsub_publish_custom_attributes] + + +def publish_messages_with_futures(project_id, topic_name): + """Publishes multiple messages to a Pub/Sub topic and prints their + message IDs.""" + # [START pubsub_publisher_concurrency_control] + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO topic_name = "Your Pub/Sub topic name" + + publisher = pubsub_v1.PublisherClient() + topic_path = publisher.topic_path(project_id, topic_name) + + # When you publish a message, the client returns a Future. This Future + # can be used to track when the message is published. + futures = [] + + for n in range(1, 10): + data = u'Message number {}'.format(n) + # Data must be a bytestring + data = data.encode('utf-8') + message_future = publisher.publish(topic_path, data=data) + futures.append(message_future) + + print('Published message IDs:') + for future in futures: + # result() blocks until the message is published. + print(future.result()) + # [END pubsub_publisher_concurrency_control] + + +def publish_messages_with_error_handler(project_id, topic_name): + """Publishes multiple messages to a Pub/Sub topic with an error handler.""" + # [START pubsub_publish_messages_error_handler] + import time + + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO topic_name = "Your Pub/Sub topic name" + + publisher = pubsub_v1.PublisherClient() + topic_path = publisher.topic_path(project_id, topic_name) + + def callback(message_future): + # When timeout is unspecified, the exception method waits indefinitely. + if message_future.exception(timeout=30): + print('Publishing message on {} threw an Exception {}.'.format( + topic_name, message_future.exception())) + else: + print(message_future.result()) + + for n in range(1, 10): + data = u'Message number {}'.format(n) + # Data must be a bytestring + data = data.encode('utf-8') + # When you publish a message, the client returns a Future. + message_future = publisher.publish(topic_path, data=data) + message_future.add_done_callback(callback) + + print('Published message IDs:') + + # We must keep the main thread from exiting to allow it to process + # messages in the background. + while True: + time.sleep(60) + # [END pubsub_publish_messages_error_handler] + + +def publish_messages_with_batch_settings(project_id, topic_name): + """Publishes multiple messages to a Pub/Sub topic with batch settings.""" + # [START pubsub_publisher_batch_settings] + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO topic_name = "Your Pub/Sub topic name" + + # Configure the batch to publish as soon as there is one kilobyte + # of data or one second has passed. + batch_settings = pubsub_v1.types.BatchSettings( + max_bytes=1024, # One kilobyte + max_latency=1, # One second + ) + publisher = pubsub_v1.PublisherClient(batch_settings) + topic_path = publisher.topic_path(project_id, topic_name) + + for n in range(1, 10): + data = u'Message number {}'.format(n) + # Data must be a bytestring + data = data.encode('utf-8') + publisher.publish(topic_path, data=data) + + print('Published messages.') + # [END pubsub_publisher_batch_settings] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument('project_id', help='Your Google Cloud project ID') + + subparsers = parser.add_subparsers(dest='command') + subparsers.add_parser('list', help=list_topics.__doc__) + + create_parser = subparsers.add_parser('create', help=create_topic.__doc__) + create_parser.add_argument('topic_name') + + delete_parser = subparsers.add_parser('delete', help=delete_topic.__doc__) + delete_parser.add_argument('topic_name') + + publish_parser = subparsers.add_parser( + 'publish', help=publish_messages.__doc__) + publish_parser.add_argument('topic_name') + + publish_with_custom_attributes_parser = subparsers.add_parser( + 'publish-with-custom-attributes', + help=publish_messages_with_custom_attributes.__doc__) + publish_with_custom_attributes_parser.add_argument('topic_name') + + publish_with_futures_parser = subparsers.add_parser( + 'publish-with-futures', + help=publish_messages_with_futures.__doc__) + publish_with_futures_parser.add_argument('topic_name') + + publish_with_error_handler_parser = subparsers.add_parser( + 'publish-with-error-handler', + help=publish_messages_with_error_handler.__doc__) + publish_with_error_handler_parser.add_argument('topic_name') + + publish_with_batch_settings_parser = subparsers.add_parser( + 'publish-with-batch-settings', + help=publish_messages_with_batch_settings.__doc__) + publish_with_batch_settings_parser.add_argument('topic_name') + + args = parser.parse_args() + + if args.command == 'list': + list_topics(args.project_id) + elif args.command == 'create': + create_topic(args.project_id, args.topic_name) + elif args.command == 'delete': + delete_topic(args.project_id, args.topic_name) + elif args.command == 'publish': + publish_messages(args.project_id, args.topic_name) + elif args.command == 'publish-with-custom-attributes': + publish_messages_with_custom_attributes( + args.project_id, args.topic_name) + elif args.command == 'publish-with-futures': + publish_messages_with_futures(args.project_id, args.topic_name) + elif args.command == 'publish-with-error-handler': + publish_messages_with_error_handler(args.project_id, args.topic_name) + elif args.command == 'publish-with-batch-settings': + publish_messages_with_batch_settings(args.project_id, args.topic_name) diff --git a/pubsub/cloud-client/publisher_test.py b/pubsub/cloud-client/publisher_test.py new file mode 100644 index 00000000000..cdb4d0e0e76 --- /dev/null +++ b/pubsub/cloud-client/publisher_test.py @@ -0,0 +1,128 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time + +from gcp_devrel.testing import eventually_consistent +from google.cloud import pubsub_v1 +import mock +import pytest + +import publisher + +PROJECT = os.environ['GCLOUD_PROJECT'] +TOPIC = 'publisher-test-topic' + + +@pytest.fixture +def client(): + yield pubsub_v1.PublisherClient() + + +@pytest.fixture +def topic(client): + topic_path = client.topic_path(PROJECT, TOPIC) + + try: + client.delete_topic(topic_path) + except Exception: + pass + + client.create_topic(topic_path) + + yield topic_path + + +def _make_sleep_patch(): + real_sleep = time.sleep + + def new_sleep(period): + if period == 60: + real_sleep(5) + raise RuntimeError('sigil') + else: + real_sleep(period) + + return mock.patch('time.sleep', new=new_sleep) + + +def test_list(client, topic, capsys): + @eventually_consistent.call + def _(): + publisher.list_topics(PROJECT) + out, _ = capsys.readouterr() + assert topic in out + + +def test_create(client): + topic_path = client.topic_path(PROJECT, TOPIC) + try: + client.delete_topic(topic_path) + except Exception: + pass + + publisher.create_topic(PROJECT, TOPIC) + + @eventually_consistent.call + def _(): + assert client.get_topic(topic_path) + + +def test_delete(client, topic): + publisher.delete_topic(PROJECT, TOPIC) + + @eventually_consistent.call + def _(): + with pytest.raises(Exception): + client.get_topic(client.topic_path(PROJECT, TOPIC)) + + +def test_publish(topic, capsys): + publisher.publish_messages(PROJECT, TOPIC) + + out, _ = capsys.readouterr() + assert 'Published' in out + + +def test_publish_with_custom_attributes(topic, capsys): + publisher.publish_messages_with_custom_attributes(PROJECT, TOPIC) + + out, _ = capsys.readouterr() + assert 'Published' in out + + +def test_publish_with_batch_settings(topic, capsys): + publisher.publish_messages_with_batch_settings(PROJECT, TOPIC) + + out, _ = capsys.readouterr() + assert 'Published' in out + + +def test_publish_with_error_handler(topic, capsys): + + with _make_sleep_patch(): + with pytest.raises(RuntimeError, match='sigil'): + publisher.publish_messages_with_error_handler( + PROJECT, TOPIC) + + out, _ = capsys.readouterr() + assert 'Published' in out + + +def test_publish_with_futures(topic, capsys): + publisher.publish_messages_with_futures(PROJECT, TOPIC) + + out, _ = capsys.readouterr() + assert 'Published' in out diff --git a/pubsub/cloud-client/quickstart.py b/pubsub/cloud-client/quickstart.py new file mode 100644 index 00000000000..f48d085e06b --- /dev/null +++ b/pubsub/cloud-client/quickstart.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse + + +def end_to_end(project_id, topic_name, subscription_name, num_messages): + # [START pubsub_end_to_end] + import time + + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO topic_name = "Your Pub/Sub topic name" + # TODO num_messages = number of messages to test end-to-end + + # Instantiates a publisher and subscriber client + publisher = pubsub_v1.PublisherClient() + subscriber = pubsub_v1.SubscriberClient() + + # The `topic_path` method creates a fully qualified identifier + # in the form `projects/{project_id}/topics/{topic_name}` + topic_path = subscriber.topic_path(project_id, topic_name) + + # The `subscription_path` method creates a fully qualified identifier + # in the form `projects/{project_id}/subscriptions/{subscription_name}` + subscription_path = subscriber.subscription_path( + project_id, subscription_name) + + # Create the topic. + topic = publisher.create_topic(topic_path) + print('\nTopic created: {}'.format(topic.name)) + + # Create a subscription. + subscription = subscriber.create_subscription( + subscription_path, topic_path) + print('\nSubscription created: {}\n'.format(subscription.name)) + + publish_begin = time.time() + + # Publish messages. + for n in range(num_messages): + data = u'Message number {}'.format(n) + # Data must be a bytestring + data = data.encode('utf-8') + # When you publish a message, the client returns a future. + future = publisher.publish(topic_path, data=data) + print('Published {} of message ID {}.'.format(data, future.result())) + + publish_time = time.time() - publish_begin + + messages = set() + + def callback(message): + print('Received message: {}'.format(message)) + # Unacknowledged messages will be sent again. + message.ack() + messages.add(message) + + subscribe_begin = time.time() + + # Receive messages. The subscriber is nonblocking. + subscriber.subscribe(subscription_path, callback=callback) + + print('\nListening for messages on {}...\n'.format(subscription_path)) + + while True: + if len(messages) == num_messages: + subscribe_time = time.time() - subscribe_begin + print("\nReceived all messages.") + print("Publish time lapsed: {:.2f}s.".format(publish_time)) + print("Subscribe time lapsed: {:.2f}s.".format(subscribe_time)) + break + else: + # Sleeps the thread at 50Hz to save on resources. + time.sleep(1. / 50) + # [END pubsub_end_to_end] + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument('project_id', help='Your Google Cloud project ID') + parser.add_argument('topic_name', help='Your topic name') + parser.add_argument('subscription_name', help='Your subscription name') + parser.add_argument('num_msgs', type=int, help='Number of test messages') + + args = parser.parse_args() + + end_to_end(args.project_id, args.topic_name, args.subscription_name, + args.num_msgs) diff --git a/pubsub/cloud-client/quickstart/pub.py b/pubsub/cloud-client/quickstart/pub.py new file mode 100644 index 00000000000..9617b34ea84 --- /dev/null +++ b/pubsub/cloud-client/quickstart/pub.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START pubsub_quickstart_pub_all] +import argparse +import time +# [START pubsub_quickstart_pub_deps] +from google.cloud import pubsub_v1 +# [END pubsub_quickstart_pub_deps] + + +def get_callback(api_future, data): + """Wrap message data in the context of the callback function.""" + + def callback(api_future): + try: + print("Published message {} now has message ID {}".format( + data, api_future.result())) + except Exception: + print("A problem occurred when publishing {}: {}\n".format( + data, api_future.exception())) + raise + return callback + + +def pub(project_id, topic_name): + """Publishes a message to a Pub/Sub topic.""" + # [START pubsub_quickstart_pub_client] + # Initialize a Publisher client + client = pubsub_v1.PublisherClient() + # [END pubsub_quickstart_pub_client] + # Create a fully qualified identifier in the form of + # `projects/{project_id}/topics/{topic_name}` + topic_path = client.topic_path(project_id, topic_name) + + # Data sent to Cloud Pub/Sub must be a bytestring + data = b"Hello, World!" + + # When you publish a message, the client returns a future. + api_future = client.publish(topic_path, data=data) + api_future.add_done_callback(get_callback(api_future, data)) + + # Keep the main thread from exiting until background message + # is processed. + while api_future.running(): + time.sleep(0.1) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument('project_id', help='Google Cloud project ID') + parser.add_argument('topic_name', help='Pub/Sub topic name') + + args = parser.parse_args() + + pub(args.project_id, args.topic_name) +# [END pubsub_quickstart_pub_all] diff --git a/pubsub/cloud-client/quickstart/pub_test.py b/pubsub/cloud-client/quickstart/pub_test.py new file mode 100644 index 00000000000..09443364a3f --- /dev/null +++ b/pubsub/cloud-client/quickstart/pub_test.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pytest + +from google.api_core.exceptions import AlreadyExists +from google.cloud import pubsub_v1 + +import pub + +PROJECT = os.environ['GCLOUD_PROJECT'] +TOPIC = 'quickstart-pub-test-topic' + + +@pytest.fixture(scope='module') +def publisher_client(): + yield pubsub_v1.PublisherClient() + + +@pytest.fixture(scope='module') +def topic(publisher_client): + topic_path = publisher_client.topic_path(PROJECT, TOPIC) + + try: + publisher_client.create_topic(topic_path) + except AlreadyExists: + pass + + yield TOPIC + + +@pytest.fixture +def to_delete(publisher_client): + doomed = [] + yield doomed + for item in doomed: + publisher_client.delete_topic(item) + + +def test_pub(publisher_client, topic, to_delete, capsys): + pub.pub(PROJECT, topic) + + to_delete.append('projects/{}/topics/{}'.format(PROJECT, TOPIC)) + + out, _ = capsys.readouterr() + + assert "Published message b'Hello, World!'" in out diff --git a/pubsub/cloud-client/quickstart/sub.py b/pubsub/cloud-client/quickstart/sub.py new file mode 100644 index 00000000000..520803d70a5 --- /dev/null +++ b/pubsub/cloud-client/quickstart/sub.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START pubsub_quickstart_sub_all] +import argparse +import time +# [START pubsub_quickstart_sub_deps] +from google.cloud import pubsub_v1 +# [END pubsub_quickstart_sub_deps] + + +def sub(project_id, subscription_name): + """Receives messages from a Pub/Sub subscription.""" + # [START pubsub_quickstart_sub_client] + # Initialize a Subscriber client + client = pubsub_v1.SubscriberClient() + # [END pubsub_quickstart_sub_client] + # Create a fully qualified identifier in the form of + # `projects/{project_id}/subscriptions/{subscription_name}` + subscription_path = client.subscription_path( + project_id, subscription_name) + + def callback(message): + print('Received message {} of message ID {}'.format( + message, message.message_id)) + # Acknowledge the message. Unack'ed messages will be redelivered. + message.ack() + print('Acknowledged message of message ID {}\n'.format( + message.message_id)) + + client.subscribe(subscription_path, callback=callback) + print('Listening for messages on {}..\n'.format(subscription_path)) + + # Keep the main thread from exiting so the subscriber can + # process messages in the background. + while True: + time.sleep(60) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument('project_id', help='Google Cloud project ID') + parser.add_argument('subscription_name', help='Pub/Sub subscription name') + + args = parser.parse_args() + + sub(args.project_id, args.subscription_name) +# [END pubsub_quickstart_sub_all] diff --git a/pubsub/cloud-client/quickstart/sub_test.py b/pubsub/cloud-client/quickstart/sub_test.py new file mode 100644 index 00000000000..9c70384ed69 --- /dev/null +++ b/pubsub/cloud-client/quickstart/sub_test.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import os +import pytest +import time + +from google.api_core.exceptions import AlreadyExists +from google.cloud import pubsub_v1 + +import sub + + +PROJECT = os.environ['GCLOUD_PROJECT'] +TOPIC = 'quickstart-sub-test-topic' +SUBSCRIPTION = 'quickstart-sub-test-topic-sub' + + +@pytest.fixture(scope='module') +def publisher_client(): + yield pubsub_v1.PublisherClient() + + +@pytest.fixture(scope='module') +def topic_path(publisher_client): + topic_path = publisher_client.topic_path(PROJECT, TOPIC) + + try: + publisher_client.create_topic(topic_path) + except AlreadyExists: + pass + + yield topic_path + + +@pytest.fixture(scope='module') +def subscriber_client(): + yield pubsub_v1.SubscriberClient() + + +@pytest.fixture(scope='module') +def subscription(subscriber_client, topic_path): + subscription_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION) + + try: + subscriber_client.create_subscription(subscription_path, topic_path) + except AlreadyExists: + pass + + yield SUBSCRIPTION + + +@pytest.fixture +def to_delete(publisher_client, subscriber_client): + doomed = [] + yield doomed + for client, item in doomed: + if 'topics' in item: + publisher_client.delete_topic(item) + if 'subscriptions' in item: + subscriber_client.delete_subscription(item) + + +def _make_sleep_patch(): + real_sleep = time.sleep + + def new_sleep(period): + if period == 60: + real_sleep(10) + raise RuntimeError('sigil') + else: + real_sleep(period) + + return mock.patch('time.sleep', new=new_sleep) + + +def test_sub(publisher_client, + topic_path, + subscriber_client, + subscription, + to_delete, + capsys): + + publisher_client.publish(topic_path, data=b'Hello, World!') + + to_delete.append((publisher_client, topic_path)) + + with _make_sleep_patch(): + with pytest.raises(RuntimeError, match='sigil'): + sub.sub(PROJECT, subscription) + + to_delete.append((subscriber_client, + 'projects/{}/subscriptions/{}'.format(PROJECT, + SUBSCRIPTION))) + + out, _ = capsys.readouterr() + assert "Received message" in out + assert "Acknowledged message" in out diff --git a/pubsub/cloud-client/quickstart_test.py b/pubsub/cloud-client/quickstart_test.py new file mode 100644 index 00000000000..ee6f7d4b21a --- /dev/null +++ b/pubsub/cloud-client/quickstart_test.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python + +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from google.cloud import pubsub_v1 +import pytest +import quickstart + +PROJECT = os.environ['GCLOUD_PROJECT'] +TOPIC = 'end-to-end-test-topic' +SUBSCRIPTION = 'end-to-end-test-topic-sub' +N = 10 + + +@pytest.fixture(scope='module') +def publisher_client(): + yield pubsub_v1.PublisherClient() + + +@pytest.fixture(scope='module') +def topic(publisher_client): + topic_path = publisher_client.topic_path(PROJECT, TOPIC) + + try: + publisher_client.delete_topic(topic_path) + except Exception: + pass + + yield TOPIC + + +@pytest.fixture(scope='module') +def subscriber_client(): + yield pubsub_v1.SubscriberClient() + + +@pytest.fixture(scope='module') +def subscription(subscriber_client, topic): + subscription_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION) + + try: + subscriber_client.delete_subscription(subscription_path) + except Exception: + pass + + yield SUBSCRIPTION + + +def test_end_to_end(topic, subscription, capsys): + + quickstart.end_to_end(PROJECT, topic, subscription, N) + out, _ = capsys.readouterr() + + assert "Received all messages" in out + assert "Publish time lapsed" in out + assert "Subscribe time lapsed" in out diff --git a/pubsub/cloud-client/requirements.txt b/pubsub/cloud-client/requirements.txt new file mode 100644 index 00000000000..d8470ecf937 --- /dev/null +++ b/pubsub/cloud-client/requirements.txt @@ -0,0 +1 @@ +google-cloud-pubsub==0.39.1 diff --git a/pubsub/cloud-client/subscriber.py b/pubsub/cloud-client/subscriber.py new file mode 100644 index 00000000000..5802218b499 --- /dev/null +++ b/pubsub/cloud-client/subscriber.py @@ -0,0 +1,494 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to perform basic operations on +subscriptions with the Cloud Pub/Sub API. + +For more information, see the README.md under /pubsub and the documentation +at https://cloud.google.com/pubsub/docs. +""" + +import argparse + + +def list_subscriptions_in_topic(project_id, topic_name): + """Lists all subscriptions for a given topic.""" + # [START pubsub_list_topic_subscriptions] + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO topic_name = "Your Pub/Sub topic name" + + publisher = pubsub_v1.PublisherClient() + topic_path = publisher.topic_path(project_id, topic_name) + + for subscription in publisher.list_topic_subscriptions(topic_path): + print(subscription) + # [END pubsub_list_topic_subscriptions] + + +def list_subscriptions_in_project(project_id): + """Lists all subscriptions in the current project.""" + # [START pubsub_list_subscriptions] + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + + subscriber = pubsub_v1.SubscriberClient() + project_path = subscriber.project_path(project_id) + + for subscription in subscriber.list_subscriptions(project_path): + print(subscription.name) + # [END pubsub_list_subscriptions] + + +def create_subscription(project_id, topic_name, subscription_name): + """Create a new pull subscription on the given topic.""" + # [START pubsub_create_pull_subscription] + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO topic_name = "Your Pub/Sub topic name" + # TODO subscription_name = "Your Pub/Sub subscription name" + + subscriber = pubsub_v1.SubscriberClient() + topic_path = subscriber.topic_path(project_id, topic_name) + subscription_path = subscriber.subscription_path( + project_id, subscription_name) + + subscription = subscriber.create_subscription( + subscription_path, topic_path) + + print('Subscription created: {}'.format(subscription)) + # [END pubsub_create_pull_subscription] + + +def create_push_subscription(project_id, + topic_name, + subscription_name, + endpoint): + """Create a new push subscription on the given topic.""" + # [START pubsub_create_push_subscription] + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO topic_name = "Your Pub/Sub topic name" + # TODO subscription_name = "Your Pub/Sub subscription name" + # TODO endpoint = "https://my-test-project.appspot.com/push" + + subscriber = pubsub_v1.SubscriberClient() + topic_path = subscriber.topic_path(project_id, topic_name) + subscription_path = subscriber.subscription_path( + project_id, subscription_name) + + push_config = pubsub_v1.types.PushConfig( + push_endpoint=endpoint) + + subscription = subscriber.create_subscription( + subscription_path, topic_path, push_config) + + print('Push subscription created: {}'.format(subscription)) + print('Endpoint for subscription is: {}'.format(endpoint)) + # [END pubsub_create_push_subscription] + + +def delete_subscription(project_id, subscription_name): + """Deletes an existing Pub/Sub topic.""" + # [START pubsub_delete_subscription] + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO subscription_name = "Your Pub/Sub subscription name" + + subscriber = pubsub_v1.SubscriberClient() + subscription_path = subscriber.subscription_path( + project_id, subscription_name) + + subscriber.delete_subscription(subscription_path) + + print('Subscription deleted: {}'.format(subscription_path)) + # [END pubsub_delete_subscription] + + +def update_subscription(project_id, subscription_name, endpoint): + """ + Updates an existing Pub/Sub subscription's push endpoint URL. + Note that certain properties of a subscription, such as + its topic, are not modifiable. + """ + # [START pubsub_update_push_configuration] + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO topic_name = "Your Pub/Sub topic name" + # TODO subscription_name = "Your Pub/Sub subscription name" + # TODO endpoint = "https://my-test-project.appspot.com/push" + + subscriber = pubsub_v1.SubscriberClient() + subscription_path = subscriber.subscription_path( + project_id, subscription_name) + + push_config = pubsub_v1.types.PushConfig( + push_endpoint=endpoint) + + subscription = pubsub_v1.types.Subscription( + name=subscription_path, + push_config=push_config) + + update_mask = { + 'paths': { + 'push_config', + } + } + + subscriber.update_subscription(subscription, update_mask) + result = subscriber.get_subscription(subscription_path) + + print('Subscription updated: {}'.format(subscription_path)) + print('New endpoint for subscription is: {}'.format( + result.push_config)) + # [END pubsub_update_push_configuration] + + +def receive_messages(project_id, subscription_name): + """Receives messages from a pull subscription.""" + # [START pubsub_subscriber_async_pull] + # [START pubsub_quickstart_subscriber] + import time + + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO subscription_name = "Your Pub/Sub subscription name" + + subscriber = pubsub_v1.SubscriberClient() + # The `subscription_path` method creates a fully qualified identifier + # in the form `projects/{project_id}/subscriptions/{subscription_name}` + subscription_path = subscriber.subscription_path( + project_id, subscription_name) + + def callback(message): + print('Received message: {}'.format(message)) + message.ack() + + subscriber.subscribe(subscription_path, callback=callback) + + # The subscriber is non-blocking. We must keep the main thread from + # exiting to allow it to process messages asynchronously in the background. + print('Listening for messages on {}'.format(subscription_path)) + while True: + time.sleep(60) + # [END pubsub_subscriber_async_pull] + # [END pubsub_quickstart_subscriber] + + +def receive_messages_with_custom_attributes(project_id, subscription_name): + """Receives messages from a pull subscription.""" + # [START pubsub_subscriber_sync_pull_custom_attributes] + import time + + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO subscription_name = "Your Pub/Sub subscription name" + + subscriber = pubsub_v1.SubscriberClient() + subscription_path = subscriber.subscription_path( + project_id, subscription_name) + + def callback(message): + print('Received message: {}'.format(message.data)) + if message.attributes: + print('Attributes:') + for key in message.attributes: + value = message.attributes.get(key) + print('{}: {}'.format(key, value)) + message.ack() + + subscriber.subscribe(subscription_path, callback=callback) + + # The subscriber is non-blocking, so we must keep the main thread from + # exiting to allow it to process messages in the background. + print('Listening for messages on {}'.format(subscription_path)) + while True: + time.sleep(60) + # [END pubsub_subscriber_sync_pull_custom_attributes] + + +def receive_messages_with_flow_control(project_id, subscription_name): + """Receives messages from a pull subscription with flow control.""" + # [START pubsub_subscriber_flow_settings] + import time + + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO subscription_name = "Your Pub/Sub subscription name" + + subscriber = pubsub_v1.SubscriberClient() + subscription_path = subscriber.subscription_path( + project_id, subscription_name) + + def callback(message): + print('Received message: {}'.format(message.data)) + message.ack() + + # Limit the subscriber to only have ten outstanding messages at a time. + flow_control = pubsub_v1.types.FlowControl(max_messages=10) + subscriber.subscribe( + subscription_path, callback=callback, flow_control=flow_control) + + # The subscriber is non-blocking, so we must keep the main thread from + # exiting to allow it to process messages in the background. + print('Listening for messages on {}'.format(subscription_path)) + while True: + time.sleep(60) + # [END pubsub_subscriber_flow_settings] + + +def synchronous_pull(project_id, subscription_name): + """Pulling messages synchronously.""" + # [START pubsub_subscriber_sync_pull] + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO subscription_name = "Your Pub/Sub subscription name" + + subscriber = pubsub_v1.SubscriberClient() + subscription_path = subscriber.subscription_path( + project_id, subscription_name) + + NUM_MESSAGES = 3 + + # The subscriber pulls a specific number of messages. + response = subscriber.pull(subscription_path, max_messages=NUM_MESSAGES) + + ack_ids = [] + for received_message in response.received_messages: + print("Received: {}".format(received_message.message.data)) + ack_ids.append(received_message.ack_id) + + # Acknowledges the received messages so they will not be sent again. + subscriber.acknowledge(subscription_path, ack_ids) + + print("Received and acknowledged {} messages. Done.".format(NUM_MESSAGES)) + # [END pubsub_subscriber_sync_pull] + + +def synchronous_pull_with_lease_management(project_id, subscription_name): + """Pulling messages synchronously with lease management""" + # [START pubsub_subscriber_sync_pull_with_lease] + import logging + import multiprocessing + import random + import time + + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO subscription_name = "Your Pub/Sub subscription name" + + subscriber = pubsub_v1.SubscriberClient() + subscription_path = subscriber.subscription_path( + project_id, subscription_name) + + NUM_MESSAGES = 2 + ACK_DEADLINE = 30 + SLEEP_TIME = 10 + + # The subscriber pulls a specific number of messages. + response = subscriber.pull(subscription_path, max_messages=NUM_MESSAGES) + + multiprocessing.log_to_stderr() + logger = multiprocessing.get_logger() + logger.setLevel(logging.INFO) + + def worker(msg): + """Simulates a long-running process.""" + RUN_TIME = random.randint(1, 60) + logger.info('{}: Running {} for {}s'.format( + time.strftime("%X", time.gmtime()), msg.message.data, RUN_TIME)) + time.sleep(RUN_TIME) + + # `processes` stores process as key and ack id and message as values. + processes = dict() + for message in response.received_messages: + process = multiprocessing.Process(target=worker, args=(message,)) + processes[process] = (message.ack_id, message.message.data) + process.start() + + while processes: + for process in list(processes): + ack_id, msg_data = processes[process] + # If the process is still running, reset the ack deadline as + # specified by ACK_DEADLINE once every while as specified + # by SLEEP_TIME. + if process.is_alive(): + # `ack_deadline_seconds` must be between 10 to 600. + subscriber.modify_ack_deadline( + subscription_path, + [ack_id], + ack_deadline_seconds=ACK_DEADLINE) + logger.info('{}: Reset ack deadline for {} for {}s'.format( + time.strftime("%X", time.gmtime()), + msg_data, ACK_DEADLINE)) + + # If the processs is finished, acknowledges using `ack_id`. + else: + subscriber.acknowledge(subscription_path, [ack_id]) + logger.info("{}: Acknowledged {}".format( + time.strftime("%X", time.gmtime()), msg_data)) + processes.pop(process) + + # If there are still processes running, sleeps the thread. + if processes: + time.sleep(SLEEP_TIME) + + print("Received and acknowledged {} messages. Done.".format(NUM_MESSAGES)) + # [END pubsub_subscriber_sync_pull_with_lease] + + +def listen_for_errors(project_id, subscription_name): + """Receives messages and catches errors from a pull subscription.""" + # [START pubsub_subscriber_error_listener] + from google.cloud import pubsub_v1 + + # TODO project = "Your Google Cloud Project ID" + # TODO subscription_name = "Your Pubsub subscription name" + + subscriber = pubsub_v1.SubscriberClient() + subscription_path = subscriber.subscription_path( + project_id, subscription_name) + + def callback(message): + print('Received message: {}'.format(message)) + message.ack() + + future = subscriber.subscribe(subscription_path, callback=callback) + + # Blocks the thread while messages are coming in through the stream. Any + # exceptions that crop up on the thread will be set on the future. + try: + # When timeout is unspecified, the result method waits indefinitely. + future.result(timeout=30) + except Exception as e: + print( + 'Listening for messages on {} threw an Exception: {}.'.format( + subscription_name, e)) + # [END pubsub_subscriber_error_listener] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument('project_id', help='Your Google Cloud project ID') + + subparsers = parser.add_subparsers(dest='command') + list_in_topic_parser = subparsers.add_parser( + 'list_in_topic', help=list_subscriptions_in_topic.__doc__) + list_in_topic_parser.add_argument('topic_name') + + list_in_project_parser = subparsers.add_parser( + 'list_in_project', help=list_subscriptions_in_project.__doc__) + + create_parser = subparsers.add_parser( + 'create', help=create_subscription.__doc__) + create_parser.add_argument('topic_name') + create_parser.add_argument('subscription_name') + + create_push_parser = subparsers.add_parser( + 'create-push', help=create_push_subscription.__doc__) + create_push_parser.add_argument('topic_name') + create_push_parser.add_argument('subscription_name') + create_push_parser.add_argument('endpoint') + + delete_parser = subparsers.add_parser( + 'delete', help=delete_subscription.__doc__) + delete_parser.add_argument('subscription_name') + + update_parser = subparsers.add_parser( + 'update', help=update_subscription.__doc__) + update_parser.add_argument('subscription_name') + update_parser.add_argument('endpoint') + + receive_parser = subparsers.add_parser( + 'receive', help=receive_messages.__doc__) + receive_parser.add_argument('subscription_name') + + receive_with_custom_attributes_parser = subparsers.add_parser( + 'receive-custom-attributes', + help=receive_messages_with_custom_attributes.__doc__) + receive_with_custom_attributes_parser.add_argument('subscription_name') + + receive_with_flow_control_parser = subparsers.add_parser( + 'receive-flow-control', + help=receive_messages_with_flow_control.__doc__) + receive_with_flow_control_parser.add_argument('subscription_name') + + synchronous_pull_parser = subparsers.add_parser( + 'receive-synchronously', + help=synchronous_pull.__doc__) + synchronous_pull_parser.add_argument('subscription_name') + + synchronous_pull_with_lease_management_parser = subparsers.add_parser( + 'receive-synchronously-with-lease', + help=synchronous_pull_with_lease_management.__doc__) + synchronous_pull_with_lease_management_parser.add_argument( + 'subscription_name') + + listen_for_errors_parser = subparsers.add_parser( + 'listen_for_errors', help=listen_for_errors.__doc__) + listen_for_errors_parser.add_argument('subscription_name') + + args = parser.parse_args() + + if args.command == 'list_in_topic': + list_subscriptions_in_topic(args.project_id, args.topic_name) + elif args.command == 'list_in_project': + list_subscriptions_in_project(args.project_id) + elif args.command == 'create': + create_subscription( + args.project_id, args.topic_name, args.subscription_name) + elif args.command == 'create-push': + create_push_subscription( + args.project_id, + args.topic_name, + args.subscription_name, + args.endpoint) + elif args.command == 'delete': + delete_subscription( + args.project_id, args.subscription_name) + elif args.command == 'update': + update_subscription( + args.project_id, args.subscription_name, args.endpoint) + elif args.command == 'receive': + receive_messages(args.project_id, args.subscription_name) + elif args.command == 'receive-custom-attributes': + receive_messages_with_custom_attributes( + args.project_id, args.subscription_name) + elif args.command == 'receive-flow-control': + receive_messages_with_flow_control( + args.project_id, args.subscription_name) + elif args.command == 'receive-synchronously': + synchronous_pull( + args.project_id, args.subscription_name) + elif args.command == 'receive-synchronously-with-lease': + synchronous_pull_with_lease_management( + args.project_id, args.subscription_name) + elif args.command == 'listen_for_errors': + listen_for_errors(args.project_id, args.subscription_name) diff --git a/pubsub/cloud-client/subscriber_test.py b/pubsub/cloud-client/subscriber_test.py new file mode 100644 index 00000000000..df5b1092bad --- /dev/null +++ b/pubsub/cloud-client/subscriber_test.py @@ -0,0 +1,255 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time + +from gcp_devrel.testing import eventually_consistent +from google.cloud import pubsub_v1 +import google.api_core.exceptions +import mock +import pytest + +import subscriber + +PROJECT = os.environ['GCLOUD_PROJECT'] +TOPIC = 'subscription-test-topic' +SUBSCRIPTION = 'subscription-test-subscription' +SUBSCRIPTION_SYNC1 = 'subscription-test-subscription-sync1' +SUBSCRIPTION_SYNC2 = 'subscription-test-subscription-sync2' +ENDPOINT = 'https://{}.appspot.com/push'.format(PROJECT) +NEW_ENDPOINT = 'https://{}.appspot.com/push2'.format(PROJECT) + + +@pytest.fixture(scope='module') +def publisher_client(): + yield pubsub_v1.PublisherClient() + + +@pytest.fixture(scope='module') +def topic(publisher_client): + topic_path = publisher_client.topic_path(PROJECT, TOPIC) + + try: + publisher_client.delete_topic(topic_path) + except Exception: + pass + + publisher_client.create_topic(topic_path) + + yield topic_path + + +@pytest.fixture(scope='module') +def subscriber_client(): + yield pubsub_v1.SubscriberClient() + + +@pytest.fixture +def subscription(subscriber_client, topic): + subscription_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION) + + try: + subscriber_client.delete_subscription(subscription_path) + except Exception: + pass + + try: + subscriber_client.create_subscription(subscription_path, topic=topic) + except google.api_core.exceptions.AlreadyExists: + pass + + yield subscription_path + + +@pytest.fixture +def subscription_sync1(subscriber_client, topic): + subscription_sync_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION_SYNC1) + + try: + subscriber_client.delete_subscription(subscription_sync_path) + except Exception: + pass + + subscriber_client.create_subscription(subscription_sync_path, topic=topic) + + yield subscription_sync_path + + +@pytest.fixture +def subscription_sync2(subscriber_client, topic): + subscription_sync_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION_SYNC2) + + try: + subscriber_client.delete_subscription(subscription_sync_path) + except Exception: + pass + + subscriber_client.create_subscription(subscription_sync_path, topic=topic) + + yield subscription_sync_path + + +def test_list_in_topic(subscription, capsys): + @eventually_consistent.call + def _(): + subscriber.list_subscriptions_in_topic(PROJECT, TOPIC) + out, _ = capsys.readouterr() + assert subscription in out + + +def test_list_in_project(subscription, capsys): + @eventually_consistent.call + def _(): + subscriber.list_subscriptions_in_project(PROJECT) + out, _ = capsys.readouterr() + assert subscription in out + + +def test_create(subscriber_client): + subscription_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION) + try: + subscriber_client.delete_subscription(subscription_path) + except Exception: + pass + + subscriber.create_subscription(PROJECT, TOPIC, SUBSCRIPTION) + + @eventually_consistent.call + def _(): + assert subscriber_client.get_subscription(subscription_path) + + +def test_create_push(subscriber_client): + subscription_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION) + try: + subscriber_client.delete_subscription(subscription_path) + except Exception: + pass + + subscriber.create_push_subscription(PROJECT, TOPIC, SUBSCRIPTION, ENDPOINT) + + @eventually_consistent.call + def _(): + assert subscriber_client.get_subscription(subscription_path) + + +def test_delete(subscriber_client, subscription): + subscriber.delete_subscription(PROJECT, SUBSCRIPTION) + + @eventually_consistent.call + def _(): + with pytest.raises(Exception): + subscriber_client.get_subscription(subscription) + + +def test_update(subscriber_client, subscription, capsys): + subscriber.update_subscription(PROJECT, SUBSCRIPTION, NEW_ENDPOINT) + + out, _ = capsys.readouterr() + assert 'Subscription updated' in out + + +def _publish_messages(publisher_client, topic): + for n in range(5): + data = u'Message {}'.format(n).encode('utf-8') + publisher_client.publish( + topic, data=data) + + +def _publish_messages_with_custom_attributes(publisher_client, topic): + data = u'Test message'.encode('utf-8') + publisher_client.publish(topic, data=data, origin='python-sample') + + +def _make_sleep_patch(): + real_sleep = time.sleep + + def new_sleep(period): + if period == 60: + real_sleep(5) + raise RuntimeError('sigil') + else: + real_sleep(period) + + return mock.patch('time.sleep', new=new_sleep) + + +def test_receive(publisher_client, topic, subscription, capsys): + _publish_messages(publisher_client, topic) + + with _make_sleep_patch(): + with pytest.raises(RuntimeError, match='sigil'): + subscriber.receive_messages(PROJECT, SUBSCRIPTION) + + out, _ = capsys.readouterr() + assert 'Listening' in out + assert subscription in out + assert 'Message 1' in out + + +def test_receive_synchronously( + publisher_client, topic, subscription_sync1, capsys): + _publish_messages(publisher_client, topic) + + subscriber.synchronous_pull(PROJECT, SUBSCRIPTION_SYNC1) + + out, _ = capsys.readouterr() + assert 'Done.' in out + + +def test_receive_synchronously_with_lease( + publisher_client, topic, subscription_sync2, capsys): + _publish_messages(publisher_client, topic) + + subscriber.synchronous_pull_with_lease_management( + PROJECT, SUBSCRIPTION_SYNC2) + + out, _ = capsys.readouterr() + assert 'Done.' in out + + +def test_receive_with_custom_attributes( + publisher_client, topic, subscription, capsys): + _publish_messages_with_custom_attributes(publisher_client, topic) + + with _make_sleep_patch(): + with pytest.raises(RuntimeError, match='sigil'): + subscriber.receive_messages_with_custom_attributes( + PROJECT, SUBSCRIPTION) + + out, _ = capsys.readouterr() + assert 'Test message' in out + assert 'origin' in out + assert 'python-sample' in out + + +def test_receive_with_flow_control( + publisher_client, topic, subscription, capsys): + _publish_messages(publisher_client, topic) + + with _make_sleep_patch(): + with pytest.raises(RuntimeError, match='sigil'): + subscriber.receive_messages_with_flow_control( + PROJECT, SUBSCRIPTION) + + out, _ = capsys.readouterr() + assert 'Listening' in out + assert subscription in out + assert 'Message 1' in out diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000000..2008cdfaf07 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +addopts = + -v + --no-success-flaky-report + --tb=native +norecursedirs = .git env lib .tox .nox diff --git a/requirements-python2.7-dev.txt b/requirements-python2.7-dev.txt deleted file mode 100644 index f551457c1d9..00000000000 --- a/requirements-python2.7-dev.txt +++ /dev/null @@ -1,30 +0,0 @@ -gcloud==0.10.1 -google-api-python-client==1.5.0 -oauth2client==2.0.1 -requests[security]==2.9.1 -beautifulsoup4==4.4.1 -coverage==4.1b2 -Flask==0.10.1 -funcsigs==0.4 -itsdangerous==0.24 -Jinja2==2.8 -MarkupSafe==0.23 -mock==1.3.0 -pbr==1.8.1 -PyYAML==3.11 -waitress==0.8.10 -WebOb==1.6.0a0 -WebTest==2.0.20 -Werkzeug==0.11.4 -Flask-SQLAlchemy==2.1 -PyMySQL==0.7.2 -pymemcache==1.3.5 -PyCrypto==2.6.1 -flaky==3.1.0 -Django==1.9.3 -twilio==6.3.dev0 -sendgrid==2.2.1 -Flask-Sockets==0.2.0 -mysql-python==1.2.5 -pytest==2.9.0 -pytest-cov==2.2.1 diff --git a/requirements-python3.4-dev.txt b/requirements-python3.4-dev.txt deleted file mode 100644 index 2c490ffe5ec..00000000000 --- a/requirements-python3.4-dev.txt +++ /dev/null @@ -1,28 +0,0 @@ -gcloud==0.10.1 -google-api-python-client==1.5.0 -oauth2client==2.0.1 -requests[security]==2.9.1 -beautifulsoup4==4.4.1 -coverage==4.1b2 -Flask==0.10.1 -funcsigs==0.4 -itsdangerous==0.24 -Jinja2==2.8 -MarkupSafe==0.23 -mock==1.3.0 -pbr==1.8.1 -PyYAML==3.11 -waitress==0.8.10 -WebOb==1.6.0a0 -WebTest==2.0.20 -Werkzeug==0.11.4 -Flask-SQLAlchemy==2.1 -PyMySQL==0.7.2 -pymemcache==1.3.5 -PyCrypto==2.6.1 -flaky==3.1.0 -Django==1.9.3 -twilio==6.3.dev0 -sendgrid==2.2.1 -pytest==2.9.0 -pytest-cov==2.2.1 diff --git a/scripts/README.md b/scripts/README.md old mode 100644 new mode 100755 index 5b071c5ebdd..21c84b3ab3b --- a/scripts/README.md +++ b/scripts/README.md @@ -1 +1,6 @@ These scripts are used for the maintenance of this repository and are not necessarily meant as samples. + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=scripts/README.md diff --git a/scripts/auto_link_to_docs.py b/scripts/auto_link_to_docs.py index 3b5e437cf68..68f4188d345 100755 --- a/scripts/auto_link_to_docs.py +++ b/scripts/auto_link_to_docs.py @@ -133,12 +133,14 @@ def update_readme(readme_path, docs): def main(): docs_links = json.load(open( - os.path.join(REPO_ROOT, 'scripts', 'docs-links.json'), 'r')) + os.path.join( + REPO_ROOT, 'scripts', 'resources', 'docs-links.json'), 'r')) files_to_docs = invert_docs_link_map(docs_links) readmes_to_docs = collect_docs_for_readmes(files_to_docs) for readme, docs in readmes_to_docs.iteritems(): update_readme(readme, docs) + if __name__ == '__main__': main() diff --git a/scripts/decrypt-secrets.sh b/scripts/decrypt-secrets.sh new file mode 100755 index 00000000000..dda0e163fe9 --- /dev/null +++ b/scripts/decrypt-secrets.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOT=$( dirname "$DIR" ) + +# Work from the project root. +cd $ROOT + +openssl aes-256-cbc -k "$1" -in testing/secrets.tar.enc -out secrets.tar -d +tar xvf secrets.tar +rm secrets.tar diff --git a/scripts/encrypt-secrets.sh b/scripts/encrypt-secrets.sh index 6bdbf9421d4..b1d6d9451f1 100755 --- a/scripts/encrypt-secrets.sh +++ b/scripts/encrypt-secrets.sh @@ -14,11 +14,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -read -s -p "Enter password for encryption: " password +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOT=$( dirname "$DIR" ) + +# Work from the project root. +cd $ROOT + +read -s -p "Enter password for encryption: " PASSWORD echo -tar cvf secrets.tar testing/resources/{service-account.json,client-secrets.json,test-env.sh} -openssl aes-256-cbc -k "$password" -in secrets.tar -out secrets.tar.enc +tar cvf secrets.tar testing/{service-account.json,client-secrets.json,test-env.sh} +openssl aes-256-cbc -k "$PASSWORD" -in secrets.tar -out testing/secrets.tar.enc rm secrets.tar -travis encrypt "secrets_password=$password" --add +travis encrypt "SECRETS_PASSWORD=$PASSWORD" --add --override diff --git a/scripts/prepare-testing-project.sh b/scripts/prepare-testing-project.sh index 1b62ea0f7f8..482011e0087 100755 --- a/scripts/prepare-testing-project.sh +++ b/scripts/prepare-testing-project.sh @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -e + GCLOUD_PROJECT=$(gcloud config list project --format="value(core.project)" 2>/dev/null) echo "Configuring project $GCLOUD_PROJECT for system tests." @@ -22,20 +24,30 @@ echo "Creating cloud storage bucket." gsutil mb gs://$GCLOUD_PROJECT gsutil defacl set public-read gs://$GCLOUD_PROJECT +echo "Creating bigtable resources." +gcloud alpha bigtable clusters create bigtable-test \ + --description="Test cluster" \ + --nodes=3 \ + --zone=us-central1-c + echo "Creating bigquery resources." -gcloud alpha bigquery datasets create test_dataset -gcloud alpha bigquery datasets create ephemeral_test_dataset -gsutil cp tests/resources/data.csv gs://$GCLOUD_PROJECT/data.csv -gcloud alpha bigquery import \ +bq mk test_dataset +bq mk --schema bigquery/api/resources/schema.json test_dataset.test_import_table +bq mk ephemeral_test_dataset +gsutil cp bigquery/api/resources/data.csv gs://$GCLOUD_PROJECT/data.csv +bq load \ + test_dataset.test_table \ gs://$GCLOUD_PROJECT/data.csv \ - test_dataset/test_table \ - --schema-file tests/resources/schema.json + bigquery/api/resources/schema.json echo "Creating datastore indexes." -gcloud preview app deploy -q datastore/api/index.yaml +gcloud app deploy -q datastore/api/index.yaml echo "Creating pubsub resources." gcloud alpha pubsub topics create gae-mvm-pubsub-topic +echo "Creating speech resources." +gsutil cp speech/api-client/resources/audio.raw gs://$GCLOUD_PROJECT/speech/ + echo "To finish setup, follow this link to enable APIs." -echo "https://console.cloud.google.com/flows/enableapi?apiid=datastore,pubsub,storage_api,logging,plus,bigquery,cloudmonitoring,compute_component" +echo "https://console.cloud.google.com/flows/enableapi?project=${GCLOUD_PROJECT}&apiid=bigtable.googleapis.com,bigtableadmin.googleapis.com,bigquery,bigquerydatatransfer.googleapis.com,cloudmonitoring,compute_component,datastore,datastore.googleapis.com,dataproc,dns,plus,pubsub,logging,storage_api,texttospeech.googleapis.com,vision.googleapis.com" diff --git a/scripts/readme-gen/readme_gen.py b/scripts/readme-gen/readme_gen.py new file mode 100644 index 00000000000..d309d6e9751 --- /dev/null +++ b/scripts/readme-gen/readme_gen.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generates READMEs using configuration defined in yaml.""" + +import argparse +import io +import os +import subprocess + +import jinja2 +import yaml + + +jinja_env = jinja2.Environment( + trim_blocks=True, + loader=jinja2.FileSystemLoader( + os.path.abspath(os.path.join(os.path.dirname(__file__), 'templates')))) + +README_TMPL = jinja_env.get_template('README.tmpl.rst') + + +def get_help(file): + return subprocess.check_output(['python', file, '--help']).decode() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('source') + parser.add_argument('--destination', default='README.rst') + + args = parser.parse_args() + + source = os.path.abspath(args.source) + root = os.path.dirname(source) + destination = os.path.join(root, args.destination) + + jinja_env.globals['get_help'] = get_help + + with io.open(source, 'r') as f: + config = yaml.load(f) + + # This allows get_help to execute in the right directory. + os.chdir(root) + + output = README_TMPL.render(config) + + with io.open(destination, 'w') as f: + f.write(output) + + +if __name__ == '__main__': + main() diff --git a/scripts/readme-gen/templates/README.tmpl.rst b/scripts/readme-gen/templates/README.tmpl.rst new file mode 100644 index 00000000000..6bdd6538fea --- /dev/null +++ b/scripts/readme-gen/templates/README.tmpl.rst @@ -0,0 +1,75 @@ +{# The following line is a lie. BUT! Once jinja2 is done with it, it will + become truth! #} +.. This file is automatically generated. Do not edit this file directly. + +{{product.name}} Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor={{folder}}/README.rst + + +This directory contains samples for {{product.name}}. {{product.description}} + +{{description}} + +.. _{{product.name}}: {{product.url}} + +{% if setup %} +Setup +------------------------------------------------------------------------------- + +{% for section in setup %} + +{% include section + '.tmpl.rst' %} + +{% endfor %} +{% endif %} + +{% if samples %} +Samples +------------------------------------------------------------------------------- + +{% for sample in samples %} +{{sample.name}} ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor={{folder}}/{{sample.file}},{{folder}}/README.rst + + +{{sample.description}} + +To run this sample: + +.. code-block:: bash + + $ python {{sample.file}} +{% if sample.show_help %} + + {{get_help(sample.file)|indent}} +{% endif %} + + +{% endfor %} +{% endif %} + +{% if cloud_client_library %} + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + +{% endif %} + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ diff --git a/scripts/readme-gen/templates/auth.tmpl.rst b/scripts/readme-gen/templates/auth.tmpl.rst new file mode 100644 index 00000000000..1446b94a5e3 --- /dev/null +++ b/scripts/readme-gen/templates/auth.tmpl.rst @@ -0,0 +1,9 @@ +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started diff --git a/scripts/readme-gen/templates/auth_api_key.tmpl.rst b/scripts/readme-gen/templates/auth_api_key.tmpl.rst new file mode 100644 index 00000000000..11957ce2714 --- /dev/null +++ b/scripts/readme-gen/templates/auth_api_key.tmpl.rst @@ -0,0 +1,14 @@ +Authentication +++++++++++++++ + +Authentication for this service is done via an `API Key`_. To obtain an API +Key: + +1. Open the `Cloud Platform Console`_ +2. Make sure that billing is enabled for your project. +3. From the **Credentials** page, create a new **API Key** or use an existing + one for your project. + +.. _API Key: + https://developers.google.com/api-client-library/python/guide/aaa_apikeys +.. _Cloud Console: https://console.cloud.google.com/project?_ diff --git a/scripts/readme-gen/templates/install_deps.tmpl.rst b/scripts/readme-gen/templates/install_deps.tmpl.rst new file mode 100644 index 00000000000..a0406dba8c8 --- /dev/null +++ b/scripts/readme-gen/templates/install_deps.tmpl.rst @@ -0,0 +1,29 @@ +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ diff --git a/scripts/readme-gen/templates/install_portaudio.tmpl.rst b/scripts/readme-gen/templates/install_portaudio.tmpl.rst new file mode 100644 index 00000000000..5ea33d18c00 --- /dev/null +++ b/scripts/readme-gen/templates/install_portaudio.tmpl.rst @@ -0,0 +1,35 @@ +Install PortAudio ++++++++++++++++++ + +Install `PortAudio`_. This is required by the `PyAudio`_ library to stream +audio from your computer's microphone. PyAudio depends on PortAudio for cross-platform compatibility, and is installed differently depending on the +platform. + +* For Mac OS X, you can use `Homebrew`_:: + + brew install portaudio + + **Note**: if you encounter an error when running `pip install` that indicates + it can't find `portaudio.h`, try running `pip install` with the following + flags:: + + pip install --global-option='build_ext' \ + --global-option='-I/usr/local/include' \ + --global-option='-L/usr/local/lib' \ + pyaudio + +* For Debian / Ubuntu Linux:: + + apt-get install portaudio19-dev python-all-dev + +* Windows may work without having to install PortAudio explicitly (it will get + installed with PyAudio). + +For more details, see the `PyAudio installation`_ page. + + +.. _PyAudio: https://people.csail.mit.edu/hubert/pyaudio/ +.. _PortAudio: http://www.portaudio.com/ +.. _PyAudio installation: + https://people.csail.mit.edu/hubert/pyaudio/#downloads +.. _Homebrew: http://brew.sh diff --git a/scripts/resources/docs-links.json b/scripts/resources/docs-links.json index be1ed7e1798..19a77f6abdc 100644 --- a/scripts/resources/docs-links.json +++ b/scripts/resources/docs-links.json @@ -1,87 +1,377 @@ { - "/appengine/docs/python/ndb/": [ - "appengine/ndb/overview/main.py" + "/storage/docs/encryption": [ + "storage/api/customer_supplied_keys.py" ], - "/storage/transfer/create-client": [ - "storage/transfer_service/create_client.py" + "/appengine/docs/python/datastore/entity-property-reference": [ + "appengine/ndb/properties/snippets.py", + "appengine/ndb/properties/snippets_test.py" ], - "/storage/docs/json_api/v1/json-api-python-samples": [ - "storage/api/list_objects.py" + "/appengine/docs/python/images/usingimages": [ + "appengine/images/guestbook/main.py" + ], + "/appengine/docs/flexible/python/config/intro": [ + "managed_vms/hello_world/app.yaml" + ], + "/python/django/flexible-environment": [ + "managed_vms/django_cloudsql/mysite/settings.py", + "managed_vms/django_cloudsql/app.yaml" + ], + "/appengine/docs/python/datastore/creating-entity-keys": [ + "appengine/ndb/entities/snippets.py" + ], + "/appengine/docs/python/ndb/queries": [ + "appengine/ndb/queries/snippets.py", + "appengine/ndb/queries/snippets_models.py", + "appengine/ndb/queries/guestbook.py" + ], + "/appengine/docs/python/datastore/projectionqueries": [ + "appengine/ndb/projection_queries/snippets.py" + ], + "/compute/docs/metadata": [ + "compute/metadata/main.py" + ], + "/compute/docs/access/create-enable-service-accounts-for-instances": [ + "compute/auth/access_token.py", + "compute/auth/application_default.py" + ], + "/appengine/docs/flexible/python/serving-static-files": [ + "managed_vms/static_files/static/main.css", + "managed_vms/static_files/templates/index.html", + "managed_vms/static_files/main.py" + ], + "/dns/monitoring": [ + "dns/api/main.py" + ], + "/bigquery/authentication": [ + "bigquery/api/getting_started.py" + ], + "/appengine/docs/python/mail/sending-mail-with-mail-api": [ + "appengine/mail/send_message.py", + "appengine/mail/user_signup.py", + "appengine/mail/send_mail.py" + ], + "/appengine/docs/flexible/python/using-cloud-storage": [ + "managed_vms/storage/app.yaml", + "managed_vms/storage/requirements.txt", + "managed_vms/storage/main.py" + ], + "/appengine/docs/python/ndb/async": [ + "appengine/ndb/async/app_sync.py", + "appengine/ndb/async/app_async.py", + "appengine/ndb/async/shopping_cart.py", + "appengine/ndb/async/guestbook.py", + "appengine/ndb/async/app_toplevel/app_toplevel.py" + ], + "/dataproc/tutorials/python-library-example": [ + "dataproc/create_cluster_and_submit_job.py", + "dataproc/list_clusters.py", + "dataproc/pyspark_sort.py" + ], + "/bigquery/loading-data-post-request": [ + "bigquery/api/load_data_by_post.py" + ], + "/monitoring/demos/": [ + "monitoring/api/v3/list_resources.py", + "monitoring/api/v3/custom_metric.py", + "monitoring/api/v2/auth.py" + ], + "/appengine/docs/python/search/options": [ + "appengine/search/snippets/snippets.py" + ], + "/appengine/docs/python/runtime": [ + "appengine/background/main.py" + ], + "/appengine/docs/flexible/python/integrating-with-analytics": [ + "managed_vms/analytics/main.py", + "managed_vms/analytics/app.yaml" + ], + "/speech/docs/non-streaming-rest-tutorial": [ + "speech/api/speech_rest.py" + ], + "/appengine/docs/flexible/python/using-sms-and-voice-services-via-twilio": [ + "managed_vms/twilio/requirements.txt", + "managed_vms/twilio/main.py", + "managed_vms/twilio/app.yaml" + ], + "/appengine/docs/python/endpoints/required_files": [ + "appengine/endpoints/backend/app.yaml" + ], + "/appengine/docs/python/mail/headers": [ + "appengine/mail/header.py" + ], + "/appengine/docs/python/appidentity/": [ + "appengine/app_identity/signing/main.py", + "appengine/app_identity/asserting/main.py", + "appengine/app_identity/incoming/main.py" + ], + "/appengine/docs/python/ndb/cache": [ + "appengine/ndb/cache/snippets.py" ], "/bigquery/querying-data": [ "bigquery/api/async_query.py", "bigquery/api/sync_query.py" ], - "/bigquery/bigquery-api-quickstart": [ - "bigquery/api/getting_started.py" + "/appengine/docs/python/ndb/": [ + "appengine/ndb/overview/main.py" ], - "/bigquery/docs/managing_jobs_datasets_projects": [ - "bigquery/api/list_datasets_projects.py" + "/appengine/docs/python/datastore/creating-entity-models": [ + "appengine/ndb/entities/snippets.py" ], - "/appengine/docs/python/blobstore/": [ - "appengine/blobstore/main.py" + "/appengine/docs/python/images/": [ + "appengine/images/api/blobstore.py", + "appengine/images/api/main.py" ], - "/appengine/docs/python/images/usingimages": [ - "appengine/images/main.py" - ], - "/compute/docs/tutorials/python-guide": [ - "compute/api/create_instance.py", - "compute/api/startup-script.sh" + "/logging/docs/api/tasks/creating-logs": [ + "cloud_logging/api/list_logs.py" ], - "/docs/authentication": [ - "storage/api/list_objects.py" + "/dns/zones/": [ + "dns/api/main.py" ], - "/bigquery/loading-data-into-bigquery": [ - "bigquery/api/load_data_from_csv.py", - "bigquery/api/load_data_by_post.py" + "/appengine/docs/python/taskqueue/push/creating-handlers": [ + "appengine/taskqueue/counter/worker.yaml", + "appengine/taskqueue/counter/worker.py" ], - "/bigquery/streaming-data-into-bigquery": [ - "bigquery/api/streaming.py" + "/bigquery/bigquery-api-quickstart": [ + "bigquery/api/getting_started.py" ], - "/appengine/docs/python/memcache/usingmemcache": [ - "appengine/memcache/guestbook/main.py" + "/compute/docs/api/how-tos/api-requests-responses": [ + "compute/api/create_instance.py" ], - "/monitoring/demos/": [ - "monitoring/api/auth.py" + "/appengine/docs/python/endpoints/api_server": [ + "appengine/endpoints/backend/main.py", + "appengine/endpoints/backend/app.yaml" ], "/appengine/docs/python/multitenancy/namespaces": [ - "appengine/multitenancy/memcache.py", "appengine/multitenancy/datastore.py", + "appengine/multitenancy/memcache.py", "appengine/multitenancy/taskqueue.py" ], - "/bigquery/docs/data": [ - "bigquery/api/sync_query.py" + "/appengine/docs/python/search/facet_search": [ + "appengine/search/snippets/snippets.py" ], - "/bigquery/authentication": [ - "appengine/bigquery/main.py", - "bigquery/api/getting_started.py" + "/appengine/docs/python/users/": [ + "appengine/users/main.py" ], - "/bigquery/exporting-data-from-bigquery": [ - "bigquery/api/export_data_to_cloud_storage.py" + "/storage/transfer/create-client": [ + "storage/transfer_service/create_client.py" + ], + "/appengine/docs/python/channel/": [ + "appengine/channel/chatactoe.py", + "appengine/channel/index.html" + ], + "/appengine/docs/python/tools/remoteapi": [ + "appengine/remote_api/app.yaml", + "appengine/remote_api/client.py" ], "/appengine/docs/python/ndb/transactions": [ "appengine/ndb/transactions/main.py" ], - "/monitoring/api/authentication": [ - "appengine/bigquery/app.yaml", - "monitoring/api/auth.py", - "appengine/bigquery/main.py" + "/appengine/docs/python/users/userobjects": [ + "appengine/users/main.py", + "appengine/ndb/entities/snippets.py" ], - "/storage/transfer/create-transfer": [ - "storage/transfer_service/nearline_request.py", - "storage/transfer_service/aws_request.py", - "storage/transfer_service/transfer_check.py" + "/appengine/docs/python/tools/webapp/blobstorehandlers": [ + "appengine/blobstore/main.py" + ], + "/appengine/docs/python/mail/attachments": [ + "appengine/mail/attachment.py" + ], + "/appengine/docs/python/mail/mailgun": [ + "appengine/mailgun/main.py" + ], + "/python/django/container-engine": [ + "container_engine/django_tutorial/mysite/settings.py", + "container_engine/django_tutorial/polls.yaml" + ], + "/bigquery/exporting-data-from-bigquery": [ + "bigquery/api/export_data_to_cloud_storage.py" + ], + "/appengine/docs/flexible/python/runtime": [ + "managed_vms/websockets/main.py" ], "/storage/docs/authentication": [ "storage/api/list_objects.py" ], + "/appengine/docs/python/users/loginurls": [ + "appengine/users/main.py" + ], + "/appengine/docs/python/cloud-sql/": [ + "appengine/cloudsql/main.py", + "appengine/cloudsql/app.yaml" + ], + "/appengine/docs/python/xmpp/": [ + "appengine/xmpp/app.yaml", + "appengine/xmpp/xmpp.py" + ], "/appengine/docs/python/tools/localunittesting": [ "appengine/localtesting/runner.py", - "appengine/localtesting/test_task_queue.py", - "appengine/localtesting/test_login.py", + "appengine/localtesting/mail_test.py", "appengine/localtesting/queue.yaml", - "appengine/localtesting/test_mail.py", - "appengine/localtesting/test_env_vars.py", - "appengine/localtesting/test_datastore.py" + "appengine/localtesting/task_queue_test.py", + "appengine/localtesting/login_test.py", + "appengine/localtesting/env_vars_test.py", + "appengine/localtesting/datastore_test.py" + ], + "/appengine/docs/python/mail/bounce": [ + "appengine/mail/app.yaml", + "appengine/mail/handle_bounced_email.py" + ], + "/appengine/docs/python/endpoints/getstarted/backend/write_api": [ + "appengine/endpoints/backend/main.py", + "appengine/endpoints/backend/app.yaml" + ], + "/appengine/docs/python/mail/sendgrid": [ + "appengine/sendgrid/main.py" + ], + "/appengine/docs/python/mail/receiving-mail-with-mail-api": [ + "appengine/mail/app.yaml", + "appengine/mail/handle_incoming_email.py" + ], + "/appengine/docs/python/datastore/creating-entities": [ + "appengine/ndb/entities/snippets.py" + ], + "/appengine/docs/flexible/python/sending-emails-with-sendgrid": [ + "managed_vms/sendgrid/app.yaml", + "managed_vms/sendgrid/requirements.txt", + "managed_vms/sendgrid/main.py" + ], + "/appengine/docs/python/ndb/subclassprop": [ + "appengine/ndb/property_subclasses/my_models.py", + "appengine/ndb/property_subclasses/snippets.py" + ], + "/appengine/docs/python/endpoints/getstarted/backend/auth": [ + "appengine/endpoints/backend/main.py" + ], + "/appengine/docs/python/search/": [ + "appengine/search/snippets/snippets.py" + ], + "/compute/docs/disks/customer-supplied-encryption": [ + "compute/encryption/generate_wrapped_rsa_key.py" + ], + "/appengine/docs/python/logs/": [ + "appengine/logging/writing_logs/main.py", + "appengine/logging/reading_logs/main.py" + ], + "/appengine/docs/python/users/adminusers": [ + "appengine/users/main.py" + ], + "/error-reporting/docs/setting-up-on-compute-engine": [ + "error_reporting/main.py" + ], + "/docs/authentication": [ + "storage/api/list_objects.py" + ], + "/bigquery/streaming-data-into-bigquery": [ + "bigquery/api/streaming.py" + ], + "/appengine/docs/python/endpoints/create_api": [ + "appengine/endpoints/backend/main.py", + "appengine/endpoints/multiapi/main.py" + ], + "/appengine/docs/flexible/python/using-cloud-sql": [ + "managed_vms/cloudsql/requirements.txt", + "managed_vms/cloudsql/main.py", + "managed_vms/cloudsql/app.yaml", + "managed_vms/cloudsql/create_tables.py" + ], + "/appengine/docs/python/how-requests-are-handled": [ + "appengine/logging/writing_logs/main.py", + "appengine/requests/main.py" + ], + "/appengine/docs/python/issue-requests": [ + "appengine/urlfetch/requests/main.py", + "appengine/urlfetch/snippets/main.py", + "appengine/urlfetch/async/rpc.py" + ], + "/appengine/docs/flexible/python/writing-and-responding-to-pub-sub-messages": [ + "managed_vms/pubsub/main.py", + "managed_vms/pubsub/app.yaml" + ], + "/compute/docs/instances/create-start-instance": [ + "compute/api/create_instance.py" + ], + "/appengine/docs/flexible/python/quickstart": [ + "managed_vms/hello_world/main.py", + "managed_vms/hello_world/requirements.txt", + "managed_vms/hello_world/app.yaml" + ], + "/storage/docs/json_api/v1/json-api-python-samples": [ + "storage/api/crud_object.py", + "storage/api/list_objects.py" + ], + "/dns/records/": [ + "dns/api/main.py" + ], + "/compute/docs/tutorials/python-guide": [ + "compute/api/startup-script.sh", + "compute/api/create_instance.py" + ], + "/appengine/docs/flexible/python/memcache-proxy": [ + "managed_vms/memcache/requirements.txt", + "managed_vms/memcache/main.py" + ], + "/bigquery/docs/managing_jobs_datasets_projects": [ + "bigquery/api/list_datasets_projects.py" + ], + "/bigquery/create-simple-app-api": [ + "bigquery/api/getting_started.py" + ], + "/appengine/articles/best-practices-for-app-engine-memcache": [ + "appengine/memcache/best_practices/migration_step1/migration1.py", + "appengine/memcache/best_practices/batch/batch.py", + "appengine/memcache/best_practices/migration_step2/migration2.py", + "appengine/memcache/best_practices/failure/failure.py", + "appengine/memcache/best_practices/sharing/sharing.py" + ], + "/appengine/docs/python/endpoints/getstarted/backend/write_api_post": [ + "appengine/endpoints/backend/main.py" + ], + "/appengine/docs/python/using-the-modules-api": [ + "appengine/modules/main.py" + ], + "/appengine/docs/python/tools/appstats": [ + "appengine/appstats/app.yaml", + "appengine/appstats/appengine_config.py" + ], + "/appengine/docs/python/blobstore/": [ + "appengine/blobstore/main.py" + ], + "/storage/transfer/create-manage-transfer-program": [ + "storage/transfer_service/aws_request.py", + "storage/transfer_service/transfer_check.py", + "storage/transfer_service/nearline_request.py" + ], + "/appengine/docs/flexible/python/using-cloud-datastore": [ + "managed_vms/datastore/main.py", + "managed_vms/datastore/app.yaml", + "managed_vms/datastore/requirements.txt" + ], + "/appengine/docs/flexible/python/sending-emails-with-mailgun": [ + "managed_vms/mailgun/main.py" + ], + "/bigquery/docs/loading-data-cloud-storage": [ + "bigquery/api/load_data_from_csv.py" + ], + "/appengine/docs/python/taskqueue/push/creating-tasks": [ + "appengine/taskqueue/counter/application.py" + ], + "/appengine/docs/python/search/results": [ + "appengine/search/snippets/snippets.py" + ], + "/bigquery/docs/data": [ + "bigquery/api/sync_query.py" + ], + "bigquery/docs/loading-data-sql-dml": [ + "bigquery/dml/insert_sql.py" + ], + "/appengine/docs/python/memcache/examples": [ + "appengine/memcache/snippets/snippets.py", + "appengine/memcache/guestbook/main.py" + ], + "/bigtable/docs/samples-python-hello": [ + "bigtable/hello/main.py" + ], + "/bigtable/docs/samples-python-hello-happybase": [ + "bigtable/hello_happybase/main.py" ] } diff --git a/scripts/run-tests.py b/scripts/run-tests.py deleted file mode 100755 index e5f6fbd04dd..00000000000 --- a/scripts/run-tests.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python - -# Copyright (C) 2013 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import os -import subprocess -import sys - -import pytest # flake8: noqa -import _pytest.main - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--junit', action='store_true', help='Output junit test report.') - parser.add_argument('--run-slow', action='store_true', help='Run slow tests.') - parser.add_argument( - 'directories', nargs='+', help='Directories to run py.test on.') - args = parser.parse_args() - - extra_args = [] - - if not args.run_slow: - extra_args.append('-m not slow and not flaky') - extra_args.append('--no-flaky-report') - - for directory in args.directories: - per_directory_args = [] - - if args.junit: - if os.path.isdir(directory): - per_directory_args.append( - '--junitxml={}/junit.xml'.format(directory)) - - # We could use pytest.main, however, we need import isolatation between - # test runs. Without using subprocess, any test files that are named - # the same will cause pytest to fail. Rather than do sys.module magic - # between runs, it's cleaner to just do a subprocess. - code = subprocess.call( - ['py.test'] + extra_args + per_directory_args + [directory]) - - if code not in ( - _pytest.main.EXIT_OK, _pytest.main.EXIT_NOTESTSCOLLECTED): - sys.exit(code) - -if __name__ == '__main__': - main() diff --git a/scripts/travis.sh b/scripts/travis.sh new file mode 100755 index 00000000000..a53fe9b3525 --- /dev/null +++ b/scripts/travis.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# Decrypt secrets and run tests if not on an external PR. +if [[ $TRAVIS_SECURE_ENV_VARS == "true" ]]; then + scripts/decrypt-secrets.sh "$SECRETS_PASSWORD" + source ${TRAVIS_BUILD_DIR}/testing/test-env.sh; + export GOOGLE_APPLICATION_CREDENTIALS=${TRAVIS_BUILD_DIR}/testing/service-account.json + export GOOGLE_CLIENT_SECRETS=${TRAVIS_BUILD_DIR}/testing/client-secrets.json + nox --envdir /tmp --stop-on-first-error -s lint gae py36 -- -m "not slow"; +else + # only run lint on external PRs + echo 'External PR: only running lint.' + nox --envdir /tmp --stop-on-first-error -s lint; +fi diff --git a/secrets.tar.enc b/secrets.tar.enc deleted file mode 100644 index 3dc345619bd..00000000000 Binary files a/secrets.tar.enc and /dev/null differ diff --git a/spanner/cloud-client/README.rst b/spanner/cloud-client/README.rst new file mode 100644 index 00000000000..89a59329143 --- /dev/null +++ b/spanner/cloud-client/README.rst @@ -0,0 +1,201 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Spanner Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=spanner/cloud-client/README.rst + + +This directory contains samples for Google Cloud Spanner. `Google Cloud Spanner`_ is a highly scalable, transactional, managed, NewSQL database service. Cloud Spanner solves the need for a horizontally-scaling database with consistent global transactions and SQL semantics. + + + + +.. _Google Cloud Spanner: https://cloud.google.com/spanner/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Snippets ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=spanner/cloud-client/snippets.py,spanner/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python snippets.py + + usage: snippets.py [-h] [--database-id DATABASE_ID] + instance_id + {create_database,insert_data,query_data,read_data,read_stale_data,add_column,update_data,query_data_with_new_column,read_write_transaction,read_only_transaction,add_index,query_data_with_index,read_data_with_index,add_storing_index,read_data_with_storing_index,create_table_with_timestamp,insert_data_with_timestamp,add_timestamp_column,update_data_with_timestamp,query_data_with_timestamp} + ... + + This application demonstrates how to do basic operations using Cloud + Spanner. + + For more information, see the README.rst under /spanner. + + positional arguments: + instance_id Your Cloud Spanner instance ID. + {create_database,insert_data,query_data,read_data,read_stale_data,add_column,update_data,query_data_with_new_column,read_write_transaction,read_only_transaction,add_index,query_data_with_index,read_data_with_index,add_storing_index,read_data_with_storing_index,create_table_with_timestamp,insert_data_with_timestamp,add_timestamp_column,update_data_with_timestamp,query_data_with_timestamp} + create_database Creates a database and tables for sample data. + insert_data Inserts sample data into the given database. The + database and table must already exist and can be + created using `create_database`. + query_data Queries sample data from the database using SQL. + read_data Reads sample data from the database. + read_stale_data Reads sample data from the database. The data is + exactly 15 seconds stale. + add_column Adds a new column to the Albums table in the example + database. + update_data Updates sample data in the database. This updates the + `MarketingBudget` column which must be created before + running this sample. You can add the column by running + the `add_column` sample or by running this DDL + statement against your database: ALTER TABLE Albums + ADD COLUMN MarketingBudget INT64 + query_data_with_new_column + Queries sample data from the database using SQL. This + sample uses the `MarketingBudget` column. You can add + the column by running the `add_column` sample or by + running this DDL statement against your database: + ALTER TABLE Albums ADD COLUMN MarketingBudget INT64 + read_write_transaction + Performs a read-write transaction to update two sample + records in the database. This will transfer 200,000 + from the `MarketingBudget` field for the second Album + to the first Album. If the `MarketingBudget` is too + low, it will raise an exception. Before running this + sample, you will need to run the `update_data` sample + to populate the fields. + read_only_transaction + Reads data inside of a read-only transaction. Within + the read-only transaction, or "snapshot", the + application sees consistent view of the database at a + particular timestamp. + add_index Adds a simple index to the example database. + query_data_with_index + Queries sample data from the database using SQL and an + index. The index must exist before running this + sample. You can add the index by running the + `add_index` sample or by running this DDL statement + against your database: CREATE INDEX AlbumsByAlbumTitle + ON Albums(AlbumTitle) This sample also uses the + `MarketingBudget` column. You can add the column by + running the `add_column` sample or by running this DDL + statement against your database: ALTER TABLE Albums + ADD COLUMN MarketingBudget INT64 + read_data_with_index + Inserts sample data into the given database. The + database and table must already exist and can be + created using `create_database`. + add_storing_index Adds an storing index to the example database. + read_data_with_storing_index + Inserts sample data into the given database. The + database and table must already exist and can be + created using `create_database`. + create_table_with_timestamp + Creates a table with a COMMIT_TIMESTAMP column. + insert_data_with_timestamp + Inserts data with a COMMIT_TIMESTAMP field into a + table. + add_timestamp_column + Adds a new TIMESTAMP column to the Albums table in the + example database. + update_data_with_timestamp + Updates Performances tables in the database with the + COMMIT_TIMESTAMP column. This updates the + `MarketingBudget` column which must be created before + running this sample. You can add the column by running + the `add_column` sample or by running this DDL + statement against your database: ALTER TABLE Albums + ADD COLUMN MarketingBudget INT64 In addition this + update expects the LastUpdateTime column added by + applying this DDL statement against your database: + ALTER TABLE Albums ADD COLUMN LastUpdateTime TIMESTAMP + OPTIONS(allow_commit_timestamp=true) + query_data_with_timestamp + Queries sample data from the database using SQL. This + updates the `LastUpdateTime` column which must be + created before running this sample. You can add the + column by running the `add_timestamp_column` sample or + by running this DDL statement against your database: + ALTER TABLE Performances ADD COLUMN LastUpdateTime + TIMESTAMP OPTIONS (allow_commit_timestamp=true) + + optional arguments: + -h, --help show this help message and exit + --database-id DATABASE_ID + Your Cloud Spanner database ID. + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/spanner/cloud-client/README.rst.in b/spanner/cloud-client/README.rst.in new file mode 100644 index 00000000000..542becb9a7f --- /dev/null +++ b/spanner/cloud-client/README.rst.in @@ -0,0 +1,24 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Spanner + short_name: Cloud Spanner + url: https://cloud.google.com/spanner/docs + description: > + `Google Cloud Spanner`_ is a highly scalable, transactional, managed, + NewSQL database service. Cloud Spanner solves the need for a + horizontally-scaling database with consistent global transactions and + SQL semantics. + +setup: +- auth +- install_deps + +samples: +- name: Snippets + file: snippets.py + show_help: true + +cloud_client_library: true + +folder: spanner/cloud-client \ No newline at end of file diff --git a/spanner/cloud-client/batch_sample.py b/spanner/cloud-client/batch_sample.py new file mode 100644 index 00000000000..e54581853a9 --- /dev/null +++ b/spanner/cloud-client/batch_sample.py @@ -0,0 +1,89 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to do batch operations using Cloud +Spanner. + +For more information, see the README.rst under /spanner. +""" + +import argparse +import concurrent.futures +import time + +from google.cloud import spanner + + +# [START spanner_batch_client] +def run_batch_query(instance_id, database_id): + """Runs an example batch query.""" + + # Expected Table Format: + # CREATE TABLE Singers ( + # SingerId INT64 NOT NULL, + # FirstName STRING(1024), + # LastName STRING(1024), + # SingerInfo BYTES(MAX), + # ) PRIMARY KEY (SingerId); + + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + # Create the batch transaction and generate partitions + snapshot = database.batch_snapshot() + partitions = snapshot.generate_read_batches( + table='Singers', + columns=('SingerId', 'FirstName', 'LastName',), + keyset=spanner.KeySet(all_=True) + ) + + # Create a pool of workers for the tasks + start = time.time() + with concurrent.futures.ThreadPoolExecutor() as executor: + futures = [executor.submit(process, snapshot, p) for p in partitions] + + for future in concurrent.futures.as_completed(futures, timeout=3600): + finish, row_ct = future.result() + elapsed = finish - start + print(u'Completed {} rows in {} seconds'.format(row_ct, elapsed)) + + # Clean up + snapshot.close() + + +def process(snapshot, partition): + """Processes the requests of a query in an separate process.""" + print('Started processing partition.') + row_ct = 0 + for row in snapshot.process_read_batch(partition): + print(u'SingerId: {}, AlbumId: {}, AlbumTitle: {}'.format(*row)) + row_ct += 1 + return time.time(), row_ct +# [END spanner_batch_client] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + 'instance_id', help='Your Cloud Spanner instance ID.') + parser.add_argument( + 'database_id', help='Your Cloud Spanner database ID.', + default='example_db') + + args = parser.parse_args() + + run_batch_query(args.instance_id, args.database_id) diff --git a/spanner/cloud-client/quickstart.py b/spanner/cloud-client/quickstart.py new file mode 100644 index 00000000000..75125839d20 --- /dev/null +++ b/spanner/cloud-client/quickstart.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def run_quickstart(): + # [START spanner_quickstart] + # Imports the Google Cloud Client Library. + from google.cloud import spanner + + # Instantiate a client. + spanner_client = spanner.Client() + + # Your Cloud Spanner instance ID. + instance_id = 'my-instance-id' + + # Get a Cloud Spanner instance by ID. + instance = spanner_client.instance(instance_id) + + # Your Cloud Spanner database ID. + database_id = 'my-database-id' + + # Get a Cloud Spanner database by ID. + database = instance.database(database_id) + + # Execute a simple SQL statement. + with database.snapshot() as snapshot: + results = snapshot.execute_sql('SELECT 1') + + for row in results: + print(row) + # [END spanner_quickstart] + + +if __name__ == '__main__': + run_quickstart() diff --git a/spanner/cloud-client/quickstart_test.py b/spanner/cloud-client/quickstart_test.py new file mode 100644 index 00000000000..de208725031 --- /dev/null +++ b/spanner/cloud-client/quickstart_test.py @@ -0,0 +1,57 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from google.cloud import spanner +import mock +import pytest + +import quickstart + +SPANNER_INSTANCE = os.environ['SPANNER_INSTANCE'] + + +@pytest.fixture +def patch_instance(): + original_instance = spanner.Client.instance + + def new_instance(self, unused_instance_name): + return original_instance(self, SPANNER_INSTANCE) + + instance_patch = mock.patch( + 'google.cloud.spanner.Client.instance', + side_effect=new_instance, + autospec=True) + + with instance_patch: + yield + + +@pytest.fixture +def example_database(): + spanner_client = spanner.Client() + instance = spanner_client.instance(SPANNER_INSTANCE) + database = instance.database('my-database-id') + + if not database.exists(): + database.create() + + yield + + +def test_quickstart(capsys, patch_instance, example_database): + quickstart.run_quickstart() + out, _ = capsys.readouterr() + assert '[1]' in out diff --git a/spanner/cloud-client/requirements.txt b/spanner/cloud-client/requirements.txt new file mode 100644 index 00000000000..1bb49a8c19a --- /dev/null +++ b/spanner/cloud-client/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-spanner==1.7.1 +futures==3.2.0; python_version < "3" diff --git a/spanner/cloud-client/snippets.py b/spanner/cloud-client/snippets.py new file mode 100644 index 00000000000..9bbe0fbd8c2 --- /dev/null +++ b/spanner/cloud-client/snippets.py @@ -0,0 +1,1237 @@ +#!/usr/bin/env python + +# Copyright 2016 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to do basic operations using Cloud +Spanner. + +For more information, see the README.rst under /spanner. +""" + +import argparse + +from google.cloud import spanner +from google.cloud.spanner_v1 import param_types + + +# [START spanner_create_database] +def create_database(instance_id, database_id): + """Creates a database and tables for sample data.""" + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + + database = instance.database(database_id, ddl_statements=[ + """CREATE TABLE Singers ( + SingerId INT64 NOT NULL, + FirstName STRING(1024), + LastName STRING(1024), + SingerInfo BYTES(MAX) + ) PRIMARY KEY (SingerId)""", + """CREATE TABLE Albums ( + SingerId INT64 NOT NULL, + AlbumId INT64 NOT NULL, + AlbumTitle STRING(MAX) + ) PRIMARY KEY (SingerId, AlbumId), + INTERLEAVE IN PARENT Singers ON DELETE CASCADE""" + ]) + + operation = database.create() + + print('Waiting for operation to complete...') + operation.result() + + print('Created database {} on instance {}'.format( + database_id, instance_id)) +# [END spanner_create_database] + + +# [START spanner_insert_data] +def insert_data(instance_id, database_id): + """Inserts sample data into the given database. + + The database and table must already exist and can be created using + `create_database`. + """ + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + with database.batch() as batch: + batch.insert( + table='Singers', + columns=('SingerId', 'FirstName', 'LastName',), + values=[ + (1, u'Marc', u'Richards'), + (2, u'Catalina', u'Smith'), + (3, u'Alice', u'Trentor'), + (4, u'Lea', u'Martin'), + (5, u'David', u'Lomond')]) + + batch.insert( + table='Albums', + columns=('SingerId', 'AlbumId', 'AlbumTitle',), + values=[ + (1, 1, u'Total Junk'), + (1, 2, u'Go, Go, Go'), + (2, 1, u'Green'), + (2, 2, u'Forever Hold Your Peace'), + (2, 3, u'Terrified')]) + + print('Inserted data.') +# [END spanner_insert_data] + + +# [START spanner_delete_data] +def delete_data(instance_id, database_id): + """Deletes sample data from the given database. + + The database, table, and data must already exist and can be created using + `create_database` and `insert_data`. + """ + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + singers_to_delete = spanner.KeySet( + keys=[[1], [2], [3], [4], [5]]) + albums_to_delete = spanner.KeySet( + keys=[[1, 1], [1, 2], [2, 1], [2, 2], [2, 3]]) + + with database.batch() as batch: + batch.delete('Albums', albums_to_delete) + batch.delete('Singers', singers_to_delete) + + print('Deleted data.') +# [END spanner_delete_data] + + +# [START spanner_query_data] +def query_data(instance_id, database_id): + """Queries sample data from the database using SQL.""" + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + with database.snapshot() as snapshot: + results = snapshot.execute_sql( + 'SELECT SingerId, AlbumId, AlbumTitle FROM Albums') + + for row in results: + print(u'SingerId: {}, AlbumId: {}, AlbumTitle: {}'.format(*row)) +# [END spanner_query_data] + + +# [START spanner_read_data] +def read_data(instance_id, database_id): + """Reads sample data from the database.""" + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + with database.snapshot() as snapshot: + keyset = spanner.KeySet(all_=True) + results = snapshot.read( + table='Albums', + columns=('SingerId', 'AlbumId', 'AlbumTitle',), + keyset=keyset,) + + for row in results: + print(u'SingerId: {}, AlbumId: {}, AlbumTitle: {}'.format(*row)) +# [END spanner_read_data] + + +# [START spanner_read_stale_data] +def read_stale_data(instance_id, database_id): + """Reads sample data from the database. The data is exactly 15 seconds + stale.""" + import datetime + + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + staleness = datetime.timedelta(seconds=15) + + with database.snapshot(exact_staleness=staleness) as snapshot: + keyset = spanner.KeySet(all_=True) + results = snapshot.read( + table='Albums', + columns=('SingerId', 'AlbumId', 'MarketingBudget',), + keyset=keyset) + + for row in results: + print(u'SingerId: {}, AlbumId: {}, MarketingBudget: {}'.format( + *row)) +# [END spanner_read_stale_data] + + +# [START spanner_query_data_with_new_column] +def query_data_with_new_column(instance_id, database_id): + """Queries sample data from the database using SQL. + + This sample uses the `MarketingBudget` column. You can add the column + by running the `add_column` sample or by running this DDL statement against + your database: + + ALTER TABLE Albums ADD COLUMN MarketingBudget INT64 + """ + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + with database.snapshot() as snapshot: + results = snapshot.execute_sql( + 'SELECT SingerId, AlbumId, MarketingBudget FROM Albums') + + for row in results: + print( + u'SingerId: {}, AlbumId: {}, MarketingBudget: {}'.format(*row)) +# [END spanner_query_data_with_new_column] + + +# [START spanner_create_index] +def add_index(instance_id, database_id): + """Adds a simple index to the example database.""" + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + operation = database.update_ddl([ + 'CREATE INDEX AlbumsByAlbumTitle ON Albums(AlbumTitle)']) + + print('Waiting for operation to complete...') + operation.result() + + print('Added the AlbumsByAlbumTitle index.') +# [END spanner_create_index] + + +# [START spanner_query_data_with_index] +def query_data_with_index( + instance_id, database_id, start_title='Aardvark', end_title='Goo'): + """Queries sample data from the database using SQL and an index. + + The index must exist before running this sample. You can add the index + by running the `add_index` sample or by running this DDL statement against + your database: + + CREATE INDEX AlbumsByAlbumTitle ON Albums(AlbumTitle) + + This sample also uses the `MarketingBudget` column. You can add the column + by running the `add_column` sample or by running this DDL statement against + your database: + + ALTER TABLE Albums ADD COLUMN MarketingBudget INT64 + + """ + from google.cloud.spanner_v1.proto import type_pb2 + + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + params = { + 'start_title': start_title, + 'end_title': end_title + } + param_types = { + 'start_title': type_pb2.Type(code=type_pb2.STRING), + 'end_title': type_pb2.Type(code=type_pb2.STRING) + } + + with database.snapshot() as snapshot: + results = snapshot.execute_sql( + "SELECT AlbumId, AlbumTitle, MarketingBudget " + "FROM Albums@{FORCE_INDEX=AlbumsByAlbumTitle} " + "WHERE AlbumTitle >= @start_title AND AlbumTitle < @end_title", + params=params, param_types=param_types) + + for row in results: + print( + u'AlbumId: {}, AlbumTitle: {}, ' + 'MarketingBudget: {}'.format(*row)) +# [END spanner_query_data_with_index] + + +# [START spanner_read_data_with_index] +def read_data_with_index(instance_id, database_id): + """Reads sample data from the database using an index. + + The index must exist before running this sample. You can add the index + by running the `add_index` sample or by running this DDL statement against + your database: + + CREATE INDEX AlbumsByAlbumTitle ON Albums(AlbumTitle) + + """ + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + with database.snapshot() as snapshot: + keyset = spanner.KeySet(all_=True) + results = snapshot.read( + table='Albums', + columns=('AlbumId', 'AlbumTitle'), + keyset=keyset, + index='AlbumsByAlbumTitle') + + for row in results: + print('AlbumId: {}, AlbumTitle: {}'.format(*row)) +# [END spanner_read_data_with_index] + + +# [START spanner_create_storing_index] +def add_storing_index(instance_id, database_id): + """Adds an storing index to the example database.""" + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + operation = database.update_ddl([ + 'CREATE INDEX AlbumsByAlbumTitle2 ON Albums(AlbumTitle)' + 'STORING (MarketingBudget)']) + + print('Waiting for operation to complete...') + operation.result() + + print('Added the AlbumsByAlbumTitle2 index.') +# [END spanner_create_storing_index] + + +# [START spanner_read_data_with_storing_index] +def read_data_with_storing_index(instance_id, database_id): + """Reads sample data from the database using an index with a storing + clause. + + The index must exist before running this sample. You can add the index + by running the `add_soring_index` sample or by running this DDL statement + against your database: + + CREATE INDEX AlbumsByAlbumTitle2 ON Albums(AlbumTitle) + STORING (MarketingBudget) + + """ + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + with database.snapshot() as snapshot: + keyset = spanner.KeySet(all_=True) + results = snapshot.read( + table='Albums', + columns=('AlbumId', 'AlbumTitle', 'MarketingBudget'), + keyset=keyset, + index='AlbumsByAlbumTitle2') + + for row in results: + print( + u'AlbumId: {}, AlbumTitle: {}, ' + 'MarketingBudget: {}'.format(*row)) +# [END spanner_read_data_with_storing_index] + + +# [START spanner_add_column] +def add_column(instance_id, database_id): + """Adds a new column to the Albums table in the example database.""" + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + operation = database.update_ddl([ + 'ALTER TABLE Albums ADD COLUMN MarketingBudget INT64']) + + print('Waiting for operation to complete...') + operation.result() + + print('Added the MarketingBudget column.') +# [END spanner_add_column] + + +# [START spanner_update_data] +def update_data(instance_id, database_id): + """Updates sample data in the database. + + This updates the `MarketingBudget` column which must be created before + running this sample. You can add the column by running the `add_column` + sample or by running this DDL statement against your database: + + ALTER TABLE Albums ADD COLUMN MarketingBudget INT64 + + """ + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + with database.batch() as batch: + batch.update( + table='Albums', + columns=( + 'SingerId', 'AlbumId', 'MarketingBudget'), + values=[ + (1, 1, 100000), + (2, 2, 500000)]) + + print('Updated data.') +# [END spanner_update_data] + + +# [START spanner_read_write_transaction] +def read_write_transaction(instance_id, database_id): + """Performs a read-write transaction to update two sample records in the + database. + + This will transfer 200,000 from the `MarketingBudget` field for the second + Album to the first Album. If the `MarketingBudget` is too low, it will + raise an exception. + + Before running this sample, you will need to run the `update_data` sample + to populate the fields. + """ + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + def update_albums(transaction): + # Read the second album budget. + second_album_keyset = spanner.KeySet(keys=[(2, 2)]) + second_album_result = transaction.read( + table='Albums', columns=('MarketingBudget',), + keyset=second_album_keyset, limit=1) + second_album_row = list(second_album_result)[0] + second_album_budget = second_album_row[0] + + transfer_amount = 200000 + + if second_album_budget < 300000: + # Raising an exception will automatically roll back the + # transaction. + raise ValueError( + 'The second album doesn\'t have enough funds to transfer') + + # Read the first album's budget. + first_album_keyset = spanner.KeySet(keys=[(1, 1)]) + first_album_result = transaction.read( + table='Albums', columns=('MarketingBudget',), + keyset=first_album_keyset, limit=1) + first_album_row = list(first_album_result)[0] + first_album_budget = first_album_row[0] + + # Update the budgets. + second_album_budget -= transfer_amount + first_album_budget += transfer_amount + print( + 'Setting first album\'s budget to {} and the second album\'s ' + 'budget to {}.'.format( + first_album_budget, second_album_budget)) + + # Update the rows. + transaction.update( + table='Albums', + columns=( + 'SingerId', 'AlbumId', 'MarketingBudget'), + values=[ + (1, 1, first_album_budget), + (2, 2, second_album_budget)]) + + database.run_in_transaction(update_albums) + + print('Transaction complete.') +# [END spanner_read_write_transaction] + + +# [START spanner_read_only_transaction] +def read_only_transaction(instance_id, database_id): + """Reads data inside of a read-only transaction. + + Within the read-only transaction, or "snapshot", the application sees + consistent view of the database at a particular timestamp. + """ + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + with database.snapshot(multi_use=True) as snapshot: + # Read using SQL. + results = snapshot.execute_sql( + 'SELECT SingerId, AlbumId, AlbumTitle FROM Albums') + + print('Results from first read:') + for row in results: + print(u'SingerId: {}, AlbumId: {}, AlbumTitle: {}'.format(*row)) + + # Perform another read using the `read` method. Even if the data + # is updated in-between the reads, the snapshot ensures that both + # return the same data. + keyset = spanner.KeySet(all_=True) + results = snapshot.read( + table='Albums', + columns=('SingerId', 'AlbumId', 'AlbumTitle',), + keyset=keyset,) + + print('Results from second read:') + for row in results: + print(u'SingerId: {}, AlbumId: {}, AlbumTitle: {}'.format(*row)) +# [END spanner_read_only_transaction] + + +# [START spanner_create_table_with_timestamp_column] +def create_table_with_timestamp(instance_id, database_id): + """Creates a table with a COMMIT_TIMESTAMP column.""" + + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + operation = database.update_ddl([ + """CREATE TABLE Performances ( + SingerId INT64 NOT NULL, + VenueId INT64 NOT NULL, + EventDate Date, + Revenue INT64, + LastUpdateTime TIMESTAMP NOT NULL + OPTIONS(allow_commit_timestamp=true) + ) PRIMARY KEY (SingerId, VenueId, EventDate), + INTERLEAVE IN PARENT Singers ON DELETE CASCADE""" + ]) + + print('Waiting for operation to complete...') + operation.result() + + print('Created Performances table on database {} on instance {}'.format( + database_id, instance_id)) +# [END spanner_create_table_with_timestamp_column] + + +# [START spanner_insert_data_with_timestamp_column] +def insert_data_with_timestamp(instance_id, database_id): + """Inserts data with a COMMIT_TIMESTAMP field into a table. """ + + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + + database = instance.database(database_id) + + with database.batch() as batch: + batch.insert( + table='Performances', + columns=( + 'SingerId', 'VenueId', 'EventDate', + 'Revenue', 'LastUpdateTime',), + values=[ + (1, 4, "2017-10-05", 11000, spanner.COMMIT_TIMESTAMP), + (1, 19, "2017-11-02", 15000, spanner.COMMIT_TIMESTAMP), + (2, 42, "2017-12-23", 7000, spanner.COMMIT_TIMESTAMP)]) + + print('Inserted data.') +# [END spanner_insert_data_with_timestamp_column] + + +# [START spanner_add_timestamp_column] +def add_timestamp_column(instance_id, database_id): + """ Adds a new TIMESTAMP column to the Albums table in the example database. + """ + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + + database = instance.database(database_id) + + operation = database.update_ddl([ + 'ALTER TABLE Albums ADD COLUMN LastUpdateTime TIMESTAMP ' + 'OPTIONS(allow_commit_timestamp=true)']) + + print('Waiting for operation to complete...') + operation.result() + + print('Altered table "Albums" on database {} on instance {}.'.format( + database_id, instance_id)) +# [END spanner_add_timestamp_column] + + +# [START spanner_update_data_with_timestamp_column] +def update_data_with_timestamp(instance_id, database_id): + """Updates Performances tables in the database with the COMMIT_TIMESTAMP + column. + + This updates the `MarketingBudget` column which must be created before + running this sample. You can add the column by running the `add_column` + sample or by running this DDL statement against your database: + + ALTER TABLE Albums ADD COLUMN MarketingBudget INT64 + + In addition this update expects the LastUpdateTime column added by + applying this DDL statement against your database: + + ALTER TABLE Albums ADD COLUMN LastUpdateTime TIMESTAMP + OPTIONS(allow_commit_timestamp=true) + """ + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + + database = instance.database(database_id) + + with database.batch() as batch: + batch.update( + table='Albums', + columns=( + 'SingerId', 'AlbumId', 'MarketingBudget', 'LastUpdateTime'), + values=[ + (1, 1, 1000000, spanner.COMMIT_TIMESTAMP), + (2, 2, 750000, spanner.COMMIT_TIMESTAMP)]) + + print('Updated data.') +# [END spanner_update_data_with_timestamp_column] + + +# [START spanner_query_data_with_timestamp_column] +def query_data_with_timestamp(instance_id, database_id): + """Queries sample data from the database using SQL. + + This updates the `LastUpdateTime` column which must be created before + running this sample. You can add the column by running the + `add_timestamp_column` sample or by running this DDL statement + against your database: + + ALTER TABLE Performances ADD COLUMN LastUpdateTime TIMESTAMP + OPTIONS (allow_commit_timestamp=true) + + """ + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + + database = instance.database(database_id) + + with database.snapshot() as snapshot: + results = snapshot.execute_sql( + 'SELECT SingerId, AlbumId, MarketingBudget FROM Albums ' + 'ORDER BY LastUpdateTime DESC') + + for row in results: + print(u'SingerId: {}, AlbumId: {}, MarketingBudget: {}'.format(*row)) +# [END spanner_query_data_with_timestamp_column] + + +# [START spanner_write_data_for_struct_queries] +def write_struct_data(instance_id, database_id): + """Inserts sample data that can be used to test STRUCT parameters + in queries. + """ + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + with database.batch() as batch: + batch.insert( + table='Singers', + columns=('SingerId', 'FirstName', 'LastName',), + values=[ + (6, u'Elena', u'Campbell'), + (7, u'Gabriel', u'Wright'), + (8, u'Benjamin', u'Martinez'), + (9, u'Hannah', u'Harris')]) + + print('Inserted sample data for STRUCT queries') +# [END spanner_write_data_for_struct_queries] + + +def query_with_struct(instance_id, database_id): + """Query a table using STRUCT parameters. """ + # [START spanner_create_struct_with_data] + record_type = param_types.Struct([ + param_types.StructField('FirstName', param_types.STRING), + param_types.StructField('LastName', param_types.STRING) + ]) + record_value = ('Elena', 'Campbell') + # [END spanner_create_struct_with_data] + + # [START spanner_query_data_with_struct] + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + + database = instance.database(database_id) + + with database.snapshot() as snapshot: + results = snapshot.execute_sql( + "SELECT SingerId FROM Singers WHERE " + "(FirstName, LastName) = @name", + params={'name': record_value}, + param_types={'name': record_type}) + + for row in results: + print(u'SingerId: {}'.format(*row)) + # [END spanner_query_data_with_struct] + + +def query_with_array_of_struct(instance_id, database_id): + """Query a table using an array of STRUCT parameters. """ + # [START spanner_create_user_defined_struct] + name_type = param_types.Struct([ + param_types.StructField('FirstName', param_types.STRING), + param_types.StructField('LastName', param_types.STRING)]) + # [END spanner_create_user_defined_struct] + + # [START spanner_create_array_of_struct_with_data] + band_members = [("Elena", "Campbell"), + ("Gabriel", "Wright"), + ("Benjamin", "Martinez")] + # [END spanner_create_array_of_struct_with_data] + + # [START spanner_query_data_with_array_of_struct] + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + with database.snapshot() as snapshot: + results = snapshot.execute_sql( + "SELECT SingerId FROM Singers WHERE " + "STRUCT" + "(FirstName, LastName) IN UNNEST(@names)", + params={'names': band_members}, + param_types={'names': param_types.Array(name_type)}) + + for row in results: + print(u'SingerId: {}'.format(*row)) + # [END spanner_query_data_with_array_of_struct] + + +# [START spanner_field_access_on_struct_parameters] +def query_struct_field(instance_id, database_id): + """Query a table using field access on a STRUCT parameter. """ + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + name_type = param_types.Struct([ + param_types.StructField('FirstName', param_types.STRING), + param_types.StructField('LastName', param_types.STRING) + ]) + + with database.snapshot() as snapshot: + results = snapshot.execute_sql( + "SELECT SingerId FROM Singers " + "WHERE FirstName = @name.FirstName", + params={'name': ("Elena", "Campbell")}, + param_types={'name': name_type}) + + for row in results: + print(u'SingerId: {}'.format(*row)) +# [START spanner_field_access_on_struct_parameters] + + +# [START spanner_field_access_on_nested_struct_parameters] +def query_nested_struct_field(instance_id, database_id): + """Query a table using nested field access on a STRUCT parameter. """ + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + song_info_type = param_types.Struct([ + param_types.StructField('SongName', param_types.STRING), + param_types.StructField( + 'ArtistNames', param_types.Array( + param_types.Struct([ + param_types.StructField( + 'FirstName', param_types.STRING), + param_types.StructField( + 'LastName', param_types.STRING) + ]) + ) + ) + ]) + + song_info = ('Imagination', [('Elena', 'Campbell'), ('Hannah', 'Harris')]) + + with database.snapshot() as snapshot: + results = snapshot.execute_sql( + "SELECT SingerId, @song_info.SongName " + "FROM Singers WHERE " + "STRUCT" + "(FirstName, LastName) " + "IN UNNEST(@song_info.ArtistNames)", + params={ + 'song_info': song_info + }, + param_types={ + 'song_info': song_info_type + } + ) + + for row in results: + print(u'SingerId: {} SongName: {}'.format(*row)) +# [END spanner_field_access_on_nested_struct_parameters] + + +def insert_data_with_dml(instance_id, database_id): + """Inserts sample data into the given database using a DML statement. """ + # [START spanner_dml_standard_insert] + # instance_id = "your-spanner-instance" + # database_id = "your-spanner-db-id" + + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + def insert_singers(transaction): + row_ct = transaction.execute_update( + "INSERT Singers (SingerId, FirstName, LastName) " + " VALUES (10, 'Virginia', 'Watson')" + ) + + print("{} record(s) inserted.".format(row_ct)) + + database.run_in_transaction(insert_singers) + # [END spanner_dml_standard_insert] + + +def update_data_with_dml(instance_id, database_id): + """Updates sample data from the database using a DML statement. """ + # [START spanner_dml_standard_update] + # instance_id = "your-spanner-instance" + # database_id = "your-spanner-db-id" + + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + def update_albums(transaction): + row_ct = transaction.execute_update( + "UPDATE Albums " + "SET MarketingBudget = MarketingBudget * 2 " + "WHERE SingerId = 1 and AlbumId = 1" + ) + + print("{} record(s) updated.".format(row_ct)) + + database.run_in_transaction(update_albums) + # [END spanner_dml_standard_update] + + +def delete_data_with_dml(instance_id, database_id): + """Deletes sample data from the database using a DML statement. """ + # [START spanner_dml_standard_delete] + # instance_id = "your-spanner-instance" + # database_id = "your-spanner-db-id" + + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + def delete_singers(transaction): + row_ct = transaction.execute_update( + "DELETE Singers WHERE FirstName = 'Alice'" + ) + + print("{} record(s) deleted.".format(row_ct)) + + database.run_in_transaction(delete_singers) + # [END spanner_dml_standard_delete] + + +def update_data_with_dml_timestamp(instance_id, database_id): + """Updates data with Timestamp from the database using a DML statement. """ + # [START spanner_dml_standard_update_with_timestamp] + # instance_id = "your-spanner-instance" + # database_id = "your-spanner-db-id" + + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + def update_albums(transaction): + row_ct = transaction.execute_update( + "UPDATE Albums " + "SET LastUpdateTime = PENDING_COMMIT_TIMESTAMP() " + "WHERE SingerId = 1" + ) + + print("{} record(s) updated.".format(row_ct)) + + database.run_in_transaction(update_albums) + # [END spanner_dml_standard_update_with_timestamp] + + +def dml_write_read_transaction(instance_id, database_id): + """First inserts data then reads it from within a transaction using DML.""" + # [START spanner_dml_write_then_read] + # instance_id = "your-spanner-instance" + # database_id = "your-spanner-db-id" + + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + def write_then_read(transaction): + # Insert record. + row_ct = transaction.execute_update( + "INSERT Singers (SingerId, FirstName, LastName) " + " VALUES (11, 'Timothy', 'Campbell')" + ) + print("{} record(s) inserted.".format(row_ct)) + + # Read newly inserted record. + results = transaction.execute_sql( + "SELECT FirstName, LastName FROM Singers WHERE SingerId = 11" + ) + for result in results: + print("FirstName: {}, LastName: {}".format(*result)) + + database.run_in_transaction(write_then_read) + # [END spanner_dml_write_then_read] + + +def update_data_with_dml_struct(instance_id, database_id): + """Updates data with a DML statement and STRUCT parameters. """ + # [START spanner_dml_structs] + # instance_id = "your-spanner-instance" + # database_id = "your-spanner-db-id" + + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + record_type = param_types.Struct([ + param_types.StructField('FirstName', param_types.STRING), + param_types.StructField('LastName', param_types.STRING) + ]) + record_value = ('Timothy', 'Campbell') + + def write_with_struct(transaction): + row_ct = transaction.execute_update( + "UPDATE Singers SET LastName = 'Grant' " + "WHERE STRUCT" + "(FirstName, LastName) = @name", + params={'name': record_value}, + param_types={'name': record_type} + ) + print("{} record(s) updated.".format(row_ct)) + + database.run_in_transaction(write_with_struct) + # [END spanner_dml_structs] + + +def insert_with_dml(instance_id, database_id): + """Inserts data with a DML statement into the database. """ + # [START spanner_dml_getting_started_insert] + # instance_id = "your-spanner-instance" + # database_id = "your-spanner-db-id" + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + def insert_singers(transaction): + row_ct = transaction.execute_update( + "INSERT Singers (SingerId, FirstName, LastName) VALUES " + "(12, 'Melissa', 'Garcia'), " + "(13, 'Russell', 'Morales'), " + "(14, 'Jacqueline', 'Long'), " + "(15, 'Dylan', 'Shaw')" + ) + print("{} record(s) inserted.".format(row_ct)) + + database.run_in_transaction(insert_singers) + # [END spanner_dml_getting_started_insert] + + +def write_with_dml_transaction(instance_id, database_id): + """ Transfers a marketing budget from one album to another. """ + # [START spanner_dml_getting_started_update] + # instance_id = "your-spanner-instance" + # database_id = "your-spanner-db-id" + + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + def transfer_budget(transaction): + # Transfer marketing budget from one album to another. Performed in a + # single transaction to ensure that the transfer is atomic. + first_album_result = transaction.execute_sql( + "SELECT MarketingBudget from Albums " + "WHERE SingerId = 1 and AlbumId = 1" + ) + first_album_row = list(first_album_result)[0] + first_album_budget = first_album_row[0] + + transfer_amount = 300000 + + # Transaction will only be committed if this condition still holds at + # the time of commit. Otherwise it will be aborted and the callable + # will be rerun by the client library + if first_album_budget >= transfer_amount: + second_album_result = transaction.execute_sql( + "SELECT MarketingBudget from Albums " + "WHERE SingerId = 1 and AlbumId = 1" + ) + second_album_row = list(second_album_result)[0] + second_album_budget = second_album_row[0] + + first_album_budget -= transfer_amount + second_album_budget += transfer_amount + + # Update first album + transaction.execute_update( + "UPDATE Albums " + "SET MarketingBudget = @AlbumBudget " + "WHERE SingerId = 1 and AlbumId = 1", + params={"AlbumBudget": first_album_budget}, + param_types={"AlbumBudget": spanner.param_types.INT64} + ) + + # Update second album + transaction.execute_update( + "UPDATE Albums " + "SET MarketingBudget = @AlbumBudget " + "WHERE SingerId = 2 and AlbumId = 2", + params={"AlbumBudget": second_album_budget}, + param_types={"AlbumBudget": spanner.param_types.INT64} + ) + + print("Transferred {} from Album1's budget to Album2's".format( + transfer_amount)) + + database.run_in_transaction(transfer_budget) + # [END spanner_dml_getting_started_update] + + +def update_data_with_partitioned_dml(instance_id, database_id): + """ Update sample data with a partitioned DML statement. """ + # [START spanner_dml_partitioned_update] + # instance_id = "your-spanner-instance" + # database_id = "your-spanner-db-id" + + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + row_ct = database.execute_partitioned_dml( + "UPDATE Albums SET MarketingBudget = 100000 WHERE SingerId > 1" + ) + + print("{} records updated.".format(row_ct)) + # [END spanner_dml_partitioned_update] + + +def delete_data_with_partitioned_dml(instance_id, database_id): + """ Delete sample data with a partitioned DML statement. """ + # [START spanner_dml_partitioned_delete] + # instance_id = "your-spanner-instance" + # database_id = "your-spanner-db-id" + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + row_ct = database.execute_partitioned_dml( + "DELETE Singers WHERE SingerId > 10" + ) + + print("{} record(s) deleted.".format(row_ct)) + # [END spanner_dml_partitioned_delete] + + +def update_with_batch_dml(instance_id, database_id): + """Updates sample data in the database using Batch DML. """ + # [START spanner_dml_batch_update] + # instance_id = "your-spanner-instance" + # database_id = "your-spanner-db-id" + + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + insert_statement = ( + "INSERT INTO Albums " + "(SingerId, AlbumId, AlbumTitle, MarketingBudget) " + "VALUES (1, 3, 'Test Album Title', 10000)" + ) + + update_statement = ( + "UPDATE Albums " + "SET MarketingBudget = MarketingBudget * 2 " + "WHERE SingerId = 1 and AlbumId = 3" + ) + + def update_albums(transaction): + row_cts = transaction.batch_update([ + insert_statement, + update_statement, + ]) + + print("Executed {} SQL statements using Batch DML.".format( + len(row_cts))) + + database.run_in_transaction(update_albums) + # [END spanner_dml_batch_update] + + +if __name__ == '__main__': # noqa: C901 + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + 'instance_id', help='Your Cloud Spanner instance ID.') + parser.add_argument( + '--database-id', help='Your Cloud Spanner database ID.', + default='example_db') + + subparsers = parser.add_subparsers(dest='command') + subparsers.add_parser('create_database', help=create_database.__doc__) + subparsers.add_parser('delete_data', help=delete_data.__doc__) + subparsers.add_parser('insert_data', help=insert_data.__doc__) + subparsers.add_parser('query_data', help=query_data.__doc__) + subparsers.add_parser('read_data', help=read_data.__doc__) + subparsers.add_parser('read_stale_data', help=read_stale_data.__doc__) + subparsers.add_parser('add_column', help=add_column.__doc__) + subparsers.add_parser('update_data', help=update_data.__doc__) + subparsers.add_parser( + 'query_data_with_new_column', help=query_data_with_new_column.__doc__) + subparsers.add_parser( + 'read_write_transaction', help=read_write_transaction.__doc__) + subparsers.add_parser( + 'read_only_transaction', help=read_only_transaction.__doc__) + subparsers.add_parser('add_index', help=add_index.__doc__) + query_data_with_index_parser = subparsers.add_parser( + 'query_data_with_index', help=query_data_with_index.__doc__) + query_data_with_index_parser.add_argument( + '--start_title', default='Aardvark') + query_data_with_index_parser.add_argument( + '--end_title', default='Goo') + subparsers.add_parser('read_data_with_index', help=insert_data.__doc__) + subparsers.add_parser('add_storing_index', help=add_storing_index.__doc__) + subparsers.add_parser( + 'read_data_with_storing_index', help=insert_data.__doc__) + subparsers.add_parser( + 'create_table_with_timestamp', + help=create_table_with_timestamp.__doc__) + subparsers.add_parser( + 'insert_data_with_timestamp', help=insert_data_with_timestamp.__doc__) + subparsers.add_parser( + 'add_timestamp_column', help=add_timestamp_column.__doc__) + subparsers.add_parser( + 'update_data_with_timestamp', help=update_data_with_timestamp.__doc__) + subparsers.add_parser( + 'query_data_with_timestamp', help=query_data_with_timestamp.__doc__) + subparsers.add_parser('write_struct_data', help=write_struct_data.__doc__) + subparsers.add_parser('query_with_struct', help=query_with_struct.__doc__) + subparsers.add_parser( + 'query_with_array_of_struct', help=query_with_array_of_struct.__doc__) + subparsers.add_parser( + 'query_struct_field', help=query_struct_field.__doc__) + subparsers.add_parser( + 'query_nested_struct_field', help=query_nested_struct_field.__doc__) + subparsers.add_parser( + 'insert_data_with_dml', help=insert_data_with_dml.__doc__) + subparsers.add_parser( + 'update_data_with_dml', help=update_data_with_dml.__doc__) + subparsers.add_parser( + 'delete_data_with_dml', help=delete_data_with_dml.__doc__) + subparsers.add_parser( + 'update_data_with_dml_timestamp', + help=update_data_with_dml_timestamp.__doc__) + subparsers.add_parser( + 'dml_write_read_transaction', + help=dml_write_read_transaction.__doc__) + subparsers.add_parser( + 'update_data_with_dml_struct', + help=update_data_with_dml_struct.__doc__) + subparsers.add_parser('insert_with_dml', help=insert_with_dml.__doc__) + subparsers.add_parser( + 'write_with_dml_transaction', help=write_with_dml_transaction.__doc__) + subparsers.add_parser( + 'update_data_with_partitioned_dml', + help=update_data_with_partitioned_dml.__doc__) + subparsers.add_parser( + 'delete_data_with_partitioned_dml', + help=delete_data_with_partitioned_dml.__doc__) + subparsers.add_parser( + 'update_with_batch_dml', + help=update_with_batch_dml.__doc__) + + args = parser.parse_args() + + if args.command == 'create_database': + create_database(args.instance_id, args.database_id) + elif args.command == 'insert_data': + insert_data(args.instance_id, args.database_id) + elif args.command == 'delete_data': + delete_data(args.instance_id, args.database_id) + elif args.command == 'query_data': + query_data(args.instance_id, args.database_id) + elif args.command == 'read_data': + read_data(args.instance_id, args.database_id) + elif args.command == 'read_stale_data': + read_stale_data(args.instance_id, args.database_id) + elif args.command == 'add_column': + add_column(args.instance_id, args.database_id) + elif args.command == 'update_data': + update_data(args.instance_id, args.database_id) + elif args.command == 'query_data_with_new_column': + query_data_with_new_column(args.instance_id, args.database_id) + elif args.command == 'read_write_transaction': + read_write_transaction(args.instance_id, args.database_id) + elif args.command == 'read_only_transaction': + read_only_transaction(args.instance_id, args.database_id) + elif args.command == 'add_index': + add_index(args.instance_id, args.database_id) + elif args.command == 'query_data_with_index': + query_data_with_index( + args.instance_id, args.database_id, + args.start_title, args.end_title) + elif args.command == 'read_data_with_index': + read_data_with_index(args.instance_id, args.database_id) + elif args.command == 'add_storing_index': + add_storing_index(args.instance_id, args.database_id) + elif args.command == 'read_data_with_storing_index': + read_data_with_storing_index(args.instance_id, args.database_id) + elif args.command == 'create_table_with_timestamp': + create_table_with_timestamp(args.instance_id, args.database_id) + elif args.command == 'insert_data_with_timestamp': + insert_data_with_timestamp(args.instance_id, args.database_id) + elif args.command == 'add_timestamp_column': + add_timestamp_column(args.instance_id, args.database_id) + elif args.command == 'update_data_with_timestamp': + update_data_with_timestamp(args.instance_id, args.database_id) + elif args.command == 'query_data_with_timestamp': + query_data_with_timestamp(args.instance_id, args.database_id) + elif args.command == 'write_struct_data': + write_struct_data(args.instance_id, args.database_id) + elif args.command == 'query_with_struct': + query_with_struct(args.instance_id, args.database_id) + elif args.command == 'query_with_array_of_struct': + query_with_array_of_struct(args.instance_id, args.database_id) + elif args.command == 'query_struct_field': + query_struct_field(args.instance_id, args.database_id) + elif args.command == 'query_nested_struct_field': + query_nested_struct_field(args.instance_id, args.database_id) + elif args.command == 'insert_data_with_dml': + insert_data_with_dml(args.instance_id, args.database_id) + elif args.command == 'update_data_with_dml': + update_data_with_dml(args.instance_id, args.database_id) + elif args.command == 'delete_data_with_dml': + delete_data_with_dml(args.instance_id, args.database_id) + elif args.command == 'update_data_with_dml_timestamp': + update_data_with_dml_timestamp(args.instance_id, args.database_id) + elif args.command == 'dml_write_read_transaction': + dml_write_read_transaction(args.instance_id, args.database_id) + elif args.command == 'update_data_with_dml_struct': + update_data_with_dml_struct(args.instance_id, args.database_id) + elif args.command == 'insert_with_dml': + insert_with_dml(args.instance_id, args.database_id) + elif args.command == 'write_with_dml_transaction': + write_with_dml_transaction(args.instance_id, args.database_id) + elif args.command == 'update_data_with_partitioned_dml': + update_data_with_partitioned_dml(args.instance_id, args.database_id) + elif args.command == 'delete_data_with_partitioned_dml': + delete_data_with_partitioned_dml(args.instance_id, args.database_id) + elif args.command == 'update_with_batch_dml': + update_with_batch_dml(args.instance_id, args.database_id) diff --git a/spanner/cloud-client/snippets_test.py b/spanner/cloud-client/snippets_test.py new file mode 100644 index 00000000000..9bd39d27a57 --- /dev/null +++ b/spanner/cloud-client/snippets_test.py @@ -0,0 +1,285 @@ +# Copyright 2016 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import random +import string +import time + +from google.cloud import spanner +import pytest + +import snippets + + +def unique_database_id(): + """ Creates a unique id for the database. """ + return 'test-db-{}'.format(''.join(random.choice( + string.ascii_lowercase + string.digits) for _ in range(5))) + + +INSTANCE_ID = os.environ['SPANNER_INSTANCE'] +DATABASE_ID = unique_database_id() + + +@pytest.fixture(scope='module') +def spanner_instance(): + spanner_client = spanner.Client() + return spanner_client.instance(INSTANCE_ID) + + +@pytest.fixture(scope='module') +def database(spanner_instance): + """ Creates a temporary database that is removed after testing. """ + snippets.create_database(INSTANCE_ID, DATABASE_ID) + db = spanner_instance.database(DATABASE_ID) + yield db + db.drop() + + +def test_create_database(database): + # Reload will only succeed if the database exists. + database.reload() + + +def test_insert_data(capsys): + snippets.insert_data(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert 'Inserted data' in out + + +def test_delete_data(capsys): + snippets.delete_data(INSTANCE_ID, DATABASE_ID) + snippets.insert_data(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert 'Deleted data' in out + + +def test_query_data(capsys): + snippets.query_data(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert 'SingerId: 1, AlbumId: 1, AlbumTitle: Total Junk' in out + + +def test_add_column(capsys): + snippets.add_column(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert 'Added the MarketingBudget column.' in out + + +def test_read_data(capsys): + snippets.read_data(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert 'SingerId: 1, AlbumId: 1, AlbumTitle: Total Junk' in out + + +def test_update_data(capsys): + # Sleep for 15 seconds to ensure previous inserts will be + # 'stale' by the time test_read_stale_data is run. + time.sleep(15) + + snippets.update_data(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert 'Updated data.' in out + + +def test_read_stale_data(capsys): + # This snippet relies on test_update_data inserting data + # at least 15 seconds after the previous insert + snippets.read_stale_data(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert 'SingerId: 1, AlbumId: 1, MarketingBudget: None' in out + + +def test_read_write_transaction(capsys): + snippets.read_write_transaction(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert 'Transaction complete' in out + + +def test_query_data_with_new_column(capsys): + snippets.query_data_with_new_column(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert 'SingerId: 1, AlbumId: 1, MarketingBudget: 300000' in out + assert 'SingerId: 2, AlbumId: 2, MarketingBudget: 300000' in out + + +def test_add_index(capsys): + snippets.add_index(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert 'Added the AlbumsByAlbumTitle index' in out + + +def test_query_data_with_index(capsys): + snippets.query_data_with_index(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert 'Go, Go, Go' in out + assert 'Forever Hold Your Peace' in out + assert 'Green' not in out + + +def test_read_data_with_index(capsys): + snippets.read_data_with_index(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert 'Go, Go, Go' in out + assert 'Forever Hold Your Peace' in out + assert 'Green' in out + + +def test_add_storing_index(capsys): + snippets.add_storing_index(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert 'Added the AlbumsByAlbumTitle2 index.' in out + + +def test_read_data_with_storing_index(capsys): + snippets.read_data_with_storing_index(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert '300000' in out + + +def test_read_only_transaction(capsys): + snippets.read_only_transaction(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + # Snippet does two reads, so entry should be listed twice + assert out.count('SingerId: 1, AlbumId: 1, AlbumTitle: Total Junk') == 2 + + +def test_add_timestamp_column(capsys): + snippets.add_timestamp_column(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert 'Altered table "Albums" on database ' in out + + +def test_update_data_with_timestamp(capsys): + snippets.update_data_with_timestamp(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert 'Updated data' in out + + +def test_query_data_with_timestamp(capsys): + snippets.query_data_with_timestamp(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert 'SingerId: 1, AlbumId: 1, MarketingBudget: 1000000' in out + assert 'SingerId: 2, AlbumId: 2, MarketingBudget: 750000' in out + + +def test_create_table_with_timestamp(capsys): + snippets.create_table_with_timestamp(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert 'Created Performances table on database' in out + + +def test_insert_data_with_timestamp(capsys): + snippets.insert_data_with_timestamp(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert 'Inserted data.' in out + + +def test_write_struct_data(capsys): + snippets.write_struct_data(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert 'Inserted sample data for STRUCT queries' in out + + +def test_query_with_struct(capsys): + snippets.query_with_struct(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert 'SingerId: 6' in out + + +def test_query_with_array_of_struct(capsys): + snippets.query_with_array_of_struct(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert 'SingerId: 6\nSingerId: 7' in out + + +def test_query_struct_field(capsys): + snippets.query_struct_field(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert 'SingerId: 6' in out + + +def test_query_nested_struct_field(capsys): + snippets.query_nested_struct_field(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert 'SingerId: 6 SongName: Imagination' in out + assert 'SingerId: 9 SongName: Imagination' in out + + +def test_insert_data_with_dml(capsys): + snippets.insert_data_with_dml(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert '1 record(s) inserted.' in out + + +def test_update_data_with_dml(capsys): + snippets.update_data_with_dml(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert '1 record(s) updated.' in out + + +def test_delete_data_with_dml(capsys): + snippets.delete_data_with_dml(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert '1 record(s) deleted.' in out + + +def test_update_data_with_dml_timestamp(capsys): + snippets.update_data_with_dml_timestamp(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert '2 record(s) updated.' in out + + +def test_dml_write_read_transaction(capsys): + snippets.dml_write_read_transaction(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert '1 record(s) inserted.' in out + assert 'FirstName: Timothy, LastName: Campbell' in out + + +def test_update_data_with_dml_struct(capsys): + snippets.update_data_with_dml_struct(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert '1 record(s) updated' in out + + +def test_insert_with_dml(capsys): + snippets.insert_with_dml(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert '4 record(s) inserted' in out + + +def test_write_with_dml_transaction(capsys): + snippets.write_with_dml_transaction(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert "Transferred 300000 from Album1's budget to Album2's" in out + + +def update_data_with_partitioned_dml(capsys): + snippets.update_data_with_partitioned_dml(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert "3 record(s) updated" in out + + +def delete_data_with_partitioned_dml(capsys): + snippets.delete_data_with_partitioned_dml(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert "5 record(s) deleted" in out + + +def update_with_batch_dml(capsys): + snippets.update_with_batch_dml(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert "Executed 2 SQL statements using Batch DML" in out diff --git a/speech/api/README.md b/speech/api/README.md deleted file mode 100644 index 3179f0d7107..00000000000 --- a/speech/api/README.md +++ /dev/null @@ -1,46 +0,0 @@ - -# Google Cloud Speech API Sample (REST API) - -This example demos accessing the [Google Cloud Speech API](http://cloud.google.com/speech) -via its REST API. - -## Prerequisites - -### Enable the Speech API - -If you have not already done so, -[enable the Google Cloud Speech API for your project](https://console.cloud.google.com/apis/api/speech.googleapis.com/overview). -You must be whitelisted to do this. - - -### Set Up to Authenticate With Your Project's Credentials - -The example uses a service account for OAuth2 authentication. -So next, set up to authenticate with the Speech API using your project's -service account credentials. - -Visit the [Cloud Console](https://console.cloud.google.com), and navigate to: -`API Manager > Credentials > Create credentials > -Service account key > New service account`. -Create a new service account, and download the json credentials file. - -Then, set -the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to point to your -downloaded service account credentials before running this example: - - export GOOGLE_APPLICATION_CREDENTIALS=/path/to/your/credentials-key.json - -If you do not do this, you will see an error that looks something like this when -you run the example script: -`...`. -See the -[Cloud Platform Auth Guide](https://cloud.google.com/docs/authentication#developer_workflow) -for more information. - -## Run the example - -```sh -$ python speechrest.py resources/audio.raw -``` - -You should see a response with the transcription result. diff --git a/speech/api/speech-discovery_google_rest_v1.json b/speech/api/speech-discovery_google_rest_v1.json deleted file mode 100644 index 135bd522a14..00000000000 --- a/speech/api/speech-discovery_google_rest_v1.json +++ /dev/null @@ -1,342 +0,0 @@ -{ - "kind": "discovery#restDescription", - "discoveryVersion": "v1", - "id": "speech:v1", - "name": "speech", - "version": "v1", - "revision": "0", - "title": "Google Cloud Speech API", - "description": "Google Cloud Speech API.", - "ownerDomain": "google.com", - "ownerName": "Google", - "icons": { - "x16": "http://www.google.com/images/icons/product/search-16.gif", - "x32": "http://www.google.com/images/icons/product/search-32.gif" - }, - "documentationLink": "https://cloud.google.com/speech", - "protocol": "rest", - "rootUrl": "https://speech.googleapis.com/", - "servicePath": "", - "baseUrl": "https://speech.googleapis.com/", - "batchPath": "batch", - "version_module": "True", - "parameters": { - "access_token": { - "type": "string", - "description": "OAuth access token.", - "location": "query" - }, - "alt": { - "type": "string", - "description": "Data format for response.", - "default": "json", - "enum": [ - "json", - "media", - "proto" - ], - "enumDescriptions": [ - "Responses with Content-Type of application/json", - "Media download with context-dependent Content-Type", - "Responses with Content-Type of application/x-protobuf" - ], - "location": "query" - }, - "bearer_token": { - "type": "string", - "description": "OAuth bearer token.", - "location": "query" - }, - "callback": { - "type": "string", - "description": "JSONP", - "location": "query" - }, - "fields": { - "type": "string", - "description": "Selector specifying which fields to include in a partial response.", - "location": "query" - }, - "key": { - "type": "string", - "description": "API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token.", - "location": "query" - }, - "oauth_token": { - "type": "string", - "description": "OAuth 2.0 token for the current user.", - "location": "query" - }, - "pp": { - "type": "boolean", - "description": "Pretty-print response.", - "default": "true", - "location": "query" - }, - "prettyPrint": { - "type": "boolean", - "description": "Returns response with indentations and line breaks.", - "default": "true", - "location": "query" - }, - "quotaUser": { - "type": "string", - "description": "Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters.", - "location": "query" - }, - "upload_protocol": { - "type": "string", - "description": "Upload protocol for media (e.g. \"raw\", \"multipart\").", - "location": "query" - }, - "uploadType": { - "type": "string", - "description": "Legacy upload protocol for media (e.g. \"media\", \"multipart\").", - "location": "query" - }, - "$.xgafv": { - "type": "string", - "description": "V1 error format.", - "enum": [ - "1", - "2" - ], - "enumDescriptions": [ - "v1 error format", - "v2 error format" - ], - "location": "query" - } - }, - "schemas": { - "RecognizeRequest": { - "id": "RecognizeRequest", - "description": "`RecognizeRequest` is the only message type sent by the client.\n\n`NonStreamingRecognize` sends only one `RecognizeRequest` message and it\nmust contain both an `initial_request` and an 'audio_request`.\n\nStreaming `Recognize` sends one or more `RecognizeRequest` messages. The\nfirst message must contain an `initial_request` and may contain an\n'audio_request`. Any subsequent messages must not contain an\n`initial_request` and must contain an 'audio_request`.", - "type": "object", - "properties": { - "initialRequest": { - "description": "The `initial_request` message provides information to the recognizer\nthat specifies how to process the request.\n\nThe first `RecognizeRequest` message must contain an `initial_request`.\nAny subsequent `RecognizeRequest` messages must not contain an\n`initial_request`.", - "$ref": "InitialRecognizeRequest" - }, - "audioRequest": { - "description": "The audio data to be recognized. For `NonStreamingRecognize`, all the\naudio data must be contained in the first (and only) `RecognizeRequest`\n message. For streaming `Recognize`, sequential chunks of audio data are\nsent in sequential `RecognizeRequest` messages.", - "$ref": "AudioRequest" - } - } - }, - "InitialRecognizeRequest": { - "id": "InitialRecognizeRequest", - "description": "The `InitialRecognizeRequest` message provides information to the recognizer\nthat specifies how to process the request.", - "type": "object", - "properties": { - "encoding": { - "description": "[Required] Encoding of audio data sent in all `AudioRequest` messages.", - "enumDescriptions": [ - "Not specified. Will return result `INVALID_ARGUMENT`.", - "Uncompressed 16-bit signed little-endian samples.\nThis is the simplest encoding format, useful for getting started.\nHowever, because it is uncompressed, it is not recommended for deployed\nclients.", - "This is the recommended encoding format because it uses lossless\ncompression; therefore recognition accuracy is not compromised by a lossy\ncodec.\n\nThe stream FLAC format is specified at:\nhttp:\/\/flac.sourceforge.net\/documentation.html.\nOnly 16-bit samples are supported.\nNot all fields in STREAMINFO are supported.", - "8-bit samples that compand 14-bit audio samples using PCMU\/mu-law.", - "Adaptive Multi-Rate Narrowband codec. `sample_rate` must be 8000 Hz.", - "Adaptive Multi-Rate Wideband codec. `sample_rate` must be 16000 Hz." - ], - "type": "string", - "enum": [ - "ENCODING_UNSPECIFIED", - "LINEAR16", - "FLAC", - "MULAW", - "AMR", - "AMR_WB" - ] - }, - "sampleRate": { - "description": "[Required] Sample rate in Hertz of the audio data sent in all\nAudioRequest messages.\n16000 is optimal. Valid values are: 8000-48000.", - "type": "integer", - "format": "int32" - }, - "languageCode": { - "description": "[Optional] The language of the supplied audio as a BCP-47 language tag.\nExample: \"en-GB\" https:\/\/www.rfc-editor.org\/rfc\/bcp\/bcp47.txt\nIf omitted, defaults to \"en-US\".", - "type": "string" - }, - "maxAlternatives": { - "description": "[Optional] Maximum number of recognition hypotheses to be returned.\nSpecifically, the maximum number of `SpeechRecognitionAlternative` messages\nwithin each `SpeechRecognitionResult`.\nThe server may return fewer than `max_alternatives`.\nValid values are `0`-`30`. A value of `0` or `1` will return a maximum of\n`1`. If omitted, defaults to `1`.", - "type": "integer", - "format": "int32" - }, - "profanityFilter": { - "description": "[Optional] If set to `true`, the server will attempt to filter out\nprofanities, replacing all but the initial character in each filtered word\nwith asterisks, e.g. \"f***\". If set to `false` or omitted, profanities\nwon't be filtered out.\nNote that profanity filtering is not implemented for all languages.\nIf the language is not supported, this setting has no effect.", - "type": "boolean" - }, - "continuous": { - "description": "[Optional] If `false` or omitted, the recognizer will detect a single\nspoken utterance, and it will cease recognition when the user stops\nspeaking. If `enable_endpointer_events` is `true`, it will return\n`END_OF_UTTERANCE` when it detects that the user has stopped speaking.\nIn all cases, it will return no more than one `SpeechRecognitionResult`,\nand set the `is_final` flag to `true`.\n\nIf `true`, the recognizer will continue recognition (even if the user\npauses speaking) until the client sends an `end_of_data` message or when\nthe maximum time limit has been reached. Multiple\n`SpeechRecognitionResult`s with the `is_final` flag set to `true` may be\nreturned to indicate that the recognizer will not return any further\nhypotheses for this portion of the transcript.", - "type": "boolean" - }, - "interimResults": { - "description": "[Optional] If this parameter is `true`, interim results may be returned as\nthey become available.\nIf `false` or omitted, only `is_final=true` result(s) are returned.", - "type": "boolean" - }, - "enableEndpointerEvents": { - "description": "[Optional] If this parameter is `true`, `EndpointerEvents` may be returned\nas they become available.\nIf `false` or omitted, no `EndpointerEvents` are returned.", - "type": "boolean" - } - } - }, - "AudioRequest": { - "id": "AudioRequest", - "description": "Contains audio data in the format specified in the `InitialRecognizeRequest`.", - "type": "object", - "properties": { - "content": { - "description": "[Required] The audio data bytes encoded as specified in\n`InitialRecognizeRequest`.", - "type": "string", - "format": "byte" - } - } - }, - "NonStreamingRecognizeResponse": { - "id": "NonStreamingRecognizeResponse", - "description": "`NonStreamingRecognizeResponse` is the only message returned to the client by\n`NonStreamingRecognize`. It contains the result as zero or more sequential\n`RecognizeResponse` messages.\n\nNote that streaming `Recognize` will also return multiple `RecognizeResponse`\nmessages, but each message is individually streamed.", - "type": "object", - "properties": { - "responses": { - "description": "[Output-only] Sequential list of messages returned by the recognizer.", - "type": "array", - "items": { - "$ref": "RecognizeResponse" - } - } - } - }, - "RecognizeResponse": { - "id": "RecognizeResponse", - "description": "`RecognizeResponse` is the only message type returned to the client.", - "type": "object", - "properties": { - "error": { - "description": "[Output-only] If set, returns a google.rpc.Status message that\nspecifies the error for the operation.", - "$ref": "Status" - }, - "results": { - "description": "[Output-only] May contain zero or one `is_final=true` result (the newly\nsettled portion). May also contain zero or more `is_final=false` results.", - "type": "array", - "items": { - "$ref": "SpeechRecognitionResult" - } - }, - "resultIndex": { - "description": "[Output-only] Indicates the lowest index in the `results` array that has\nchanged. The repeated `SpeechRecognitionResult` results overwrite past\nresults at this index and higher.", - "type": "integer", - "format": "int32" - }, - "endpoint": { - "description": "[Output-only] Indicates the type of endpointer event.", - "enumDescriptions": [ - "No endpointer event specified.", - "Speech has been detected in the audio stream.", - "Speech has ceased to be detected in the audio stream.", - "The end of the audio stream has been reached. and it is being processed.", - "This event is only sent when continuous is `false`. It indicates that the\nserver has detected the end of the user's speech utterance and expects no\nadditional speech. Therefore, the server will not process additional\naudio. The client should stop sending additional audio data." - ], - "type": "string", - "enum": [ - "ENDPOINTER_EVENT_UNSPECIFIED", - "START_OF_SPEECH", - "END_OF_SPEECH", - "END_OF_AUDIO", - "END_OF_UTTERANCE" - ] - } - } - }, - "Status": { - "id": "Status", - "description": "The `Status` type defines a logical error model that is suitable for different\nprogramming environments, including REST APIs and RPC APIs. It is used by\n[gRPC](https:\/\/github.com\/grpc). The error model is designed to be:\n\n- Simple to use and understand for most users\n- Flexible enough to meet unexpected needs\n\n# Overview\n\nThe `Status` message contains three pieces of data: error code, error message,\nand error details. The error code should be an enum value of\ngoogle.rpc.Code, but it may accept additional error codes if needed. The\nerror message should be a developer-facing English message that helps\ndevelopers *understand* and *resolve* the error. If a localized user-facing\nerror message is needed, put the localized message in the error details or\nlocalize it in the client. The optional error details may contain arbitrary\ninformation about the error. There is a predefined set of error detail types\nin the package `google.rpc` which can be used for common error conditions.\n\n# Language mapping\n\nThe `Status` message is the logical representation of the error model, but it\nis not necessarily the actual wire format. When the `Status` message is\nexposed in different client libraries and different wire protocols, it can be\nmapped differently. For example, it will likely be mapped to some exceptions\nin Java, but more likely mapped to some error codes in C.\n\n# Other uses\n\nThe error model and the `Status` message can be used in a variety of\nenvironments, either with or without APIs, to provide a\nconsistent developer experience across different environments.\n\nExample uses of this error model include:\n\n- Partial errors. If a service needs to return partial errors to the client,\n it may embed the `Status` in the normal response to indicate the partial\n errors.\n\n- Workflow errors. A typical workflow has multiple steps. Each step may\n have a `Status` message for error reporting purpose.\n\n- Batch operations. If a client uses batch request and batch response, the\n `Status` message should be used directly inside batch response, one for\n each error sub-response.\n\n- Asynchronous operations. If an API call embeds asynchronous operation\n results in its response, the status of those operations should be\n represented directly using the `Status` message.\n\n- Logging. If some API errors are stored in logs, the message `Status` could\n be used directly after any stripping needed for security\/privacy reasons.", - "type": "object", - "properties": { - "code": { - "description": "The status code, which should be an enum value of google.rpc.Code.", - "type": "integer", - "format": "int32" - }, - "message": { - "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\ngoogle.rpc.Status.details field, or localized by the client.", - "type": "string" - }, - "details": { - "description": "A list of messages that carry the error details. There will be a\ncommon set of message types for APIs to use.", - "type": "array", - "items": { - "type": "object", - "additionalProperties": { - "type": "any", - "description": "Properties of the object. Contains field @ype with type URL." - } - } - } - } - }, - "SpeechRecognitionResult": { - "id": "SpeechRecognitionResult", - - "type": "object", - "properties": { - "alternatives": { - - "type": "array", - "items": { - "$ref": "SpeechRecognitionAlternative" - } - }, - "isFinal": { - "description": "[Output-only] Set `true` if this is the final time the speech service will\nreturn this particular `SpeechRecognitionResult`. If `false`, this\nrepresents an interim result that may change.", - "type": "boolean" - }, - "stability": { - "description": "[Output-only] An estimate of the probability that the recognizer will not\nchange its guess about this interim result. Values range from 0.0\n(completely unstable) to 1.0 (completely stable). Note that this is not the\nsame as `confidence`, which estimates the probability that a recognition\nresult is correct.\nThis field is only provided for interim results (`is_final=false`).\nThe default of 0.0 is a sentinel value indicating stability was not set.", - "type": "number", - "format": "float" - } - } - }, - "SpeechRecognitionAlternative": { - "id": "SpeechRecognitionAlternative", - "description": "Alternative hypotheses (a.k.a. n-best list).", - "type": "object", - "properties": { - "transcript": { - "description": "[Output-only] Transcript text representing the words that the user spoke.", - "type": "string" - }, - "confidence": { - "description": "[Output-only] The confidence estimate between 0.0 and 1.0. A higher number\nmeans the system is more confident that the recognition is correct.\nThis field is typically provided only for the top hypothesis. and only for\n`is_final=true` results.\nThe default of 0.0 is a sentinel value indicating confidence was not set.", - "type": "number", - "format": "float" - } - } - } - }, - "resources": { - "speech": { - "methods": { - "recognize": { - "id": "speech.speech.recognize", - "path": "v1/speech:recognize", - "flatPath": "v1/speech:recognize", - "httpMethod": "POST", - "description": "Perform non-streaming speech recognition on audio using HTTPS.", - "parameters": { - }, - "parameterOrder": [ - ], - "request": { - "$ref": "RecognizeRequest" - }, - "response": { - "$ref": "NonStreamingRecognizeResponse" - } - } - } - } - }, - "basePath": "" -} diff --git a/speech/api/speechrest.py b/speech/api/speechrest.py deleted file mode 100644 index 43a8e39f65c..00000000000 --- a/speech/api/speechrest.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Google Cloud Speech API sample application using the REST API for batch -processing.""" - -# [START import_libraries] -import argparse -import base64 -import json -import os - -from googleapiclient import discovery - -import httplib2 - -from oauth2client.client import GoogleCredentials -# [END import_libraries] - -# Path to local discovery file -# [START discovery_doc] -API_DISCOVERY_FILE = os.path.join( - os.path.dirname(__file__), 'speech-discovery_google_rest_v1.json') -# [END discovery_doc] - - -# Application default credentials provided by env variable -# GOOGLE_APPLICATION_CREDENTIALS -def get_speech_service(): - # [START authenticating] - credentials = GoogleCredentials.get_application_default().create_scoped( - ['https://www.googleapis.com/auth/xapi.zoo']) - with open(API_DISCOVERY_FILE, 'r') as f: - doc = f.read() - - return discovery.build_from_document( - doc, credentials=credentials, http=httplib2.Http()) - # [END authenticating] - - -def main(speech_file): - """Transcribe the given audio file. - - Args: - speech_file: the name of the audio file. - """ - # [START construct_request] - with open(speech_file, 'rb') as speech: - # Base64 encode the binary audio file for inclusion in the JSON - # request. - speech_content = base64.b64encode(speech.read()) - - service = get_speech_service() - service_request = service.speech().recognize( - body={ - 'initialRequest': { - 'encoding': 'LINEAR16', - 'sampleRate': 16000 - }, - 'audioRequest': { - 'content': speech_content.decode('UTF-8') - } - }) - # [END construct_request] - # [START send_request] - response = service_request.execute() - print(json.dumps(response)) - # [END send_request] - -# [START run_application] -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument( - 'speech_file', help='Full path of audio file to be recognized') - args = parser.parse_args() - main(args.speech_file) - # [END run_application] diff --git a/speech/api/speechrest_test.py b/speech/api/speechrest_test.py deleted file mode 100644 index 8f1d2851bfa..00000000000 --- a/speech/api/speechrest_test.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2016, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from speechrest import main - - -def test_main(resource, capsys): - main(resource('audio.raw')) - out, err = capsys.readouterr() - - assert re.search(r'how old is the Brooklyn Bridge', out, re.DOTALL | re.I) diff --git a/speech/cloud-client/README.rst b/speech/cloud-client/README.rst new file mode 100644 index 00000000000..f43716a5789 --- /dev/null +++ b/speech/cloud-client/README.rst @@ -0,0 +1,361 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Speech API Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=speech/cloud-client/README.rst + + +This directory contains samples for Google Cloud Speech API. The `Google Cloud Speech API`_ enables easy integration of Google speech recognition technologies into developer applications. Send audio and receive a text transcription from the Cloud Speech API service. + +- See the `migration guide`_ for information about migrating to Python client library v0.27. + +.. _migration guide: https://cloud.google.com/speech/docs/python-client-migration + + + + +.. _Google Cloud Speech API: https://cloud.google.com/speech/docs/ + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Quickstart ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=speech/cloud-client/quickstart.py,speech/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python quickstart.py + + +Transcribe ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=speech/cloud-client/transcribe.py,speech/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python transcribe.py + + usage: transcribe.py [-h] path + + Google Cloud Speech API sample application using the REST API for batch + processing. + + Example usage: + python transcribe.py resources/audio.raw + python transcribe.py gs://cloud-samples-tests/speech/brooklyn.flac + + positional arguments: + path File or GCS path for audio file to be recognized + + optional arguments: + -h, --help show this help message and exit + + + +Transcribe async ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=speech/cloud-client/transcribe_async.py,speech/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python transcribe_async.py + + usage: transcribe_async.py [-h] path + + Google Cloud Speech API sample application using the REST API for async + batch processing. + + Example usage: + python transcribe_async.py resources/audio.raw + python transcribe_async.py gs://cloud-samples-tests/speech/vr.flac + + positional arguments: + path File or GCS path for audio file to be recognized + + optional arguments: + -h, --help show this help message and exit + + + +Transcribe with word time offsets ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=speech/cloud-client/transcribe_word_time_offsets.py,speech/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python transcribe_word_time_offsets.py + + usage: transcribe_word_time_offsets.py [-h] path + + Google Cloud Speech API sample that demonstrates word time offsets. + + Example usage: + python transcribe_word_time_offsets.py resources/audio.raw + python transcribe_word_time_offsets.py gs://cloud-samples-tests/speech/vr.flac + + positional arguments: + path File or GCS path for audio file to be recognized + + optional arguments: + -h, --help show this help message and exit + + + +Transcribe Streaming ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=speech/cloud-client/transcribe_streaming.py,speech/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python transcribe_streaming.py + + usage: transcribe_streaming.py [-h] stream + + Google Cloud Speech API sample application using the streaming API. + + Example usage: + python transcribe_streaming.py resources/audio.raw + + positional arguments: + stream File to stream to the API + + optional arguments: + -h, --help show this help message and exit + + + +Transcribe Enhanced Models ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=speech/cloud-client/transcribe_enhanced_model.py,speech/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python transcribe_enhanced_model.py + + usage: transcribe_enhanced_model.py [-h] path + + Google Cloud Speech API sample that demonstrates enhanced models + and recognition metadata. + + Example usage: + python transcribe_enhanced_model.py resources/commercial_mono.wav + + positional arguments: + path File to stream to the API + + optional arguments: + -h, --help show this help message and exit + + + +Transcribe Automatic Punctuation ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=speech/cloud-client/transcribe_auto_punctuation.py,speech/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python transcribe_auto_punctuation.py + + usage: transcribe_auto_punctuation.py [-h] path + + Google Cloud Speech API sample that demonstrates auto punctuation + and recognition metadata. + + Example usage: + python transcribe_auto_punctuation.py resources/commercial_mono.wav + + positional arguments: + path File to stream to the API + + optional arguments: + -h, --help show this help message and exit + + + +Transcribe with Model Selection ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=speech/cloud-client/transcribe_auto_punctuation.py,speech/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python transcribe_model_selection.py + + usage: transcribe_model_selection.py [-h] + [--model {command_and_search,phone_call,video,default}] + path + + Google Cloud Speech API sample that demonstrates how to select the model + used for speech recognition. + + Example usage: + python transcribe_model_selection.py resources/Google_Gnome.wav --model video + python transcribe_model_selection.py gs://cloud-samples-tests/speech/Google_Gnome.wav --model video + + positional arguments: + path File or GCS path for audio file to be recognized + + optional arguments: + -h, --help show this help message and exit + --model {command_and_search,phone_call,video,default} + The speech recognition model to use + + + +Beta Samples ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=speech/cloud-client/beta_snippets.py,speech/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python beta_snippets.py + + usage: beta_snippets.py [-h] command + + Google Cloud Speech API sample that demonstrates enhanced models + and recognition metadata. + + Example usage: + python beta_snippets.py enhanced-model + python beta_snippets.py metadata + python beta_snippets.py punctuation + python beta_snippets.py diarization + python beta_snippets.py multi-channel + python beta_snippets.py multi-language + python beta_snippets.py word-level-conf + + positional arguments: + command + + optional arguments: + -h, --help show this help message and exit + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ diff --git a/speech/cloud-client/README.rst.in b/speech/cloud-client/README.rst.in new file mode 100644 index 00000000000..9447e48548d --- /dev/null +++ b/speech/cloud-client/README.rst.in @@ -0,0 +1,49 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Speech API + short_name: Cloud Speech API + url: https://cloud.google.com/speech/docs/ + description: > + The `Google Cloud Speech API`_ enables easy integration of Google speech + recognition technologies into developer applications. Send audio and receive + a text transcription from the Cloud Speech API service. + + + - See the `migration guide`_ for information about migrating to Python client library v0.27. + + + .. _migration guide: https://cloud.google.com/speech/docs/python-client-migration + +setup: +- auth +- install_deps + +samples: +- name: Quickstart + file: quickstart.py +- name: Transcribe + file: transcribe.py + show_help: true +- name: Transcribe async + file: transcribe_async.py + show_help: true +- name: Transcribe with word time offsets + file: transcribe_word_time_offsets.py + show_help: true +- name: Transcribe Streaming + file: transcribe_streaming.py + show_help: true +- name: Transcribe Enhanced Models + file: transcribe_enhanced_model.py + show_help: true +- name: Transcribe Automatic Punctuation + file: transcribe_auto_punctuation.py + show_help: true +- name: Beta Samples + file: beta_snippets.py + show_help: true + +cloud_client_library: true + +folder: speech/cloud-client \ No newline at end of file diff --git a/speech/cloud-client/beta_snippets.py b/speech/cloud-client/beta_snippets.py new file mode 100644 index 00000000000..51bd0d16949 --- /dev/null +++ b/speech/cloud-client/beta_snippets.py @@ -0,0 +1,297 @@ +#!/usr/bin/env python + +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Google Cloud Speech API sample that demonstrates enhanced models +and recognition metadata. + +Example usage: + python beta_snippets.py enhanced-model + python beta_snippets.py metadata + python beta_snippets.py punctuation + python beta_snippets.py diarization + python beta_snippets.py multi-channel + python beta_snippets.py multi-language + python beta_snippets.py word-level-conf +""" + +import argparse +import io + + +def transcribe_file_with_enhanced_model(): + """Transcribe the given audio file using an enhanced model.""" + # [START speech_transcribe_enhanced_model_beta] + from google.cloud import speech_v1p1beta1 as speech + client = speech.SpeechClient() + + speech_file = 'resources/commercial_mono.wav' + + with io.open(speech_file, 'rb') as audio_file: + content = audio_file.read() + + audio = speech.types.RecognitionAudio(content=content) + config = speech.types.RecognitionConfig( + encoding=speech.enums.RecognitionConfig.AudioEncoding.LINEAR16, + sample_rate_hertz=8000, + language_code='en-US', + # Enhanced models are only available to projects that + # opt in for audio data collection. + use_enhanced=True, + # A model must be specified to use enhanced model. + model='phone_call') + + response = client.recognize(config, audio) + + for i, result in enumerate(response.results): + alternative = result.alternatives[0] + print('-' * 20) + print('First alternative of result {}'.format(i)) + print('Transcript: {}'.format(alternative.transcript)) + # [END speech_transcribe_enhanced_model_beta] + + +def transcribe_file_with_metadata(): + """Send a request that includes recognition metadata.""" + # [START speech_transcribe_recognition_metadata_beta] + from google.cloud import speech_v1p1beta1 as speech + client = speech.SpeechClient() + + speech_file = 'resources/commercial_mono.wav' + + with io.open(speech_file, 'rb') as audio_file: + content = audio_file.read() + + # Here we construct a recognition metadata object. + # Most metadata fields are specified as enums that can be found + # in speech.enums.RecognitionMetadata + metadata = speech.types.RecognitionMetadata() + metadata.interaction_type = ( + speech.enums.RecognitionMetadata.InteractionType.DISCUSSION) + metadata.microphone_distance = ( + speech.enums.RecognitionMetadata.MicrophoneDistance.NEARFIELD) + metadata.recording_device_type = ( + speech.enums.RecognitionMetadata.RecordingDeviceType.SMARTPHONE) + # Some metadata fields are free form strings + metadata.recording_device_name = "Pixel 2 XL" + # And some are integers, for instance the 6 digit NAICS code + # https://www.naics.com/search/ + metadata.industry_naics_code_of_audio = 519190 + + audio = speech.types.RecognitionAudio(content=content) + config = speech.types.RecognitionConfig( + encoding=speech.enums.RecognitionConfig.AudioEncoding.LINEAR16, + sample_rate_hertz=8000, + language_code='en-US', + # Add this in the request to send metadata. + metadata=metadata) + + response = client.recognize(config, audio) + + for i, result in enumerate(response.results): + alternative = result.alternatives[0] + print('-' * 20) + print('First alternative of result {}'.format(i)) + print('Transcript: {}'.format(alternative.transcript)) + # [END speech_transcribe_recognition_metadata_beta] + + +def transcribe_file_with_auto_punctuation(): + """Transcribe the given audio file with auto punctuation enabled.""" + # [START speech_transcribe_auto_punctuation_beta] + from google.cloud import speech_v1p1beta1 as speech + client = speech.SpeechClient() + + speech_file = 'resources/commercial_mono.wav' + + with io.open(speech_file, 'rb') as audio_file: + content = audio_file.read() + + audio = speech.types.RecognitionAudio(content=content) + config = speech.types.RecognitionConfig( + encoding=speech.enums.RecognitionConfig.AudioEncoding.LINEAR16, + sample_rate_hertz=8000, + language_code='en-US', + # Enable automatic punctuation + enable_automatic_punctuation=True) + + response = client.recognize(config, audio) + + for i, result in enumerate(response.results): + alternative = result.alternatives[0] + print('-' * 20) + print('First alternative of result {}'.format(i)) + print('Transcript: {}'.format(alternative.transcript)) + # [END speech_transcribe_auto_punctuation_beta] + + +def transcribe_file_with_diarization(): + """Transcribe the given audio file synchronously with diarization.""" + # [START speech_transcribe_diarization_beta] + from google.cloud import speech_v1p1beta1 as speech + client = speech.SpeechClient() + + speech_file = 'resources/commercial_mono.wav' + + with open(speech_file, 'rb') as audio_file: + content = audio_file.read() + + audio = speech.types.RecognitionAudio(content=content) + + config = speech.types.RecognitionConfig( + encoding=speech.enums.RecognitionConfig.AudioEncoding.LINEAR16, + sample_rate_hertz=8000, + language_code='en-US', + enable_speaker_diarization=True, + diarization_speaker_count=2) + + print('Waiting for operation to complete...') + response = client.recognize(config, audio) + + # The transcript within each result is separate and sequential per result. + # However, the words list within an alternative includes all the words + # from all the results thus far. Thus, to get all the words with speaker + # tags, you only have to take the words list from the last result: + result = response.results[-1] + + words_info = result.alternatives[0].words + + # Printing out the output: + for word_info in words_info: + print("word: '{}', speaker_tag: {}".format(word_info.word, + word_info.speaker_tag)) + # [END speech_transcribe_diarization_beta] + + +def transcribe_file_with_multichannel(): + """Transcribe the given audio file synchronously with + multi channel.""" + # [START speech_transcribe_multichannel_beta] + from google.cloud import speech_v1p1beta1 as speech + client = speech.SpeechClient() + + speech_file = 'resources/Google_Gnome.wav' + + with open(speech_file, 'rb') as audio_file: + content = audio_file.read() + + audio = speech.types.RecognitionAudio(content=content) + + config = speech.types.RecognitionConfig( + encoding=speech.enums.RecognitionConfig.AudioEncoding.LINEAR16, + sample_rate_hertz=16000, + language_code='en-US', + audio_channel_count=1, + enable_separate_recognition_per_channel=True) + + response = client.recognize(config, audio) + + for i, result in enumerate(response.results): + alternative = result.alternatives[0] + print('-' * 20) + print('First alternative of result {}'.format(i)) + print(u'Transcript: {}'.format(alternative.transcript)) + print(u'Channel Tag: {}'.format(result.channel_tag)) + # [END speech_transcribe_multichannel_beta] + + +def transcribe_file_with_multilanguage(): + """Transcribe the given audio file synchronously with + multi language.""" + # [START speech_transcribe_multilanguage_beta] + from google.cloud import speech_v1p1beta1 as speech + client = speech.SpeechClient() + + speech_file = 'resources/multi.wav' + first_lang = 'en-US' + second_lang = 'es' + + with open(speech_file, 'rb') as audio_file: + content = audio_file.read() + + audio = speech.types.RecognitionAudio(content=content) + + config = speech.types.RecognitionConfig( + encoding=speech.enums.RecognitionConfig.AudioEncoding.LINEAR16, + sample_rate_hertz=44100, + audio_channel_count=2, + language_code=first_lang, + alternative_language_codes=[second_lang]) + + print('Waiting for operation to complete...') + response = client.recognize(config, audio) + + for i, result in enumerate(response.results): + alternative = result.alternatives[0] + print('-' * 20) + print('First alternative of result {}: {}'.format(i, alternative)) + print(u'Transcript: {}'.format(alternative.transcript)) + # [END speech_transcribe_multilanguage_beta] + + +def transcribe_file_with_word_level_confidence(): + """Transcribe the given audio file synchronously with + word level confidence.""" + # [START speech_transcribe_word_level_confidence_beta] + from google.cloud import speech_v1p1beta1 as speech + client = speech.SpeechClient() + + speech_file = 'resources/Google_Gnome.wav' + + with open(speech_file, 'rb') as audio_file: + content = audio_file.read() + + audio = speech.types.RecognitionAudio(content=content) + + config = speech.types.RecognitionConfig( + encoding=speech.enums.RecognitionConfig.AudioEncoding.LINEAR16, + sample_rate_hertz=16000, + language_code='en-US', + enable_word_confidence=True) + + response = client.recognize(config, audio) + + for i, result in enumerate(response.results): + alternative = result.alternatives[0] + print('-' * 20) + print('First alternative of result {}'.format(i)) + print(u'Transcript: {}'.format(alternative.transcript)) + print(u'First Word and Confidence: ({}, {})'.format( + alternative.words[0].word, alternative.words[0].confidence)) + # [END speech_transcribe_word_level_confidence_beta] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('command') + + args = parser.parse_args() + + if args.command == 'enhanced-model': + transcribe_file_with_enhanced_model() + elif args.command == 'metadata': + transcribe_file_with_metadata() + elif args.command == 'punctuation': + transcribe_file_with_auto_punctuation() + elif args.command == 'diarization': + transcribe_file_with_diarization() + elif args.command == 'multi-channel': + transcribe_file_with_multichannel() + elif args.command == 'multi-language': + transcribe_file_with_multilanguage() + elif args.command == 'word-level-conf': + transcribe_file_with_word_level_confidence() diff --git a/speech/cloud-client/beta_snippets_test.py b/speech/cloud-client/beta_snippets_test.py new file mode 100644 index 00000000000..44b421bb575 --- /dev/null +++ b/speech/cloud-client/beta_snippets_test.py @@ -0,0 +1,75 @@ +# Copyright 2018, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from beta_snippets import ( + transcribe_file_with_auto_punctuation, + transcribe_file_with_diarization, + transcribe_file_with_enhanced_model, + transcribe_file_with_metadata, + transcribe_file_with_multichannel, + transcribe_file_with_multilanguage, + transcribe_file_with_word_level_confidence) + +RESOURCES = os.path.join(os.path.dirname(__file__), 'resources') + + +def test_transcribe_file_with_enhanced_model(capsys): + transcribe_file_with_enhanced_model() + out, _ = capsys.readouterr() + + assert 'Chrome' in out + + +def test_transcribe_file_with_metadata(capsys): + transcribe_file_with_metadata() + out, _ = capsys.readouterr() + + assert 'Chrome' in out + + +def test_transcribe_file_with_auto_punctuation(capsys): + transcribe_file_with_auto_punctuation() + out, _ = capsys.readouterr() + + assert 'Okay. Sure.' in out + + +def test_transcribe_diarization(capsys): + transcribe_file_with_diarization() + out, err = capsys.readouterr() + + assert "word:" in out + assert "speaker_tag:" in out + + +def test_transcribe_multichannel_file(capsys): + transcribe_file_with_multichannel() + out, err = capsys.readouterr() + + assert 'OK Google stream stranger things from Netflix to my TV' in out + + +def test_transcribe_multilanguage_file(capsys): + transcribe_file_with_multilanguage() + out, err = capsys.readouterr() + + assert 'how are you doing estoy bien e tu' in out + + +def test_transcribe_word_level_confidence(capsys): + transcribe_file_with_word_level_confidence() + out, err = capsys.readouterr() + + assert 'OK Google stream stranger things from Netflix to my TV' in out diff --git a/speech/cloud-client/quickstart.py b/speech/cloud-client/quickstart.py new file mode 100644 index 00000000000..f90f52fb04b --- /dev/null +++ b/speech/cloud-client/quickstart.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def run_quickstart(): + # [START speech_quickstart] + import io + import os + + # Imports the Google Cloud client library + # [START speech_python_migration_imports] + from google.cloud import speech + from google.cloud.speech import enums + from google.cloud.speech import types + # [END speech_python_migration_imports] + + # Instantiates a client + # [START speech_python_migration_client] + client = speech.SpeechClient() + # [END speech_python_migration_client] + + # The name of the audio file to transcribe + file_name = os.path.join( + os.path.dirname(__file__), + 'resources', + 'audio.raw') + + # Loads the audio into memory + with io.open(file_name, 'rb') as audio_file: + content = audio_file.read() + audio = types.RecognitionAudio(content=content) + + config = types.RecognitionConfig( + encoding=enums.RecognitionConfig.AudioEncoding.LINEAR16, + sample_rate_hertz=16000, + language_code='en-US') + + # Detects speech in the audio file + response = client.recognize(config, audio) + + for result in response.results: + print('Transcript: {}'.format(result.alternatives[0].transcript)) + # [END speech_quickstart] + + +if __name__ == '__main__': + run_quickstart() diff --git a/speech/cloud-client/quickstart_test.py b/speech/cloud-client/quickstart_test.py new file mode 100644 index 00000000000..0675ad195d3 --- /dev/null +++ b/speech/cloud-client/quickstart_test.py @@ -0,0 +1,22 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import quickstart + + +def test_quickstart(capsys): + quickstart.run_quickstart() + out, _ = capsys.readouterr() + assert 'Transcript: how old is the Brooklyn Bridge' in out diff --git a/speech/cloud-client/requirements.txt b/speech/cloud-client/requirements.txt new file mode 100644 index 00000000000..e4805f68751 --- /dev/null +++ b/speech/cloud-client/requirements.txt @@ -0,0 +1 @@ +google-cloud-speech==0.36.3 diff --git a/speech/cloud-client/resources/Google_Gnome.wav b/speech/cloud-client/resources/Google_Gnome.wav new file mode 100644 index 00000000000..2f497b7fbe7 Binary files /dev/null and b/speech/cloud-client/resources/Google_Gnome.wav differ diff --git a/speech/api/resources/audio.raw b/speech/cloud-client/resources/audio.raw similarity index 100% rename from speech/api/resources/audio.raw rename to speech/cloud-client/resources/audio.raw diff --git a/speech/api/resources/audio2.raw b/speech/cloud-client/resources/audio2.raw similarity index 100% rename from speech/api/resources/audio2.raw rename to speech/cloud-client/resources/audio2.raw diff --git a/speech/cloud-client/resources/commercial_mono.wav b/speech/cloud-client/resources/commercial_mono.wav new file mode 100644 index 00000000000..e6b9ed434f9 Binary files /dev/null and b/speech/cloud-client/resources/commercial_mono.wav differ diff --git a/speech/cloud-client/resources/multi.wav b/speech/cloud-client/resources/multi.wav new file mode 100644 index 00000000000..7f71d74b951 Binary files /dev/null and b/speech/cloud-client/resources/multi.wav differ diff --git a/speech/cloud-client/transcribe.py b/speech/cloud-client/transcribe.py new file mode 100644 index 00000000000..26b61d69f1c --- /dev/null +++ b/speech/cloud-client/transcribe.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Google Cloud Speech API sample application using the REST API for batch +processing. + +Example usage: + python transcribe.py resources/audio.raw + python transcribe.py gs://cloud-samples-tests/speech/brooklyn.flac +""" + +import argparse +import io + + +# [START speech_transcribe_sync] +def transcribe_file(speech_file): + """Transcribe the given audio file.""" + from google.cloud import speech + from google.cloud.speech import enums + from google.cloud.speech import types + client = speech.SpeechClient() + + # [START speech_python_migration_sync_request] + # [START speech_python_migration_config] + with io.open(speech_file, 'rb') as audio_file: + content = audio_file.read() + + audio = types.RecognitionAudio(content=content) + config = types.RecognitionConfig( + encoding=enums.RecognitionConfig.AudioEncoding.LINEAR16, + sample_rate_hertz=16000, + language_code='en-US') + # [END speech_python_migration_config] + + # [START speech_python_migration_sync_response] + response = client.recognize(config, audio) + # [END speech_python_migration_sync_request] + # Each result is for a consecutive portion of the audio. Iterate through + # them to get the transcripts for the entire audio file. + for result in response.results: + # The first alternative is the most likely one for this portion. + print(u'Transcript: {}'.format(result.alternatives[0].transcript)) + # [END speech_python_migration_sync_response] +# [END speech_transcribe_sync] + + +# [START speech_transcribe_sync_gcs] +def transcribe_gcs(gcs_uri): + """Transcribes the audio file specified by the gcs_uri.""" + from google.cloud import speech + from google.cloud.speech import enums + from google.cloud.speech import types + client = speech.SpeechClient() + + # [START speech_python_migration_config_gcs] + audio = types.RecognitionAudio(uri=gcs_uri) + config = types.RecognitionConfig( + encoding=enums.RecognitionConfig.AudioEncoding.FLAC, + sample_rate_hertz=16000, + language_code='en-US') + # [END speech_python_migration_config_gcs] + + response = client.recognize(config, audio) + # Each result is for a consecutive portion of the audio. Iterate through + # them to get the transcripts for the entire audio file. + for result in response.results: + # The first alternative is the most likely one for this portion. + print(u'Transcript: {}'.format(result.alternatives[0].transcript)) +# [END speech_transcribe_sync_gcs] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + 'path', help='File or GCS path for audio file to be recognized') + args = parser.parse_args() + if args.path.startswith('gs://'): + transcribe_gcs(args.path) + else: + transcribe_file(args.path) diff --git a/speech/cloud-client/transcribe_async.py b/speech/cloud-client/transcribe_async.py new file mode 100644 index 00000000000..0f9f5b2dc60 --- /dev/null +++ b/speech/cloud-client/transcribe_async.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Google Cloud Speech API sample application using the REST API for async +batch processing. + +Example usage: + python transcribe_async.py resources/audio.raw + python transcribe_async.py gs://cloud-samples-tests/speech/vr.flac +""" + +import argparse +import io + + +# [START speech_transcribe_async] +def transcribe_file(speech_file): + """Transcribe the given audio file asynchronously.""" + from google.cloud import speech + from google.cloud.speech import enums + from google.cloud.speech import types + client = speech.SpeechClient() + + # [START speech_python_migration_async_request] + with io.open(speech_file, 'rb') as audio_file: + content = audio_file.read() + + audio = types.RecognitionAudio(content=content) + config = types.RecognitionConfig( + encoding=enums.RecognitionConfig.AudioEncoding.LINEAR16, + sample_rate_hertz=16000, + language_code='en-US') + + # [START speech_python_migration_async_response] + operation = client.long_running_recognize(config, audio) + # [END speech_python_migration_async_request] + + print('Waiting for operation to complete...') + response = operation.result(timeout=90) + + # Each result is for a consecutive portion of the audio. Iterate through + # them to get the transcripts for the entire audio file. + for result in response.results: + # The first alternative is the most likely one for this portion. + print(u'Transcript: {}'.format(result.alternatives[0].transcript)) + print('Confidence: {}'.format(result.alternatives[0].confidence)) + # [END speech_python_migration_async_response] +# [END speech_transcribe_async] + + +# [START speech_transcribe_async_gcs] +def transcribe_gcs(gcs_uri): + """Asynchronously transcribes the audio file specified by the gcs_uri.""" + from google.cloud import speech + from google.cloud.speech import enums + from google.cloud.speech import types + client = speech.SpeechClient() + + audio = types.RecognitionAudio(uri=gcs_uri) + config = types.RecognitionConfig( + encoding=enums.RecognitionConfig.AudioEncoding.FLAC, + sample_rate_hertz=16000, + language_code='en-US') + + operation = client.long_running_recognize(config, audio) + + print('Waiting for operation to complete...') + response = operation.result(timeout=90) + + # Each result is for a consecutive portion of the audio. Iterate through + # them to get the transcripts for the entire audio file. + for result in response.results: + # The first alternative is the most likely one for this portion. + print(u'Transcript: {}'.format(result.alternatives[0].transcript)) + print('Confidence: {}'.format(result.alternatives[0].confidence)) +# [END speech_transcribe_async_gcs] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + 'path', help='File or GCS path for audio file to be recognized') + args = parser.parse_args() + if args.path.startswith('gs://'): + transcribe_gcs(args.path) + else: + transcribe_file(args.path) diff --git a/speech/cloud-client/transcribe_async_test.py b/speech/cloud-client/transcribe_async_test.py new file mode 100644 index 00000000000..7d66747eb44 --- /dev/null +++ b/speech/cloud-client/transcribe_async_test.py @@ -0,0 +1,35 @@ +# Copyright 2016, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +import transcribe_async + +RESOURCES = os.path.join(os.path.dirname(__file__), 'resources') + + +def test_transcribe(capsys): + transcribe_async.transcribe_file( + os.path.join(RESOURCES, 'audio.raw')) + out, err = capsys.readouterr() + + assert re.search(r'how old is the Brooklyn Bridge', out, re.DOTALL | re.I) + + +def test_transcribe_gcs(capsys): + transcribe_async.transcribe_gcs( + 'gs://python-docs-samples-tests/speech/audio.flac') + out, err = capsys.readouterr() + + assert re.search(r'how old is the Brooklyn Bridge', out, re.DOTALL | re.I) diff --git a/speech/cloud-client/transcribe_auto_punctuation.py b/speech/cloud-client/transcribe_auto_punctuation.py new file mode 100644 index 00000000000..4e65afafaf4 --- /dev/null +++ b/speech/cloud-client/transcribe_auto_punctuation.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Google Cloud Speech API sample that demonstrates auto punctuation +and recognition metadata. + +Example usage: + python transcribe_auto_punctuation.py resources/commercial_mono.wav +""" + +import argparse +import io + + +def transcribe_file_with_auto_punctuation(path): + """Transcribe the given audio file with auto punctuation enabled.""" + # [START speech_transcribe_auto_punctuation] + from google.cloud import speech + client = speech.SpeechClient() + + # path = 'resources/commercial_mono.wav' + with io.open(path, 'rb') as audio_file: + content = audio_file.read() + + audio = speech.types.RecognitionAudio(content=content) + config = speech.types.RecognitionConfig( + encoding=speech.enums.RecognitionConfig.AudioEncoding.LINEAR16, + sample_rate_hertz=8000, + language_code='en-US', + # Enable automatic punctuation + enable_automatic_punctuation=True) + + response = client.recognize(config, audio) + + for i, result in enumerate(response.results): + alternative = result.alternatives[0] + print('-' * 20) + print('First alternative of result {}'.format(i)) + print('Transcript: {}'.format(alternative.transcript)) + # [END speech_transcribe_auto_punctuation] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('path', help='File to stream to the API') + + args = parser.parse_args() + + transcribe_file_with_auto_punctuation(args.path) diff --git a/speech/cloud-client/transcribe_auto_punctuation_test.py b/speech/cloud-client/transcribe_auto_punctuation_test.py new file mode 100644 index 00000000000..19db1e9c9d3 --- /dev/null +++ b/speech/cloud-client/transcribe_auto_punctuation_test.py @@ -0,0 +1,26 @@ +# Copyright 2018, Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import transcribe_auto_punctuation + +RESOURCES = os.path.join(os.path.dirname(__file__), 'resources') + + +def test_transcribe_file_with_auto_punctuation(capsys): + transcribe_auto_punctuation.transcribe_file_with_auto_punctuation( + 'resources/commercial_mono.wav') + out, _ = capsys.readouterr() + + assert 'Okay. Sure.' in out diff --git a/speech/cloud-client/transcribe_enhanced_model.py b/speech/cloud-client/transcribe_enhanced_model.py new file mode 100644 index 00000000000..1b233c52696 --- /dev/null +++ b/speech/cloud-client/transcribe_enhanced_model.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Google Cloud Speech API sample that demonstrates enhanced models +and recognition metadata. + +Example usage: + python transcribe_enhanced_model.py resources/commercial_mono.wav +""" + +import argparse + + +def transcribe_file_with_enhanced_model(path): + """Transcribe the given audio file using an enhanced model.""" + # [START speech_transcribe_enhanced_model] + import io + + from google.cloud import speech + + client = speech.SpeechClient() + + # path = 'resources/commercial_mono.wav' + with io.open(path, 'rb') as audio_file: + content = audio_file.read() + + audio = speech.types.RecognitionAudio(content=content) + config = speech.types.RecognitionConfig( + encoding=speech.enums.RecognitionConfig.AudioEncoding.LINEAR16, + sample_rate_hertz=8000, + language_code='en-US', + # Enhanced models are only available to projects that + # opt in for audio data collection. + use_enhanced=True, + # A model must be specified to use enhanced model. + model='phone_call') + + response = client.recognize(config, audio) + + for i, result in enumerate(response.results): + alternative = result.alternatives[0] + print('-' * 20) + print('First alternative of result {}'.format(i)) + print('Transcript: {}'.format(alternative.transcript)) + # [END speech_transcribe_enhanced_model] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('path', help='File to stream to the API') + + args = parser.parse_args() + + transcribe_file_with_enhanced_model(args.path) diff --git a/speech/cloud-client/transcribe_enhanced_model_test.py b/speech/cloud-client/transcribe_enhanced_model_test.py new file mode 100644 index 00000000000..6e5676cfb8f --- /dev/null +++ b/speech/cloud-client/transcribe_enhanced_model_test.py @@ -0,0 +1,26 @@ +# Copyright 2018, Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import transcribe_enhanced_model + +RESOURCES = os.path.join(os.path.dirname(__file__), 'resources') + + +def test_transcribe_file_with_enhanced_model(capsys): + transcribe_enhanced_model.transcribe_file_with_enhanced_model( + 'resources/commercial_mono.wav') + out, _ = capsys.readouterr() + + assert 'Chrome' in out diff --git a/speech/cloud-client/transcribe_model_selection.py b/speech/cloud-client/transcribe_model_selection.py new file mode 100644 index 00000000000..f81b9e72dd1 --- /dev/null +++ b/speech/cloud-client/transcribe_model_selection.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Google Cloud Speech API sample that demonstrates how to select the model +used for speech recognition. + +Example usage: + python transcribe_model_selection.py \ + resources/Google_Gnome.wav --model video + python transcribe_model_selection.py \ + gs://cloud-samples-tests/speech/Google_Gnome.wav --model video +""" + +import argparse + + +# [START speech_transcribe_model_selection] +def transcribe_model_selection(speech_file, model): + """Transcribe the given audio file synchronously with + the selected model.""" + from google.cloud import speech + client = speech.SpeechClient() + + with open(speech_file, 'rb') as audio_file: + content = audio_file.read() + + audio = speech.types.RecognitionAudio(content=content) + + config = speech.types.RecognitionConfig( + encoding=speech.enums.RecognitionConfig.AudioEncoding.LINEAR16, + sample_rate_hertz=16000, + language_code='en-US', + model=model) + + response = client.recognize(config, audio) + + for i, result in enumerate(response.results): + alternative = result.alternatives[0] + print('-' * 20) + print('First alternative of result {}'.format(i)) + print(u'Transcript: {}'.format(alternative.transcript)) +# [END speech_transcribe_model_selection] + + +# [START speech_transcribe_model_selection_gcs] +def transcribe_model_selection_gcs(gcs_uri, model): + """Transcribe the given audio file asynchronously with + the selected model.""" + from google.cloud import speech + client = speech.SpeechClient() + + audio = speech.types.RecognitionAudio(uri=gcs_uri) + + config = speech.types.RecognitionConfig( + encoding=speech.enums.RecognitionConfig.AudioEncoding.LINEAR16, + sample_rate_hertz=16000, + language_code='en-US', + model=model) + + operation = client.long_running_recognize(config, audio) + + print('Waiting for operation to complete...') + response = operation.result(timeout=90) + + for i, result in enumerate(response.results): + alternative = result.alternatives[0] + print('-' * 20) + print('First alternative of result {}'.format(i)) + print(u'Transcript: {}'.format(alternative.transcript)) +# [END speech_transcribe_model_selection_gcs] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + 'path', help='File or GCS path for audio file to be recognized') + parser.add_argument( + '--model', help='The speech recognition model to use', + choices=['command_and_search', 'phone_call', 'video', 'default'], + default='default') + + args = parser.parse_args() + + if args.path.startswith('gs://'): + transcribe_model_selection_gcs(args.path, args.model) + else: + transcribe_model_selection(args.path, args.model) diff --git a/speech/cloud-client/transcribe_model_selection_test.py b/speech/cloud-client/transcribe_model_selection_test.py new file mode 100644 index 00000000000..07bd91a4a0a --- /dev/null +++ b/speech/cloud-client/transcribe_model_selection_test.py @@ -0,0 +1,35 @@ +# Copyright 2016, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +import transcribe_model_selection + +RESOURCES = os.path.join(os.path.dirname(__file__), 'resources') + + +def test_transcribe_model_selection_file(capsys): + transcribe_model_selection.transcribe_model_selection( + os.path.join(RESOURCES, 'Google_Gnome.wav'), 'video') + out, err = capsys.readouterr() + + assert re.search(r'the weather outside is sunny', out, re.DOTALL | re.I) + + +def test_transcribe_model_selection_gcs(capsys): + transcribe_model_selection.transcribe_model_selection_gcs( + 'gs://cloud-samples-tests/speech/Google_Gnome.wav', 'video') + out, err = capsys.readouterr() + + assert re.search(r'the weather outside is sunny', out, re.DOTALL | re.I) diff --git a/speech/cloud-client/transcribe_multichannel.py b/speech/cloud-client/transcribe_multichannel.py new file mode 100644 index 00000000000..e84da59ad7b --- /dev/null +++ b/speech/cloud-client/transcribe_multichannel.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Google Cloud Speech API sample that demonstrates multichannel recognition. + +Example usage: + python transcribe_multichannel.py resources/multi.wav + python transcribe_multichannel.py \ + gs://cloud-samples-tests/speech/multi.wav +""" + +import argparse + + +def transcribe_file_with_multichannel(speech_file): + """Transcribe the given audio file synchronously with + multi channel.""" + # [START speech_transcribe_multichannel] + from google.cloud import speech + client = speech.SpeechClient() + + with open(speech_file, 'rb') as audio_file: + content = audio_file.read() + + audio = speech.types.RecognitionAudio(content=content) + + config = speech.types.RecognitionConfig( + encoding=speech.enums.RecognitionConfig.AudioEncoding.LINEAR16, + sample_rate_hertz=44100, + language_code='en-US', + audio_channel_count=2, + enable_separate_recognition_per_channel=True) + + response = client.recognize(config, audio) + + for i, result in enumerate(response.results): + alternative = result.alternatives[0] + print('-' * 20) + print('First alternative of result {}'.format(i)) + print(u'Transcript: {}'.format(alternative.transcript)) + print(u'Channel Tag: {}'.format(result.channel_tag)) + # [END speech_transcribe_multichannel] + + +def transcribe_gcs_with_multichannel(gcs_uri): + """Transcribe the given audio file on GCS with + multi channel.""" + # [START speech_transcribe_multichannel_gcs] + from google.cloud import speech + client = speech.SpeechClient() + + audio = speech.types.RecognitionAudio(uri=gcs_uri) + + config = speech.types.RecognitionConfig( + encoding=speech.enums.RecognitionConfig.AudioEncoding.LINEAR16, + sample_rate_hertz=44100, + language_code='en-US', + audio_channel_count=2, + enable_separate_recognition_per_channel=True) + + response = client.recognize(config, audio) + + for i, result in enumerate(response.results): + alternative = result.alternatives[0] + print('-' * 20) + print('First alternative of result {}'.format(i)) + print(u'Transcript: {}'.format(alternative.transcript)) + print(u'Channel Tag: {}'.format(result.channel_tag)) + # [END speech_transcribe_multichannel_gcs] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + 'path', help='File or GCS path for audio file to be recognized') + args = parser.parse_args() + if args.path.startswith('gs://'): + transcribe_gcs_with_multichannel(args.path) + else: + transcribe_file_with_multichannel(args.path) diff --git a/speech/cloud-client/transcribe_multichannel_test.py b/speech/cloud-client/transcribe_multichannel_test.py new file mode 100644 index 00000000000..de955862999 --- /dev/null +++ b/speech/cloud-client/transcribe_multichannel_test.py @@ -0,0 +1,36 @@ +# Copyright 2019, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from transcribe_multichannel import ( + transcribe_file_with_multichannel, + transcribe_gcs_with_multichannel) + +RESOURCES = os.path.join(os.path.dirname(__file__), 'resources') + + +def test_transcribe_multichannel_file(capsys): + transcribe_file_with_multichannel( + os.path.join(RESOURCES, 'multi.wav')) + out, err = capsys.readouterr() + + assert 'how are you doing' in out + + +def test_transcribe_multichannel_gcs(capsys): + transcribe_gcs_with_multichannel( + 'gs://cloud-samples-data/speech/multi.wav') + out, err = capsys.readouterr() + + assert 'how are you doing' in out diff --git a/speech/cloud-client/transcribe_streaming.py b/speech/cloud-client/transcribe_streaming.py new file mode 100644 index 00000000000..76829f5ba5a --- /dev/null +++ b/speech/cloud-client/transcribe_streaming.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Google Cloud Speech API sample application using the streaming API. + +Example usage: + python transcribe_streaming.py resources/audio.raw +""" + +import argparse +import io + + +# [START speech_transcribe_streaming] +def transcribe_streaming(stream_file): + """Streams transcription of the given audio file.""" + from google.cloud import speech + from google.cloud.speech import enums + from google.cloud.speech import types + client = speech.SpeechClient() + + # [START speech_python_migration_streaming_request] + with io.open(stream_file, 'rb') as audio_file: + content = audio_file.read() + + # In practice, stream should be a generator yielding chunks of audio data. + stream = [content] + requests = (types.StreamingRecognizeRequest(audio_content=chunk) + for chunk in stream) + + config = types.RecognitionConfig( + encoding=enums.RecognitionConfig.AudioEncoding.LINEAR16, + sample_rate_hertz=16000, + language_code='en-US') + streaming_config = types.StreamingRecognitionConfig(config=config) + + # streaming_recognize returns a generator. + # [START speech_python_migration_streaming_response] + responses = client.streaming_recognize(streaming_config, requests) + # [END speech_python_migration_streaming_request] + + for response in responses: + # Once the transcription has settled, the first result will contain the + # is_final result. The other results will be for subsequent portions of + # the audio. + for result in response.results: + print('Finished: {}'.format(result.is_final)) + print('Stability: {}'.format(result.stability)) + alternatives = result.alternatives + # The alternatives are ordered from most likely to least. + for alternative in alternatives: + print('Confidence: {}'.format(alternative.confidence)) + print(u'Transcript: {}'.format(alternative.transcript)) + # [END speech_python_migration_streaming_response] +# [END speech_transcribe_streaming] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('stream', help='File to stream to the API') + args = parser.parse_args() + transcribe_streaming(args.stream) diff --git a/speech/cloud-client/transcribe_streaming_test.py b/speech/cloud-client/transcribe_streaming_test.py new file mode 100644 index 00000000000..2b3ca8ee5c0 --- /dev/null +++ b/speech/cloud-client/transcribe_streaming_test.py @@ -0,0 +1,27 @@ +# Copyright 2017, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +import transcribe_streaming + +RESOURCES = os.path.join(os.path.dirname(__file__), 'resources') + + +def test_transcribe_streaming(capsys): + transcribe_streaming.transcribe_streaming( + os.path.join(RESOURCES, 'audio.raw')) + out, err = capsys.readouterr() + + assert re.search(r'how old is the Brooklyn Bridge', out, re.DOTALL | re.I) diff --git a/speech/cloud-client/transcribe_test.py b/speech/cloud-client/transcribe_test.py new file mode 100644 index 00000000000..d1e9f6338ea --- /dev/null +++ b/speech/cloud-client/transcribe_test.py @@ -0,0 +1,34 @@ +# Copyright 2016, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +import transcribe + +RESOURCES = os.path.join(os.path.dirname(__file__), 'resources') + + +def test_transcribe_file(capsys): + transcribe.transcribe_file(os.path.join(RESOURCES, 'audio.raw')) + out, err = capsys.readouterr() + + assert re.search(r'how old is the Brooklyn Bridge', out, re.DOTALL | re.I) + + +def test_transcribe_gcs(capsys): + transcribe.transcribe_gcs( + 'gs://python-docs-samples-tests/speech/audio.flac') + out, err = capsys.readouterr() + + assert re.search(r'how old is the Brooklyn Bridge', out, re.DOTALL | re.I) diff --git a/speech/cloud-client/transcribe_word_time_offsets.py b/speech/cloud-client/transcribe_word_time_offsets.py new file mode 100644 index 00000000000..43ddf38c9aa --- /dev/null +++ b/speech/cloud-client/transcribe_word_time_offsets.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Google Cloud Speech API sample that demonstrates word time offsets. + +Example usage: + python transcribe_word_time_offsets.py resources/audio.raw + python transcribe_word_time_offsets.py \ + gs://cloud-samples-tests/speech/vr.flac +""" + +import argparse +import io + + +def transcribe_file_with_word_time_offsets(speech_file): + """Transcribe the given audio file synchronously and output the word time + offsets.""" + from google.cloud import speech + from google.cloud.speech import enums + from google.cloud.speech import types + client = speech.SpeechClient() + + with io.open(speech_file, 'rb') as audio_file: + content = audio_file.read() + + audio = types.RecognitionAudio(content=content) + config = types.RecognitionConfig( + encoding=enums.RecognitionConfig.AudioEncoding.LINEAR16, + sample_rate_hertz=16000, + language_code='en-US', + enable_word_time_offsets=True) + + response = client.recognize(config, audio) + + for result in response.results: + alternative = result.alternatives[0] + print(u'Transcript: {}'.format(alternative.transcript)) + + for word_info in alternative.words: + word = word_info.word + start_time = word_info.start_time + end_time = word_info.end_time + print('Word: {}, start_time: {}, end_time: {}'.format( + word, + start_time.seconds + start_time.nanos * 1e-9, + end_time.seconds + end_time.nanos * 1e-9)) + + +# [START speech_transcribe_async_word_time_offsets_gcs] +def transcribe_gcs_with_word_time_offsets(gcs_uri): + """Transcribe the given audio file asynchronously and output the word time + offsets.""" + from google.cloud import speech + from google.cloud.speech import enums + from google.cloud.speech import types + client = speech.SpeechClient() + + audio = types.RecognitionAudio(uri=gcs_uri) + config = types.RecognitionConfig( + encoding=enums.RecognitionConfig.AudioEncoding.FLAC, + sample_rate_hertz=16000, + language_code='en-US', + enable_word_time_offsets=True) + + operation = client.long_running_recognize(config, audio) + + print('Waiting for operation to complete...') + result = operation.result(timeout=90) + + for result in result.results: + alternative = result.alternatives[0] + print(u'Transcript: {}'.format(alternative.transcript)) + print('Confidence: {}'.format(alternative.confidence)) + + for word_info in alternative.words: + word = word_info.word + start_time = word_info.start_time + end_time = word_info.end_time + print('Word: {}, start_time: {}, end_time: {}'.format( + word, + start_time.seconds + start_time.nanos * 1e-9, + end_time.seconds + end_time.nanos * 1e-9)) +# [END speech_transcribe_async_word_time_offsets_gcs] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + 'path', help='File or GCS path for audio file to be recognized') + args = parser.parse_args() + if args.path.startswith('gs://'): + transcribe_gcs_with_word_time_offsets(args.path) + else: + transcribe_file_with_word_time_offsets(args.path) diff --git a/speech/cloud-client/transcribe_word_time_offsets_test.py b/speech/cloud-client/transcribe_word_time_offsets_test.py new file mode 100644 index 00000000000..e894385f1e6 --- /dev/null +++ b/speech/cloud-client/transcribe_word_time_offsets_test.py @@ -0,0 +1,43 @@ +# Copyright 2016, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +import transcribe_word_time_offsets + +RESOURCES = os.path.join(os.path.dirname(__file__), 'resources') + + +def test_transcribe_file_with_word_time_offsets(capsys): + transcribe_word_time_offsets.transcribe_file_with_word_time_offsets( + os.path.join(RESOURCES, 'audio.raw')) + out, _ = capsys.readouterr() + + print(out) + match = re.search(r'Bridge, start_time: ([0-9.]+)', out, re.DOTALL | re.I) + time = float(match.group(1)) + + assert time > 0 + + +def test_transcribe_gcs_with_word_time_offsets(capsys): + transcribe_word_time_offsets.transcribe_gcs_with_word_time_offsets( + 'gs://python-docs-samples-tests/speech/audio.flac') + out, _ = capsys.readouterr() + + print(out) + match = re.search(r'Bridge, start_time: ([0-9.]+)', out, re.DOTALL | re.I) + time = float(match.group(1)) + + assert time > 0 diff --git a/speech/microphone/README.rst b/speech/microphone/README.rst new file mode 100644 index 00000000000..6363b5738a9 --- /dev/null +++ b/speech/microphone/README.rst @@ -0,0 +1,82 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Speech API Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=speech/microphone/README.rst + + +This directory contains samples for Google Cloud Speech API. The `Google Cloud Speech API`_ enables easy integration of Google speech recognition technologies into developer applications. Send audio and receive a text transcription from the Cloud Speech API service. + +- See the `migration guide`_ for information about migrating to Python client library v0.27. + +.. _migration guide: https://cloud.google.com/speech/docs/python-client-migration + + + + +.. _Google Cloud Speech API: https://cloud.google.com/speech/docs/ + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/speech/microphone/README.rst.in b/speech/microphone/README.rst.in new file mode 100644 index 00000000000..11831cca2df --- /dev/null +++ b/speech/microphone/README.rst.in @@ -0,0 +1,24 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Speech API + short_name: Cloud Speech API + url: https://cloud.google.com/speech/docs/ + description: > + The `Google Cloud Speech API`_ enables easy integration of Google speech + recognition technologies into developer applications. Send audio and receive + a text transcription from the Cloud Speech API service. + + + - See the `migration guide`_ for information about migrating to Python client library v0.27. + + + .. _migration guide: https://cloud.google.com/speech/docs/python-client-migration + +setup: +- auth +- install_deps + +cloud_client_library: true + +folder: speech/microphone \ No newline at end of file diff --git a/speech/microphone/requirements.txt b/speech/microphone/requirements.txt new file mode 100644 index 00000000000..88d0bd85cb1 --- /dev/null +++ b/speech/microphone/requirements.txt @@ -0,0 +1,3 @@ +google-cloud-speech==0.36.3 +pyaudio==0.2.11 +six==1.12.0 diff --git a/speech/microphone/resources/quit.raw b/speech/microphone/resources/quit.raw new file mode 100644 index 00000000000..a01dfc45a59 Binary files /dev/null and b/speech/microphone/resources/quit.raw differ diff --git a/speech/microphone/transcribe_streaming_indefinite.py b/speech/microphone/transcribe_streaming_indefinite.py new file mode 100644 index 00000000000..f1adb2247f1 --- /dev/null +++ b/speech/microphone/transcribe_streaming_indefinite.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Google Cloud Speech API sample application using the streaming API. + +NOTE: This module requires the additional dependency `pyaudio`. To install +using pip: + + pip install pyaudio + +Example usage: + python transcribe_streaming_indefinite.py +""" + +# [START speech_transcribe_infinite_streaming] +from __future__ import division + +import time +import re +import sys + +from google.cloud import speech + +import pyaudio +from six.moves import queue + +# Audio recording parameters +STREAMING_LIMIT = 55000 +SAMPLE_RATE = 16000 +CHUNK_SIZE = int(SAMPLE_RATE / 10) # 100ms + + +def get_current_time(): + return int(round(time.time() * 1000)) + + +def duration_to_secs(duration): + return duration.seconds + (duration.nanos / float(1e9)) + + +class ResumableMicrophoneStream: + """Opens a recording stream as a generator yielding the audio chunks.""" + def __init__(self, rate, chunk_size): + self._rate = rate + self._chunk_size = chunk_size + self._num_channels = 1 + self._max_replay_secs = 5 + + # Create a thread-safe buffer of audio data + self._buff = queue.Queue() + self.closed = True + self.start_time = get_current_time() + + # 2 bytes in 16 bit samples + self._bytes_per_sample = 2 * self._num_channels + self._bytes_per_second = self._rate * self._bytes_per_sample + + self._bytes_per_chunk = (self._chunk_size * self._bytes_per_sample) + self._chunks_per_second = ( + self._bytes_per_second // self._bytes_per_chunk) + + def __enter__(self): + self.closed = False + + self._audio_interface = pyaudio.PyAudio() + self._audio_stream = self._audio_interface.open( + format=pyaudio.paInt16, + channels=self._num_channels, + rate=self._rate, + input=True, + frames_per_buffer=self._chunk_size, + # Run the audio stream asynchronously to fill the buffer object. + # This is necessary so that the input device's buffer doesn't + # overflow while the calling thread makes network requests, etc. + stream_callback=self._fill_buffer, + ) + + return self + + def __exit__(self, type, value, traceback): + self._audio_stream.stop_stream() + self._audio_stream.close() + self.closed = True + # Signal the generator to terminate so that the client's + # streaming_recognize method will not block the process termination. + self._buff.put(None) + self._audio_interface.terminate() + + def _fill_buffer(self, in_data, *args, **kwargs): + """Continuously collect data from the audio stream, into the buffer.""" + self._buff.put(in_data) + return None, pyaudio.paContinue + + def generator(self): + while not self.closed: + if get_current_time() - self.start_time > STREAMING_LIMIT: + self.start_time = get_current_time() + break + # Use a blocking get() to ensure there's at least one chunk of + # data, and stop iteration if the chunk is None, indicating the + # end of the audio stream. + chunk = self._buff.get() + if chunk is None: + return + data = [chunk] + + # Now consume whatever other data's still buffered. + while True: + try: + chunk = self._buff.get(block=False) + if chunk is None: + return + data.append(chunk) + except queue.Empty: + break + + yield b''.join(data) + + +def listen_print_loop(responses, stream): + """Iterates through server responses and prints them. + + The responses passed is a generator that will block until a response + is provided by the server. + + Each response may contain multiple results, and each result may contain + multiple alternatives; for details, see https://goo.gl/tjCPAU. Here we + print only the transcription for the top alternative of the top result. + + In this case, responses are provided for interim results as well. If the + response is an interim one, print a line feed at the end of it, to allow + the next result to overwrite it, until the response is a final one. For the + final one, print a newline to preserve the finalized transcription. + """ + responses = (r for r in responses if ( + r.results and r.results[0].alternatives)) + + num_chars_printed = 0 + for response in responses: + if not response.results: + continue + + # The `results` list is consecutive. For streaming, we only care about + # the first result being considered, since once it's `is_final`, it + # moves on to considering the next utterance. + result = response.results[0] + if not result.alternatives: + continue + + # Display the transcription of the top alternative. + top_alternative = result.alternatives[0] + transcript = top_alternative.transcript + + # Display interim results, but with a carriage return at the end of the + # line, so subsequent lines will overwrite them. + # + # If the previous result was longer than this one, we need to print + # some extra spaces to overwrite the previous result + overwrite_chars = ' ' * (num_chars_printed - len(transcript)) + + if not result.is_final: + sys.stdout.write(transcript + overwrite_chars + '\r') + sys.stdout.flush() + + num_chars_printed = len(transcript) + else: + print(transcript + overwrite_chars) + + # Exit recognition if any of the transcribed phrases could be + # one of our keywords. + if re.search(r'\b(exit|quit)\b', transcript, re.I): + print('Exiting..') + stream.closed = True + break + + num_chars_printed = 0 + + +def main(): + client = speech.SpeechClient() + config = speech.types.RecognitionConfig( + encoding=speech.enums.RecognitionConfig.AudioEncoding.LINEAR16, + sample_rate_hertz=SAMPLE_RATE, + language_code='en-US', + max_alternatives=1, + enable_word_time_offsets=True) + streaming_config = speech.types.StreamingRecognitionConfig( + config=config, + interim_results=True) + + mic_manager = ResumableMicrophoneStream(SAMPLE_RATE, CHUNK_SIZE) + + print('Say "Quit" or "Exit" to terminate the program.') + + with mic_manager as stream: + while not stream.closed: + audio_generator = stream.generator() + requests = (speech.types.StreamingRecognizeRequest( + audio_content=content) + for content in audio_generator) + + responses = client.streaming_recognize(streaming_config, + requests) + # Now, put the transcription responses to use. + listen_print_loop(responses, stream) + + +if __name__ == '__main__': + main() +# [END speech_transcribe_infinite_streaming] diff --git a/speech/microphone/transcribe_streaming_mic.py b/speech/microphone/transcribe_streaming_mic.py new file mode 100644 index 00000000000..3ca7b709412 --- /dev/null +++ b/speech/microphone/transcribe_streaming_mic.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Google Cloud Speech API sample application using the streaming API. + +NOTE: This module requires the additional dependency `pyaudio`. To install +using pip: + + pip install pyaudio + +Example usage: + python transcribe_streaming_mic.py +""" + +# [START speech_transcribe_streaming_mic] +from __future__ import division + +import re +import sys + +from google.cloud import speech +from google.cloud.speech import enums +from google.cloud.speech import types +import pyaudio +from six.moves import queue + +# Audio recording parameters +RATE = 16000 +CHUNK = int(RATE / 10) # 100ms + + +class MicrophoneStream(object): + """Opens a recording stream as a generator yielding the audio chunks.""" + def __init__(self, rate, chunk): + self._rate = rate + self._chunk = chunk + + # Create a thread-safe buffer of audio data + self._buff = queue.Queue() + self.closed = True + + def __enter__(self): + self._audio_interface = pyaudio.PyAudio() + self._audio_stream = self._audio_interface.open( + format=pyaudio.paInt16, + # The API currently only supports 1-channel (mono) audio + # https://goo.gl/z757pE + channels=1, rate=self._rate, + input=True, frames_per_buffer=self._chunk, + # Run the audio stream asynchronously to fill the buffer object. + # This is necessary so that the input device's buffer doesn't + # overflow while the calling thread makes network requests, etc. + stream_callback=self._fill_buffer, + ) + + self.closed = False + + return self + + def __exit__(self, type, value, traceback): + self._audio_stream.stop_stream() + self._audio_stream.close() + self.closed = True + # Signal the generator to terminate so that the client's + # streaming_recognize method will not block the process termination. + self._buff.put(None) + self._audio_interface.terminate() + + def _fill_buffer(self, in_data, frame_count, time_info, status_flags): + """Continuously collect data from the audio stream, into the buffer.""" + self._buff.put(in_data) + return None, pyaudio.paContinue + + def generator(self): + while not self.closed: + # Use a blocking get() to ensure there's at least one chunk of + # data, and stop iteration if the chunk is None, indicating the + # end of the audio stream. + chunk = self._buff.get() + if chunk is None: + return + data = [chunk] + + # Now consume whatever other data's still buffered. + while True: + try: + chunk = self._buff.get(block=False) + if chunk is None: + return + data.append(chunk) + except queue.Empty: + break + + yield b''.join(data) + + +def listen_print_loop(responses): + """Iterates through server responses and prints them. + + The responses passed is a generator that will block until a response + is provided by the server. + + Each response may contain multiple results, and each result may contain + multiple alternatives; for details, see https://goo.gl/tjCPAU. Here we + print only the transcription for the top alternative of the top result. + + In this case, responses are provided for interim results as well. If the + response is an interim one, print a line feed at the end of it, to allow + the next result to overwrite it, until the response is a final one. For the + final one, print a newline to preserve the finalized transcription. + """ + num_chars_printed = 0 + for response in responses: + if not response.results: + continue + + # The `results` list is consecutive. For streaming, we only care about + # the first result being considered, since once it's `is_final`, it + # moves on to considering the next utterance. + result = response.results[0] + if not result.alternatives: + continue + + # Display the transcription of the top alternative. + transcript = result.alternatives[0].transcript + + # Display interim results, but with a carriage return at the end of the + # line, so subsequent lines will overwrite them. + # + # If the previous result was longer than this one, we need to print + # some extra spaces to overwrite the previous result + overwrite_chars = ' ' * (num_chars_printed - len(transcript)) + + if not result.is_final: + sys.stdout.write(transcript + overwrite_chars + '\r') + sys.stdout.flush() + + num_chars_printed = len(transcript) + + else: + print(transcript + overwrite_chars) + + # Exit recognition if any of the transcribed phrases could be + # one of our keywords. + if re.search(r'\b(exit|quit)\b', transcript, re.I): + print('Exiting..') + break + + num_chars_printed = 0 + + +def main(): + # See http://g.co/cloud/speech/docs/languages + # for a list of supported languages. + language_code = 'en-US' # a BCP-47 language tag + + client = speech.SpeechClient() + config = types.RecognitionConfig( + encoding=enums.RecognitionConfig.AudioEncoding.LINEAR16, + sample_rate_hertz=RATE, + language_code=language_code) + streaming_config = types.StreamingRecognitionConfig( + config=config, + interim_results=True) + + with MicrophoneStream(RATE, CHUNK) as stream: + audio_generator = stream.generator() + requests = (types.StreamingRecognizeRequest(audio_content=content) + for content in audio_generator) + + responses = client.streaming_recognize(streaming_config, requests) + + # Now, put the transcription responses to use. + listen_print_loop(responses) + + +if __name__ == '__main__': + main() +# [END speech_transcribe_streaming_mic] diff --git a/speech/microphone/transcribe_streaming_mic_test.py b/speech/microphone/transcribe_streaming_mic_test.py new file mode 100644 index 00000000000..dd5e7ea6f5e --- /dev/null +++ b/speech/microphone/transcribe_streaming_mic_test.py @@ -0,0 +1,69 @@ +# Copyright 2017, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +import threading +import time + +import mock + +RESOURCES = os.path.join(os.path.dirname(__file__), 'resources') + + +class MockPyAudio(object): + def __init__(self, audio_filename): + self.audio_filename = audio_filename + + def __call__(self, *args): + return self + + def open(self, stream_callback, rate, *args, **kwargs): + self.rate = rate + self.closed = threading.Event() + self.stream_thread = threading.Thread( + target=self.stream_audio, args=( + self.audio_filename, stream_callback, self.closed)) + self.stream_thread.start() + return self + + def close(self): + self.closed.set() + + def stop_stream(self): + pass + + def terminate(self): + pass + + def stream_audio(self, audio_filename, callback, closed, num_frames=512): + with open(audio_filename, 'rb') as audio_file: + while not closed.is_set(): + # Approximate realtime by sleeping for the appropriate time for + # the requested number of frames + time.sleep(num_frames / float(self.rate)) + # audio is 16-bit samples, whereas python byte is 8-bit + num_bytes = 2 * num_frames + chunk = audio_file.read(num_bytes) or b'\0' * num_bytes + callback(chunk, None, None, None) + + +@mock.patch.dict('sys.modules', pyaudio=mock.MagicMock( + PyAudio=MockPyAudio(os.path.join(RESOURCES, 'quit.raw')))) +def test_main(capsys): + import transcribe_streaming_mic + + transcribe_streaming_mic.main() + out, err = capsys.readouterr() + + assert re.search(r'quit', out, re.DOTALL | re.I) diff --git a/storage/README.md b/storage/README.md deleted file mode 100644 index 7476a656d5a..00000000000 --- a/storage/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# Google Cloud Storage Samples - -This section contains samples for [Google Cloud Storage](https://cloud.google.com/storage). - -## Running the samples - -1. Your environment must be setup with [authentication -information](https://developers.google.com/identity/protocols/application-default-credentials#howtheywork). If you're running in your local development environment and you have the [Google Cloud SDK](https://cloud.google.com/sdk/) installed, you can do this easily by running: - - $ gcloud init - -2. Install dependencies from `requirements.txt`: - - $ pip install -r requirements.txt - -3. Depending on the sample, you may also need to create resources on the [Google Developers Console](https://console.developers.google.com). Refer to the sample description and associated documentation page. - -## Additional resources - -For more information on Cloud Storage you can visit: - -> https://cloud.google.com/storage - -For more information on the Cloud Storage API Python library surface you -can visit: - -> https://developers.google.com/resources/api-libraries/documentation/storage/v1/python/latest/ - -For information on the Python Client Library visit: - -> https://developers.google.com/api-client-library/python diff --git a/storage/api/README.md b/storage/api/README.md deleted file mode 100644 index 287f60f010c..00000000000 --- a/storage/api/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# Cloud Storage API Samples - - -These samples are used on the following documentation pages: - -> -* https://cloud.google.com/storage/docs/json_api/v1/json-api-python-samples -* https://cloud.google.com/storage/docs/authentication -* https://cloud.google.com/docs/authentication - - diff --git a/storage/api/README.rst b/storage/api/README.rst new file mode 100644 index 00000000000..46e9323cd9f --- /dev/null +++ b/storage/api/README.rst @@ -0,0 +1,217 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Storage Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/api/README.rst + + +This directory contains samples for Google Cloud Storage. `Google Cloud Storage`_ allows world-wide storage and retrieval of any amount of data at any time. + + + + +.. _Google Cloud Storage: https://cloud.google.com/storage/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +List Objects ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/api/list_objects.py,storage/api/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python list_objects.py + + usage: list_objects.py [-h] bucket + + Command-line sample application for listing all objects in a bucket using + the Cloud Storage API. + + This sample is used on this page: + + https://cloud.google.com/storage/docs/json_api/v1/json-api-python-samples + + For more information, see the README.md under /storage. + + positional arguments: + bucket Your Cloud Storage bucket. + + optional arguments: + -h, --help show this help message and exit + + + +CRUD Objects ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/api/crud_object.py,storage/api/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python crud_object.py + + usage: crud_object.py [-h] [--reader READER] [--owner OWNER] filename bucket + + Application for uploading an object using the Cloud Storage API. + + This sample is used on this page: + + https://cloud.google.com/storage/docs/json_api/v1/json-api-python-samples + + For more information, see the README.md under /storage. + + positional arguments: + filename The name of the file to upload + bucket Your Cloud Storage bucket. + + optional arguments: + -h, --help show this help message and exit + --reader READER Your Cloud Storage bucket. + --owner OWNER Your Cloud Storage bucket. + + + +Compose objects ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/api/compose_objects.py,storage/api/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python compose_objects.py + + usage: compose_objects.py [-h] bucket destination sources [sources ...] + + Command-line sample application for composing objects using the Cloud + Storage API. + + This sample is used on this page: + + https://cloud.google.com/storage/docs/json_api/v1/json-api-python-samples + + For more information, see the README.md under /storage. + + To run, create a least two sample files: + $ echo "File 1" > file1.txt + $ echo "File 2" > file2.txt + + Example invocation: + $ python compose_objects.py my-bucket destination.txt file1.txt file2.txt + + positional arguments: + bucket Your Cloud Storage bucket. + destination Destination file name. + sources Source files to compose. + + optional arguments: + -h, --help show this help message and exit + + + +Customer-Supplied Encryption ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/api/customer_supplied_keys.py,storage/api/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python customer_supplied_keys.py + + usage: customer_supplied_keys.py [-h] bucket filename + + Command-line sample app demonstrating customer-supplied encryption keys. + + This sample demonstrates uploading an object while supplying an encryption key, + retrieving that object's contents, and finally rotating that key to a new + value. + + This sample is used on this page: + + https://cloud.google.com/storage/docs/json_api/v1/json-api-python-samples + + For more information, see the README.md under /storage. + + positional arguments: + bucket Your Cloud Storage bucket. + filename A file to upload and download. + + optional arguments: + -h, --help show this help message and exit + + + + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/storage/api/README.rst.in b/storage/api/README.rst.in new file mode 100644 index 00000000000..6bd64f097df --- /dev/null +++ b/storage/api/README.rst.in @@ -0,0 +1,29 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Storage + short_name: Cloud Storage + url: https://cloud.google.com/storage/docs + description: > + `Google Cloud Storage`_ allows world-wide storage and retrieval of any + amount of data at any time. + +setup: +- auth +- install_deps + +samples: +- name: List Objects + file: list_objects.py + show_help: true +- name: CRUD Objects + file: crud_object.py + show_help: true +- name: Compose objects + file: compose_objects.py + show_help: true +- name: Customer-Supplied Encryption + file: customer_supplied_keys.py + show_help: true + +folder: storage/api \ No newline at end of file diff --git a/storage/api/compose_objects.py b/storage/api/compose_objects.py index c542026508c..624bd1f92a4 100644 --- a/storage/api/compose_objects.py +++ b/storage/api/compose_objects.py @@ -36,19 +36,13 @@ import argparse import json -from googleapiclient import discovery -from oauth2client.client import GoogleCredentials +import googleapiclient.discovery def main(bucket, destination, sources): - # Get the application default credentials. When running locally, these are - # available after running `gcloud init`. When running on compute - # engine, these are available from the environment. - credentials = GoogleCredentials.get_application_default() - # Construct the service object for the interacting with the Cloud Storage # API. - service = discovery.build('storage', 'v1', credentials=credentials) + service = googleapiclient.discovery.build('storage', 'v1') # Upload the source files. for filename in sources: diff --git a/storage/api/compose_objects_test.py b/storage/api/compose_objects_test.py index 131adc6279d..1847096a91f 100644 --- a/storage/api/compose_objects_test.py +++ b/storage/api/compose_objects_test.py @@ -11,13 +11,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os + from compose_objects import main +RESOURCES = os.path.join(os.path.dirname(__file__), 'resources') +BUCKET = os.environ['CLOUD_STORAGE_BUCKET'] + -def test_main(cloud_config, resource): +def test_main(): main( - cloud_config.storage_bucket, + BUCKET, 'dest.txt', - [resource('file1.txt'), - resource('file2.txt')] + [os.path.join(RESOURCES, 'file1.txt'), + os.path.join(RESOURCES, 'file2.txt')] ) diff --git a/storage/api/crud_object.py b/storage/api/crud_object.py index 02f9cb8eba8..f93cbc177ad 100644 --- a/storage/api/crud_object.py +++ b/storage/api/crud_object.py @@ -24,14 +24,11 @@ """ import argparse -import filecmp import json import tempfile -from googleapiclient import discovery -from googleapiclient import http - -from oauth2client.client import GoogleCredentials +import googleapiclient.discovery +import googleapiclient.http def main(bucket, filename, readers=[], owners=[]): @@ -40,12 +37,8 @@ def main(bucket, filename, readers=[], owners=[]): print(json.dumps(resp, indent=2)) print('Fetching object..') - with tempfile.NamedTemporaryFile(mode='w+b') as tmpfile: + with tempfile.TemporaryFile(mode='w+b') as tmpfile: get_object(bucket, filename, out_file=tmpfile) - tmpfile.seek(0) - - if not filecmp.cmp(filename, tmpfile.name): - raise Exception('Downloaded file != uploaded object') print('Deleting object..') resp = delete_object(bucket, filename) @@ -55,16 +48,11 @@ def main(bucket, filename, readers=[], owners=[]): def create_service(): - # Get the application default credentials. When running locally, these are - # available after running `gcloud init`. When running on compute - # engine, these are available from the environment. - credentials = GoogleCredentials.get_application_default() - # Construct the service object for interacting with the Cloud Storage API - # the 'storage' service, at version 'v1'. # You can browse other available api services and versions here: - # http://g.co/dev/api-client-library/python/apis/ - return discovery.build('storage', 'v1', credentials=credentials) + # http://g.co/dv/api-client-library/python/apis/ + return googleapiclient.discovery.build('storage', 'v1') def upload_object(bucket, filename, readers, owners): @@ -95,14 +83,15 @@ def upload_object(bucket, filename, readers, owners): }) # Now insert them into the specified bucket as a media insertion. - # http://g.co/dev/resources/api-libraries/documentation/storage/v1/python/latest/storage_v1.objects.html#insert + # http://g.co/dv/resources/api-libraries/documentation/storage/v1/python/latest/storage_v1.objects.html#insert with open(filename, 'rb') as f: req = service.objects().insert( bucket=bucket, body=body, - # You can also just set media_body=filename, but # for the sake of + # You can also just set media_body=filename, but for the sake of # demonstration, pass in the more generic file handle, which could # very well be a StringIO or similar. - media_body=http.MediaIoBaseUpload(f, 'application/octet-stream')) + media_body=googleapiclient.http.MediaIoBaseUpload( + f, 'application/octet-stream')) resp = req.execute() return resp @@ -112,10 +101,10 @@ def get_object(bucket, filename, out_file): service = create_service() # Use get_media instead of get to get the actual contents of the object. - # http://g.co/dev/resources/api-libraries/documentation/storage/v1/python/latest/storage_v1.objects.html#get_media + # http://g.co/dv/resources/api-libraries/documentation/storage/v1/python/latest/storage_v1.objects.html#get_media req = service.objects().get_media(bucket=bucket, object=filename) - downloader = http.MediaIoBaseDownload(out_file, req) + downloader = googleapiclient.http.MediaIoBaseDownload(out_file, req) done = False while done is False: diff --git a/storage/api/crud_object_test.py b/storage/api/crud_object_test.py index 89e764b4d58..d2798a43649 100644 --- a/storage/api/crud_object_test.py +++ b/storage/api/crud_object_test.py @@ -11,13 +11,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import re from crud_object import main +BUCKET = os.environ['CLOUD_STORAGE_BUCKET'] -def test_main(cloud_config, capsys): - main(cloud_config.storage_bucket, __file__) + +def test_main(capsys): + main(BUCKET, __file__) out, err = capsys.readouterr() assert not re.search(r'Downloaded file [!]=', out) diff --git a/storage/api/customer_supplied_keys.py b/storage/api/customer_supplied_keys.py index 5b5551e8daa..bff738b4deb 100644 --- a/storage/api/customer_supplied_keys.py +++ b/storage/api/customer_supplied_keys.py @@ -30,9 +30,8 @@ import filecmp import tempfile -from googleapiclient import discovery -from googleapiclient import http -from oauth2client.client import GoogleCredentials +import googleapiclient.discovery +import googleapiclient.http # You can (and should) generate your own encryption key. Here's a good way to @@ -43,7 +42,7 @@ # is a bad idea to store your encryption keys in your source code. ENCRYPTION_KEY = '4RzDI0TeWa9M/nAvYH05qbCskPaSU/CFV5HeCxk0IUA=' -# You can use openssl to quicly calculate the hash of any key. +# You can use openssl to quickly calculate the hash of any key. # Try running this: # openssl base64 -d <<< ENCRYPTION_KEY | openssl dgst -sha256 -binary \ # | openssl base64 @@ -55,16 +54,11 @@ def create_service(): """Creates the service object for calling the Cloud Storage API.""" - # Get the application default credentials. When running locally, these are - # available after running `gcloud init`. When running on compute - # engine, these are available from the environment. - credentials = GoogleCredentials.get_application_default() - # Construct the service object for interacting with the Cloud Storage API - # the 'storage' service, at version 'v1'. # You can browse other available api services and versions here: # https://developers.google.com/api-client-library/python/apis/ - return discovery.build('storage', 'v1', credentials=credentials) + return googleapiclient.discovery.build('storage', 'v1') def upload_object(bucket, filename, encryption_key, key_hash): @@ -77,7 +71,8 @@ def upload_object(bucket, filename, encryption_key, key_hash): # You can also just set media_body=filename, but for the sake of # demonstration, pass in the more generic file handle, which could # very well be a StringIO or similar. - media_body=http.MediaIoBaseUpload(f, 'application/octet-stream')) + media_body=googleapiclient.http.MediaIoBaseUpload( + f, 'application/octet-stream')) request.headers['x-goog-encryption-algorithm'] = 'AES256' request.headers['x-goog-encryption-key'] = encryption_key request.headers['x-goog-encryption-key-sha256'] = key_hash diff --git a/storage/api/customer_supplied_keys_test.py b/storage/api/customer_supplied_keys_test.py index ec4ec0b5031..d6ff3cefd40 100644 --- a/storage/api/customer_supplied_keys_test.py +++ b/storage/api/customer_supplied_keys_test.py @@ -11,13 +11,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import re +from gcp_devrel.testing.flaky import flaky + from customer_supplied_keys import main +BUCKET = os.environ['CLOUD_STORAGE_BUCKET'] + -def test_main(cloud_config, capsys): - main(cloud_config.storage_bucket, __file__) +@flaky +def test_main(capsys): + main(BUCKET, __file__) out, err = capsys.readouterr() assert not re.search(r'Downloaded file [!]=', out) diff --git a/storage/api/list_objects.py b/storage/api/list_objects.py index ca0d80f0e10..99223e6608a 100644 --- a/storage/api/list_objects.py +++ b/storage/api/list_objects.py @@ -26,23 +26,16 @@ import argparse import json -from googleapiclient import discovery - -from oauth2client.client import GoogleCredentials +import googleapiclient.discovery def create_service(): """Creates the service object for calling the Cloud Storage API.""" - # Get the application default credentials. When running locally, these are - # available after running `gcloud init`. When running on compute - # engine, these are available from the environment. - credentials = GoogleCredentials.get_application_default() - # Construct the service object for interacting with the Cloud Storage API - # the 'storage' service, at version 'v1'. # You can browse other available api services and versions here: # https://developers.google.com/api-client-library/python/apis/ - return discovery.build('storage', 'v1', credentials=credentials) + return googleapiclient.discovery.build('storage', 'v1') def get_bucket_metadata(bucket): diff --git a/storage/api/list_objects_test.py b/storage/api/list_objects_test.py index 9910bba57e8..374dcb8f53a 100644 --- a/storage/api/list_objects_test.py +++ b/storage/api/list_objects_test.py @@ -11,8 +11,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os + from list_objects import main +BUCKET = os.environ['CLOUD_STORAGE_BUCKET'] + -def test_main(cloud_config): - main(cloud_config.storage_bucket) +def test_main(): + main(BUCKET) diff --git a/storage/api/requirements.txt b/storage/api/requirements.txt index c3b2784ce87..7e4359ce08d 100644 --- a/storage/api/requirements.txt +++ b/storage/api/requirements.txt @@ -1 +1,3 @@ -google-api-python-client==1.5.0 +google-api-python-client==1.7.8 +google-auth==1.6.2 +google-auth-httplib2==0.0.3 diff --git a/storage/cloud-client/README.rst b/storage/cloud-client/README.rst new file mode 100644 index 00000000000..dc8b7fdca1e --- /dev/null +++ b/storage/cloud-client/README.rst @@ -0,0 +1,412 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Storage Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/cloud-client/README.rst + + +This directory contains samples for Google Cloud Storage. `Google Cloud Storage`_ allows world-wide storage and retrieval of any amount of data at any time. + + + + +.. _Google Cloud Storage: https://cloud.google.com/storage/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Quickstart ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/cloud-client/quickstart.py,storage/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python quickstart.py + + +Snippets ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/cloud-client/snippets.py,storage/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python snippets.py + + usage: snippets.py [-h] + bucket_name + {create-bucket,delete-bucket,get-bucket-labels,add-bucket-label,remove-bucket-label,list,list-with-prefix,upload,enable-default-kms-key,upload-with-kms-key,download,delete,metadata,make-public,signed-url,rename,copy} + ... + + This application demonstrates how to perform basic operations on blobs + (objects) in a Google Cloud Storage bucket. + + For more information, see the README.md under /storage and the documentation + at https://cloud.google.com/storage/docs. + + positional arguments: + bucket_name Your cloud storage bucket. + {create-bucket,delete-bucket,get-bucket-labels,add-bucket-label,remove-bucket-label,list,list-with-prefix,upload,enable-default-kms-key,upload-with-kms-key,download,delete,metadata,make-public,signed-url,rename,copy} + create-bucket Creates a new bucket. + delete-bucket Deletes a bucket. The bucket must be empty. + get-bucket-labels Prints out a bucket's labels. + add-bucket-label Add a label to a bucket. + remove-bucket-label + Remove a label from a bucket. + list Lists all the blobs in the bucket. + list-with-prefix Lists all the blobs in the bucket that begin with the + prefix. This can be used to list all blobs in a + "folder", e.g. "public/". The delimiter argument can + be used to restrict the results to only the "files" in + the given "folder". Without the delimiter, the entire + tree under the prefix is returned. For example, given + these blobs: /a/1.txt /a/b/2.txt If you just specify + prefix = '/a', you'll get back: /a/1.txt /a/b/2.txt + However, if you specify prefix='/a' and delimiter='/', + you'll get back: /a/1.txt + upload Uploads a file to the bucket. + enable-default-kms-key + Sets a bucket's default KMS key. + upload-with-kms-key + Uploads a file to the bucket, encrypting it with the + given KMS key. + download Downloads a blob from the bucket. + delete Deletes a blob from the bucket. + metadata Prints out a blob's metadata. + make-public Makes a blob publicly accessible. + signed-url Generates a signed URL for a blob. Note that this + method requires a service account key file. You can + not use this if you are using Application Default + Credentials from Google Compute Engine or from the + Google Cloud SDK. + rename Renames a blob. + copy Renames a blob. + + optional arguments: + -h, --help show this help message and exit + + + +Access Control Lists ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/cloud-client/acl.py,storage/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python acl.py + + usage: acl.py [-h] + {print-bucket-acl,print-bucket-acl-for-user,add-bucket-owner,remove-bucket-owner,add-bucket-default-owner,remove-bucket-default-owner,print-blob-acl,print-blob-acl-for-user,add-blob-owner,remove-blob-owner} + ... + + This application demonstrates how to manage access control lists (acls) in + Google Cloud Storage. + + For more information, see the README.md under /storage and the documentation + at https://cloud.google.com/storage/docs/encryption. + + positional arguments: + {print-bucket-acl,print-bucket-acl-for-user,add-bucket-owner,remove-bucket-owner,add-bucket-default-owner,remove-bucket-default-owner,print-blob-acl,print-blob-acl-for-user,add-blob-owner,remove-blob-owner} + print-bucket-acl Prints out a bucket's access control list. + print-bucket-acl-for-user + Prints out a bucket's access control list. + add-bucket-owner Adds a user as an owner on the given bucket. + remove-bucket-owner + Removes a user from the access control list of the + given bucket. + add-bucket-default-owner + Adds a user as an owner in the given bucket's default + object access control list. + remove-bucket-default-owner + Removes a user from the access control list of the + given bucket's default object access control list. + print-blob-acl Prints out a blob's access control list. + print-blob-acl-for-user + Prints out a blob's access control list for a given + user. + add-blob-owner Adds a user as an owner on the given blob. + remove-blob-owner Removes a user from the access control list of the + given blob in the given bucket. + + optional arguments: + -h, --help show this help message and exit + + + +Customer-Supplied Encryption ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/cloud-client/encryption.py,storage/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python encryption.py + + usage: encryption.py [-h] {generate-encryption-key,upload,download,rotate} ... + + This application demonstrates how to upload and download encrypted blobs + (objects) in Google Cloud Storage. + + Use `generate-encryption-key` to generate an example key: + + python encryption.py generate-encryption-key + + Then use the key to upload and download files encrypted with a custom key. + + For more information, see the README.md under /storage and the documentation + at https://cloud.google.com/storage/docs/encryption. + + positional arguments: + {generate-encryption-key,upload,download,rotate} + generate-encryption-key + Generates a 256 bit (32 byte) AES encryption key and + prints the base64 representation. This is included for + demonstration purposes. You should generate your own + key. Please remember that encryption keys should be + handled with a comprehensive security policy. + upload Uploads a file to a Google Cloud Storage bucket using + a custom encryption key. The file will be encrypted by + Google Cloud Storage and only retrievable using the + provided encryption key. + download Downloads a previously-encrypted blob from Google + Cloud Storage. The encryption key provided must be the + same key provided when uploading the blob. + rotate Performs a key rotation by re-writing an encrypted + blob with a new encryption key. + + optional arguments: + -h, --help show this help message and exit + + + +Bucket Lock ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/cloud-client/bucket_lock.py,storage/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python bucket_lock.py + + usage: bucket_lock.py [-h] + {set-retention-policy,remove-retention-policy,lock-retention-policy,get-retention-policy,set-temporary-hold,release-temporary-hold,set-event-based-hold,release-event-based-hold,enable-default-event-based-hold,disable-default-event-based-hold,get-default-event-based-hold} + ... + + positional arguments: + {set-retention-policy,remove-retention-policy,lock-retention-policy,get-retention-policy,set-temporary-hold,release-temporary-hold,set-event-based-hold,release-event-based-hold,enable-default-event-based-hold,disable-default-event-based-hold,get-default-event-based-hold} + set-retention-policy + Defines a retention policy on a given bucket + remove-retention-policy + Removes the retention policy on a given bucket + lock-retention-policy + Locks the retention policy on a given bucket + get-retention-policy + Gets the retention policy on a given bucket + set-temporary-hold Sets a temporary hold on a given blob + release-temporary-hold + Releases the temporary hold on a given blob + set-event-based-hold + Sets a event based hold on a given blob + release-event-based-hold + Releases the event based hold on a given blob + enable-default-event-based-hold + Enables the default event based hold on a given bucket + disable-default-event-based-hold + Disables the default event based hold on a given + bucket + get-default-event-based-hold + Gets the default event based hold on a given bucket + + optional arguments: + -h, --help show this help message and exit + + + +Bucket Policy Only ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/cloud-client/bucket_policy_only.py,storage/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python bucket_policy_only.py + + usage: bucket_policy_only.py [-h] + {enable-bucket-policy-only,disable-bucket-policy-only,get-bucket-policy-only} + ... + + positional arguments: + {enable-bucket-policy-only,disable-bucket-policy-only,get-bucket-policy-only} + enable-bucket-policy-only + Enable Bucket Policy Only for a bucket + disable-bucket-policy-only + Disable Bucket Policy Only for a bucket + get-bucket-policy-only + Get Bucket Policy Only for a bucket + + optional arguments: + -h, --help show this help message and exit + + + +Notification Polling ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/cloud-client/notification_polling.py,storage/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python notification_polling.py + + usage: notification_polling.py [-h] project subscription + + This application demonstrates how to poll for GCS notifications from a + Cloud Pub/Sub subscription, parse the incoming message, and acknowledge the + successful processing of the message. + + This application will work with any subscription configured for pull rather + than push notifications. If you do not already have notifications configured, + you may consult the docs at + https://cloud.google.com/storage/docs/reporting-changes or follow the steps + below: + + 1. First, follow the common setup steps for these snippets, specically + configuring auth and installing dependencies. See the README's "Setup" + section. + + 2. Activate the Google Cloud Pub/Sub API, if you have not already done so. + https://console.cloud.google.com/flows/enableapi?apiid=pubsub + + 3. Create a Google Cloud Storage bucket: + $ gsutil mb gs://testbucket + + 4. Create a Cloud Pub/Sub topic and publish bucket notifications there: + $ gsutil notification create -f json -t testtopic gs://testbucket + + 5. Create a subscription for your new topic: + $ gcloud beta pubsub subscriptions create testsubscription --topic=testtopic + + 6. Run this program: + $ python notification_polling.py my-project-id testsubscription + + 7. While the program is running, upload and delete some files in the testbucket + bucket (you could use the console or gsutil) and watch as changes scroll by + in the app. + + positional arguments: + project The ID of the project that owns the subscription + subscription The ID of the Pub/Sub subscription + + optional arguments: + -h, --help show this help message and exit + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/storage/cloud-client/README.rst.in b/storage/cloud-client/README.rst.in new file mode 100644 index 00000000000..3b8f33af7f5 --- /dev/null +++ b/storage/cloud-client/README.rst.in @@ -0,0 +1,39 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Storage + short_name: Cloud Storage + url: https://cloud.google.com/storage/docs + description: > + `Google Cloud Storage`_ allows world-wide storage and retrieval of any + amount of data at any time. + +setup: +- auth +- install_deps + +samples: +- name: Quickstart + file: quickstart.py +- name: Snippets + file: snippets.py + show_help: true +- name: Access Control Lists + file: acl.py + show_help: true +- name: Customer-Supplied Encryption + file: encryption.py + show_help: true +- name: Bucket Lock + file: bucket_lock.py + show_help: true +- name: Bucket Policy Only + file: bucket_policy_only.py + show_help: true +- name: Notification Polling + file: notification_polling.py + show_help: true + +cloud_client_library: true + +folder: storage/cloud-client \ No newline at end of file diff --git a/storage/cloud-client/acl.py b/storage/cloud-client/acl.py new file mode 100644 index 00000000000..d742ae42849 --- /dev/null +++ b/storage/cloud-client/acl.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python + +# Copyright 2016 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to manage access control lists (acls) in +Google Cloud Storage. + +For more information, see the README.md under /storage and the documentation +at https://cloud.google.com/storage/docs/encryption. +""" + +import argparse + +from google.cloud import storage + + +def print_bucket_acl(bucket_name): + """Prints out a bucket's access control list.""" + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + for entry in bucket.acl: + print('{}: {}'.format(entry['role'], entry['entity'])) + + +def print_bucket_acl_for_user(bucket_name, user_email): + """Prints out a bucket's access control list for a given user.""" + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + # Reload fetches the current ACL from Cloud Storage. + bucket.acl.reload() + + # You can also use `group`, `domain`, `all_authenticated` and `all` to + # get the roles for different types of entities. + roles = bucket.acl.user(user_email).get_roles() + + print(roles) + + +def add_bucket_owner(bucket_name, user_email): + """Adds a user as an owner on the given bucket.""" + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + # Reload fetches the current ACL from Cloud Storage. + bucket.acl.reload() + + # You can also use `group()`, `domain()`, `all_authenticated()` and `all()` + # to grant access to different types of entities. + # You can also use `grant_read()` or `grant_write()` to grant different + # roles. + bucket.acl.user(user_email).grant_owner() + bucket.acl.save() + + print('Added user {} as an owner on bucket {}.'.format( + user_email, bucket_name)) + + +def remove_bucket_owner(bucket_name, user_email): + """Removes a user from the access control list of the given bucket.""" + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + # Reload fetches the current ACL from Cloud Storage. + bucket.acl.reload() + + # You can also use `group`, `domain`, `all_authenticated` and `all` to + # remove access for different types of entities. + bucket.acl.user(user_email).revoke_read() + bucket.acl.user(user_email).revoke_write() + bucket.acl.user(user_email).revoke_owner() + bucket.acl.save() + + print('Removed user {} from bucket {}.'.format( + user_email, bucket_name)) + + +def add_bucket_default_owner(bucket_name, user_email): + """Adds a user as an owner in the given bucket's default object access + control list.""" + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + # Reload fetches the current ACL from Cloud Storage. + bucket.acl.reload() + + # You can also use `group`, `domain`, `all_authenticated` and `all` to + # grant access to different types of entities. You can also use + # `grant_read` or `grant_write` to grant different roles. + bucket.default_object_acl.user(user_email).grant_owner() + bucket.default_object_acl.save() + + print('Added user {} as an owner in the default acl on bucket {}.'.format( + user_email, bucket_name)) + + +def remove_bucket_default_owner(bucket_name, user_email): + """Removes a user from the access control list of the given bucket's + default object access control list.""" + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + # Reload fetches the current ACL from Cloud Storage. + bucket.acl.reload() + + # You can also use `group`, `domain`, `all_authenticated` and `all` to + # remove access for different types of entities. + bucket.default_object_acl.user(user_email).revoke_read() + bucket.default_object_acl.user(user_email).revoke_write() + bucket.default_object_acl.user(user_email).revoke_owner() + bucket.default_object_acl.save() + + print('Removed user {} from the default acl of bucket {}.'.format( + user_email, bucket_name)) + + +def print_blob_acl(bucket_name, blob_name): + """Prints out a blob's access control list.""" + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + for entry in blob.acl: + print('{}: {}'.format(entry['role'], entry['entity'])) + + +def print_blob_acl_for_user(bucket_name, blob_name, user_email): + """Prints out a blob's access control list for a given user.""" + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + # Reload fetches the current ACL from Cloud Storage. + blob.acl.reload() + + # You can also use `group`, `domain`, `all_authenticated` and `all` to + # get the roles for different types of entities. + roles = blob.acl.user(user_email).get_roles() + + print(roles) + + +def add_blob_owner(bucket_name, blob_name, user_email): + """Adds a user as an owner on the given blob.""" + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + # Reload fetches the current ACL from Cloud Storage. + blob.acl.reload() + + # You can also use `group`, `domain`, `all_authenticated` and `all` to + # grant access to different types of entities. You can also use + # `grant_read` or `grant_write` to grant different roles. + blob.acl.user(user_email).grant_owner() + blob.acl.save() + + print('Added user {} as an owner on blob {} in bucket {}.'.format( + user_email, blob_name, bucket_name)) + + +def remove_blob_owner(bucket_name, blob_name, user_email): + """Removes a user from the access control list of the given blob in the + given bucket.""" + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + # You can also use `group`, `domain`, `all_authenticated` and `all` to + # remove access for different types of entities. + blob.acl.user(user_email).revoke_read() + blob.acl.user(user_email).revoke_write() + blob.acl.user(user_email).revoke_owner() + blob.acl.save() + + print('Removed user {} from blob {} in bucket {}.'.format( + user_email, blob_name, bucket_name)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + subparsers = parser.add_subparsers(dest='command') + + print_bucket_acl_parser = subparsers.add_parser( + 'print-bucket-acl', help=print_bucket_acl.__doc__) + print_bucket_acl_parser.add_argument('bucket_name') + + print_bucket_acl_for_user_parser = subparsers.add_parser( + 'print-bucket-acl-for-user', help=print_bucket_acl.__doc__) + print_bucket_acl_for_user_parser.add_argument('bucket_name') + print_bucket_acl_for_user_parser.add_argument('user_email') + + add_bucket_owner_parser = subparsers.add_parser( + 'add-bucket-owner', help=add_bucket_owner.__doc__) + add_bucket_owner_parser.add_argument('bucket_name') + add_bucket_owner_parser.add_argument('user_email') + + remove_bucket_owner_parser = subparsers.add_parser( + 'remove-bucket-owner', help=remove_bucket_owner.__doc__) + remove_bucket_owner_parser.add_argument('bucket_name') + remove_bucket_owner_parser.add_argument('user_email') + + add_bucket_default_owner_parser = subparsers.add_parser( + 'add-bucket-default-owner', help=add_bucket_default_owner.__doc__) + add_bucket_default_owner_parser.add_argument('bucket_name') + add_bucket_default_owner_parser.add_argument('user_email') + + remove_bucket_default_owner_parser = subparsers.add_parser( + 'remove-bucket-default-owner', + help=remove_bucket_default_owner.__doc__) + remove_bucket_default_owner_parser.add_argument('bucket_name') + remove_bucket_default_owner_parser.add_argument('user_email') + + print_blob_acl_parser = subparsers.add_parser( + 'print-blob-acl', help=print_blob_acl.__doc__) + print_blob_acl_parser.add_argument('bucket_name') + print_blob_acl_parser.add_argument('blob_name') + + print_blob_acl_for_user_parser = subparsers.add_parser( + 'print-blob-acl-for-user', help=print_blob_acl_for_user.__doc__) + print_blob_acl_for_user_parser.add_argument('bucket_name') + print_blob_acl_for_user_parser.add_argument('blob_name') + print_blob_acl_for_user_parser.add_argument('user_email') + + add_blob_owner_parser = subparsers.add_parser( + 'add-blob-owner', help=add_blob_owner.__doc__) + add_blob_owner_parser.add_argument('bucket_name') + add_blob_owner_parser.add_argument('blob_name') + add_blob_owner_parser.add_argument('user_email') + + remove_blob_owner_parser = subparsers.add_parser( + 'remove-blob-owner', help=remove_blob_owner.__doc__) + remove_blob_owner_parser.add_argument('bucket_name') + remove_blob_owner_parser.add_argument('blob_name') + remove_blob_owner_parser.add_argument('user_email') + + args = parser.parse_args() + + if args.command == 'print-bucket-acl': + print_bucket_acl(args.bucket_name) + elif args.command == 'print-bucket-acl-for-user': + print_bucket_acl_for_user(args.bucket_name, args.user_email) + elif args.command == 'add-bucket-owner': + add_bucket_owner(args.bucket_name, args.user_email) + elif args.command == 'remove-bucket-owner': + remove_bucket_owner(args.bucket_name, args.user_email) + elif args.command == 'add-bucket-default-owner': + add_bucket_default_owner(args.bucket_name, args.user_email) + elif args.command == 'remove-bucket-default-owner': + remove_bucket_default_owner(args.bucket_name, args.user_email) + elif args.command == 'print-blob-acl': + print_blob_acl(args.bucket_name, args.blob_name) + elif args.command == 'print-blob-acl-for-user': + print_blob_acl_for_user( + args.bucket_name, args.blob_name, args.user_email) + elif args.command == 'add-blob-owner': + add_blob_owner(args.bucket_name, args.blob_name, args.user_email) + elif args.command == 'remove-blob-owner': + remove_blob_owner(args.bucket_name, args.blob_name, args.user_email) diff --git a/storage/cloud-client/acl_test.py b/storage/cloud-client/acl_test.py new file mode 100644 index 00000000000..aeb1312ee17 --- /dev/null +++ b/storage/cloud-client/acl_test.py @@ -0,0 +1,145 @@ +# Copyright 2016 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time + +from google.cloud import storage +import google.cloud.storage.acl +import pytest + +import acl + +BUCKET = os.environ['CLOUD_STORAGE_BUCKET'] +# Typically we'd use a @example.com address, but GCS requires a real Google +# account. +TEST_EMAIL = ( + 'google-auth-system-tests' + '@python-docs-samples-tests.iam.gserviceaccount.com') + + +@pytest.fixture +def test_bucket(): + """Yields a bucket that resets its acl after the test completes.""" + bucket = storage.Client().bucket(BUCKET) + acl = google.cloud.storage.acl.BucketACL(bucket) + object_default_acl = google.cloud.storage.acl.DefaultObjectACL(bucket) + acl.reload() + object_default_acl.reload() + time.sleep(1) # bucket ops rate limited 1 update per second + yield bucket + time.sleep(1) # bucket ops rate limited 1 update per second + acl.save() + object_default_acl.save() + + +@pytest.fixture +def test_blob(): + """Yields a blob that resets its acl after the test completes.""" + bucket = storage.Client().bucket(BUCKET) + blob = bucket.blob('storage_acl_test_sigil') + blob.upload_from_string('Hello, is it me you\'re looking for?') + acl = google.cloud.storage.acl.ObjectACL(blob) + acl.reload() # bucket ops rate limited 1 update per second + time.sleep(1) + yield blob # bucket ops rate limited 1 update per second + time.sleep(1) + acl.save() + + +def test_print_bucket_acl(capsys): + acl.print_bucket_acl(BUCKET) + out, _ = capsys.readouterr() + assert out + + +def test_print_bucket_acl_for_user(test_bucket, capsys): + test_bucket.acl.user(TEST_EMAIL).grant_owner() + test_bucket.acl.save() + + acl.print_bucket_acl_for_user(BUCKET, TEST_EMAIL) + + out, _ = capsys.readouterr() + assert 'OWNER' in out + + +def test_add_bucket_owner(test_bucket): + acl.add_bucket_owner(BUCKET, TEST_EMAIL) + + test_bucket.acl.reload() + assert 'OWNER' in test_bucket.acl.user(TEST_EMAIL).get_roles() + + +def test_remove_bucket_owner(test_bucket): + test_bucket.acl.user(TEST_EMAIL).grant_owner() + test_bucket.acl.save() + + acl.remove_bucket_owner(BUCKET, TEST_EMAIL) + + test_bucket.acl.reload() + assert 'OWNER' not in test_bucket.acl.user(TEST_EMAIL).get_roles() + + +def test_add_bucket_default_owner(test_bucket): + acl.add_bucket_default_owner(BUCKET, TEST_EMAIL) + + test_bucket.default_object_acl.reload() + roles = test_bucket.default_object_acl.user(TEST_EMAIL).get_roles() + assert 'OWNER' in roles + + +def test_remove_bucket_default_owner(test_bucket): + test_bucket.acl.user(TEST_EMAIL).grant_owner() + test_bucket.acl.save() + + acl.remove_bucket_default_owner(BUCKET, TEST_EMAIL) + + test_bucket.default_object_acl.reload() + roles = test_bucket.default_object_acl.user(TEST_EMAIL).get_roles() + assert 'OWNER' not in roles + + +def test_print_blob_acl(test_blob, capsys): + acl.print_blob_acl(BUCKET, test_blob.name) + out, _ = capsys.readouterr() + assert out + + +def test_print_blob_acl_for_user(test_blob, capsys): + test_blob.acl.user(TEST_EMAIL).grant_owner() + test_blob.acl.save() + + acl.print_blob_acl_for_user( + BUCKET, test_blob.name, TEST_EMAIL) + + out, _ = capsys.readouterr() + assert 'OWNER' in out + + +def test_add_blob_owner(test_blob): + acl.add_blob_owner(BUCKET, test_blob.name, TEST_EMAIL) + + test_blob.acl.reload() + assert 'OWNER' in test_blob.acl.user(TEST_EMAIL).get_roles() + + +def test_remove_blob_owner(test_blob): + test_blob.acl.user(TEST_EMAIL).grant_owner() + test_blob.acl.save() + + acl.remove_blob_owner( + BUCKET, test_blob.name, TEST_EMAIL) + + test_blob.acl.reload() + assert 'OWNER' not in test_blob.acl.user(TEST_EMAIL).get_roles() diff --git a/storage/cloud-client/bucket_lock.py b/storage/cloud-client/bucket_lock.py new file mode 100644 index 00000000000..497176f9fcf --- /dev/null +++ b/storage/cloud-client/bucket_lock.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python + +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +from google.cloud import storage + + +def set_retention_policy(bucket_name, retention_period): + """Defines a retention policy on a given bucket""" + # [START storage_set_retention_policy] + # bucket_name = "my-bucket" + # retention_period = 10 + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + bucket.retention_period = retention_period + bucket.patch() + + print('Bucket {} retention period set for {} seconds'.format( + bucket.name, + bucket.retention_period)) + # [END storage_set_retention_policy] + + +def remove_retention_policy(bucket_name): + """Removes the retention policy on a given bucket""" + # [START storage_remove_retention_policy] + # bucket_name = "my-bucket" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + bucket.reload() + + if bucket.retention_policy_locked: + print( + 'Unable to remove retention period as retention policy is locked.') + return + + bucket.retention_period = None + bucket.patch() + + print('Removed bucket {} retention policy'.format(bucket.name)) + # [END storage_remove_retention_policy] + + +def lock_retention_policy(bucket_name): + """Locks the retention policy on a given bucket""" + # [START storage_lock_retention_policy] + # bucket_name = "my-bucket" + + storage_client = storage.Client() + # get_bucket gets the current metageneration value for the bucket, + # required by lock_retention_policy. + bucket = storage_client.get_bucket(bucket_name) + + # Warning: Once a retention policy is locked it cannot be unlocked + # and retention period can only be increased. + bucket.lock_retention_policy() + + print('Retention policy for {} is now locked'.format(bucket_name)) + print('Retention policy effective as of {}'.format( + bucket.retention_policy_effective_time)) + # [END storage_lock_retention_policy] + + +def get_retention_policy(bucket_name): + """Gets the retention policy on a given bucket""" + # [START storage_get_retention_policy] + # bucket_name = "my-bucket" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + bucket.reload() + + print('Retention Policy for {}'.format(bucket_name)) + print('Retention Period: {}'.format(bucket.retention_period)) + if bucket.retention_policy_locked: + print('Retention Policy is locked') + + if bucket.retention_policy_effective_time: + print('Effective Time: {}' + .format(bucket.retention_policy_effective_time)) + # [END storage_get_retention_policy] + + +def set_temporary_hold(bucket_name, blob_name): + """Sets a temporary hold on a given blob""" + # [START storage_set_temporary_hold] + # bucket_name = "my-bucket" + # blob_name = "my-blob" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + blob.temporary_hold = True + blob.patch() + + print("Temporary hold was set for #{blob_name}") + # [END storage_set_temporary_hold] + + +def release_temporary_hold(bucket_name, blob_name): + """Releases the temporary hold on a given blob""" + # [START storage_release_temporary_hold] + # bucket_name = "my-bucket" + # blob_name = "my-blob" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + blob.temporary_hold = False + blob.patch() + + print("Temporary hold was release for #{blob_name}") + # [END storage_release_temporary_hold] + + +def set_event_based_hold(bucket_name, blob_name): + """Sets a event based hold on a given blob""" + # [START storage_set_event_based_hold] + # bucket_name = "my-bucket" + # blob_name = "my-blob" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + blob.event_based_hold = True + blob.patch() + + print('Event based hold was set for {}'.format(blob_name)) + # [END storage_set_event_based_hold] + + +def release_event_based_hold(bucket_name, blob_name): + """Releases the event based hold on a given blob""" + # [START storage_release_event_based_hold] + # bucket_name = "my-bucket" + # blob_name = "my-blob" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + blob.event_based_hold = False + blob.patch() + + print('Event based hold was released for {}'.format(blob_name)) + # [END storage_release_event_based_hold] + + +def enable_default_event_based_hold(bucket_name): + """Enables the default event based hold on a given bucket""" + # [START storage_enable_default_event_based_hold] + # bucket_name = "my-bucket" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + bucket.default_event_based_hold = True + bucket.patch() + + print('Default event based hold was enabled for {}'.format(bucket_name)) + # [END storage_enable_default_event_based_hold] + + +def disable_default_event_based_hold(bucket_name): + """Disables the default event based hold on a given bucket""" + # [START storage_disable_default_event_based_hold] + # bucket_name = "my-bucket" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + bucket.default_event_based_hold = False + bucket.patch() + + print("Default event based hold was disabled for {}".format(bucket_name)) + # [END storage_disable_default_event_based_hold] + + +def get_default_event_based_hold(bucket_name): + """Gets the default event based hold on a given bucket""" + # [START storage_get_default_event_based_hold] + # bucket_name = "my-bucket" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + bucket.reload() + + if bucket.default_event_based_hold: + print('Default event-based hold is enabled for {}'.format(bucket_name)) + else: + print('Default event-based hold is not enabled for {}' + .format(bucket_name)) + # [END storage_get_default_event_based_hold] + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + subparsers = parser.add_subparsers(dest='command') + + set_retention_policy_parser = subparsers.add_parser( + 'set-retention-policy', help=set_retention_policy.__doc__) + set_retention_policy_parser.add_argument('bucket_name') + set_retention_policy_parser.add_argument('retention_period') + + remove_retention_policy_parser = subparsers.add_parser( + 'remove-retention-policy', help=remove_retention_policy.__doc__) + remove_retention_policy_parser.add_argument('bucket_name') + + lock_retention_policy_parser = subparsers.add_parser( + 'lock-retention-policy', help=lock_retention_policy.__doc__) + lock_retention_policy_parser.add_argument('bucket_name') + + get_retention_policy_parser = subparsers.add_parser( + 'get-retention-policy', help=get_retention_policy.__doc__) + get_retention_policy_parser.add_argument('bucket_name') + + set_temporary_hold_parser = subparsers.add_parser( + 'set-temporary-hold', help=set_temporary_hold.__doc__) + set_temporary_hold_parser.add_argument('bucket_name') + set_temporary_hold_parser.add_argument('blob_name') + + release_temporary_hold_parser = subparsers.add_parser( + 'release-temporary-hold', help=release_temporary_hold.__doc__) + release_temporary_hold_parser.add_argument('bucket_name') + release_temporary_hold_parser.add_argument('blob_name') + + set_event_based_hold_parser = subparsers.add_parser( + 'set-event-based-hold', help=set_event_based_hold.__doc__) + set_event_based_hold_parser.add_argument('bucket_name') + set_event_based_hold_parser.add_argument('blob_name') + + release_event_based_hold_parser = subparsers.add_parser( + 'release-event-based-hold', help=release_event_based_hold.__doc__) + release_event_based_hold_parser.add_argument('bucket_name') + release_event_based_hold_parser.add_argument('blob_name') + + enable_default_event_based_hold_parser = subparsers.add_parser( + 'enable-default-event-based-hold', + help=enable_default_event_based_hold.__doc__) + enable_default_event_based_hold_parser.add_argument('bucket_name') + + disable_default_event_based_hold_parser = subparsers.add_parser( + 'disable-default-event-based-hold', + help=disable_default_event_based_hold.__doc__) + disable_default_event_based_hold_parser.add_argument('bucket_name') + + get_default_event_based_hold_parser = subparsers.add_parser( + 'get-default-event-based-hold', + help=get_default_event_based_hold.__doc__) + get_default_event_based_hold_parser.add_argument('bucket_name') + + args = parser.parse_args() + + if args.command == 'set-retention-policy': + set_retention_policy(args.bucket_name, args.retention_period) + elif args.command == 'remove-retention-policy': + remove_retention_policy(args.bucket_name) + elif args.command == 'lock-retention-policy': + lock_retention_policy(args.bucket_name) + elif args.command == 'get-retention-policy': + get_retention_policy(args.bucket_name) + elif args.command == 'set-temporary-hold': + set_temporary_hold(args.bucket_name, args.blob_name) + elif args.command == 'release-temporary-hold': + release_temporary_hold(args.bucket_name, args.blob_name) + elif args.command == 'set-event-based-hold': + set_event_based_hold(args.bucket_name, args.blob_name) + elif args.command == 'release-event-based-hold': + release_event_based_hold(args.bucket_name, args.blob_name) + elif args.command == 'enable-default-event-based-hold': + enable_default_event_based_hold(args.bucket_name) + elif args.command == 'disable-default-event-based-hold': + disable_default_event_based_hold(args.bucket_name) + elif args.command == 'get-default-event-based-hold': + get_default_event_based_hold(args.bucket_name) diff --git a/storage/cloud-client/bucket_lock_test.py b/storage/cloud-client/bucket_lock_test.py new file mode 100644 index 00000000000..63e1afbaaca --- /dev/null +++ b/storage/cloud-client/bucket_lock_test.py @@ -0,0 +1,136 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +from google.cloud import storage + +import pytest + +import bucket_lock + +BLOB_NAME = 'storage_snippets_test_sigil' +BLOB_CONTENT = 'Hello, is it me you\'re looking for?' +# Retention policy for 5 seconds +RETENTION_POLICY = 5 + + +@pytest.fixture() +def bucket(): + """Creates a test bucket and deletes it upon completion.""" + client = storage.Client() + bucket_name = 'bucket-lock-' + str(int(time.time())) + bucket = client.create_bucket(bucket_name) + yield bucket + bucket.delete(force=True) + + +def test_retention_policy_no_lock(bucket, capsys): + bucket_lock.set_retention_policy(bucket.name, RETENTION_POLICY) + bucket.reload() + + assert bucket.retention_period is RETENTION_POLICY + assert bucket.retention_policy_effective_time is not None + assert bucket.retention_policy_locked is None + + bucket_lock.get_retention_policy(bucket.name) + out, _ = capsys.readouterr() + assert 'Retention Policy for {}'.format(bucket.name) in out + assert 'Retention Period: 5' in out + assert 'Effective Time: ' in out + assert 'Retention Policy is locked' not in out + + blob = bucket.blob(BLOB_NAME) + blob.upload_from_string(BLOB_CONTENT) + + assert blob.retention_expiration_time is not None + + bucket_lock.remove_retention_policy(bucket.name) + bucket.reload() + assert bucket.retention_period is None + + time.sleep(RETENTION_POLICY) + + +def test_retention_policy_lock(bucket, capsys): + bucket_lock.set_retention_policy(bucket.name, RETENTION_POLICY) + bucket.reload() + assert bucket.retention_policy_locked is None + + bucket_lock.lock_retention_policy(bucket.name) + bucket.reload() + assert bucket.retention_policy_locked is True + + bucket_lock.get_retention_policy(bucket.name) + out, _ = capsys.readouterr() + assert 'Retention Policy is locked' in out + + +def test_enable_disable_bucket_default_event_based_hold(bucket, capsys): + bucket_lock.get_default_event_based_hold(bucket.name) + out, _ = capsys.readouterr() + assert 'Default event-based hold is not enabled for {}'.format( + bucket.name) in out + assert 'Default event-based hold is enabled for {}'.format( + bucket.name) not in out + + bucket_lock.enable_default_event_based_hold(bucket.name) + bucket.reload() + + assert bucket.default_event_based_hold is True + + bucket_lock.get_default_event_based_hold(bucket.name) + out, _ = capsys.readouterr() + assert 'Default event-based hold is enabled for {}'.format( + bucket.name) in out + + blob = bucket.blob(BLOB_NAME) + blob.upload_from_string(BLOB_CONTENT) + assert blob.event_based_hold is True + + bucket_lock.release_event_based_hold(bucket.name, blob.name) + blob.reload() + assert blob.event_based_hold is False + + bucket_lock.disable_default_event_based_hold(bucket.name) + bucket.reload() + assert bucket.default_event_based_hold is False + + +def test_enable_disable_temporary_hold(bucket): + blob = bucket.blob(BLOB_NAME) + blob.upload_from_string(BLOB_CONTENT) + assert blob.temporary_hold is None + + bucket_lock.set_temporary_hold(bucket.name, blob.name) + blob.reload() + assert blob.temporary_hold is True + + bucket_lock.release_temporary_hold(bucket.name, blob.name) + blob.reload() + assert blob.temporary_hold is False + + +def test_enable_disable_event_based_hold(bucket): + blob = bucket.blob(BLOB_NAME) + blob.upload_from_string(BLOB_CONTENT) + assert blob.event_based_hold is None + + bucket_lock.set_event_based_hold(bucket.name, blob.name) + blob.reload() + assert blob.event_based_hold is True + + bucket_lock.release_event_based_hold(bucket.name, blob.name) + blob.reload() + assert blob.event_based_hold is False diff --git a/storage/cloud-client/bucket_policy_only.py b/storage/cloud-client/bucket_policy_only.py new file mode 100644 index 00000000000..53057454471 --- /dev/null +++ b/storage/cloud-client/bucket_policy_only.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +from google.cloud import storage + + +def enable_bucket_policy_only(bucket_name): + """Enable Bucket Policy Only for a bucket""" + # [START storage_enable_bucket_policy_only] + # bucket_name = "my-bucket" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + bucket.iam_configuration.bucket_policy_only_enabled = True + bucket.patch() + + print('Bucket Policy Only was enabled for {}.'.format(bucket.name)) + # [END storage_enable_bucket_policy_only] + + +def disable_bucket_policy_only(bucket_name): + """Disable Bucket Policy Only for a bucket""" + # [START storage_disable_bucket_policy_only] + # bucket_name = "my-bucket" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + bucket.iam_configuration.bucket_policy_only_enabled = False + bucket.patch() + + print('Bucket Policy Only was disabled for {}.'.format(bucket.name)) + # [END storage_disable_bucket_policy_only] + + +def get_bucket_policy_only(bucket_name): + """Get Bucket Policy Only for a bucket""" + # [START storage_get_bucket_policy_only] + # bucket_name = "my-bucket" + + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + iam_configuration = bucket.iam_configuration + + if iam_configuration.bucket_policy_only_enabled: + print('Bucket Policy Only is enabled for {}.'.format(bucket.name)) + print('Bucket will be locked on {}.'.format( + iam_configuration.bucket_policy_only_locked_time)) + else: + print('Bucket Policy Only is disabled for {}.'.format(bucket.name)) + # [END storage_get_bucket_policy_only] + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + subparsers = parser.add_subparsers(dest='command') + + enable_bucket_policy_only_parser = subparsers.add_parser( + 'enable-bucket-policy-only', help=enable_bucket_policy_only.__doc__) + enable_bucket_policy_only_parser.add_argument('bucket_name') + + disable_bucket_policy_only_parser = subparsers.add_parser( + 'disable-bucket-policy-only', help=disable_bucket_policy_only.__doc__) + disable_bucket_policy_only_parser.add_argument('bucket_name') + + get_bucket_policy_only_parser = subparsers.add_parser( + 'get-bucket-policy-only', help=get_bucket_policy_only.__doc__) + get_bucket_policy_only_parser.add_argument('bucket_name') + + args = parser.parse_args() + + if args.command == 'enable-bucket-policy-only': + enable_bucket_policy_only(args.bucket_name) + elif args.command == 'disable-bucket-policy-only': + disable_bucket_policy_only(args.bucket_name) + elif args.command == 'get-bucket-policy-only': + get_bucket_policy_only(args.bucket_name) diff --git a/storage/cloud-client/bucket_policy_only_test.py b/storage/cloud-client/bucket_policy_only_test.py new file mode 100644 index 00000000000..64a9dad10b3 --- /dev/null +++ b/storage/cloud-client/bucket_policy_only_test.py @@ -0,0 +1,52 @@ +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +from google.cloud import storage + +import pytest + +import bucket_policy_only + + +@pytest.fixture() +def bucket(): + """Creates a test bucket and deletes it upon completion.""" + client = storage.Client() + bucket_name = 'bucket-policy-only-' + str(int(time.time())) + bucket = client.create_bucket(bucket_name) + yield bucket + bucket.delete(force=True) + + +def test_get_bucket_policy_only(bucket, capsys): + bucket_policy_only.get_bucket_policy_only(bucket.name) + out, _ = capsys.readouterr() + assert 'Bucket Policy Only is disabled for {}.'.format( + bucket.name) in out + + +def test_enable_bucket_policy_only(bucket, capsys): + bucket_policy_only.enable_bucket_policy_only(bucket.name) + out, _ = capsys.readouterr() + assert 'Bucket Policy Only was enabled for {}.'.format( + bucket.name) in out + + +def test_disable_bucket_policy_only(bucket, capsys): + bucket_policy_only.disable_bucket_policy_only(bucket.name) + out, _ = capsys.readouterr() + assert 'Bucket Policy Only was disabled for {}.'.format( + bucket.name) in out diff --git a/storage/cloud-client/encryption.py b/storage/cloud-client/encryption.py new file mode 100644 index 00000000000..04718cc9df1 --- /dev/null +++ b/storage/cloud-client/encryption.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python + +# Copyright 2016 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to upload and download encrypted blobs +(objects) in Google Cloud Storage. + +Use `generate-encryption-key` to generate an example key: + + python encryption.py generate-encryption-key + +Then use the key to upload and download files encrypted with a custom key. + +For more information, see the README.md under /storage and the documentation +at https://cloud.google.com/storage/docs/encryption. +""" + +import argparse +import base64 +import os + +from google.cloud import storage +from google.cloud.storage import Blob + + +def generate_encryption_key(): + """Generates a 256 bit (32 byte) AES encryption key and prints the + base64 representation. + + This is included for demonstration purposes. You should generate your own + key. Please remember that encryption keys should be handled with a + comprehensive security policy. + """ + key = os.urandom(32) + encoded_key = base64.b64encode(key).decode('utf-8') + print('Base 64 encoded encryption key: {}'.format(encoded_key)) + + +def upload_encrypted_blob(bucket_name, source_file_name, + destination_blob_name, base64_encryption_key): + """Uploads a file to a Google Cloud Storage bucket using a custom + encryption key. + + The file will be encrypted by Google Cloud Storage and only + retrievable using the provided encryption key. + """ + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + # Encryption key must be an AES256 key represented as a bytestring with + # 32 bytes. Since it's passed in as a base64 encoded string, it needs + # to be decoded. + encryption_key = base64.b64decode(base64_encryption_key) + blob = Blob(destination_blob_name, bucket, encryption_key=encryption_key) + + blob.upload_from_filename(source_file_name) + + print('File {} uploaded to {}.'.format( + source_file_name, + destination_blob_name)) + + +def download_encrypted_blob(bucket_name, source_blob_name, + destination_file_name, base64_encryption_key): + """Downloads a previously-encrypted blob from Google Cloud Storage. + + The encryption key provided must be the same key provided when uploading + the blob. + """ + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + # Encryption key must be an AES256 key represented as a bytestring with + # 32 bytes. Since it's passed in as a base64 encoded string, it needs + # to be decoded. + encryption_key = base64.b64decode(base64_encryption_key) + blob = Blob(source_blob_name, bucket, encryption_key=encryption_key) + + blob.download_to_filename(destination_file_name) + + print('Blob {} downloaded to {}.'.format( + source_blob_name, + destination_file_name)) + + +def rotate_encryption_key(bucket_name, blob_name, base64_encryption_key, + base64_new_encryption_key): + """Performs a key rotation by re-writing an encrypted blob with a new + encryption key.""" + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + current_encryption_key = base64.b64decode(base64_encryption_key) + new_encryption_key = base64.b64decode(base64_new_encryption_key) + + # Both source_blob and destination_blob refer to the same storage object, + # but destination_blob has the new encryption key. + source_blob = Blob( + blob_name, bucket, encryption_key=current_encryption_key) + destination_blob = Blob( + blob_name, bucket, encryption_key=new_encryption_key) + + token = None + + while True: + token, bytes_rewritten, total_bytes = destination_blob.rewrite( + source_blob, token=token) + if token is None: + break + + print('Key rotation complete for Blob {}'.format(blob_name)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + subparsers = parser.add_subparsers(dest='command') + + subparsers.add_parser( + 'generate-encryption-key', help=generate_encryption_key.__doc__) + + upload_parser = subparsers.add_parser( + 'upload', help=upload_encrypted_blob.__doc__) + upload_parser.add_argument( + 'bucket_name', help='Your cloud storage bucket.') + upload_parser.add_argument('source_file_name') + upload_parser.add_argument('destination_blob_name') + upload_parser.add_argument('base64_encryption_key') + + download_parser = subparsers.add_parser( + 'download', help=download_encrypted_blob.__doc__) + download_parser.add_argument( + 'bucket_name', help='Your cloud storage bucket.') + download_parser.add_argument('source_blob_name') + download_parser.add_argument('destination_file_name') + download_parser.add_argument('base64_encryption_key') + + rotate_parser = subparsers.add_parser( + 'rotate', help=rotate_encryption_key.__doc__) + rotate_parser.add_argument( + 'bucket_name', help='Your cloud storage bucket.') + rotate_parser.add_argument('blob_name') + rotate_parser.add_argument('base64_encryption_key') + rotate_parser.add_argument('base64_new_encryption_key') + + args = parser.parse_args() + + if args.command == 'generate-encryption-key': + generate_encryption_key() + elif args.command == 'upload': + upload_encrypted_blob( + args.bucket_name, + args.source_file_name, + args.destination_blob_name, + args.base64_encryption_key) + elif args.command == 'download': + download_encrypted_blob( + args.bucket_name, + args.source_blob_name, + args.destination_file_name, + args.base64_encryption_key) + elif args.command == 'rotate': + rotate_encryption_key( + args.bucket_name, + args.blob_name, + args.base64_encryption_key, + args.base64_new_encryption_key) diff --git a/storage/cloud-client/encryption_test.py b/storage/cloud-client/encryption_test.py new file mode 100644 index 00000000000..4db6e6cb0f0 --- /dev/null +++ b/storage/cloud-client/encryption_test.py @@ -0,0 +1,93 @@ +# Copyright 2016 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import os +import tempfile + +from google.cloud import storage +from google.cloud.storage import Blob +import pytest + +import encryption + +BUCKET = os.environ['CLOUD_STORAGE_BUCKET'] + +TEST_ENCRYPTION_KEY = 'brtJUWneL92g5q0N2gyDSnlPSYAiIVZ/cWgjyZNeMy0=' +TEST_ENCRYPTION_KEY_DECODED = base64.b64decode(TEST_ENCRYPTION_KEY) + +TEST_ENCRYPTION_KEY_2 = 'o4OD7SWCaPjfeEGhAY+YCgMdY9UW+OJ8mvfWD9lNtO4=' +TEST_ENCRYPTION_KEY_2_DECODED = base64.b64decode(TEST_ENCRYPTION_KEY_2) + + +def test_generate_encryption_key(capsys): + encryption.generate_encryption_key() + out, _ = capsys.readouterr() + encoded_key = out.split(':', 1).pop().strip() + key = base64.b64decode(encoded_key) + assert len(key) == 32, 'Returned key should be 32 bytes' + + +def test_upload_encrypted_blob(): + with tempfile.NamedTemporaryFile() as source_file: + source_file.write(b'test') + + encryption.upload_encrypted_blob( + BUCKET, + source_file.name, + 'test_encrypted_upload_blob', + TEST_ENCRYPTION_KEY) + + +@pytest.fixture +def test_blob(): + """Provides a pre-existing blob in the test bucket.""" + bucket = storage.Client().bucket(BUCKET) + blob = Blob('encryption_test_sigil', + bucket, encryption_key=TEST_ENCRYPTION_KEY_DECODED) + content = 'Hello, is it me you\'re looking for?' + blob.upload_from_string(content) + return blob.name, content + + +def test_download_blob(test_blob): + test_blob_name, test_blob_content = test_blob + with tempfile.NamedTemporaryFile() as dest_file: + encryption.download_encrypted_blob( + BUCKET, + test_blob_name, + dest_file.name, + TEST_ENCRYPTION_KEY) + + downloaded_content = dest_file.read().decode('utf-8') + assert downloaded_content == test_blob_content + + +def test_rotate_encryption_key(test_blob): + test_blob_name, test_blob_content = test_blob + encryption.rotate_encryption_key( + BUCKET, + test_blob_name, + TEST_ENCRYPTION_KEY, + TEST_ENCRYPTION_KEY_2) + + with tempfile.NamedTemporaryFile() as dest_file: + encryption.download_encrypted_blob( + BUCKET, + test_blob_name, + dest_file.name, + TEST_ENCRYPTION_KEY_2) + + downloaded_content = dest_file.read().decode('utf-8') + assert downloaded_content == test_blob_content diff --git a/storage/cloud-client/iam.py b/storage/cloud-client/iam.py new file mode 100644 index 00000000000..ba20bc1dde2 --- /dev/null +++ b/storage/cloud-client/iam.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python + +# Copyright 2017 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to get and set IAM policies on Google +Cloud Storage buckets. + +For more information, see the documentation at +https://cloud.google.com/storage/docs/access-control/using-iam-permissions. +""" + +import argparse + +from google.cloud import storage + + +def view_bucket_iam_members(bucket_name): + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + policy = bucket.get_iam_policy() + + for role in policy: + members = policy[role] + print('Role: {}, Members: {}'.format(role, members)) + + +def add_bucket_iam_member(bucket_name, role, member): + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + policy = bucket.get_iam_policy() + + policy[role].add(member) + + bucket.set_iam_policy(policy) + + print('Added {} with role {} to {}.'.format( + member, role, bucket_name)) + + +def remove_bucket_iam_member(bucket_name, role, member): + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + policy = bucket.get_iam_policy() + + policy[role].discard(member) + + bucket.set_iam_policy(policy) + + print('Removed {} with role {} from {}.'.format( + member, role, bucket_name)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('bucket_name', help='Your Cloud Storage bucket name.') + subparsers = parser.add_subparsers(dest='command') + + subparsers.add_parser( + 'view-bucket-iam-members', help=view_bucket_iam_members.__doc__) + + add_member_parser = subparsers.add_parser( + 'add-bucket-iam-member', help=add_bucket_iam_member.__doc__) + add_member_parser.add_argument('role') + add_member_parser.add_argument('member') + + remove_member_parser = subparsers.add_parser( + 'remove-bucket-iam-member', help=remove_bucket_iam_member.__doc__) + remove_member_parser.add_argument('role') + remove_member_parser.add_argument('member') + + args = parser.parse_args() + + if args.command == 'view-bucket-iam-members': + view_bucket_iam_members(args.bucket_name) + elif args.command == 'add-bucket-iam-member': + add_bucket_iam_member(args.bucket_name, args.role, args.member) + elif args.command == 'remove-bucket-iam-member': + remove_bucket_iam_member(args.bucket_name, args.role, args.member) diff --git a/storage/cloud-client/iam_test.py b/storage/cloud-client/iam_test.py new file mode 100644 index 00000000000..0c823afa00e --- /dev/null +++ b/storage/cloud-client/iam_test.py @@ -0,0 +1,45 @@ +# Copyright 2017 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from google.cloud import storage +import pytest + +import iam + +BUCKET = os.environ['CLOUD_STORAGE_BUCKET'] +MEMBER = 'group:dpebot@google.com' +ROLE = 'roles/storage.legacyBucketReader' + + +@pytest.fixture +def bucket(): + yield storage.Client().bucket(BUCKET) + + +def test_view_bucket_iam_members(): + iam.view_bucket_iam_members(BUCKET) + + +def test_add_bucket_iam_member(bucket): + iam.add_bucket_iam_member( + BUCKET, ROLE, MEMBER) + assert MEMBER in bucket.get_iam_policy()[ROLE] + + +def test_remove_bucket_iam_member(bucket): + iam.remove_bucket_iam_member( + BUCKET, ROLE, MEMBER) + assert MEMBER not in bucket.get_iam_policy()[ROLE] diff --git a/storage/cloud-client/notification_polling.py b/storage/cloud-client/notification_polling.py new file mode 100644 index 00000000000..88868f9481c --- /dev/null +++ b/storage/cloud-client/notification_polling.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python + +# Copyright 2017 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to poll for GCS notifications from a +Cloud Pub/Sub subscription, parse the incoming message, and acknowledge the +successful processing of the message. + +This application will work with any subscription configured for pull rather +than push notifications. If you do not already have notifications configured, +you may consult the docs at +https://cloud.google.com/storage/docs/reporting-changes or follow the steps +below: + +1. First, follow the common setup steps for these snippets, specically + configuring auth and installing dependencies. See the README's "Setup" + section. + +2. Activate the Google Cloud Pub/Sub API, if you have not already done so. + https://console.cloud.google.com/flows/enableapi?apiid=pubsub + +3. Create a Google Cloud Storage bucket: + $ gsutil mb gs://testbucket + +4. Create a Cloud Pub/Sub topic and publish bucket notifications there: + $ gsutil notification create -f json -t testtopic gs://testbucket + +5. Create a subscription for your new topic: + $ gcloud beta pubsub subscriptions create testsubscription --topic=testtopic + +6. Run this program: + $ python notification_polling.py my-project-id testsubscription + +7. While the program is running, upload and delete some files in the testbucket + bucket (you could use the console or gsutil) and watch as changes scroll by + in the app. +""" + +import argparse +import json +import time + +from google.cloud import pubsub_v1 + + +def summarize(message): + # [START parse_message] + data = message.data.decode('utf-8') + attributes = message.attributes + + event_type = attributes['eventType'] + bucket_id = attributes['bucketId'] + object_id = attributes['objectId'] + generation = attributes['objectGeneration'] + description = ( + '\tEvent type: {event_type}\n' + '\tBucket ID: {bucket_id}\n' + '\tObject ID: {object_id}\n' + '\tGeneration: {generation}\n').format( + event_type=event_type, + bucket_id=bucket_id, + object_id=object_id, + generation=generation) + + if 'overwroteGeneration' in attributes: + description += '\tOverwrote generation: %s\n' % ( + attributes['overwroteGeneration']) + if 'overwrittenByGeneration' in attributes: + description += '\tOverwritten by generation: %s\n' % ( + attributes['overwrittenByGeneration']) + + payload_format = attributes['payloadFormat'] + if payload_format == 'JSON_API_V1': + object_metadata = json.loads(data) + size = object_metadata['size'] + content_type = object_metadata['contentType'] + metageneration = object_metadata['metageneration'] + description += ( + '\tContent type: {content_type}\n' + '\tSize: {object_size}\n' + '\tMetageneration: {metageneration}\n').format( + content_type=content_type, + object_size=size, + metageneration=metageneration) + return description + # [END parse_message] + + +def poll_notifications(project, subscription_name): + """Polls a Cloud Pub/Sub subscription for new GCS events for display.""" + # [BEGIN poll_notifications] + subscriber = pubsub_v1.SubscriberClient() + subscription_path = subscriber.subscription_path( + project, subscription_name) + + def callback(message): + print('Received message:\n{}'.format(summarize(message))) + message.ack() + + subscriber.subscribe(subscription_path, callback=callback) + + # The subscriber is non-blocking, so we must keep the main thread from + # exiting to allow it to process messages in the background. + print('Listening for messages on {}'.format(subscription_path)) + while True: + time.sleep(60) + # [END poll_notifications] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + 'project', + help='The ID of the project that owns the subscription') + parser.add_argument('subscription', + help='The ID of the Pub/Sub subscription') + args = parser.parse_args() + poll_notifications(args.project, args.subscription) diff --git a/storage/cloud-client/notification_polling_test.py b/storage/cloud-client/notification_polling_test.py new file mode 100644 index 00000000000..b816bd9df48 --- /dev/null +++ b/storage/cloud-client/notification_polling_test.py @@ -0,0 +1,51 @@ +# Copyright 2017 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from google.cloud.pubsub_v1.subscriber.message import Message +import mock + +from notification_polling import summarize + + +MESSAGE_ID = 12345 + + +def test_parse_json_message(): + attributes = { + 'eventType': 'OBJECT_FINALIZE', + 'bucketId': 'mybucket', + 'objectId': 'myobject', + 'objectGeneration': 1234567, + 'resource': 'projects/_/buckets/mybucket/objects/myobject#1234567', + 'notificationConfig': ('projects/_/buckets/mybucket/' + 'notificationConfigs/5'), + 'payloadFormat': 'JSON_API_V1'} + data = (b'{' + b' "size": 12345,' + b' "contentType": "text/html",' + b' "metageneration": 1' + b'}') + message = Message( + mock.Mock(data=data, attributes=attributes), + MESSAGE_ID, + mock.Mock()) + assert summarize(message) == ( + '\tEvent type: OBJECT_FINALIZE\n' + '\tBucket ID: mybucket\n' + '\tObject ID: myobject\n' + '\tGeneration: 1234567\n' + '\tContent type: text/html\n' + '\tSize: 12345\n' + '\tMetageneration: 1\n') diff --git a/storage/cloud-client/quickstart.py b/storage/cloud-client/quickstart.py new file mode 100644 index 00000000000..9aff9b21492 --- /dev/null +++ b/storage/cloud-client/quickstart.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def run_quickstart(): + # [START storage_quickstart] + # Imports the Google Cloud client library + from google.cloud import storage + + # Instantiates a client + storage_client = storage.Client() + + # The name for the new bucket + bucket_name = 'my-new-bucket' + + # Creates the new bucket + bucket = storage_client.create_bucket(bucket_name) + + print('Bucket {} created.'.format(bucket.name)) + # [END storage_quickstart] + + +if __name__ == '__main__': + run_quickstart() diff --git a/storage/cloud-client/quickstart_test.py b/storage/cloud-client/quickstart_test.py new file mode 100644 index 00000000000..cb0503972ff --- /dev/null +++ b/storage/cloud-client/quickstart_test.py @@ -0,0 +1,28 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +import quickstart + + +@mock.patch('google.cloud.storage.client.Client.create_bucket') +def test_quickstart(create_bucket_mock, capsys): + # Unlike other quickstart tests, this one mocks out the creation + # because buckets are expensive, globally-namespaced object. + create_bucket_mock.return_value = mock.sentinel.bucket + + quickstart.run_quickstart() + + create_bucket_mock.assert_called_with('my-new-bucket') diff --git a/storage/cloud-client/requester_pays.py b/storage/cloud-client/requester_pays.py new file mode 100644 index 00000000000..b98ba96f188 --- /dev/null +++ b/storage/cloud-client/requester_pays.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python + +# Copyright 2017 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to use requester pays features on Google +Cloud Storage buckets. + +For more information, see the documentation at +https://cloud.google.com/storage/docs/using-requester-pays. +""" + +import argparse + +from google.cloud import storage + + +def get_requester_pays_status(bucket_name): + """Get a bucket's requester pays metadata""" + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + requester_pays_status = bucket.requester_pays + if requester_pays_status: + print('Requester Pays is enabled for {}'.format(bucket_name)) + else: + print('Requester Pays is disabled for {}'.format(bucket_name)) + + +def enable_requester_pays(bucket_name): + """Enable a bucket's requesterpays metadata""" + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + bucket.requester_pays = True + bucket.patch() + print('Requester Pays has been enabled for {}'.format(bucket_name)) + + +def disable_requester_pays(bucket_name): + """Disable a bucket's requesterpays metadata""" + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + bucket.requester_pays = False + bucket.patch() + print('Requester Pays has been disabled for {}'.format(bucket_name)) + + +def download_file_requester_pays( + bucket_name, project_id, source_blob_name, destination_file_name): + """Download file using specified project as the requester""" + storage_client = storage.Client() + user_project = project_id + bucket = storage_client.bucket(bucket_name, user_project) + blob = bucket.blob(source_blob_name) + blob.download_to_filename(destination_file_name) + + print('Blob {} downloaded to {} using a requester-pays request.'.format( + source_blob_name, + destination_file_name)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('bucket_name', help='Your Cloud Storage bucket name.') + subparsers = parser.add_subparsers(dest='command') + + subparsers.add_parser( + 'check-status', help=get_requester_pays_status.__doc__) + + subparsers.add_parser( + 'enable', help=enable_requester_pays.__doc__) + + subparsers.add_parser( + 'disable', help=disable_requester_pays.__doc__) + + download_parser = subparsers.add_parser( + 'download', help=download_file_requester_pays.__doc__) + download_parser.add_argument('project') + download_parser.add_argument('source_blob_name') + download_parser.add_argument('destination_file_name') + + args = parser.parse_args() + + if args.command == 'check-status': + get_requester_pays_status(args.bucket_name) + elif args.command == 'enable': + enable_requester_pays(args.bucket_name) + elif args.command == 'disable': + disable_requester_pays(args.bucket_name) + elif args.command == 'download': + download_file_requester_pays( + args.bucket_name, args.project, args.source_blob_name, + args.destination_file_name) diff --git a/storage/cloud-client/requester_pays_test.py b/storage/cloud-client/requester_pays_test.py new file mode 100644 index 00000000000..05c9a2275b1 --- /dev/null +++ b/storage/cloud-client/requester_pays_test.py @@ -0,0 +1,62 @@ +# Copyright 2017 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import tempfile + +from google.cloud import storage +import pytest + +import requester_pays + +BUCKET = os.environ['CLOUD_STORAGE_BUCKET'] +PROJECT = os.environ['GCLOUD_PROJECT'] + + +def test_enable_requester_pays(capsys): + requester_pays.enable_requester_pays(BUCKET) + out, _ = capsys.readouterr() + assert 'Requester Pays has been enabled for {}'.format(BUCKET) in out + + +def test_disable_requester_pays(capsys): + requester_pays.disable_requester_pays(BUCKET) + out, _ = capsys.readouterr() + assert 'Requester Pays has been disabled for {}'.format(BUCKET) in out + + +def test_get_requester_pays_status(capsys): + requester_pays.get_requester_pays_status(BUCKET) + out, _ = capsys.readouterr() + assert 'Requester Pays is disabled for {}'.format(BUCKET) in out + + +@pytest.fixture +def test_blob(): + """Provides a pre-existing blob in the test bucket.""" + bucket = storage.Client().bucket(BUCKET) + blob = bucket.blob('storage_snippets_test_sigil') + blob.upload_from_string('Hello, is it me you\'re looking for?') + return blob + + +def test_download_file_requester_pays(test_blob, capsys): + with tempfile.NamedTemporaryFile() as dest_file: + requester_pays.download_file_requester_pays( + BUCKET, + PROJECT, + test_blob.name, + dest_file.name) + + assert dest_file.read() diff --git a/storage/cloud-client/requirements.txt b/storage/cloud-client/requirements.txt new file mode 100644 index 00000000000..4cb3be26a0b --- /dev/null +++ b/storage/cloud-client/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-pubsub==0.39.1 +google-cloud-storage==1.14.0 diff --git a/storage/cloud-client/snippets.py b/storage/cloud-client/snippets.py new file mode 100644 index 00000000000..fcc0ca060ff --- /dev/null +++ b/storage/cloud-client/snippets.py @@ -0,0 +1,411 @@ +#!/usr/bin/env python + +# Copyright 2016 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to perform basic operations on blobs +(objects) in a Google Cloud Storage bucket. + +For more information, see the README.md under /storage and the documentation +at https://cloud.google.com/storage/docs. +""" + +import argparse +import datetime +import pprint + +# [START storage_upload_file] +from google.cloud import storage + +# [END storage_upload_file] + + +def create_bucket(bucket_name): + """Creates a new bucket.""" + storage_client = storage.Client() + bucket = storage_client.create_bucket(bucket_name) + print('Bucket {} created'.format(bucket.name)) + + +def delete_bucket(bucket_name): + """Deletes a bucket. The bucket must be empty.""" + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + bucket.delete() + print('Bucket {} deleted'.format(bucket.name)) + + +def enable_default_kms_key(bucket_name, kms_key_name): + # [START storage_set_bucket_default_kms_key] + """Sets a bucket's default KMS key.""" + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + bucket.default_kms_key_name = kms_key_name + bucket.patch() + + print('Set default KMS key for bucket {} to {}.'.format( + bucket.name, + bucket.default_kms_key_name)) + # [END storage_set_bucket_default_kms_key] + + +def get_bucket_labels(bucket_name): + """Prints out a bucket's labels.""" + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + labels = bucket.labels + pprint.pprint(labels) + + +def add_bucket_label(bucket_name): + """Add a label to a bucket.""" + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + + labels = bucket.labels + labels['example'] = 'label' + bucket.labels = labels + bucket.patch() + + print('Updated labels on {}.'.format(bucket.name)) + pprint.pprint(bucket.labels) + + +def remove_bucket_label(bucket_name): + """Remove a label from a bucket.""" + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + + labels = bucket.labels + + if 'example' in labels: + del labels['example'] + + bucket.labels = labels + bucket.patch() + + print('Updated labels on {}.'.format(bucket.name)) + pprint.pprint(bucket.labels) + + +def list_blobs(bucket_name): + """Lists all the blobs in the bucket.""" + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + + blobs = bucket.list_blobs() + + for blob in blobs: + print(blob.name) + + +def list_blobs_with_prefix(bucket_name, prefix, delimiter=None): + """Lists all the blobs in the bucket that begin with the prefix. + + This can be used to list all blobs in a "folder", e.g. "public/". + + The delimiter argument can be used to restrict the results to only the + "files" in the given "folder". Without the delimiter, the entire tree under + the prefix is returned. For example, given these blobs: + + /a/1.txt + /a/b/2.txt + + If you just specify prefix = '/a', you'll get back: + + /a/1.txt + /a/b/2.txt + + However, if you specify prefix='/a' and delimiter='/', you'll get back: + + /a/1.txt + + """ + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + + blobs = bucket.list_blobs(prefix=prefix, delimiter=delimiter) + + print('Blobs:') + for blob in blobs: + print(blob.name) + + if delimiter: + print('Prefixes:') + for prefix in blobs.prefixes: + print(prefix) + + +# [START storage_upload_file] +def upload_blob(bucket_name, source_file_name, destination_blob_name): + """Uploads a file to the bucket.""" + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + blob = bucket.blob(destination_blob_name) + + blob.upload_from_filename(source_file_name) + + print('File {} uploaded to {}.'.format( + source_file_name, + destination_blob_name)) +# [END storage_upload_file] + + +def upload_blob_with_kms(bucket_name, source_file_name, destination_blob_name, + kms_key_name): + # [START storage_upload_with_kms_key] + """Uploads a file to the bucket, encrypting it with the given KMS key.""" + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + blob = bucket.blob(destination_blob_name, kms_key_name=kms_key_name) + blob.upload_from_filename(source_file_name) + + print('File {} uploaded to {} with encryption key {}.'.format( + source_file_name, + destination_blob_name, + kms_key_name)) + # [END storage_upload_with_kms_key] + + +def download_blob(bucket_name, source_blob_name, destination_file_name): + """Downloads a blob from the bucket.""" + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + blob = bucket.blob(source_blob_name) + + blob.download_to_filename(destination_file_name) + + print('Blob {} downloaded to {}.'.format( + source_blob_name, + destination_file_name)) + + +def delete_blob(bucket_name, blob_name): + """Deletes a blob from the bucket.""" + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + blob = bucket.blob(blob_name) + + blob.delete() + + print('Blob {} deleted.'.format(blob_name)) + + +def blob_metadata(bucket_name, blob_name): + """Prints out a blob's metadata.""" + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + blob = bucket.get_blob(blob_name) + + print('Blob: {}'.format(blob.name)) + print('Bucket: {}'.format(blob.bucket.name)) + print('Storage class: {}'.format(blob.storage_class)) + print('ID: {}'.format(blob.id)) + print('Size: {} bytes'.format(blob.size)) + print('Updated: {}'.format(blob.updated)) + print('Generation: {}'.format(blob.generation)) + print('Metageneration: {}'.format(blob.metageneration)) + print('Etag: {}'.format(blob.etag)) + print('Owner: {}'.format(blob.owner)) + print('Component count: {}'.format(blob.component_count)) + print('Crc32c: {}'.format(blob.crc32c)) + print('md5_hash: {}'.format(blob.md5_hash)) + print('Cache-control: {}'.format(blob.cache_control)) + print('Content-type: {}'.format(blob.content_type)) + print('Content-disposition: {}'.format(blob.content_disposition)) + print('Content-encoding: {}'.format(blob.content_encoding)) + print('Content-language: {}'.format(blob.content_language)) + print('Metadata: {}'.format(blob.metadata)) + print("Temporary hold: ", + 'enabled' if blob.temporary_hold else 'disabled') + print("Event based hold: ", + 'enabled' if blob.event_based_hold else 'disabled') + if blob.retention_expiration_time: + print("retentionExpirationTime: {}" + .format(blob.retention_expiration_time)) + + +def make_blob_public(bucket_name, blob_name): + """Makes a blob publicly accessible.""" + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + blob = bucket.blob(blob_name) + + blob.make_public() + + print('Blob {} is publicly accessible at {}'.format( + blob.name, blob.public_url)) + + +def generate_signed_url(bucket_name, blob_name): + """Generates a signed URL for a blob. + + Note that this method requires a service account key file. You can not use + this if you are using Application Default Credentials from Google Compute + Engine or from the Google Cloud SDK. + """ + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + blob = bucket.blob(blob_name) + + url = blob.generate_signed_url( + # This URL is valid for 1 hour + expiration=datetime.timedelta(hours=1), + # Allow GET requests using this URL. + method='GET') + + print('The signed url for {} is {}'.format(blob.name, url)) + return url + + +def rename_blob(bucket_name, blob_name, new_name): + """Renames a blob.""" + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + blob = bucket.blob(blob_name) + + new_blob = bucket.rename_blob(blob, new_name) + + print('Blob {} has been renamed to {}'.format( + blob.name, new_blob.name)) + + +def copy_blob(bucket_name, blob_name, new_bucket_name, new_blob_name): + """Copies a blob from one bucket to another with a new name.""" + storage_client = storage.Client() + source_bucket = storage_client.get_bucket(bucket_name) + source_blob = source_bucket.blob(blob_name) + destination_bucket = storage_client.get_bucket(new_bucket_name) + + new_blob = source_bucket.copy_blob( + source_blob, destination_bucket, new_blob_name) + + print('Blob {} in bucket {} copied to blob {} in bucket {}.'.format( + source_blob.name, source_bucket.name, new_blob.name, + destination_bucket.name)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('bucket_name', help='Your cloud storage bucket.') + + subparsers = parser.add_subparsers(dest='command') + subparsers.add_parser('create-bucket', help=create_bucket.__doc__) + subparsers.add_parser('delete-bucket', help=delete_bucket.__doc__) + subparsers.add_parser('get-bucket-labels', help=get_bucket_labels.__doc__) + subparsers.add_parser('add-bucket-label', help=add_bucket_label.__doc__) + subparsers.add_parser( + 'remove-bucket-label', help=remove_bucket_label.__doc__) + subparsers.add_parser('list', help=list_blobs.__doc__) + + list_with_prefix_parser = subparsers.add_parser( + 'list-with-prefix', help=list_blobs_with_prefix.__doc__) + list_with_prefix_parser.add_argument('prefix') + list_with_prefix_parser.add_argument('--delimiter', default=None) + + upload_parser = subparsers.add_parser('upload', help=upload_blob.__doc__) + upload_parser.add_argument('source_file_name') + upload_parser.add_argument('destination_blob_name') + + enable_default_kms_parser = subparsers.add_parser( + 'enable-default-kms-key', help=enable_default_kms_key.__doc__) + enable_default_kms_parser.add_argument('kms_key_name') + + upload_kms_parser = subparsers.add_parser( + 'upload-with-kms-key', help=upload_blob_with_kms.__doc__) + upload_kms_parser.add_argument('source_file_name') + upload_kms_parser.add_argument('destination_blob_name') + upload_kms_parser.add_argument('kms_key_name') + + download_parser = subparsers.add_parser( + 'download', help=download_blob.__doc__) + download_parser.add_argument('source_blob_name') + download_parser.add_argument('destination_file_name') + + delete_parser = subparsers.add_parser('delete', help=delete_blob.__doc__) + delete_parser.add_argument('blob_name') + + metadata_parser = subparsers.add_parser( + 'metadata', help=blob_metadata.__doc__) + metadata_parser.add_argument('blob_name') + + make_public_parser = subparsers.add_parser( + 'make-public', help=make_blob_public.__doc__) + make_public_parser.add_argument('blob_name') + + signed_url_parser = subparsers.add_parser( + 'signed-url', help=generate_signed_url.__doc__) + signed_url_parser.add_argument('blob_name') + + rename_parser = subparsers.add_parser('rename', help=rename_blob.__doc__) + rename_parser.add_argument('blob_name') + rename_parser.add_argument('new_name') + + copy_parser = subparsers.add_parser('copy', help=rename_blob.__doc__) + copy_parser.add_argument('blob_name') + copy_parser.add_argument('new_bucket_name') + copy_parser.add_argument('new_blob_name') + + args = parser.parse_args() + + if args.command == 'create-bucket': + create_bucket(args.bucket_name) + if args.command == 'enable-default-kms-key': + enable_default_kms_key(args.bucket_name, args.kms_key_name) + elif args.command == 'delete-bucket': + delete_bucket(args.bucket_name) + if args.command == 'get-bucket-labels': + get_bucket_labels(args.bucket_name) + if args.command == 'add-bucket-label': + add_bucket_label(args.bucket_name) + if args.command == 'remove-bucket-label': + remove_bucket_label(args.bucket_name) + elif args.command == 'list': + list_blobs(args.bucket_name) + elif args.command == 'list-with-prefix': + list_blobs_with_prefix(args.bucket_name, args.prefix, args.delimiter) + elif args.command == 'upload': + upload_blob( + args.bucket_name, + args.source_file_name, + args.destination_blob_name) + elif args.command == 'upload-with-kms-key': + upload_blob_with_kms( + args.bucket_name, + args.source_file_name, + args.destination_blob_name, + args.kms_key_name) + elif args.command == 'download': + download_blob( + args.bucket_name, + args.source_blob_name, + args.destination_file_name) + elif args.command == 'delete': + delete_blob(args.bucket_name, args.blob_name) + elif args.command == 'metadata': + blob_metadata(args.bucket_name, args.blob_name) + elif args.command == 'make-public': + make_blob_public(args.bucket_name, args.blob_name) + elif args.command == 'signed-url': + generate_signed_url(args.bucket_name, args.blob_name) + elif args.command == 'rename': + rename_blob(args.bucket_name, args.blob_name, args.new_name) + elif args.command == 'copy': + copy_blob( + args.bucket_name, + args.blob_name, + args.new_bucket_name, + args.new_blob_name) diff --git a/storage/cloud-client/snippets_test.py b/storage/cloud-client/snippets_test.py new file mode 100644 index 00000000000..ec444b36b43 --- /dev/null +++ b/storage/cloud-client/snippets_test.py @@ -0,0 +1,174 @@ +# Copyright 2016 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import tempfile +import time + +from google.cloud import storage +import google.cloud.exceptions +import pytest +import requests + +import snippets + +BUCKET = os.environ['CLOUD_STORAGE_BUCKET'] +KMS_KEY = os.environ['CLOUD_KMS_KEY'] + + +def test_enable_default_kms_key(): + snippets.enable_default_kms_key( + bucket_name=BUCKET, + kms_key_name=KMS_KEY) + time.sleep(2) # Let change propagate as needed + bucket = storage.Client().get_bucket(BUCKET) + assert bucket.default_kms_key_name.startswith(KMS_KEY) + bucket.default_kms_key_name = None + bucket.patch() + + +def test_get_bucket_labels(): + snippets.get_bucket_labels(BUCKET) + + +def test_add_bucket_label(capsys): + snippets.add_bucket_label(BUCKET) + out, _ = capsys.readouterr() + assert 'example' in out + + +@pytest.mark.xfail( + reason=( + 'https://github.com/GoogleCloudPlatform' + '/google-cloud-python/issues/3711')) +def test_remove_bucket_label(capsys): + snippets.add_bucket_label(BUCKET) + snippets.remove_bucket_label(BUCKET) + out, _ = capsys.readouterr() + assert '{}' in out + + +@pytest.fixture +def test_blob(): + """Provides a pre-existing blob in the test bucket.""" + bucket = storage.Client().bucket(BUCKET) + blob = bucket.blob('storage_snippets_test_sigil') + blob.upload_from_string('Hello, is it me you\'re looking for?') + return blob + + +def test_list_blobs(test_blob, capsys): + snippets.list_blobs(BUCKET) + out, _ = capsys.readouterr() + assert test_blob.name in out + + +def test_list_blobs_with_prefix(test_blob, capsys): + snippets.list_blobs_with_prefix( + BUCKET, + prefix='storage_snippets') + out, _ = capsys.readouterr() + assert test_blob.name in out + + +def test_upload_blob(): + with tempfile.NamedTemporaryFile() as source_file: + source_file.write(b'test') + + snippets.upload_blob( + BUCKET, + source_file.name, + 'test_upload_blob') + + +def test_upload_blob_with_kms(): + with tempfile.NamedTemporaryFile() as source_file: + source_file.write(b'test') + snippets.upload_blob_with_kms( + BUCKET, + source_file.name, + 'test_upload_blob_encrypted', + KMS_KEY) + bucket = storage.Client().bucket(BUCKET) + kms_blob = bucket.get_blob('test_upload_blob_encrypted') + assert kms_blob.kms_key_name.startswith(KMS_KEY) + + +def test_download_blob(test_blob): + with tempfile.NamedTemporaryFile() as dest_file: + snippets.download_blob( + BUCKET, + test_blob.name, + dest_file.name) + + assert dest_file.read() + + +def test_blob_metadata(test_blob, capsys): + snippets.blob_metadata(BUCKET, test_blob.name) + out, _ = capsys.readouterr() + assert test_blob.name in out + + +def test_delete_blob(test_blob): + snippets.delete_blob( + BUCKET, + test_blob.name) + + +def test_make_blob_public(test_blob): + snippets.make_blob_public( + BUCKET, + test_blob.name) + + r = requests.get(test_blob.public_url) + assert r.text == 'Hello, is it me you\'re looking for?' + + +def test_generate_signed_url(test_blob, capsys): + url = snippets.generate_signed_url( + BUCKET, + test_blob.name) + + r = requests.get(url) + assert r.text == 'Hello, is it me you\'re looking for?' + + +def test_rename_blob(test_blob): + bucket = storage.Client().bucket(BUCKET) + + try: + bucket.delete_blob('test_rename_blob') + except google.cloud.exceptions.exceptions.NotFound: + pass + + snippets.rename_blob(bucket.name, test_blob.name, 'test_rename_blob') + + assert bucket.get_blob('test_rename_blob') is not None + assert bucket.get_blob(test_blob.name) is None + + +def test_copy_blob(test_blob): + bucket = storage.Client().bucket(BUCKET) + + try: + bucket.delete_blob('test_copy_blob') + except google.cloud.exceptions.NotFound: + pass + + snippets.copy_blob( + bucket.name, test_blob.name, bucket.name, 'test_copy_blob') + + assert bucket.get_blob('test_copy_blob') is not None + assert bucket.get_blob(test_blob.name) is not None diff --git a/storage/s3-sdk/README.rst b/storage/s3-sdk/README.rst new file mode 100644 index 00000000000..42cf0f3b7aa --- /dev/null +++ b/storage/s3-sdk/README.rst @@ -0,0 +1,114 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Storage Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/s3-sdk/README.rst + + +This directory contains samples for Google Cloud Storage. `Google Cloud Storage`_ provides support for S3 API users through the GCS XML API. +Learn more about migrating data from S3 to GCS at https://cloud.google.com/storage/docs/migrating. + + + + +.. _Google Cloud Storage: https://cloud.google.com/storage/docs + +Setup +------------------------------------------------------------------------------- + + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +List GCS Buckets using S3 SDK ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/s3-sdk/list_gcs_buckets.py,storage/s3-sdk/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python list_gcs_buckets.py + + usage: list_gcs_buckets.py [-h] google_access_key_id google_access_key_secret + + positional arguments: + google_access_key_id Your Cloud Storage HMAC Access Key ID. + google_access_key_secret + Your Cloud Storage HMAC Access Key Secret. + + optional arguments: + -h, --help show this help message and exit + + + +List GCS Objects using S3 SDK ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/s3-sdk/list_gcs_objects.py,storage/s3-sdk/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python list_gcs_objects.py + + usage: list_gcs_objects.py [-h] + google_access_key_id google_access_key_secret + bucket_name + + positional arguments: + google_access_key_id Your Cloud Storage HMAC Access Key ID. + google_access_key_secret + Your Cloud Storage HMAC Access Key Secret. + bucket_name Your Cloud Storage bucket name + + optional arguments: + -h, --help show this help message and exit + + + + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/storage/s3-sdk/README.rst.in b/storage/s3-sdk/README.rst.in new file mode 100644 index 00000000000..d5b3a2c276e --- /dev/null +++ b/storage/s3-sdk/README.rst.in @@ -0,0 +1,26 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Storage + short_name: Cloud Storage + url: https://cloud.google.com/storage/docs + description: > + `Google Cloud Storage`_ provides support for S3 API users through the + GCS XML API. + + Learn more about migrating data from S3 to GCS at https://cloud.google.com/storage/docs/migrating. + +setup: +- install_deps + +samples: +- name: List GCS Buckets using S3 SDK + file: list_gcs_buckets.py + show_help: true +- name: List GCS Objects using S3 SDK + file: list_gcs_objects.py + show_help: true + +cloud_client_library: false + +folder: storage/s3-sdk diff --git a/storage/s3-sdk/list_gcs_buckets.py b/storage/s3-sdk/list_gcs_buckets.py new file mode 100644 index 00000000000..8f3373db1dc --- /dev/null +++ b/storage/s3-sdk/list_gcs_buckets.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +# [START storage_s3_sdk_list_buckets] +import boto3 + + +def list_gcs_buckets(google_access_key_id, google_access_key_secret): + """Lists all GCS buckets using boto3 SDK""" + # Create a new client and do the following: + # 1. Change the endpoint URL to use the + # Google Cloud Storage XML API endpoint. + # 2. Use Cloud Storage HMAC Credentials. + client = boto3.client("s3", region_name="auto", + endpoint_url="https://storage.googleapis.com", + aws_access_key_id=google_access_key_id, + aws_secret_access_key=google_access_key_secret) + + # Call GCS to list current buckets + response = client.list_buckets() + + # Print bucket names + print("Buckets:") + for bucket in response["Buckets"]: + print(bucket["Name"]) +# [END storage_s3_sdk_list_buckets] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("google_access_key_id", + help="Your Cloud Storage HMAC Access Key ID.") + parser.add_argument("google_access_key_secret", + help="Your Cloud Storage HMAC Access Key Secret.") + + args = parser.parse_args() + + list_gcs_buckets(google_access_key_id=args.google_access_key_id, + google_access_key_secret=args.google_access_key_secret) diff --git a/storage/s3-sdk/list_gcs_buckets_test.py b/storage/s3-sdk/list_gcs_buckets_test.py new file mode 100644 index 00000000000..1d29f23a389 --- /dev/null +++ b/storage/s3-sdk/list_gcs_buckets_test.py @@ -0,0 +1,28 @@ +# Copyright 2019 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import list_gcs_buckets + +BUCKET = os.environ["GOOGLE_CLOUD_PROJECT_S3_SDK"] +KEY_ID = os.environ["STORAGE_HMAC_ACCESS_KEY_ID"] +SECRET_KEY = os.environ["STORAGE_HMAC_ACCESS_SECRET_KEY"] + + +def test_list_blobs(capsys): + list_gcs_buckets.list_gcs_buckets(google_access_key_id=KEY_ID, + google_access_key_secret=SECRET_KEY) + out, _ = capsys.readouterr() + assert BUCKET in out diff --git a/storage/s3-sdk/list_gcs_objects.py b/storage/s3-sdk/list_gcs_objects.py new file mode 100644 index 00000000000..3747a164084 --- /dev/null +++ b/storage/s3-sdk/list_gcs_objects.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +# [START storage_s3_sdk_list_objects] +import boto3 + + +def list_gcs_objects(google_access_key_id, google_access_key_secret, + bucket_name): + """Lists GCS objects using boto3 SDK""" + # Create a new client and do the following: + # 1. Change the endpoint URL to use the + # Google Cloud Storage XML API endpoint. + # 2. Use Cloud Storage HMAC Credentials. + + client = boto3.client("s3", region_name="auto", + endpoint_url="https://storage.googleapis.com", + aws_access_key_id=google_access_key_id, + aws_secret_access_key=google_access_key_secret) + + # Call GCS to list objects in bucket_name + response = client.list_objects(Bucket=bucket_name) + + # Print object names + print("Objects:") + for blob in response["Contents"]: + print(blob["Key"]) +# [END storage_s3_sdk_list_objects] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("google_access_key_id", + help="Your Cloud Storage HMAC Access Key ID.") + parser.add_argument("google_access_key_secret", + help="Your Cloud Storage HMAC Access Key Secret.") + parser.add_argument('bucket_name', + help="Your Cloud Storage bucket name") + + args = parser.parse_args() + + list_gcs_objects(google_access_key_id=args.google_access_key_id, + google_access_key_secret=args.google_access_key_secret, + bucket_name=args.bucket_name) diff --git a/storage/s3-sdk/list_gcs_objects_test.py b/storage/s3-sdk/list_gcs_objects_test.py new file mode 100644 index 00000000000..ab915b5ca1e --- /dev/null +++ b/storage/s3-sdk/list_gcs_objects_test.py @@ -0,0 +1,29 @@ +# Copyright 2019 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import list_gcs_objects + +BUCKET = os.environ["GOOGLE_CLOUD_PROJECT_S3_SDK"] +KEY_ID = os.environ["STORAGE_HMAC_ACCESS_KEY_ID"] +SECRET_KEY = os.environ["STORAGE_HMAC_ACCESS_SECRET_KEY"] + + +def test_list_blobs(capsys): + list_gcs_objects.list_gcs_objects(google_access_key_id=KEY_ID, + google_access_key_secret=SECRET_KEY, + bucket_name=BUCKET) + out, _ = capsys.readouterr() + assert "Objects:" in out diff --git a/storage/s3-sdk/requirements.txt b/storage/s3-sdk/requirements.txt new file mode 100644 index 00000000000..d65147f7ed3 --- /dev/null +++ b/storage/s3-sdk/requirements.txt @@ -0,0 +1 @@ +boto3==1.9.38 \ No newline at end of file diff --git a/storage/signed_urls/README.rst b/storage/signed_urls/README.rst new file mode 100644 index 00000000000..9f00f00fdf2 --- /dev/null +++ b/storage/signed_urls/README.rst @@ -0,0 +1,97 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Storage Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/signed_urls/README.rst + + +This directory contains samples for Google Cloud Storage. `Google Cloud Storage`_ allows world-wide storage and retrieval of any amount of data at any time. + + + + +.. _Google Cloud Storage: https://cloud.google.com/storage/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Generate Signed URLs in Python ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/signed_urls/generate_signed_urls.py,storage/signed_urls/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python generate_signed_urls.py + + usage: generate_signed_urls.py [-h] + service_account_file request_method bucket_name + object_name expiration + + positional arguments: + service_account_file Path to your Google service account. + request_method A request method, e.g GET, POST. + bucket_name Your Cloud Storage bucket name. + object_name Your Cloud Storage object name. + expiration Expiration Time. + + optional arguments: + -h, --help show this help message and exit + + + + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/storage/signed_urls/README.rst.in b/storage/signed_urls/README.rst.in new file mode 100644 index 00000000000..fdc427f6617 --- /dev/null +++ b/storage/signed_urls/README.rst.in @@ -0,0 +1,22 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Storage + short_name: Cloud Storage + url: https://cloud.google.com/storage/docs + description: > + `Google Cloud Storage`_ allows world-wide storage and retrieval of any + amount of data at any time. + +setup: +- auth +- install_deps + +samples: +- name: Generate Signed URLs in Python + file: generate_signed_urls.py + show_help: true + +cloud_client_library: false + +folder: storage/signed_urls \ No newline at end of file diff --git a/storage/signed_urls/generate_signed_urls.py b/storage/signed_urls/generate_signed_urls.py new file mode 100644 index 00000000000..7cee36461b6 --- /dev/null +++ b/storage/signed_urls/generate_signed_urls.py @@ -0,0 +1,167 @@ +# Copyright 2018 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +"""This application demonstrates how to construct a Signed URL for objects in + Google Cloud Storage. + +For more information, see the README.md under /storage and the documentation +at https://cloud.google.com/storage/docs/access-control/signing-urls-manually. +""" + +# [START storage_signed_url_all] +# [START storage_signed_url_dependencies] +import binascii +import collections +import datetime +import hashlib +import sys + +# pip install six +from six.moves.urllib.parse import quote + +# [START storage_signed_url_signer] +# pip install google-auth +from google.oauth2 import service_account + +# [END storage_signed_url_signer] +# [END storage_signed_url_dependencies] + + +def generate_signed_url(service_account_file, bucket_name, object_name, + expiration, http_method='GET', query_parameters=None, + headers=None): + + if expiration > 604800: + print('Expiration Time can\'t be longer than 604800 seconds (7 days).') + sys.exit(1) + + # [START storage_signed_url_canonical_uri] + escaped_object_name = quote(object_name, safe='') + canonical_uri = '/{}/{}'.format(bucket_name, escaped_object_name) + # [END storage_signed_url_canonical_uri] + + # [START storage_signed_url_canonical_datetime] + datetime_now = datetime.datetime.utcnow() + request_timestamp = datetime_now.strftime('%Y%m%dT%H%M%SZ') + datestamp = datetime_now.strftime('%Y%m%d') + # [END storage_signed_url_canonical_datetime] + + # [START storage_signed_url_credentials] + # [START storage_signed_url_signer] + google_credentials = service_account.Credentials.from_service_account_file( + service_account_file) + # [END storage_signed_url_signer] + client_email = google_credentials.service_account_email + credential_scope = '{}/auto/storage/goog4_request'.format(datestamp) + credential = '{}/{}'.format(client_email, credential_scope) + # [END storage_signed_url_credentials] + + if headers is None: + headers = dict() + # [START storage_signed_url_canonical_headers] + headers['host'] = 'storage.googleapis.com' + + canonical_headers = '' + ordered_headers = collections.OrderedDict(sorted(headers.items())) + for k, v in ordered_headers.items(): + lower_k = str(k).lower() + strip_v = str(v).lower() + canonical_headers += '{}:{}\n'.format(lower_k, strip_v) + # [END storage_signed_url_canonical_headers] + + # [START storage_signed_url_signed_headers] + signed_headers = '' + for k, _ in ordered_headers.items(): + lower_k = str(k).lower() + signed_headers += '{};'.format(lower_k) + signed_headers = signed_headers[:-1] # remove trailing ';' + # [END storage_signed_url_signed_headers] + + if query_parameters is None: + query_parameters = dict() + # [START storage_signed_url_canonical_query_parameters] + query_parameters['X-Goog-Algorithm'] = 'GOOG4-RSA-SHA256' + query_parameters['X-Goog-Credential'] = credential + query_parameters['X-Goog-Date'] = request_timestamp + query_parameters['X-Goog-Expires'] = expiration + query_parameters['X-Goog-SignedHeaders'] = signed_headers + + canonical_query_string = '' + ordered_query_parameters = collections.OrderedDict( + sorted(query_parameters.items())) + for k, v in ordered_query_parameters.items(): + encoded_k = quote(str(k), safe='') + encoded_v = quote(str(v), safe='') + canonical_query_string += '{}={}&'.format(encoded_k, encoded_v) + canonical_query_string = canonical_query_string[:-1] # remove trailing ';' + # [END storage_signed_url_canonical_query_parameters] + + # [START storage_signed_url_canonical_request] + canonical_request = '\n'.join([http_method, + canonical_uri, + canonical_query_string, + canonical_headers, + signed_headers, + 'UNSIGNED-PAYLOAD']) + # [END storage_signed_url_canonical_request] + + # [START storage_signed_url_hash] + canonical_request_hash = hashlib.sha256( + canonical_request.encode()).hexdigest() + # [END storage_signed_url_hash] + + # [START storage_signed_url_string_to_sign] + string_to_sign = '\n'.join(['GOOG4-RSA-SHA256', + request_timestamp, + credential_scope, + canonical_request_hash]) + # [END storage_signed_url_string_to_sign] + + # [START storage_signed_url_signer] + signature = binascii.hexlify( + google_credentials.signer.sign(string_to_sign) + ).decode() + # [END storage_signed_url_signer] + + # [START storage_signed_url_construction] + host_name = 'https://storage.googleapis.com' + signed_url = '{}{}?{}&X-Goog-Signature={}'.format(host_name, canonical_uri, + canonical_query_string, + signature) + # [END storage_signed_url_construction] + return signed_url +# [END storage_signed_url_all] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('service_account_file', + help='Path to your Google service account.') + parser.add_argument( + 'request_method', help='A request method, e.g GET, POST.') + parser.add_argument('bucket_name', help='Your Cloud Storage bucket name.') + parser.add_argument('object_name', help='Your Cloud Storage object name.') + parser.add_argument('expiration', help='Expiration Time.') + + args = parser.parse_args() + signed_url = generate_signed_url( + service_account_file=args.service_account_file, + http_method=args.request_method, bucket_name=args.bucket_name, + object_name=args.object_name, expiration=int(args.expiration)) + + print(signed_url) diff --git a/storage/signed_urls/generate_signed_urls_test.py b/storage/signed_urls/generate_signed_urls_test.py new file mode 100644 index 00000000000..2f3c426a6f6 --- /dev/null +++ b/storage/signed_urls/generate_signed_urls_test.py @@ -0,0 +1,42 @@ +# Copyright 2018 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from google.cloud import storage +import pytest +import requests + +import generate_signed_urls + +BUCKET = os.environ['CLOUD_STORAGE_BUCKET'] +GOOGLE_APPLICATION_CREDENTIALS = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] + + +@pytest.fixture +def test_blob(): + """Provides a pre-existing blob in the test bucket.""" + bucket = storage.Client().bucket(BUCKET) + blob = bucket.blob('storage_snippets_test_sigil') + blob.upload_from_string('Hello, is it me you\'re looking for?') + return blob + + +def test_generate_get_signed_url(test_blob, capsys): + get_signed_url = generate_signed_urls.generate_signed_url( + service_account_file=GOOGLE_APPLICATION_CREDENTIALS, + bucket_name=BUCKET, object_name=test_blob.name, + expiration=60) + response = requests.get(get_signed_url) + assert response.ok diff --git a/storage/signed_urls/requirements.txt b/storage/signed_urls/requirements.txt new file mode 100644 index 00000000000..3b50e357261 --- /dev/null +++ b/storage/signed_urls/requirements.txt @@ -0,0 +1,3 @@ +google-cloud-storage==1.13.2 +google-auth==1.6.2 +six==1.12.0 diff --git a/storage/transfer_service/README.md b/storage/transfer_service/README.md deleted file mode 100644 index 7d36a64c731..00000000000 --- a/storage/transfer_service/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# Transfer Service sample using Python - -This app creates two types of transfers using the Transfer Service tool. - - -These samples are used on the following documentation pages: - -> -* https://cloud.google.com/storage/transfer/create-transfer -* https://cloud.google.com/storage/transfer/create-client - - - -## Prerequisites - -1. Set up a project on Google Developers Console. - 1. Go to the [Developers Console](https://cloud.google.com/console) and create or select your project. - You will need the project ID later. -1. Within Developers Console, select APIs & auth > Credentials. - 1. Select Add credentials > Service account > JSON key. - 1. Set the environment variable GOOGLE_APPLICATION_CREDENTIALS to point to your JSON key. -1. Add the Storage Transfer service account as an editor of your project - storage-transfer-@partnercontent.gserviceaccount.com -1. Set up gcloud for application default credentials. - 1. `gcloud components update` - 1. `gcloud init` -1. Install [Google API Client Library for Python](https://developers.google.com/api-client-library/python/start/installation). - -## Transfer from Amazon S3 to Google Cloud Storage - -Creating a one-time transfer from Amazon S3 to Google Cloud Storage. -1. Set up data sink. - 1. Go to the Developers Console and create a bucket under Cloud Storage > Storage Browser. -1. Set up data source. - 1. Go to AWS Management Console and create a bucket. - 1. Under Security Credentials, create an IAM User with access to the bucket. - 1. Create an Access Key for the user. Note the Access Key ID and Secret Access Key. -1. In aws_request.py, fill in the Transfer Job JSON template with relevant values. -1. Run with `python aws_request.py` - 1. Note the job ID in the returned Transfer Job. - -## Transfer data from a standard Cloud Storage bucket to a Cloud Storage Nearline bucket - -Creating a daily transfer from a standard Cloud Storage bucket to a Cloud Storage Nearline -bucket for files untouched for 30 days. -1. Set up data sink. - 1. Go to the Developers Console and create a bucket under Cloud Storage > Storage Browser. - 1. Select Nearline for Storage Class. -1. Set up data source. - 1. Go to the Developers Console and create a bucket under Cloud Storage > Storage Browser. -1. In nearline_request.py, fill in the Transfer Job JSON template with relevant values. -1. Run with `python nearline_request.py` - 1. Note the job ID in the returned Transfer Job. - -## Checking the status of a transfer - -1. In transfer_check.py, fill in the Transfer Job JSON template with relevant values. - Use the Job Name you recorded earlier. -1. Run with `python transfer_check.py` diff --git a/storage/transfer_service/README.rst b/storage/transfer_service/README.rst new file mode 100644 index 00000000000..833afc4742f --- /dev/null +++ b/storage/transfer_service/README.rst @@ -0,0 +1,186 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Storage Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/transfer_service/README.rst + + +This directory contains samples for Google Cloud Storage. `Google Cloud Storage`_ allows world-wide storage and retrieval of any amount of data at any time. + + +These samples demonstrate how to transfer data between Google Cloud Storage and other storage systems. + + +.. _Google Cloud Storage: https://cloud.google.com/storage/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Transfer to GCS Nearline ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/transfer_service/nearline_request.py,storage/transfer_service/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python nearline_request.py + + usage: nearline_request.py [-h] + description project_id start_date start_time + source_bucket sink_bucket + + Command-line sample that creates a daily transfer from a standard + GCS bucket to a Nearline GCS bucket for objects untouched for 30 days. + + This sample is used on this page: + + https://cloud.google.com/storage/transfer/create-transfer + + For more information, see README.md. + + positional arguments: + description Transfer description. + project_id Your Google Cloud project ID. + start_date Date YYYY/MM/DD. + start_time UTC Time (24hr) HH:MM:SS. + source_bucket Standard GCS bucket name. + sink_bucket Nearline GCS bucket name. + + optional arguments: + -h, --help show this help message and exit + + + +Transfer from AWS ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/transfer_service/aws_request.py,storage/transfer_service/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python aws_request.py + + usage: aws_request.py [-h] + description project_id start_date start_time + source_bucket access_key_id secret_access_key + sink_bucket + + Command-line sample that creates a one-time transfer from Amazon S3 to + Google Cloud Storage. + + This sample is used on this page: + + https://cloud.google.com/storage/transfer/create-transfer + + For more information, see README.md. + + positional arguments: + description Transfer description. + project_id Your Google Cloud project ID. + start_date Date YYYY/MM/DD. + start_time UTC Time (24hr) HH:MM:SS. + source_bucket AWS source bucket name. + access_key_id Your AWS access key id. + secret_access_key Your AWS secret access key. + sink_bucket GCS sink bucket name. + + optional arguments: + -h, --help show this help message and exit + + + +Check transfer status ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/transfer_service/transfer_check.py,storage/transfer_service/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python transfer_check.py + + usage: transfer_check.py [-h] project_id job_name + + Command-line sample that checks the status of an in-process transfer. + + This sample is used on this page: + + https://cloud.google.com/storage/transfer/create-transfer + + For more information, see README.md. + + positional arguments: + project_id Your Google Cloud project ID. + job_name Your job name. + + optional arguments: + -h, --help show this help message and exit + + + + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/storage/transfer_service/README.rst.in b/storage/transfer_service/README.rst.in new file mode 100644 index 00000000000..5d834e70ef4 --- /dev/null +++ b/storage/transfer_service/README.rst.in @@ -0,0 +1,30 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Storage + short_name: Cloud Storage + url: https://cloud.google.com/storage/docs + description: > + `Google Cloud Storage`_ allows world-wide storage and retrieval of any + amount of data at any time. + +description: > + These samples demonstrate how to transfer data between Google Cloud Storage + and other storage systems. + +setup: +- auth +- install_deps + +samples: +- name: Transfer to GCS Nearline + file: nearline_request.py + show_help: true +- name: Transfer from AWS + file: aws_request.py + show_help: true +- name: Check transfer status + file: transfer_check.py + show_help: true + +folder: storage/transfer_service \ No newline at end of file diff --git a/storage/transfer_service/aws_request.py b/storage/transfer_service/aws_request.py index 568ba5001a8..0984ae639cc 100644 --- a/storage/transfer_service/aws_request.py +++ b/storage/transfer_service/aws_request.py @@ -28,45 +28,42 @@ import datetime import json -from googleapiclient import discovery -from oauth2client.client import GoogleCredentials +import googleapiclient.discovery # [START main] -def main(description, project_id, day, month, year, hours, minutes, - source_bucket, access_key, secret_access_key, sink_bucket): - """Create a one-off transfer from Amazon S3 to Google Cloud Storage.""" - credentials = GoogleCredentials.get_application_default() - storagetransfer = discovery.build( - 'storagetransfer', 'v1', credentials=credentials) +def main(description, project_id, start_date, start_time, source_bucket, + access_key_id, secret_access_key, sink_bucket): + """Create a one-time transfer from Amazon S3 to Google Cloud Storage.""" + storagetransfer = googleapiclient.discovery.build('storagetransfer', 'v1') # Edit this template with desired parameters. - # Specify times below using US Pacific Time Zone. transfer_job = { 'description': description, 'status': 'ENABLED', 'projectId': project_id, 'schedule': { 'scheduleStartDate': { - 'day': day, - 'month': month, - 'year': year + 'day': start_date.day, + 'month': start_date.month, + 'year': start_date.year }, 'scheduleEndDate': { - 'day': day, - 'month': month, - 'year': year + 'day': start_date.day, + 'month': start_date.month, + 'year': start_date.year }, 'startTimeOfDay': { - 'hours': hours, - 'minutes': minutes + 'hours': start_time.hour, + 'minutes': start_time.minute, + 'seconds': start_time.second } }, 'transferSpec': { 'awsS3DataSource': { 'bucketName': source_bucket, 'awsAccessKey': { - 'accessKeyId': access_key, + 'accessKeyId': access_key_id, 'secretAccessKey': secret_access_key } }, @@ -81,34 +78,34 @@ def main(description, project_id, day, month, year, hours, minutes, json.dumps(result, indent=4))) # [END main] + if __name__ == '__main__': parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('description', help='Transfer description.') parser.add_argument('project_id', help='Your Google Cloud project ID.') - parser.add_argument('date', help='Date YYYY/MM/DD.') - parser.add_argument('time', help='Time (24hr) HH:MM.') - parser.add_argument('source_bucket', help='Source bucket name.') - parser.add_argument('access_key', help='Your AWS access key id.') - parser.add_argument('secret_access_key', help='Your AWS secret access ' - 'key.') - parser.add_argument('sink_bucket', help='Sink bucket name.') + parser.add_argument('start_date', help='Date YYYY/MM/DD.') + parser.add_argument('start_time', help='UTC Time (24hr) HH:MM:SS.') + parser.add_argument('source_bucket', help='AWS source bucket name.') + parser.add_argument('access_key_id', help='Your AWS access key id.') + parser.add_argument( + 'secret_access_key', + help='Your AWS secret access key.' + ) + parser.add_argument('sink_bucket', help='GCS sink bucket name.') args = parser.parse_args() - date = datetime.datetime.strptime(args.date, '%Y/%m/%d') - time = datetime.datetime.strptime(args.time, '%H:%M') + start_date = datetime.datetime.strptime(args.start_date, '%Y/%m/%d') + start_time = datetime.datetime.strptime(args.start_time, '%H:%M:%S') main( args.description, args.project_id, - date.year, - date.month, - date.day, - time.hour, - time.minute, + start_date, + start_time, args.source_bucket, - args.access_key, + args.access_key_id, args.secret_access_key, args.sink_bucket) # [END all] diff --git a/storage/transfer_service/create_client.py b/storage/transfer_service/create_client.py index 445e9b20aa2..2a4f85ec94a 100644 --- a/storage/transfer_service/create_client.py +++ b/storage/transfer_service/create_client.py @@ -12,11 +12,9 @@ # limitations under the License. # [START all] -from googleapiclient import discovery -from oauth2client.client import GoogleCredentials +import googleapiclient.discovery def create_transfer_client(): - credentials = GoogleCredentials.get_application_default() - return discovery.build('storagetransfer', 'v1', credentials=credentials) + return googleapiclient.discovery.build('storagetransfer', 'v1') # [END all] diff --git a/storage/transfer_service/nearline_request.py b/storage/transfer_service/nearline_request.py index adfb485158c..d135094832f 100644 --- a/storage/transfer_service/nearline_request.py +++ b/storage/transfer_service/nearline_request.py @@ -13,8 +13,8 @@ # [START all] -"""Command-line sample that creates a one-time transfer from Google Cloud -Storage standard class to the Nearline storage class." +"""Command-line sample that creates a daily transfer from a standard +GCS bucket to a Nearline GCS bucket for objects untouched for 30 days. This sample is used on this page: @@ -27,34 +27,30 @@ import datetime import json -from googleapiclient import discovery -from oauth2client.client import GoogleCredentials +import googleapiclient.discovery # [START main] -def main(description, project_id, day, month, year, hours, minutes, - source_bucket, sink_bucket): - """Create a transfer from the Google Cloud Storage Standard class to the - Nearline Storage class.""" - credentials = GoogleCredentials.get_application_default() - storagetransfer = discovery.build( - 'storagetransfer', 'v1', credentials=credentials) +def main(description, project_id, start_date, start_time, source_bucket, + sink_bucket): + """Create a daily transfer from Standard to Nearline Storage class.""" + storagetransfer = googleapiclient.discovery.build('storagetransfer', 'v1') # Edit this template with desired parameters. - # Specify times below using US Pacific Time Zone. transfer_job = { 'description': description, 'status': 'ENABLED', 'projectId': project_id, 'schedule': { 'scheduleStartDate': { - 'day': day, - 'month': month, - 'year': year + 'day': start_date.day, + 'month': start_date.month, + 'year': start_date.year }, 'startTimeOfDay': { - 'hours': hours, - 'minutes': minutes + 'hours': start_time.hour, + 'minutes': start_time.minute, + 'seconds': start_time.second } }, 'transferSpec': { @@ -65,7 +61,7 @@ def main(description, project_id, day, month, year, hours, minutes, 'bucketName': sink_bucket }, 'objectConditions': { - 'minTimeElapsedSinceLastModification': '2592000s' + 'minTimeElapsedSinceLastModification': '2592000s' # 30 days }, 'transferOptions': { 'deleteObjectsFromSourceAfterTransfer': 'true' @@ -78,29 +74,27 @@ def main(description, project_id, day, month, year, hours, minutes, json.dumps(result, indent=4))) # [END main] + if __name__ == '__main__': parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('description', help='Transfer description.') parser.add_argument('project_id', help='Your Google Cloud project ID.') - parser.add_argument('date', help='Date YYYY/MM/DD.') - parser.add_argument('time', help='Time (24hr) HH:MM.') - parser.add_argument('source_bucket', help='Source bucket name.') - parser.add_argument('sink_bucket', help='Sink bucket name.') + parser.add_argument('start_date', help='Date YYYY/MM/DD.') + parser.add_argument('start_time', help='UTC Time (24hr) HH:MM:SS.') + parser.add_argument('source_bucket', help='Standard GCS bucket name.') + parser.add_argument('sink_bucket', help='Nearline GCS bucket name.') args = parser.parse_args() - date = datetime.datetime.strptime(args.date, '%Y/%m/%d') - time = datetime.datetime.strptime(args.time, '%H:%M') + start_date = datetime.datetime.strptime(args.start_date, '%Y/%m/%d') + start_time = datetime.datetime.strptime(args.start_time, '%H:%M:%S') main( args.description, args.project_id, - date.year, - date.month, - date.day, - time.hour, - time.minute, + start_date, + start_time, args.source_bucket, args.sink_bucket) # [END all] diff --git a/storage/transfer_service/requirements.txt b/storage/transfer_service/requirements.txt index c3b2784ce87..7e4359ce08d 100644 --- a/storage/transfer_service/requirements.txt +++ b/storage/transfer_service/requirements.txt @@ -1 +1,3 @@ -google-api-python-client==1.5.0 +google-api-python-client==1.7.8 +google-auth==1.6.2 +google-auth-httplib2==0.0.3 diff --git a/storage/transfer_service/transfer_check.py b/storage/transfer_service/transfer_check.py index 9ff40af741d..15c434c76ee 100644 --- a/storage/transfer_service/transfer_check.py +++ b/storage/transfer_service/transfer_check.py @@ -27,16 +27,13 @@ import argparse import json -from googleapiclient import discovery -from oauth2client.client import GoogleCredentials +import googleapiclient.discovery # [START main] def main(project_id, job_name): """Review the transfer operations associated with a transfer job.""" - credentials = GoogleCredentials.get_application_default() - storagetransfer = discovery.build( - 'storagetransfer', 'v1', credentials=credentials) + storagetransfer = googleapiclient.discovery.build('storagetransfer', 'v1') filterString = ( '{{"project_id": "{project_id}", ' @@ -50,6 +47,7 @@ def main(project_id, job_name): json.dumps(result, indent=4, sort_keys=True))) # [END main] + if __name__ == '__main__': parser = argparse.ArgumentParser( description=__doc__, diff --git a/tables/automl/notebooks/census_income_prediction/README.md b/tables/automl/notebooks/census_income_prediction/README.md new file mode 100644 index 00000000000..4c5ed03ce28 --- /dev/null +++ b/tables/automl/notebooks/census_income_prediction/README.md @@ -0,0 +1,96 @@ +AutoML Tables enables your entire team to automatically build and deploy state-of-the-art machine learning models on structured data at massively increased speed and scale. + + +## Problem Description +The model uses a real dataset from the [Census Income Dataset](https://archive.ics.uci.edu/ml/datasets/Census+Income). + + +The goal is the predict if a given individual has an income above or below 50k, given information like the person's age, education level, marital-status, occupation etc... +This is framed as a binary classification model, to label the individual as either having an income above or below 50k. + + + + + + +Dataset Details + + +The dataset consists of over 30k rows, where each row corresponds to a different person. For a given row, there are 14 features that the model conditions on to predict the income of the person. A few of the features are named above, and the exhaustive list can be found both in the dataset link above or seen in the colab. + + + + +## Solution Walkthrough +The solution has been developed using [Google Colab Notebook](https://colab.research.google.com/notebooks/welcome.ipynb). + + + + +Steps Involved + + +### 1. Set up +The first step in this process was to set up the project. We referred to the [AutoML tables documentation](https://cloud.google.com/automl-tables/docs/) and take the following steps: +* Create a Google Cloud Platform (GCP) project +* Enable billing +* Enable the AutoML API +* Enable the AutoML Tables API +* Create a service account, grant required permissions, and download the service account private key. + + +### 2. Initialize and authenticate + + +The client library installation is entirely self explanatory in the colab. + + +The authentication process is only slightly more complex: run the second code block entitled "Authenticate using service account key" and then upload the service account key you created in the set up step. + + +To make sure your colab was authenticated and has access to your project, replace the project_id with your project_id, and run the subsequent code blocks. You should see the lists of your datasets and any models you made previously in AutoML Tables. + + +### 3. Import training data + + +This section has you create a dataset and import the data. You have both the option of using the csv import from a Cloud Storage bucket, or you can upload the csv into Big Query and import it from there. + + + + +### 4. Update dataset: assign a label column and enable nullable columns + + +This section is important, as it is where you specify which column (meaning which feature) you will use as your label. This label feature will then be predicted using all other features in the row. + + +### 5. Creating a model + + +This section is where you train your model. You can specify how long you want your model to train for. + + +### 6. Make a prediction + + +This section gives you the ability to do a single online prediction. You can toggle exactly which values you want for all of the numeric features, and choose from the drop down windows which values you want for the categorical features. + + +The model takes a while to deploy online, and currently there does not exist a feedback mechanism in the sdk, so you will need to wait until the model finishes deployment to run the online prediction. +When the deployment code ```response = client.deploy_model(model_name)``` finishes, you will be able to see this on the [UI](https://console.cloud.google.com/automl-tables). + + +To see when it finishes, click on the UI link above and navitage to the dataset you just uploaded, and go to the predict tab. You should see "online prediction" text near the top, click on it, and it will take you to a view of your online prediction interface. You should see "model deployed" on the far right of the screen if the model is deployed, or a "deploying model" message if it is still deploying. + + +Once the model finishes deployment, go ahead and run the ```prediction_client.predict(model_name, payload)``` line. + + +Note: If the model has not finished deployment, the prediction will NOT work. + + +### 7. Batch Prediction + + +There is a validation csv file provided with a few rows of data not used in the training or testing for you to run a batch prediction with. The csv is linked in the text of the colab as well as [here](https://storage.cloud.google.com/cloud-ml-data/automl-tables/notebooks/census_income_batch_prediction_input.csv) . \ No newline at end of file diff --git a/tables/automl/notebooks/census_income_prediction/census_income_prediction.ipynb b/tables/automl/notebooks/census_income_prediction/census_income_prediction.ipynb new file mode 100644 index 00000000000..1e5ac840c7b --- /dev/null +++ b/tables/automl/notebooks/census_income_prediction/census_income_prediction.ipynb @@ -0,0 +1,932 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "m26YhtBMvVWA" + }, + "source": [ + "# Getting started with AutoML Tables\n", + "\n", + "To use this Colab notebook, copy it to your own Google Drive and open it with [Colaboratory](https://colab.research.google.com/) (or Colab). To run a cell hold the Shift key and press the Enter key (or Return key). Colab automatically displays the return value of the last line in each cell. Refer to [this page](https://colab.research.google.com/notebooks/welcome.ipynb) for more information on Colab.\n", + "\n", + "You can run a Colab notebook on a hosted runtime in the Cloud. The hosted VM times out after 90 minutes of inactivity and you will lose all the data stored in the memory including your authentication data. If your session gets disconnected (for example, because you closed your laptop) for less than the 90 minute inactivity timeout limit, press 'RECONNECT' on the top right corner of your notebook and resume the session. After Colab timeout, you'll need to\n", + "\n", + "1. Re-run the initialization and authentication.\n", + "2. Continue from where you left off. You may need to copy-paste the value of some variables such as the `dataset_name` from the printed output of the previous cells.\n", + "\n", + "Alternatively you can connect your Colab notebook to a [local runtime](https://research.google.com/colaboratory/local-runtimes.html).\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "b--5FDDwCG9C" + }, + "source": [ + "## 1. Project set up\n", + "\n", + "\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "AZs0ICgy4jkQ" + }, + "source": [ + "Follow the [AutoML Tables documentation](https://cloud.google.com/automl-tables/docs/) to\n", + "* Create a Google Cloud Platform (GCP) project.\n", + "* Enable billing.\n", + "* Apply to whitelist your project.\n", + "* Enable AutoML API.\n", + "* Enable AutoML Tables API.\n", + "* Create a service account, grant required permissions, and download the service account private key.\n", + "\n", + "You also need to upload your data into Google Cloud Storage (GCS) or BigQuery. For example, to use GCS as your data source\n", + "* Create a GCS bucket.\n", + "* Upload the training and batch prediction files.\n", + "\n", + "\n", + "**Warning:** Private keys must be kept secret. If you expose your private key it is recommended to revoke it immediately from the Google Cloud Console." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "xZECt1oL429r" + }, + "source": [ + "\n", + "\n", + "---\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "rstRPH9SyZj_" + }, + "source": [ + "## 2. Initialize and authenticate\n", + "This section runs intialization and authentication. It creates an authenticated session which is required for running any of the following sections." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "BR0POq2UzE7e" + }, + "source": [ + "### Install the client library\n", + "Run the following cell." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "43aXKjDRt_qZ" + }, + "outputs": [], + "source": [ + "#@title Install AutoML Tables client library { vertical-output: true }\n", + "!pip install google-cloud-automl" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "eVFsPPEociwF" + }, + "source": [ + "### Authenticate using service account key\n", + "Run the following cell. Click on the 'Choose Files' button and select the service account private key file. If your Service Account key file or folder is hidden, you can reveal it in a Mac by pressing the Command + Shift + . combo." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "u-kCqysAuaJk" + }, + "outputs": [], + "source": [ + "#@title Authenticate and create a client. { vertical-output: true }\n", + "\n", + "from google.cloud import automl_v1beta1\n", + "\n", + "# Upload service account key\n", + "keyfile_upload = files.upload()\n", + "keyfile_name = list(keyfile_upload.keys())[0]\n", + "# Authenticate and create an AutoML client.\n", + "client = automl_v1beta1.AutoMlClient.from_service_account_file(keyfile_name)\n", + "# Authenticate and create a prediction service client.\n", + "prediction_client = automl_v1beta1.PredictionServiceClient.from_service_account_file(keyfile_name)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "s3F2xbEJdDvN" + }, + "source": [ + "### Test" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "0uX4aJYUiXh5" + }, + "source": [ + "Enter your GCP project ID." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "6R4h5HF1Dtds" + }, + "outputs": [], + "source": [ + "#@title GCP project ID and location\n", + "\n", + "project_id = 'my-project-trial5' #@param {type:'string'}\n", + "location = 'us-central1'\n", + "location_path = client.location_path(project_id, location)\n", + "location_path" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "rUlBcZ3OfWcJ" + }, + "source": [ + "To test whether your project set up and authentication steps were successful, run the following cell to list your datasets." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "both", + "colab": {}, + "colab_type": "code", + "id": "sf32nKXIqYje" + }, + "outputs": [], + "source": [ + "#@title List datasets. { vertical-output: true }\n", + "\n", + "list_datasets_response = client.list_datasets(location_path)\n", + "datasets = {\n", + " dataset.display_name: dataset.name for dataset in list_datasets_response}\n", + "datasets" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "t9uE8MvMkOPd" + }, + "source": [ + "You can also print the list of your models by running the following cell." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "both", + "colab": {}, + "colab_type": "code", + "id": "j4-bYRSWj7xk" + }, + "outputs": [], + "source": [ + "#@title List models. { vertical-output: true }\n", + "\n", + "list_models_response = client.list_models(location_path)\n", + "models = {model.display_name: model.name for model in list_models_response}\n", + "models" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "qozQWMnOu48y" + }, + "source": [ + "\n", + "\n", + "---\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "ODt86YuVDZzm" + }, + "source": [ + "## 3. Import training data" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "XwjZc9Q62Fm5" + }, + "source": [ + "### Create dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "_JfZFGSceyE_" + }, + "source": [ + "Select a dataset display name and pass your table source information to create a new dataset." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "Z_JErW3cw-0J" + }, + "outputs": [], + "source": [ + "#@title Create dataset { vertical-output: true, output-height: 200 }\n", + "\n", + "dataset_display_name = 'test_deployment' #@param {type: 'string'}\n", + "\n", + "create_dataset_response = client.create_dataset(\n", + " location_path,\n", + " {'display_name': dataset_display_name, 'tables_dataset_metadata': {}})\n", + "dataset_name = create_dataset_response.name\n", + "create_dataset_response" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "35YZ9dy34VqJ" + }, + "source": [ + "### Import data" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "3c0o15gVREAw" + }, + "source": [ + "You can import your data to AutoML Tables from GCS or BigQuery. For this tutorial, you can use the [census_income dataset](https://storage.cloud.google.com/cloud-ml-data/automl-tables/notebooks/census_income.csv) \n", + "as your training data. You can create a GCS bucket and upload the data intofa your bucket. The URI for your file is `gs://BUCKET_NAME/FOLDER_NAME1/FOLDER_NAME2/.../FILE_NAME`. Alternatively you can create a BigQuery table and upload the data into the table. The URI for your table is `bq://PROJECT_ID.DATASET_ID.TABLE_ID`.\n", + "\n", + "Importing data may take a few minutes or hours depending on the size of your data. If your Colab times out, run the following command to retrieve your dataset. Replace `dataset_name` with its actual value obtained in the preceding cells.\n", + "\n", + " dataset = client.get_dataset(dataset_name)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "UIWlq3NTYhOl" + }, + "outputs": [], + "source": [ + "#@title ... if data source is GCS { vertical-output: true }\n", + "\n", + "dataset_gcs_input_uris = ['gs://cloud-ml-data/automl-tables/notebooks/census_income.csv',] #@param\n", + "# Define input configuration.\n", + "input_config = {\n", + " 'gcs_source': {\n", + " 'input_uris': dataset_gcs_input_uris\n", + " }\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "bB_GdeqCJW5i" + }, + "outputs": [], + "source": [ + "#@title ... if data source is BigQuery { vertical-output: true }\n", + "\n", + "dataset_bq_input_uri = 'bq://my-project-trial5.census_income.income_census' #@param {type: 'string'}\n", + "# Define input configuration.\n", + "input_config = {\n", + " 'bigquery_source': {\n", + " 'input_uri': dataset_bq_input_uri\n", + " }\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "FNVYfpoXJsNB" + }, + "outputs": [], + "source": [ + " #@title Import data { vertical-output: true }\n", + "\n", + "import_data_response = client.import_data(dataset_name, input_config)\n", + "print('Dataset import operation: {}'.format(import_data_response.operation))\n", + "# Wait until import is done.\n", + "import_data_result = import_data_response.result()\n", + "import_data_result" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "QdxBI4s44ZRI" + }, + "source": [ + "### Review the specs" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "RC0PWKqH4jwr" + }, + "source": [ + "Run the following command to see table specs such as row count." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "v2Vzq_gwXxo-" + }, + "outputs": [], + "source": [ + "#@title Table schema { vertical-output: true }\n", + "\n", + "import google.cloud.automl_v1beta1.proto.data_types_pb2 as data_types\n", + "import matplotlib.pyplot as plt\n", + "\n", + "# List table specs\n", + "list_table_specs_response = client.list_table_specs(dataset_name)\n", + "table_specs = [s for s in list_table_specs_response]\n", + "# List column specs\n", + "table_spec_name = table_specs[0].name\n", + "list_column_specs_response = client.list_column_specs(table_spec_name)\n", + "column_specs = {s.display_name: s for s in list_column_specs_response}\n", + "# Table schema pie chart.\n", + "type_counts = {}\n", + "for column_spec in column_specs.values():\n", + " type_name = data_types.TypeCode.Name(column_spec.data_type.type_code)\n", + " type_counts[type_name] = type_counts.get(type_name, 0) + 1\n", + "\n", + "plt.pie(x=type_counts.values(), labels=type_counts.keys(), autopct='%1.1f%%')\n", + "plt.axis('equal')\n", + "plt.show()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "vcJP7xoq4yAJ" + }, + "source": [ + "Run the following command to see column specs such inferred schema." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "FNykW_YOYt6d" + }, + "source": [ + "___" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "kNRVJqVOL8h3" + }, + "source": [ + "## 4. Update dataset: assign a label column and enable nullable columns" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "-57gehId9PQ5" + }, + "source": [ + "AutoML Tables automatically detects your data column type. For example, for the ([census_income](https://storage.cloud.google.com/cloud-ml-data/automl-tables/notebooks/census_income.csv)) it detects `income` to be categorical (as it is just either over or under 50k) and `age` to be numerical. Depending on the type of your label column, AutoML Tables chooses to run a classification or regression model. If your label column contains only numerical values, but they represent categories, change your label column type to categorical by updating your schema." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "iRqdQ7Xiq04x" + }, + "source": [ + "### Update a column: set to nullable" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "OCEUIPKegWrf" + }, + "outputs": [], + "source": [ + "#@title Update dataset { vertical-output: true }\n", + "\n", + "update_column_spec_dict = {\n", + " 'name': column_specs['income'].name,\n", + " 'data_type': {\n", + " 'type_code': 'CATEGORY',\n", + " 'nullable': False\n", + " }\n", + "}\n", + "update_column_response = client.update_column_spec(update_column_spec_dict)\n", + "update_column_response" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "GUqKi3tkqrgW" + }, + "source": [ + "**Tip:** You can use `'type_code': 'CATEGORY'` in the preceding `update_column_spec_dict` to convert the column data type from `FLOAT64` `to `CATEGORY`." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "nDMH_chybe4w" + }, + "source": [ + "### Update dataset: assign a label" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "hVIruWg0u33t" + }, + "outputs": [], + "source": [ + "#@title Update dataset { vertical-output: true }\n", + "\n", + "label_column_name = 'income' #@param {type: 'string'}\n", + "label_column_spec = column_specs[label_column_name]\n", + "label_column_id = label_column_spec.name.rsplit('/', 1)[-1]\n", + "print('Label column ID: {}'.format(label_column_id))\n", + "# Define the values of the fields to be updated.\n", + "update_dataset_dict = {\n", + " 'name': dataset_name,\n", + " 'tables_dataset_metadata': {\n", + " 'target_column_spec_id': label_column_id\n", + " }\n", + "}\n", + "update_dataset_response = client.update_dataset(update_dataset_dict)\n", + "update_dataset_response" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "z23NITLrcxmi" + }, + "source": [ + "___" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "FcKgvj1-Tbgj" + }, + "source": [ + "## 5. Creating a model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "Pnlk8vdQlO_k" + }, + "source": [ + "### Train a model\n", + "Specify the duration of the training. For example, `'train_budget_milli_node_hours': 1000` runs the training for one hour. If your Colab times out, use `client.list_models(location_path)` to check whether your model has been created. Then use model name to continue to the next steps. Run the following command to retrieve your model. Replace `model_name` with its actual value.\n", + "\n", + " model = client.get_model(model_name)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "11izNd6Fu37N" + }, + "outputs": [], + "source": [ + "#@title Create model { vertical-output: true }\n", + "\n", + "model_display_name = 'census_income_model' #@param {type:'string'}\n", + "\n", + "model_dict = {\n", + " 'display_name': model_display_name,\n", + " 'dataset_id': dataset_name.rsplit('/', 1)[-1],\n", + " 'tables_model_metadata': {'train_budget_milli_node_hours': 1000}\n", + "}\n", + "create_model_response = client.create_model(location_path, model_dict)\n", + "print('Dataset import operation: {}'.format(create_model_response.operation))\n", + "# Wait until model training is done.\n", + "create_model_result = create_model_response.result()\n", + "model_name = create_model_result.name\n", + "create_model_result" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "1wS1is9IY5nK" + }, + "source": [ + "___" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "LMYmHSiCE8om" + }, + "source": [ + "## 6. Make a prediction" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "G2WVbMFll96k" + }, + "source": [ + "### There are two different prediction modes: online and batch. The following cells show you how to make an online prediction. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "ZAGi8Co-SU-b" + }, + "source": [ + "Run the following cell, and then choose the desired test values for your online prediction." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "yt-KXEEQu3-U" + }, + "outputs": [], + "source": [ + "#@title Make an online prediction: set the categorical variables{ vertical-output: true }\n", + "from ipywidgets import interact\n", + "import ipywidgets as widgets\n", + "\n", + "workclass_ids = ['Private', 'Self-emp-not-inc', 'Self-emp-inc', 'Federal-gov', 'Local-gov', 'State-gov', 'Without-pay', 'Never-worked']\n", + "education_ids = ['Bachelors', 'Some-college', '11th', 'HS-grad', 'Prof-school', 'Assoc-acdm', 'Assoc-voc', '9th', '7th-8th', '12th', 'Masters', '1st-4th', '10th', 'Doctorate', '5th-6th', 'Preschool']\n", + "marital_status_ids = ['Married-civ-spouse', 'Divorced', 'Never-married', 'Separated', 'Widowed', 'Married-spouse-absent', 'Married-AF-spouse']\n", + "occupation_ids = ['Tech-support', 'Craft-repair', 'Other-service', 'Sales', 'Exec-managerial', 'Prof-specialty', 'Handlers-cleaners', 'Machine-op-inspct', 'Adm-clerical', 'Farming-fishing', 'Transport-moving', 'Priv-house-serv', 'Protective-serv', 'Armed-Forces']\n", + "relationship_ids = ['Wife', 'Own-child', 'Husband', 'Not-in-family', 'Other-relative', 'Unmarried']\n", + "race_ids = ['White', 'Asian-Pac-Islander', 'Amer-Indian-Eskimo', 'Other', 'Black']\n", + "sex_ids = ['Female', 'Male']\n", + "native_country_ids = ['United-States', 'Cambodia', 'England', 'Puerto-Rico', 'Canada', 'Germany', 'Outlying-US(Guam-USVI-etc)', 'India', 'Japan', 'Greece', 'South', 'China', 'Cuba', 'Iran', 'Honduras', 'Philippines', 'Italy', 'Poland', 'Jamaica', 'Vietnam', 'Mexico', 'Portugal', 'Ireland', 'France', 'Dominican-Republic', 'Laos', 'Ecuador', 'Taiwan', 'Haiti', 'Columbia', 'Hungary', 'Guatemala', 'Nicaragua', 'Scotland', 'Thailand', 'Yugoslavia', 'El-Salvador', 'Trinadad&Tobago', 'Peru', 'Hong', 'Holand-Netherlands']\n", + "workclass = widgets.Dropdown(options=workclass_ids, value=workclass_ids[0],\n", + " description='workclass:')\n", + "\n", + "education = widgets.Dropdown(options=education_ids, value=education_ids[0],\n", + " description='education:', width='500px')\n", + "\n", + "marital_status = widgets.Dropdown(options=marital_status_ids, value=marital_status_ids[0],\n", + " description='marital status:', width='500px')\n", + "\n", + "occupation = widgets.Dropdown(options=occupation_ids, value=occupation_ids[0],\n", + " description='occupation:', width='500px')\n", + "\n", + "relationship = widgets.Dropdown(options=relationship_ids, value=relationship_ids[0],\n", + " description='relationship:', width='500px')\n", + "\n", + "race = widgets.Dropdown(options=race_ids, value=race_ids[0],\n", + " description='race:', width='500px')\n", + "\n", + "sex = widgets.Dropdown(options=sex_ids, value=sex_ids[0],\n", + " description='sex:', width='500px')\n", + "\n", + "native_country = widgets.Dropdown(options=native_country_ids, value=native_country_ids[0],\n", + " description='native_country:', width='500px')\n", + "\n", + "display(workclass)\n", + "display(education)\n", + "display(marital_status)\n", + "display(occupation)\n", + "display(relationship)\n", + "display(race)\n", + "display(sex)\n", + "display(native_country)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "xGVGwgwXSZe_" + }, + "source": [ + "Adjust the slides on the right to the desired test values for your online prediction." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "bDzd5GYQSdpa" + }, + "outputs": [], + "source": [ + "#@title Make an online prediction: set the numeric variables{ vertical-output: true }\n", + "\n", + "age = 34 #@param {type:'slider', min:1, max:100, step:1}\n", + "capital_gain = 40000 #@param {type:'slider', min:0, max:100000, step:10000}\n", + "capital_loss = 3.8 #@param {type:'slider', min:0, max:4000, step:0.1}\n", + "fnlwgt = 150000 #@param {type:'slider', min:0, max:1000000, step:50000}\n", + "education_num = 9 #@param {type:'slider', min:1, max:16, step:1}\n", + "hours_per_week = 40 #@param {type:'slider', min:1, max:100, step:1}\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "n0lFAIkISf4K" + }, + "source": [ + "**IMPORTANT** : Deploy the model, then wait until the model FINISHES deployment.\n", + "Check the [UI](https://console.cloud.google.com/automl-tables?_ga=2.255483016.-1079099924.1550856636) and navigate to the predict tab of your model, and then to the online prediction portion, to see when it finishes online deployment before running the prediction cell." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "kRoHFbVnSk05" + }, + "outputs": [], + "source": [ + "response = client.deploy_model(model_name)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "0tymBrhLSnDX" + }, + "source": [ + "Run the prediction, only after the model finishes deployment" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "Kc4SKLLPSoKz" + }, + "outputs": [], + "source": [ + "payload = {\n", + " 'row': { \n", + " 'values': [\n", + " {'number_value': age},\n", + " {'string_value': workclass.value},\n", + " {'number_value': fnlwgt},\n", + " {'string_value': education.value},\n", + " {'number_value': education_num},\n", + " {'string_value': marital_status.value},\n", + " {'string_value': occupation.value},\n", + " {'string_value': relationship.value},\n", + " {'string_value': race.value},\n", + " {'string_value': sex.value},\n", + " {'number_value': capital_gain},\n", + " {'number_value': capital_loss},\n", + " {'number_value': hours_per_week},\n", + " {'string_value': native_country.value}\n", + " ]\n", + " }\n", + "}\n", + "prediction_client.predict(model_name, payload)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "O9CRdIfrS1nb" + }, + "source": [ + "Undeploy the model" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "DWa1idtOS0GE" + }, + "outputs": [], + "source": [ + "response2 = client.undeploy_model(model_name)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "TarOq84-GXch" + }, + "source": [ + "## 7. Batch prediction" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "Soy5OB8Wbp_R" + }, + "source": [ + "### Initialize prediction" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "39bIGjIlau5a" + }, + "source": [ + "Your data source for batch prediction can be GCS or BigQuery. For this tutorial, you can use [census_income_batch_prediction_input.csv](https://storage.cloud.google.com/cloud-ml-data/automl-tables/notebooks/census_income_batch_prediction_input.csv) as input source. Create a GCS bucket and upload the file into your bucket. Some of the lines in the batch prediction input file are intentionally left missing some values. The AutoML Tables logs the errors in the `errors.csv` file.\n", + "Also, enter the UI and create the bucket into which you will load your predictions. The bucket's default name here is automl-tables-pred.\n", + "\n", + "**NOTE:** The client library has a bug. If the following cell returns a `TypeError: Could not convert Any to BatchPredictResult` error, ignore it. The batch prediction output file(s) will be updated to the GCS bucket that you set in the preceding cells." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "gkF3bH0qu4DU" + }, + "outputs": [], + "source": [ + "#@title Start batch prediction { vertical-output: true, output-height: 200 }\n", + "\n", + "batch_predict_gcs_input_uris = ['gs://cloud-ml-data/automl-tables/notebooks/census_income_batch_prediction_input.csv',] #@param\n", + "batch_predict_gcs_output_uri_prefix = 'gs://automl-tables-pred1' #@param {type:'string'}\n", + "#gs://automl-tables-pred\n", + "# Define input source.\n", + "batch_prediction_input_source = {\n", + " 'gcs_source': {\n", + " 'input_uris': batch_predict_gcs_input_uris\n", + " }\n", + "}\n", + "# Define output target.\n", + "batch_prediction_output_target = {\n", + " 'gcs_destination': {\n", + " 'output_uri_prefix': batch_predict_gcs_output_uri_prefix\n", + " }\n", + "}\n", + "batch_predict_response = prediction_client.batch_predict(\n", + " model_name, batch_prediction_input_source, batch_prediction_output_target)\n", + "print('Batch prediction operation: {}'.format(batch_predict_response.operation))\n", + "# Wait until batch prediction is done.\n", + "batch_predict_result = batch_predict_response.result()\n", + "batch_predict_response.metadata" + ] + } + ], + "metadata": { + "colab": { + "collapsed_sections": [], + "name": "census_income_prediction.ipynb", + "provenance": [], + "version": "0.3.2" + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.2" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} \ No newline at end of file diff --git a/tables/automl/notebooks/energy_price_forecasting/README.md b/tables/automl/notebooks/energy_price_forecasting/README.md new file mode 100644 index 00000000000..f9612854db2 --- /dev/null +++ b/tables/automl/notebooks/energy_price_forecasting/README.md @@ -0,0 +1,112 @@ +---------------------------------------- + +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0) + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and limitations under the License. + +---------------------------------------- + +# 1. Introduction + +This guide provides a high-level overview of an energy price forecasting solution, reviewing the significance of the solution and which audiences and use cases it applies to. In this section, we outline the business case for this solution, the problem, the solution, and results. In section 2, we provide the code setup instructions. + +Solution description: Model to forecast hourly energy prices for the next 7 days. + +Significance: This is a good complement to standard demand forecasting models that typically predict N periods in the future. This model does a rolling forecast that is vital for operational decisions. It also takes into consideration historical trends, seasonal patterns, and external factors (like weather) to make more accurate forecasts. + + +## 1.1 Solution scenario + +### Challenge + +Many companies use forecasting models to predict prices, demand, sales, etc. Many of these forecasting problems have similar characteristics that can be leveraged to produce accurate predictions, like historical trends, seasonal patterns, and external factors. + +For example, think about an energy company that needs to accurately forecast the country’s hourly energy prices for the next 5 days (120 predictions) for optimal energy trading. + +At forecast time, they have access to historical energy prices as well as weather forecasts for the time period in question. + +In this particular scenario, an energy company actually hosted a competition ([http://complatt.smartwatt.net/](http://complatt.smartwatt.net/)) for developers to use the data sets to create a more accurate prediction model. + +### Solution + +We solved the energy pricing challenge by preparing a training dataset that encodes historical price trends, seasonal price patterns, and weather forecasts in a single table. We then used that table to train a deep neural network that can make accurate hourly predictions for the next 5 days. + +## 1.2 Similar applications + +Using the solution that we created for the competition, we can now show how other forecasting problems can also be solved with the same solution. + +This type of solution includes any demand forecasting model that predicts N periods in the future and takes advantage of seasonal patterns, historical trends, and external datasets to produce accurate forecasts. + +Here are some additional demand forecasting examples: + +* Sales forecasting + +* Product or service usage forecasting + +* Traffic forecasting + + +# 2. Setting up the solution in a Google Cloud Platform project + +## 2.1 Create GCP project and download raw data + +Learn how to create a GCP project and prepare it for running the solution following these steps: + +1. Create a project in GCP ([article](https://cloud.google.com/resource-manager/docs/creating-managing-projects) on how to create and manage GCP projects). + +2. Raw data for this problem: + +>[MarketPricePT](http://complatt.smartwatt.net/assets/files/historicalRealData/RealMarketPriceDataPT.csv) - Historical hourly energy prices. +>![alt text](https://storage.googleapis.com/images_public/price_schema.png) +>![alt text](https://storage.googleapis.com/images_public/price_data.png) + +>[historical_weather](http://complatt.smartwatt.net/assets/files/weatherHistoricalData/WeatherHistoricalData.zip) - Historical hourly weather forecasts. +>![alt text](https://storage.googleapis.com/images_public/weather_schema.png) +>![alt text](https://storage.googleapis.com/images_public/loc_portugal.png) +>![alt text](https://storage.googleapis.com/images_public/weather_data.png) + +*Disclaimer: The data for both tables comes from [http://complatt.smartwatt.net/](http://complatt.smartwatt.net/). This website hosts a closed competition meant to solve the energy price forecasting problem. The data was not collected or vetted by Google LLC and hence, we cannot guarantee the veracity or quality of it. + + +## 2.2 Execute script for data preparation + +Prepare the data that is going to be used by the forecaster model by following these instructions: + +1. Clone the solution code from here: [https://github.com/GoogleCloudPlatform/professional-services/tree/master/examples/cloudml-energy-price-forecasting](https://github.com/GoogleCloudPlatform/professional-services/tree/master/examples/cloudml-energy-price-forecasting). In the solution code, navigate to the "data_preparation" folder. + +2. Run script "data_preparation.data_prep" to generate training, validation, and testing data as well as the constant files needed for normalization. + +3. Export training, validation, and testing tables as CSVs (into Google Cloud Storage bucket gs://energyforecast/data/csv). + +4. Read the "README.md" file for more information. + +5. Understand which parameters can be passed to the script (to override defaults). + +Training data schema: +![alt text](https://storage.googleapis.com/images_public/training_schema.png) + +## 2.3 Execute notebook in this folder + +Train the forecasting model in AutoML tables by running all cells in the notebook in this folder! + +## 2.4 AutoML Tables Results + +The following results are from our solution to this problem. + +* MAE (Mean Absolute Error) = 0.0416 +* RMSE (Root Mean Squared Error) = 0.0524 + +![alt text](https://storage.googleapis.com/images_public/automl_test.png) + +Feature importance: +![alt text](https://storage.googleapis.com/images_public/feature_importance.png) + diff --git a/tables/automl/notebooks/energy_price_forecasting/energy_price_forecasting.ipynb b/tables/automl/notebooks/energy_price_forecasting/energy_price_forecasting.ipynb new file mode 100644 index 00000000000..fab8984c711 --- /dev/null +++ b/tables/automl/notebooks/energy_price_forecasting/energy_price_forecasting.ipynb @@ -0,0 +1,727 @@ + +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "Energy_Price_Forecasting.ipynb", + "version": "0.3.2", + "provenance": [], + "collapsed_sections": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "cells": [ + { + "metadata": { + "id": "KOAz-lD1P7Kx", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "----------------------------------------\n", + "\n", + "Copyright 2018 Google LLC \n", + "\n", + "Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "you may not use this file except in compliance with the License.\n", + "You may obtain a copy of the License at\n", + "\n", + "[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0)\n", + "\n", + "Unless required by applicable law or agreed to in writing, software\n", + "distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "See the License for the specific language governing permissions and limitations under the License.\n", + "\n", + "----------------------------------------" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "m26YhtBMvVWA" + }, + "cell_type": "markdown", + "source": [ + "# Energy Forecasting with AutoML Tables\n", + "\n", + "To use this Colab notebook, copy it to your own Google Drive and open it with [Colaboratory](https://colab.research.google.com/) (or Colab). To run a cell hold the Shift key and press the Enter key (or Return key). Colab automatically displays the return value of the last line in each cell. Refer to [this page](https://colab.research.google.com/notebooks/welcome.ipynb) for more information on Colab.\n", + "\n", + "You can run a Colab notebook on a hosted runtime in the Cloud. The hosted VM times out after 90 minutes of inactivity and you will lose all the data stored in the memory including your authentication data. If your session gets disconnected (for example, because you closed your laptop) for less than the 90 minute inactivity timeout limit, press 'RECONNECT' on the top right corner of your notebook and resume the session. After Colab timeout, you'll need to\n", + "\n", + "1. Re-run the initialization and authentication.\n", + "2. Continue from where you left off. You may need to copy-paste the value of some variables such as the `dataset_name` from the printed output of the previous cells.\n", + "\n", + "Alternatively you can connect your Colab notebook to a [local runtime](https://research.google.com/colaboratory/local-runtimes.html)." + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "b--5FDDwCG9C" + }, + "cell_type": "markdown", + "source": [ + "## 1. Project set up\n", + "\n", + "\n", + "\n" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "AZs0ICgy4jkQ" + }, + "cell_type": "markdown", + "source": [ + "Follow the [AutoML Tables documentation](https://cloud.google.com/automl-tables/docs/) to\n", + "* Create a Google Cloud Platform (GCP) project.\n", + "* Enable billing.\n", + "* Apply to whitelist your project.\n", + "* Enable AutoML API.\n", + "* Enable AutoML Talbes API.\n", + "* Create a service account, grant required permissions, and download the service account private key.\n", + "\n", + "You also need to upload your data into Google Cloud Storage (GCS) or BigQuery. For example, to use GCS as your data source\n", + "* Create a GCS bucket.\n", + "* Upload the training and batch prediction files.\n", + "\n", + "\n", + "**Warning:** Private keys must be kept secret. If you expose your private key it is recommended to revoke it immediately from the Google Cloud Console." + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "xZECt1oL429r" + }, + "cell_type": "markdown", + "source": [ + "\n", + "\n", + "---\n", + "\n" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "rstRPH9SyZj_" + }, + "cell_type": "markdown", + "source": [ + "## 2. Initialize and authenticate\n", + "This section runs intialization and authentication. It creates an authenticated session which is required for running any of the following sections." + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "BR0POq2UzE7e" + }, + "cell_type": "markdown", + "source": [ + "### Install the client library\n", + "Run the following cell to install the client library using `pip`." + ] + }, + { + "metadata": { + "id": "43aXKjDRt_qZ", + "colab_type": "code", + "colab": { + "resources": { + "http://localhost:8080/nbextensions/google.colab/files.js": { + "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=", + "ok": true, + "headers": [ + [ + "content-type", + "application/javascript" + ] + ], + "status": 200, + "status_text": "" + } + }, + "base_uri": "https://localhost:8080/", + "height": 602 + }, + "outputId": "4d3628f9-e5be-4145-f550-8eaffca97d37" + }, + "cell_type": "code", + "source": [ + "#@title Install AutoML Tables client library { vertical-output: true }\n", + "\n", + "!pip install google-cloud-automl" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "eVFsPPEociwF" + }, + "cell_type": "markdown", + "source": [ + "### Authenticate using service account key\n", + "Run the following cell. Click on the 'Choose Files' button and select the service account private key file. If your Service Account key file or folder is hidden, you can reveal it in a Mac by pressing the Command + Shift + . combo." + ] + }, + { + "metadata": { + "id": "u-kCqysAuaJk", + "colab_type": "code", + "colab": { + "resources": { + "http://localhost:8080/nbextensions/google.colab/files.js": { + "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=", + "ok": true, + "headers": [ + [ + "content-type", + "application/javascript" + ] + ], + "status": 200, + "status_text": "" + } + }, + "base_uri": "https://localhost:8080/", + "height": 71 + }, + "outputId": "06154a63-f410-435f-b565-cd1599243b88" + }, + "cell_type": "code", + "source": [ + "#@title Authenticate using service account key and create a client. { vertical-output: true }\n", + "\n", + "from google.cloud import automl_v1beta1\n", + "\n", + "# Upload service account key\n", + "keyfile_upload = files.upload()\n", + "keyfile_name = list(keyfile_upload.keys())[0]\n", + "# Authenticate and create an AutoML client.\n", + "client = automl_v1beta1.AutoMlClient.from_service_account_file(keyfile_name)\n", + "# Authenticate and create a prediction service client.\n", + "prediction_client = automl_v1beta1.PredictionServiceClient.from_service_account_file(keyfile_name)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "s3F2xbEJdDvN" + }, + "cell_type": "markdown", + "source": [ + "### Set Project and Location" + ] + }, + { + "metadata": { + "id": "0uX4aJYUiXh5", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Enter your GCP project ID." + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "6R4h5HF1Dtds", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "outputId": "1e049b34-4683-4755-ab08-aec08de2bc66" + }, + "cell_type": "code", + "source": [ + "#@title GCP project ID and location\n", + "\n", + "project_id = 'energy-forecasting' #@param {type:'string'}\n", + "location = 'us-central1' #@param {type:'string'}\n", + "location_path = client.location_path(project_id, location)\n", + "location_path" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "qozQWMnOu48y" + }, + "cell_type": "markdown", + "source": [ + "\n", + "\n", + "---\n", + "\n" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "ODt86YuVDZzm" + }, + "cell_type": "markdown", + "source": [ + "## 3. Import training data" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "XwjZc9Q62Fm5" + }, + "cell_type": "markdown", + "source": [ + "### Create dataset" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "_JfZFGSceyE_" + }, + "cell_type": "markdown", + "source": [ + "Select a dataset display name and pass your table source information to create a new dataset." + ] + }, + { + "metadata": { + "id": "Z_JErW3cw-0J", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 224 + }, + "outputId": "7fe366df-73ae-4ab1-ceaa-fd6ced4ccdd9" + }, + "cell_type": "code", + "source": [ + "#@title Create dataset { vertical-output: true, output-height: 200 }\n", + "\n", + "dataset_display_name = 'energy_forecasting_solution' #@param {type: 'string'}\n", + "\n", + "create_dataset_response = client.create_dataset(\n", + " location_path,\n", + " {'display_name': dataset_display_name, 'tables_dataset_metadata': {}})\n", + "dataset_name = create_dataset_response.name\n", + "create_dataset_response" + ], + "execution_count":0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "35YZ9dy34VqJ" + }, + "cell_type": "markdown", + "source": [ + "### Import data" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "3c0o15gVREAw" + }, + "cell_type": "markdown", + "source": [ + "You can import your data to AutoML Tables from GCS or BigQuery. For this tutorial, you can use the [iris dataset](https://storage.cloud.google.com/rostam-193618-tutorial/automl-tables-v1beta1/iris.csv) as your training data. You can create a GCS bucket and upload the data into your bucket. The URI for your file is `gs://BUCKET_NAME/FOLDER_NAME1/FOLDER_NAME2/.../FILE_NAME`. Alternatively you can create a BigQuery table and upload the data into the table. The URI for your table is `bq://PROJECT_ID.DATASET_ID.TABLE_ID`.\n", + "\n", + "Importing data may take a few minutes or hours depending on the size of your data. If your Colab times out, run the following command to retrieve your dataset. Replace `dataset_name` with its actual value obtained in the preceding cells.\n", + "\n", + " dataset = client.get_dataset(dataset_name)" + ] + }, + { + "metadata": { + "id": "bB_GdeqCJW5i", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title Datasource in BigQuery { vertical-output: true }\n", + "\n", + "dataset_bq_input_uri = 'bq://energy-forecasting.Energy.automldata' #@param {type: 'string'}\n", + "# Define input configuration.\n", + "input_config = {\n", + " 'bigquery_source': {\n", + " 'input_uri': dataset_bq_input_uri\n", + " }\n", + "}" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "FNVYfpoXJsNB", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 139 + }, + "outputId": "0ecc8d11-5bf1-4c2e-f688-b6d9be934e3c" + }, + "cell_type": "code", + "source": [ + " #@title Import data { vertical-output: true }\n", + "\n", + "import_data_response = client.import_data(dataset_name, input_config)\n", + "print('Dataset import operation: {}'.format(import_data_response.operation))\n", + "# Wait until import is done.\n", + "import_data_result = import_data_response.result()\n", + "import_data_result" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "QdxBI4s44ZRI", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Review the specs" + ] + }, + { + "metadata": { + "id": "RC0PWKqH4jwr", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Run the following command to see table specs such as row count." + ] + }, + { + "metadata": { + "id": "v2Vzq_gwXxo-", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 3247 + }, + "outputId": "c89cd7b1-4344-46d9-c4a3-1b012b5b720d" + }, + "cell_type": "code", + "source": [ + "#@title Table schema { vertical-output: true }\n", + "\n", + "import google.cloud.automl_v1beta1.proto.data_types_pb2 as data_types\n", + "\n", + "# List table specs\n", + "list_table_specs_response = client.list_table_specs(dataset_name)\n", + "table_specs = [s for s in list_table_specs_response]\n", + "# List column specs\n", + "table_spec_name = table_specs[0].name\n", + "list_column_specs_response = client.list_column_specs(table_spec_name)\n", + "column_specs = {s.display_name: s for s in list_column_specs_response}\n", + "[(x, data_types.TypeCode.Name(\n", + " column_specs[x].data_type.type_code)) for x in column_specs.keys()]" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "vcJP7xoq4yAJ", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Run the following command to see column specs such inferred schema." + ] + }, + { + "metadata": { + "id": "FNykW_YOYt6d", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "___" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "kNRVJqVOL8h3" + }, + "cell_type": "markdown", + "source": [ + "## 4. Update dataset: assign a label column and enable nullable columns" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "-57gehId9PQ5" + }, + "cell_type": "markdown", + "source": [ + "AutoML Tables automatically detects your data column type. For example, for the [Iris dataset](https://storage.cloud.google.com/rostam-193618-tutorial/automl-tables-v1beta1/iris.csv) it detects `species` to be categorical and `petal_length`, `petal_width`, `sepal_length`, and `sepal_width` to be numerical. Depending on the type of your label column, AutoML Tables chooses to run a classification or regression model. If your label column contains only numerical values, but they represent categories, change your label column type to categorical by updating your schema." + ] + }, + { + "metadata": { + "id": "iRqdQ7Xiq04x", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Update a column: set as categorical" + ] + }, + { + "metadata": { + "id": "OCEUIPKegWrf", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "outputId": "44370b2c-f3dc-46bc-cefd-8a6f29f9cabe" + }, + "cell_type": "code", + "source": [ + "#@title Update dataset { vertical-output: true }\n", + "\n", + "column_to_category = 'hour' #@param {type: 'string'}\n", + "\n", + "update_column_spec_dict = {\n", + " \"name\": column_specs[column_to_category].name,\n", + " \"data_type\": {\n", + " \"type_code\": \"CATEGORY\"\n", + " }\n", + "}\n", + "update_column_response = client.update_column_spec(update_column_spec_dict)\n", + "update_column_response.display_name , update_column_response.data_type \n" + ], + "execution_count":0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "nDMH_chybe4w" + }, + "cell_type": "markdown", + "source": [ + "### Update dataset: assign a label and split column" + ] + }, + { + "metadata": { + "id": "hVIruWg0u33t", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 360 + }, + "outputId": "eeb5f733-16ec-4191-ea59-c2fab30c8442" + }, + "cell_type": "code", + "source": [ + "#@title Update dataset { vertical-output: true }\n", + "\n", + "label_column_name = 'price' #@param {type: 'string'}\n", + "label_column_spec = column_specs[label_column_name]\n", + "label_column_id = label_column_spec.name.rsplit('/', 1)[-1]\n", + "print('Label column ID: {}'.format(label_column_id))\n", + "\n", + "split_column_name = 'split' #@param {type: 'string'}\n", + "split_column_spec = column_specs[split_column_name]\n", + "split_column_id = split_column_spec.name.rsplit('/', 1)[-1]\n", + "print('Split column ID: {}'.format(split_column_id))\n", + "# Define the values of the fields to be updated.\n", + "update_dataset_dict = {\n", + " 'name': dataset_name,\n", + " 'tables_dataset_metadata': {\n", + " 'target_column_spec_id': label_column_id,\n", + " 'ml_use_column_spec_id': split_column_id,\n", + " }\n", + "}\n", + "update_dataset_response = client.update_dataset(update_dataset_dict)\n", + "update_dataset_response" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "z23NITLrcxmi", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "___" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "FcKgvj1-Tbgj" + }, + "cell_type": "markdown", + "source": [ + "## 5. Creating a model" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "Pnlk8vdQlO_k" + }, + "cell_type": "markdown", + "source": [ + "### Train a model\n", + "Specify the duration of the training. For example, `'train_budget_milli_node_hours': 1000` runs the training for one hour. If your Colab times out, use `client.list_models(location_path)` to check whether your model has been created. Then use model name to continue to the next steps. Run the following command to retrieve your model. Replace `model_name` with its actual value.\n", + "\n", + " model = client.get_model(model_name)" + ] + }, + { + "metadata": { + "id": "11izNd6Fu37N", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 139 + }, + "outputId": "1bca25aa-eb19-4b27-a3fa-7ef137aaf4e2" + }, + "cell_type": "code", + "source": [ + "#@title Create model { vertical-output: true }\n", + "\n", + "\n", + "\n", + "model_display_name = 'energy_model' #@param {type:'string'}\n", + "model_train_hours = 12 #@param {type:'integer'}\n", + "model_optimization_objective = 'MINIMIZE_MAE' #@param {type:'string'}\n", + "column_to_ignore = 'date_utc' #@param {type:'string'}\n", + "\n", + "# Create list of features to use\n", + "feat_list = list(column_specs.keys())\n", + "feat_list.remove(label_column_name)\n", + "feat_list.remove(split_column_name)\n", + "feat_list.remove(column_to_ignore)\n", + "\n", + "model_dict = {\n", + " 'display_name': model_display_name,\n", + " 'dataset_id': dataset_name.rsplit('/', 1)[-1],\n", + " 'tables_model_metadata': {\n", + " 'train_budget_milli_node_hours':model_train_hours * 1000,\n", + " 'optimization_objective': model_optimization_objective,\n", + " 'target_column_spec': column_specs[label_column_name],\n", + " 'input_feature_column_specs': [\n", + " column_specs[x] for x in feat_list]}\n", + " }\n", + " \n", + "create_model_response = client.create_model(location_path, model_dict)\n", + "print('Dataset import operation: {}'.format(create_model_response.operation))\n", + "# Wait until model training is done.\n", + "create_model_result = create_model_response.result()\n", + "model_name = create_model_result.name\n", + "create_model_result" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "puVew1GgPfQa", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 85 + }, + "outputId": "42b9296c-d231-4787-f7fb-4aa1a6ff9bd9" + }, + "cell_type": "code", + "source": [ + "#@title Model Metrics {vertical-output: true }\n", + "\n", + "metrics= [x for x in client.list_model_evaluations(model_name)][-1]\n", + "metrics.regression_evaluation_metrics" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "YQnfEwyrSt2T", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "![alt text](https://storage.googleapis.com/images_public/automl_test.png)" + ] + }, + { + "metadata": { + "id": "Vyc8ckbpRMHp", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 272 + }, + "outputId": "931d4921-2144-4092-dab6-165c1b1c2a88" + }, + "cell_type": "code", + "source": [ + "#@title Feature Importance {vertical-output: true }\n", + "\n", + "model = client.get_model(model_name)\n", + "feat_list = [(x.feature_importance, x.column_display_name) for x in model.tables_model_metadata.tables_model_column_info]\n", + "feat_list.sort(reverse=True)\n", + "feat_list[:15]" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "__2gDQ5I5gcj", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "![alt text](https://storage.googleapis.com/images_public/feature_importance.png)\n", + "![alt text](https://storage.googleapis.com/images_public/loc_portugal.png)\n", + "![alt text](https://storage.googleapis.com/images_public/weather_schema.png)\n", + "![alt text](https://storage.googleapis.com/images_public/training_schema.png)" + ] + }, + { + "metadata": { + "id": "1wS1is9IY5nK", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "___" + ] + } + ] +} diff --git a/tables/automl/notebooks/purchase_prediction/README.md b/tables/automl/notebooks/purchase_prediction/README.md new file mode 100644 index 00000000000..b2dd7d271a5 --- /dev/null +++ b/tables/automl/notebooks/purchase_prediction/README.md @@ -0,0 +1,129 @@ +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License");you may not use this file except in compliance with the License.You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + + +# Purchase Prediction using AutoML Tables +One of the most common use cases in Marketing is to predict the likelihood of conversion. Conversion could be defined by the marketer as taking a certain action like making a purchase, signing up for a free trial, subscribing to a newsletter, etc. Knowing the likelihood that a marketing lead or prospect will ‘convert’ can enable the marketer to target the lead with the right marketing campaign. This could take the form of remarketing, targeted email campaigns, online offers or other treatments. + +Here we demonstrate how you can use Bigquery and AutoML Tables to build a supervised binary classification model for purchase prediction. + +## Problem Description +The model uses a real dataset from the [Google Merchandise store](www.googlemerchandisestore.com) consisting of Google Analytics web sessions. + +The goal here is to predict the likelihood of a web visitor visiting the online Google Merchandise Store making a purchase on the website during that Google Analytics session. Past web interactions of the user on the store website in addition to information like browser details and geography are used to make this prediction. + +This is framed as a binary classification model, to label a user during a session as either true (makes a purchase) or false (does not make a purchase). +Dataset Details +The dataset consists of a set of tables corresponding to Google Analytics sessions being tracked on the [Google Merchandise Store](https://www.googlemerchandisestore.com/). Each table is a single day of GA sessions. More details around the schema can be seen [here](https://support.google.com/analytics/answer/3437719?hl=en&ref_topic=3416089). + +You can access the data on BigQuery [here](https://bigquery.cloud.google.com/dataset/bigquery-public-data:google_analytics_sample). + +## Solution Walkthrough +The solution has been developed using [Google Colab Notebook](https://colab.research.google.com/notebooks/welcome.ipynb). Here are the thought process and specific steps that went into building the “Purchase Prediction with AutoML Tables” colab. The colab is broken into 7 parts; this write up will mirror that structure. + +Before we dive in, a few housekeeping notes about setting up the colab. + + +Steps Involved + +### 1. Set up +The first step in this process was to set up the project. We referred to the [AutoML tables documentation](https://cloud.google.com/automl-tables/docs/) and take the following steps: +* Create a Google Cloud Platform (GCP) project +* Enable billing +* Enable the AutoML API +* Enable the AutoML Tables API + +There are a few options concerning how to host the colab: default hosted runtime, local runtime, or hosting the runtime on a Virtual Machine (VM). + +##### Default Hosted Runtime: + +The hosted runtime is the simplest to use. It accesses a default VM already configured to host the colab notebook. Simply navigate to the upper right hand corner click on the connect drop down box, which will give you the option to “connect to hosted runtime”. + +##### Local Runtime: +The local runtime takes a bit more work. It involves downloading jupyter notebooks onto your local machine, likely the desktop from which you access the colab. After downloading jupyter notebooks, you can connect to the local runtime. The colab notebook will run off of your local machine. Detailed instructions can be found [here](https://research.google.com/colaboratory/local-runtimes.html). + +##### VM hosted Runtime: +Finally, the runtime hosted on the VM requires the most amount of set up, but gives you more control on the machine choice allowing you to access machines with more memory and processing.The instructions are similar to the steps taken for the local runtime, with one main distinction: the VM hosted runtime runs the colab notebook off of the VM, so you will need to set up everything on the VM rather than on your local machine. + +To achieve this, create a Compute Engine VM instance. Then make sure that you have the firewall open to allow you to ssh into the VM. + +The firewall rules can be found in the VPC Network tab on the Cloud Console. Navigate into the firewall rules, and add a rule that allows your local IP address to allow ingress on tcp: 22. To find your IP address, type into the terminal the following command: + +```curl -4 ifconfig.co``` + +Once your firewall rules are created, you should be able to ssh into your VM instance. To ssh, run the following command: + +```gcloud compute ssh --zone YOUR_ZONE YOUR_INSTANCE_NAME -- -L 8888:localhost:8888``` + +This will allow your local terminal to ssh into the VM instance you created, which simultaneously port forwarding the port 8888 from your local machine to the VM. Once in the VM, you can download jupyter notebooks and open up a notebook as seen in the instructions [here](https://research.google.com/colaboratory/local-runtimes.html). Specifically steps 2, 3. + +We recommend hosting using the VM for two main reasons: +1. The VM can be provisioned to be much, much more powerful than either your local machine or the default runtime allocated by the colab notebook. +2. The colab is currently configured to run on either your local machine or a VM. It requires you to install the AutoML client library and uplaod a service account key to the machine from which you are hosting the colab. These two actions can be done the default hosted runtime, but would require a different set of instructions not detailed in this specific colab. To see them, refer to the AutoML Tables sample colab found in the tutorials section of the [documentation](https://cloud.google.com/automl-tables/docs/). Specifically step 2. + + +### 2. Initialize and authenticate +The client library installation is entirely self explanatory in the colab. + +The authentication process is only slightly more complex: run the second code block entitled "Authenticate using service account key and create a client" and then upload the service account key you created in the set up step + Would also recommend setting a global variable + +```export GOOGLE_APPLICATION_CREDENTIALS=`` ``` + +Be sure to export whenever you boot up a new session. + + +### 3. Data Cleaning and Transformation +This step was by far the most involved. It includes a few sections that create an AutoML tables dataset, pull the Google merchandise store data from BigQuery, transform the data, and save it multiple times to csv files in google cloud storage. + +The dataset that is made viewable in the AutoML Tables UI. It will eventually hold the training data after that training data is cleaned and transformed. + +This dataset has only around 1% of its values with a positive label value of True i.e. cases when a transaction was made. This is a class imbalance problem. There are several ways to handle class imbalance. We chose to oversample the positive class by random over sampling. This resulted in an artificial increase in the sessions with the positive label of true transaction value. + +There were also many columns with either all missing or all constant values. These columns would not add any signal to our model, so we dropped them. + +There were also columns with NaN rather than 0 values. For instance, rather than having a count of 0, a column might have a null value. So we added code to change some of these null values to 0, specifically in our target column, in which null values were not allowed by AutoML Tables. However, AutoML Tables can handle null values for the features. + +### 4. Feature Engineering + +The dataset had rich information on customer location and behavior; however, it can be improved by performing feature engineering. Moreover, there was a concern about data leakage. The decision to do feature engineering, therefore, had two contributing motivations: remove data leakage without too much loss of useful data, and to improve the signal in our data. + + + +#### 4.1 Weekdays + +The date seemed like a useful piece of information to include, as it could capture seasonal effects. Unfortunately, we only had one year of data, so seasonality on an annual scale would be difficult (read impossible) to incorporate. Fortunately, we could try and detect seasonal effects on a micro, with perhaps equally informative results. We ended up creating a new column of weekdays out of dates, to denote which day of the week the session was held on. This new feature turned out to have some useful predictive power, when added as a variable into our model. + +#### 4.2 Data Leakage +The marginal gain from adding a weekday feature, was overshadowed by the concern of data leakage in our training data. In the initial naive models we trained, we got outstanding results. So outstanding that we knew that something must be going on. As it turned out, quite a few features functioned as proxies for the feature we were trying to predict: meaning some of the features we conditioned on to build the model had an almost 1:1 correlation with the target feature. Intuitively, this made sense. + +One feature that exhibited this behavior was the number of page views a customer made during a session. By conditioning on page views in a session, we could very reliably predict which customer sessions a purchase would be made in. At first this seems like the golden ticket, we can reliably predict whether or not a purchase is made! The catch: the full page view information can only be collected at the end of the session, by which point we would also have whether or not a transaction was made. Seen from this perspective, collecting page views at the same time as collecting the transaction information would make it pointless to predict the transaction information using the page views information, as we would already have both. One solution was to drop page views as a feature entirely. This would safely stop the data leakage, but we would lose some critically useful information. Another solution, (the one we ended up going with), was to track the page view information of all previous sessions for a given customer, and use it to inform the current session. This way, we could use the page view information, but only the information that we would have before the session even began. So we created a new column called previous_views, and populated it with the total count of all previous page views made by the customer in all previous sessions. We then deleted the page views feature, to stop the data leakage. + +Our rationale for this change can be boiled down to the concise heuristic: only use the information that is available to us on the first click of the session. Applying this reasoning, we performed similar data engineering on other features which we found to be proxies for the label feature. We also refined our objective in the process: For a visit to the Google Merchandise store, what is the probability that a customer will make a purchase, and can we calculate this probability the moment the customer arrives? By clarifying the question, we both made the result more powerful/useful, and eliminated the data leakage that threatened to make the predictive power trivial. + + +### 5. Train-Validation-Test Split + +To create the datasets for training, testing and validation, we first had to consider what kind of data we were dealing with. The data we had keeps track of all customer sessions with the Google Merchandise store over a year. AutoML tables does its own training and testing, and delivers a quite nice UI to view the results in. For the training and testing dataset then, we simply used the over sampled, balanced dataset created by the transformations described above. But we first partitioned the dataset to include the first 9 months in one table and the last 3 in another. This allowed us to train and test with an entirely different dataset that what we used to validate. + +Moreover, we held off on oversampling for the validation dataset, to not bias the data that we would ultimately use to judge the success of our model. + +The decision to divide the sessions along time was made to avoid the model training on future data to predict past data. (This can be avoided with a datetime variable in the dataset and by toggling a button in the UI) + +### 6. Update dataset: assign a label column and enable nullable columns + +This section is fairly self explanatory in the colab. Simply update the target column to not nullable, and update the assigned label to ‘totalTransactionRevenue’ + +### 7. Creating a Model, Make a Prediction + +These parts are mostly self explanatory. +Note that we trained on the first 9 months of data and we validate using the last 3. + +### 8. Evaluate your Prediction +In this section, we take our validation data prediction results and plot the Precision Recall Curve and the ROC curve of both the false and true predictions. \ No newline at end of file diff --git a/tables/automl/notebooks/purchase_prediction/purchase_prediction.ipynb b/tables/automl/notebooks/purchase_prediction/purchase_prediction.ipynb new file mode 100644 index 00000000000..e2082b1fc5e --- /dev/null +++ b/tables/automl/notebooks/purchase_prediction/purchase_prediction.ipynb @@ -0,0 +1,909 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "colab_C4M.ipynb", + "version": "0.3.2", + "provenance": [], + "collapsed_sections": [] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "accelerator": "GPU" + }, + "cells": [ + { + "metadata": { + "id": "OFJAWue1ss3C", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "#Purchase Prediction with AutoML Tables\n", + "\n", + "To use this Colab notebook, copy it to your own Google Drive and open it with [Colaboratory](https://colab.research.google.com/) (or Colab). To run a cell hold the Shift key and press the Enter key (or Return key). Colab automatically displays the return value of the last line in each cell. Refer to [this page](https://colab.sandbox.google.com/notebooks/welcome.ipynb) for more information on Colab.\n", + "\n", + "You can run a Colab notebook on a hosted runtime in the Cloud. The hosted VM times out after 90 minutes of inactivity and you will lose all the data stored in the memory including your authentication data. If your session gets disconnected (for example, because you closed your laptop) for less than the 90 minute inactivity timeout limit, press 'RECONNECT' on the top right corner of your notebook and resume the session. After Colab timeout, you'll need to\n", + "\n", + "1. Re-run the initialization and authentication.\n", + "2. Continue from where you left off. You may need to copy-paste the value of some variables such as the `dataset_name` from the printed output of the previous cells.\n", + "\n", + "Alternatively you can connect your Colab notebook to a [local runtime](https://research.google.com/colaboratory/local-runtimes.html). \n", + "It is recommended to run this notebook using vm, as the computational complexity is high enough that the hosted runtime becomes inconveniently slow. The local runtime link above also contains instructions for running the notebook on a vm. When using a vm, be sure to use a tensorflow vm, as it comes with the colab libraries. A standard vm of vCPUs will not work with the colab libraries required for this colab.\n" + ] + }, + { + "metadata": { + "id": "dMoTkf3BVD39", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "#1. Project Set Up\n", + "Follow the [AutoML Tables documentation](https://cloud.google.com/automl-tables/docs/) to\n", + "* Create a Google Cloud Platform (GCP) project.\n", + "* Enable billing.\n", + "* Apply to whitelist your project.\n", + "* Enable AutoML API.\n", + "* Enable AutoML Tables API.\n", + "* Create a service account, grant required permissions, and download the service account private key.\n", + "\n", + "You also need to upload your data into Google Cloud Storage (GCS) or BigQuery. For example, to use GCS as your data source\n", + "* Create a GCS bucket.\n", + "* Upload the training and batch prediction files.\n", + "\n", + "\n", + "**Warning:** Private keys must be kept secret. If you expose your private key it is recommended to revoke it immediately from the Google Cloud Console.\n", + "Extra steps, other than permission setting\n", + "1. download both the client library and the service account\n", + "2. zip up the client library and upload it to the vm\n", + "3. upload the service account key to the vm\n", + "\n" + ] + }, + { + "metadata": { + "id": "KAg-2-BQ4un6", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# 2. Initialize and authenticate\n", + " This section runs intialization and authentication. It creates an authenticated session which is required for running any of the following sections." + ] + }, + { + "metadata": { + "id": "hid7SmtS4yE_", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Install the client library\n", + "Run the following cell to install uthe client library using pip." + ] + }, + { + "metadata": { + "id": "yXZlxqICsMg2", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title Install AutoML Tables client library { vertical-output: true }\n", + "!pip install google-cloud-automl", + "\n" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "gGuRq4DI47hj", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Authenticate using service account key\n", + "Create a service account key, and download it onto either your local machine or vm. Write in the path to the service account key. If your Service Account key file or folder is hidden, you can reveal it in a Mac by pressing the Command + Shift + . combo.\n" + ] + }, + { + "metadata": { + "id": "m3j1Kl4osNaJ", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title Authenticate using service account key and create a client. \n", + "from google.cloud import automl_v1beta1\n", + "import os \n", + "path = \"my-project-trial5-e542e03e96c7.json\" #@param {type:'string'}\n", + "os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = path\n", + "\n", + "# Authenticate and create an AutoML client.\n", + "client = automl_v1beta1.AutoMlClient.from_service_account_file(path)\n", + "# Authenticate and create a prediction service client.\n", + "prediction_client = automl_v1beta1.PredictionServiceClient.from_service_account_file(path)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "9zuplbargStJ", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Enter your GCP project ID.\n" + ] + }, + { + "metadata": { + "id": "KIdmobtSsPj8", + "colab_type": "code", + "outputId": "14c234ca-5070-4301-a48c-c69d16ae4c31", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + } + }, + "cell_type": "code", + "source": [ + "#@title GCP project ID and location\n", + "\n", + "project_id = 'my-project-trial5' #@param {type:'string'}\n", + "location = 'us-central1'\n", + "location_path = client.location_path(project_id, location)\n", + "location_path\n", + "\n" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "e1fYDBjDgYEB", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# 3. Import, clean, transform and perform feature engineering on the training Data" + ] + }, + { + "metadata": { + "id": "dYoCTvaAgZK2", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Create dataset in AutoML Tables\n" + ] + }, + { + "metadata": { + "id": "uPRPqyw2gebp", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Select a dataset display name and pass your table source information to create a new dataset.\n" + ] + }, + { + "metadata": { + "id": "Iu3KNlcwsRhN", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title Create dataset { vertical-output: true, output-height: 200 }\n", + "\n", + "dataset_display_name = 'colab_trial11' #@param {type: 'string'}\n", + "\n", + "create_dataset_response = client.create_dataset(\n", + " location_path,\n", + " {'display_name': dataset_display_name, 'tables_dataset_metadata': {}})\n", + "dataset_name = create_dataset_response.name\n", + "create_dataset_response\n", + "\n" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "iTT5N97D0YPo", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Create a bucke to store the training data in" + ] + }, + { + "metadata": { + "id": "RQuGIbyGgud9", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title Create bucket to store data in { vertical-output: true, output-height: 200 }\n", + "\n", + "bucket_name = 'trial_for_c4m' #@param {type: 'string'}\n" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "IQJuy1-PpF3b", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Import Dependencies\n" + ] + }, + { + "metadata": { + "id": "zzCeDmnnQRNy", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "!sudo pip install google-cloud-bigquery google-cloud-storage pandas pandas-gbq gcsfs oauth2client\n", + "\n", + "import datetime\n", + "import pandas as pd\n", + "\n", + "import gcsfs\n", + "from google.cloud import bigquery\n", + "from google.cloud import storage" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "UR5n1crIpQuX", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Transformation and Feature Engineering Functions\n" + ] + }, + { + "metadata": { + "id": "RODZJaq4o9b5", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def balanceTable(table):\n", + "\t#class count\n", + " count_class_false, count_class_true = table.totalTransactionRevenue.value_counts()\n", + "\n", + "\t#divide by class\n", + " table_class_false = table[table[\"totalTransactionRevenue\"] == False]\n", + " table_class_true = table[table[\"totalTransactionRevenue\"] == True]\n", + "\n", + "\t#random over-sampling\n", + " table_class_true_over = table_class_true.sample(count_class_false, replace = True)\n", + " table_test_over = pd.concat([table_class_false, table_class_true_over])\n", + " return table_test_over\n", + "\n", + "\n", + "def partitionTable(table, dt=20170500):\n", + " #the automl tables model could be training on future data and implicitly learning about past data in the testing\n", + " #dataset, this would cause data leakage. To prevent this, we are training only with the first 9 months of data (table1)\n", + " #and doing validation with the last three months of data (table2).\n", + " table1 = table[table[\"date\"] <= dt]\n", + " table2 = table[table[\"date\"] > dt]\n", + " return table1, table2\n", + "\n", + "def N_updatePrevCount(table, new_column, old_column):\n", + " table = table.fillna(0)\n", + " table[new_column] = 1\n", + " table.sort_values(by=['fullVisitorId','date'])\n", + " table[new_column] = table.groupby(['fullVisitorId'])[old_column].apply(lambda x: x.cumsum())\n", + " table.drop([old_column], axis = 1, inplace = True)\n", + " return table\n", + "\n", + "\n", + "def N_updateDate(table):\n", + " table['weekday'] = 1\n", + " table['date'] = pd.to_datetime(table['date'].astype(str), format = '%Y%m%d')\n", + " table['weekday'] = table['date'].dt.dayofweek\n", + " return table\n", + "\n", + "\n", + "def change_transaction_values(table):\n", + " table['totalTransactionRevenue'] = table['totalTransactionRevenue'].fillna(0)\n", + " table['totalTransactionRevenue'] = table['totalTransactionRevenue'].apply(lambda x: x!=0)\n", + " return table\n", + "\n", + "def saveTable(table, csv_file_name, bucket_name):\n", + " table.to_csv(csv_file_name, index = False)\n", + " storage_client = storage.Client()\n", + " bucket = storage_client.get_bucket(bucket_name)\n", + " blob = bucket.blob(csv_file_name)\n", + " blob.upload_from_filename(filename = csv_file_name)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "2eGAIUmRqjqX", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "###Import training data" + ] + }, + { + "metadata": { + "id": "XTmXPMUsTgEs", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "You also have the option of just downloading the file, FULL.csv, [here](https://storage.cloud.google.com/cloud-ml-data/automl-tables/notebooks/trial_for_c4m/FULL.csv), instead of running the code below. Just be sure to move the file into the google cloud storage bucket you specified above." + ] + }, + { + "metadata": { + "id": "Bl9-DSjIqj7c", + "colab_type": "code", + "cellView": "both", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title Input name of file to save data to { vertical-output: true, output-height: 200 }\n", + "sqll = '''\n", + "SELECT\n", + "date, device, geoNetwork, totals, trafficSource, fullVisitorId \n", + "FROM `bigquery-public-data.google_analytics_sample.ga_sessions_*`\n", + "WHERE\n", + "_TABLE_SUFFIX BETWEEN FORMAT_DATE('%Y%m%d',DATE_SUB('2017-08-01', INTERVAL 366 DAY))\n", + "AND\n", + "FORMAT_DATE('%Y%m%d',DATE_SUB('2017-08-01', INTERVAL 1 DAY))\n", + "'''\n", + "df = pd.read_gbq(sqll, project_id = project_id, dialect='standard')\n", + "print(df.iloc[:3])\n", + "path_to_data_pre_transformation = \"FULL.csv\" #@param {type: 'string'}\n", + "saveTable(df, path_to_data_pre_transformation, bucket_name)\n", + "\n" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "V5WK71tiq-2b", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "###Unnest the Data" + ] + }, + { + "metadata": { + "id": "RFpgLfeNqUBk", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#some transformations on the basic dataset\n", + "#@title Input the name of file to hold the unnested data to { vertical-output: true, output-height: 200 }\n", + "unnested_file_name = \"FULL_unnested.csv\" #@param {type: 'string'}\n" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "2dyJlNAVqXUn", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "You also have the option of just downloading the file, FULL_unnested.csv, [here](https://storage.cloud.google.com/cloud-ml-data/automl-tables/notebooks/trial_for_c4m/FULL_unnested.csv), instead of running the code below. Just be sure to move the file into the google cloud storage bucket you specified above." + ] + }, + { + "metadata": { + "id": "tLPHeF2Y2l5l", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "\n", + "table = pd.read_csv(\"gs://\"+bucket_name+\"/\"+file_name)\n", + "\n", + "column_names = ['device', 'geoNetwork','totals', 'trafficSource']\n", + "\n", + "for name in column_names:\n", + " print(name)\n", + " table[name] = table[name].apply(lambda i: dict(eval(i)))\n", + " temp = table[name].apply(pd.Series)\n", + " table = pd.concat([table, temp], axis=1).drop(name, axis=1)\n", + "\n", + "#need to drop a column\n", + "table.drop(['adwordsClickInfo'], axis = 1, inplace = True)\n", + "saveTable(table, unnested_file_name, bucket_name)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "9_WC-AJLsdqo", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "###Run the Transformations" + ] + }, + { + "metadata": { + "id": "YWQ4462vnpOg", + "colab_type": "code", + "outputId": "5ca7e95a-e0f2-48c2-9b59-8f043d233bd2", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 272 + } + }, + "cell_type": "code", + "source": [ + "table = pd.read_csv(\"gs://\"+bucket_name+\"/\"+unnested_file_name)\n", + "\n", + "consts = ['transactionRevenue', 'transactions', 'adContent', 'browserSize', 'campaignCode', \n", + "'cityId', 'flashVersion', 'javaEnabled', 'language', 'latitude', 'longitude', 'mobileDeviceBranding', \n", + "'mobileDeviceInfo', 'mobileDeviceMarketingName','mobileDeviceModel','mobileInputSelector', 'networkLocation', \n", + "'operatingSystemVersion', 'screenColors', 'screenResolution', 'screenviews', 'sessionQualityDim', 'timeOnScreen',\n", + "'visits', 'uniqueScreenviews', 'browserVersion','referralPath','fullVisitorId', 'date']\n", + "\n", + "table = N_updatePrevCount(table, 'previous_views', 'pageviews')\n", + "table = N_updatePrevCount(table, 'previous_hits', 'hits')\n", + "table = N_updatePrevCount(table, 'previous_timeOnSite', 'timeOnSite')\n", + "table = N_updatePrevCount(table, 'previous_Bounces', 'bounces')\n", + "\n", + "table = change_transaction_values(table)\n", + "\n", + "table1, table2 = partitionTable(table)\n", + "table1 = N_updateDate(table1)\n", + "table2 = N_updateDate(table2)\n", + "#validation_unnested_FULL.csv = the last 3 months of data\n", + "\n", + "table.drop(consts, axis = 1, inplace = True)\n", + "\n", + "saveTable(table2,'validation_unnested_FULL.csv', bucket_name)\n", + "\n", + "table1 = balanceTable(table1)\n", + "\n", + "#training_unnested_FULL.csv = the first 9 months of data\n", + "saveTable(table1, 'training_unnested_balanced_FULL.csv', bucket_name)\n" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "LqmARBnRHWh8", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title ... take the data source from GCS { vertical-output: true } \n", + "\n", + "dataset_gcs_input_uris = ['gs://trial_for_c4m/training_unnested_balanced_FULL.csv',] #@param\n", + "# Define input configuration.\n", + "input_config = {\n", + " 'gcs_source': {\n", + " 'input_uris': dataset_gcs_input_uris\n", + " }\n", + "}" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "SfXjtAwDsYlV", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + " #@title Import data { vertical-output: true }\n", + "\n", + "import_data_response = client.import_data(dataset_name, input_config)\n", + "print('Dataset import operation: {}'.format(import_data_response.operation))\n", + "# Wait until import is done.\n", + "import_data_result = import_data_response.result()\n", + "import_data_result" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "W3SiSLS4tml9", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# 4. Update dataset: assign a label column and enable nullable columns" + ] + }, + { + "metadata": { + "id": "jVo8Z8PGtpB7", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "AutoML Tables automatically detects your data column type. Depending on the type of your label column, AutoML Tables chooses to run a classification or regression model. If your label column contains only numerical values, but they represent categories, change your label column type to categorical by updating your schema." + ] + }, + { + "metadata": { + "id": "dMdOoFsXxyxj", + "colab_type": "code", + "outputId": "e6fab957-2316-48c0-be66-1bff9dc5c23c", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 255 + } + }, + "cell_type": "code", + "source": [ + "#@title Table schema { vertical-output: true }\n", + "\n", + "import google.cloud.automl_v1beta1.proto.data_types_pb2 as data_types\n", + "import matplotlib.pyplot as plt\n", + "\n", + "# List table specs\n", + "list_table_specs_response = client.list_table_specs(dataset_name)\n", + "table_specs = [s for s in list_table_specs_response]\n", + "# List column specs\n", + "table_spec_name = table_specs[0].name\n", + "list_column_specs_response = client.list_column_specs(table_spec_name)\n", + "column_specs = {s.display_name: s for s in list_column_specs_response}\n", + "# Table schema pie chart.\n", + "type_counts = {}\n", + "for column_spec in column_specs.values():\n", + " type_name = data_types.TypeCode.Name(column_spec.data_type.type_code)\n", + " type_counts[type_name] = type_counts.get(type_name, 0) + 1\n", + "\n", + "plt.pie(x=type_counts.values(), labels=type_counts.keys(), autopct='%1.1f%%')\n", + "plt.axis('equal')\n", + "plt.show()\n" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "AfT4upKysamH", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title Update a column: set to not nullable { vertical-output: true }\n", + "\n", + "update_column_spec_dict = {\n", + " 'name': column_specs['totalTransactionRevenue'].name,\n", + " 'data_type': {\n", + " 'type_code': 'CATEGORY',\n", + " 'nullable': False\n", + " }\n", + "}\n", + "update_column_response = client.update_column_spec(update_column_spec_dict)\n", + "update_column_response" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "3O9cFko3t3ai", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "**Tip:** You can use `'type_code': 'CATEGORY'` in the preceding `update_column_spec_dict` to convert the column data type from `FLOAT64` `to `CATEGORY`.\n", + "\n", + "\n" + ] + }, + { + "metadata": { + "id": "rR2RaPP7t6y8", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Update dataset: assign a label" + ] + }, + { + "metadata": { + "id": "aTt2mIzbsduV", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title Update dataset { vertical-output: true }\n", + "\n", + "label_column_name = 'totalTransactionRevenue' #@param {type: 'string'}\n", + "label_column_spec = column_specs[label_column_name]\n", + "label_column_id = label_column_spec.name.rsplit('/', 1)[-1]\n", + "print('Label column ID: {}'.format(label_column_id))\n", + "# Define the values of the fields to be updated.\n", + "update_dataset_dict = {\n", + " 'name': dataset_name,\n", + " 'tables_dataset_metadata': {\n", + " 'target_column_spec_id': label_column_id\n", + " }\n", + "}\n", + "update_dataset_response = client.update_dataset(update_dataset_dict)\n", + "update_dataset_response" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "xajewSavt9K1", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "#5. Creating a model" + ] + }, + { + "metadata": { + "id": "dA-FE6iWt-A_", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Train a model\n", + "Training the model may take one hour or more. The following cell keeps running until the training is done. If your Colab times out, use `client.list_models(location_path)` to check whether your model has been created. Then use model name to continue to the next steps. Run the following command to retrieve your model. Replace `model_name` with its actual value.\n", + "\n", + " model = client.get_model(model_name)\n", + " " + ] + }, + { + "metadata": { + "id": "Kp0gGkp8H3zj", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title Create model { vertical-output: true }\n", + "#this will create a model that can be access through the auto ml tables colab\n", + "model_display_name = 'trial_10' #@param {type:'string'}\n", + "\n", + "model_dict = {\n", + " 'display_name': model_display_name,\n", + " 'dataset_id': dataset_name.rsplit('/', 1)[-1],\n", + " 'tables_model_metadata': {'train_budget_milli_node_hours': 1000}\n", + "}\n", + "create_model_response = client.create_model(location_path, model_dict)\n", + "print('Dataset import operation: {}'.format(create_model_response.operation))\n", + "# Wait until model training is done.\n", + "create_model_result = create_model_response.result()\n", + "model_name = create_model_result.name\n", + "print(model_name)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "tCIk1e4UuDxZ", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# 6. Make a prediction" + ] + }, + { + "metadata": { + "id": "H7Fi5f9zuG5f", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "There are two different prediction modes: online and batch. The following cell shows you how to make a batch prediction." + ] + }, + { + "metadata": { + "id": "AZ_CPff77m4e", + "colab_type": "code", + "cellView": "both", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title Start batch prediction { vertical-output: true, output-height: 200 }\n", + "print(client.list_models(location_path))\n", + "\n", + "batch_predict_gcs_input_uris = ['gs://trial_for_c4m/validation_unnested_FULL.csv',] #@param\n", + "batch_predict_gcs_output_uri_prefix = 'gs://trial_for_c4m' #@param {type:'string'}\n", + "# Define input source.\n", + "batch_prediction_input_source = {\n", + " 'gcs_source': {\n", + " 'input_uris': batch_predict_gcs_input_uris\n", + " }\n", + "}\n", + "# Define output target.\n", + "batch_prediction_output_target = {\n", + " 'gcs_destination': {\n", + " 'output_uri_prefix': batch_predict_gcs_output_uri_prefix\n", + " }\n", + "}\n", + "batch_predict_response = prediction_client.batch_predict(\n", + " model_name, batch_prediction_input_source, batch_prediction_output_target)\n", + "print('Batch prediction operation: {}'.format(batch_predict_response.operation))\n", + "# Wait until batch prediction is done.\n", + "batch_predict_result = batch_predict_response.result()\n", + "batch_predict_response.metadata\n", + "\n" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "utGPmXI-uKNr", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "#7. Evaluate your prediction" + ] + }, + { + "metadata": { + "id": "GsOdhJeauTC3", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "The follow cell creates a Precision Recall Curve and a ROC curve for both the true and false classifications.\n", + "Fill in the batch_predict_results_location with the location of the results.csv file created in the previous \"Make a prediction\" step\n" + ] + }, + { + "metadata": { + "id": "orejkh0CH4mu", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "\n", + "import numpy as np\n", + "from sklearn import metrics\n", + "import matplotlib.pyplot as plt\n", + "\n", + "def invert(x):\n", + " return 1-x\n", + "\n", + "def switch_label(x):\n", + " return(not x)\n", + "batch_predict_results_location = 'gs://trial_for_c4m/prediction-trial_10-2019-03-23T00:22:56.802Z' #@param {type:'string'}\n", + "\n", + "table = pd.read_csv(batch_predict_results_location +'/result.csv')\n", + "y = table[\"totalTransactionRevenue\"]\n", + "scores = table[\"totalTransactionRevenue_1.0_score\"]\n", + "scores_invert = table['totalTransactionRevenue_0.0_score']\n", + "\n", + "#code for ROC curve, for true values\n", + "fpr, tpr, thresholds = metrics.roc_curve(y, scores)\n", + "roc_auc = metrics.auc(fpr, tpr)\n", + "\n", + "plt.figure()\n", + "lw = 2\n", + "plt.plot(fpr, tpr, color='darkorange',\n", + " lw=lw, label='ROC curve (area = %0.2f)' % roc_auc)\n", + "plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')\n", + "plt.xlim([0.0, 1.0])\n", + "plt.ylim([0.0, 1.05])\n", + "plt.xlabel('False Positive Rate')\n", + "plt.ylabel('True Positive Rate')\n", + "plt.title('Receiver operating characteristic for True')\n", + "plt.legend(loc=\"lower right\")\n", + "plt.show()\n", + "\n", + "\n", + "#code for ROC curve, for false values\n", + "plt.figure()\n", + "lw = 2\n", + "label_invert = y.apply(switch_label)\n", + "fpr, tpr, thresholds = metrics.roc_curve(label_invert, scores_invert)\n", + "plt.plot(fpr, tpr, color='darkorange',\n", + " lw=lw, label='ROC curve (area = %0.2f)' % roc_auc)\n", + "plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')\n", + "plt.xlim([0.0, 1.0])\n", + "plt.ylim([0.0, 1.05])\n", + "plt.xlabel('False Positive Rate')\n", + "plt.ylabel('True Positive Rate')\n", + "plt.title('Receiver operating characteristic for False')\n", + "plt.legend(loc=\"lower right\")\n", + "plt.show()\n", + "\n", + "\n", + "#code for PR curve, for true values\n", + "\n", + "precision, recall, thresholds = metrics.precision_recall_curve(y, scores)\n", + "\n", + "\n", + "plt.figure()\n", + "lw = 2\n", + "plt.plot( recall, precision, color='darkorange',\n", + " lw=lw, label='Precision recall curve for True')\n", + "plt.xlim([0.0, 1.0])\n", + "plt.ylim([0.0, 1.05])\n", + "plt.xlabel('Recall')\n", + "plt.ylabel('Precision')\n", + "plt.title('Precision Recall Curve for True')\n", + "plt.legend(loc=\"lower right\")\n", + "plt.show()\n", + "\n", + "#code for PR curve, for false values\n", + "\n", + "precision, recall, thresholds = metrics.precision_recall_curve(label_invert, scores_invert)\n", + "print(precision.shape)\n", + "print(recall.shape)\n", + "\n", + "plt.figure()\n", + "lw = 2\n", + "plt.plot( recall, precision, color='darkorange',\n", + " label='Precision recall curve for False')\n", + "plt.xlim([0.0, 1.1])\n", + "plt.ylim([0.0, 1.1])\n", + "plt.xlabel('Recall')\n", + "plt.ylabel('Precision')\n", + "plt.title('Precision Recall Curve for False')\n", + "plt.legend(loc=\"lower right\")\n", + "plt.show()\n", + "\n" + ], + "execution_count": 0, + "outputs": [] + } + ] +} diff --git a/tables/automl/notebooks/result_slicing/README.md b/tables/automl/notebooks/result_slicing/README.md new file mode 100644 index 00000000000..e96ef39708d --- /dev/null +++ b/tables/automl/notebooks/result_slicing/README.md @@ -0,0 +1,55 @@ +Copyright 2019 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License");you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +### Summary: Use open source tools to slice and analyze a classification model built in AutoML Tables + + +# Result Slicing with a model built in AutoML Tables + + +AutoML Tables enables you to build machine learning models based on tables of your own data and host them on Google Cloud for scalability. This solution demonstrates how you can use open source tools to analyze a classification model's output by slicing the results to understand performance discrepancies. This should serve as an introduction to a couple of tools that make in-depth model analysis simpler for AutoML Tables users. + +Our exercise will + +1. Preprocess the output data +2. Examine the dataset in the What-If Tool +3. Use TFMA to slice the data for analysis + + +## Problem Description + +Top-level metrics don't always tell the whole story of how a model is performing. Sometimes, specific characteristics of the data may make certain subclasses of the dataset harder to predict accurately. This notebook will give some examples of how to use open source tools to slice data results from an AutoML Tables classification model, and discover potential performance discrepancies. + + +## Data Preprocessing + +### Prerequisite + +To perform this exercise, you need to have a GCP (Google Cloud Platform) account. If you don't have a GCP account, see [Create a GCP project](https://cloud.google.com/resource-manager/docs/creating-managing-projects). If you'd like to try analyzing your own model, you also need to have already built a model in AutoML Tables and exported its results to BigQuery. + +### Data + +The data we use in this exercise is a public dataset, the [Default of Credit Card Clients](https://archive.ics.uci.edu/ml/datasets/default+of+credit+card+clients) dataset, for analysis. This dataset was collected to help compare different methods of predicting credit card default. Using this colab to analyze your own dataset may require a little adaptation, but should be possible. The data was already used in AutoML Tables to train a binary classifier which attempts to predict whether or not the customer will default in the following month. + +If you'd like to try using your own data in this notebook, you'll need to [train an AutoML Tables model](https://cloud.google.com/automl-tables/docs/beginners-guide) and export the results to BigQuery using the link on the Evaluate tab. Once the BigQuery table is finished exporting, you can copy the Table ID from GCP console into the notebook's "table_name" parameter to import it. There are several other parameters you'll need to update, such as sampling rates and field names. + +### Format for Analysis + +Many of the tools we use to analyze models and data expect to find their inputs in the [tensorflow.Example](https://www.tensorflow.org/tutorials/load_data/tf_records) format. In the Colab, we'll show code to preprocess our data into tf.Examples, and also extract the predicted class from our classifier, which is binary. + + +## What-If Tool + +The [What-If Tool](https://pair-code.github.io/what-if-tool/) is a powerful visual interface to explore data, models, and predictions. Because we're reading our results from BigQuery, we aren't able to use the features of the What-If Tool that query the model directly. But we can still use many of its other features to explore our data distribution in depth. + +## Tensorflow Model Analysis + +This section of the tutorial will use [TFMA](https://github.com/tensorflow/model-analysis) model agnostic analysis capabilities. + +TFMA generates sliced metrics graphs and confusion matrices. We can use these to dig deeper into the question of how well this model performs on different classes of inputs, using the given dataset as a motivating example. + diff --git a/tables/automl/notebooks/result_slicing/slicing_eval_results.ipynb b/tables/automl/notebooks/result_slicing/slicing_eval_results.ipynb new file mode 100644 index 00000000000..bac645db406 --- /dev/null +++ b/tables/automl/notebooks/result_slicing/slicing_eval_results.ipynb @@ -0,0 +1,373 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "slicing_eval_results.ipynb", + "version": "0.3.2", + "provenance": [ + { + "file_id": "1goi268plF-1AJ77xjdMwIpapBr1ssb-q", + "timestamp": 1551899111384 + }, + { + "file_id": "/piper/depot/google3/cloud/ml/autoflow/colab/slicing_eval_results.ipynb?workspaceId=simonewu:autoflow-1::citc", + "timestamp": 1547767618990 + }, + { + "file_id": "1fjkKgZq5iMevPnfiIpSHSiSiw5XimZ1C", + "timestamp": 1547596565571 + } + ], + "collapsed_sections": [], + "last_runtime": { + "build_target": "//learning/fairness/colabs:ml_fairness_notebook", + "kind": "shared" + } + }, + "kernelspec": { + "display_name": "Python 2", + "name": "python2" + } + }, + "cells": [ + { + "metadata": { + "colab_type": "text", + "id": "jt_Hqb95fRz8" + }, + "cell_type": "markdown", + "source": [ + "# Slicing AutoML Tables Evaluation Results with BigQuery\n", + "\n", + "This colab assumes that you've created a dataset with AutoML Tables, and used that dataset to train a classification model. Once the model is done training, you also need to export the results table by using the following instructions. You'll see more detailed setup instructions below.\n", + "\n", + "This colab will walk you through the process of using BigQuery to visualize data slices, showing you one simple way to evaluate your model for bias.\n", + "\n", + "## Setup\n", + "\n", + "To use this Colab, copy it to your own Google Drive or open it in the Playground mode. Follow the instructions in the [AutoML Tables Product docs](https://cloud.google.com/automl-tables/docs/) to create a GCP project, enable the API, and create and download a service account private key, and set up required permission. You'll also need to use the AutoML Tables frontend or service to create a model and export its evaluation results to BigQuery. You should find a link on the Evaluate tab to view your evaluation results in BigQuery once you've finished training your model. Then navigate to BigQuery in your GCP console and you'll see your new results table in the list of tables to which your project has access. \n", + "\n", + "For demo purposes, we'll be using the [Default of Credit Card Clients](https://archive.ics.uci.edu/ml/datasets/default+of+credit+card+clients) dataset for analysis. This dataset was collected to help compare different methods of predicting credit card default. Using this colab to analyze your own dataset may require a little adaptation.\n", + "\n", + "The code below will sample if you want it to. Or you can set sample_count to be as large or larger than your dataset to use the whole thing for analysis. \n", + "\n", + "Note also that although the data we use in this demo is public, you'll need to enter your own Google Cloud project ID in the parameter below to authenticate to it.\n", + "\n" + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "m2oL8tO-f9rK", + "colab": {} + }, + "cell_type": "code", + "source": [ + "from __future__ import absolute_import\n", + "from __future__ import division\n", + "from __future__ import print_function\n", + "\n", + "from google.colab import auth\n", + "import numpy as np\n", + "import os\n", + "import pandas as pd\n", + "import sys\n", + "sys.path.append('./python')\n", + "from sklearn.metrics import confusion_matrix\n", + "from sklearn.metrics import accuracy_score, roc_curve, roc_auc_score\n", + "from sklearn.metrics import precision_recall_curve\n", + "# For facets\n", + "from IPython.core.display import display, HTML\n", + "import base64\n", + "!pip install --upgrade tf-nightly witwidget\n", + "import witwidget.notebook.visualization as visualization\n", + "!pip install apache-beam\n", + "!pip install --upgrade tensorflow_model_analysis\n", + "!pip install --upgrade tensorflow\n", + "\n", + "import tensorflow as tf\n", + "import tensorflow_model_analysis as tfma\n", + "print('TFMA version: {}'.format(tfma.version.VERSION_STRING))\n", + "\n", + "# https://cloud.google.com/resource-manager/docs/creating-managing-projects\n", + "project_id = '[YOUR PROJECT ID HERE]' #@param {type:\"string\"}\n", + "table_name = 'bigquery-public-data:ml_datasets.credit_card_default' #@param {type:\"string\"}\n", + "os.environ[\"GOOGLE_CLOUD_PROJECT\"]=project_id\n", + "sample_count = 3000 #@param\n", + "row_count = pd.io.gbq.read_gbq('''\n", + " SELECT \n", + " COUNT(*) as total\n", + " FROM [%s]''' % (table_name), project_id=project_id, verbose=False).total[0]\n", + "df = pd.io.gbq.read_gbq('''\n", + " SELECT\n", + " *\n", + " FROM\n", + " [%s]\n", + " WHERE RAND() < %d/%d\n", + "''' % (table_name, sample_count, row_count), project_id=project_id, verbose=False)\n", + "print('Full dataset has %d rows' % row_count)\n", + "df.describe()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "608Fe8PRtj5q" + }, + "cell_type": "markdown", + "source": [ + "##Data Preprocessing\n", + "\n", + "Many of the tools we use to analyze models and data expect to find their inputs in the [tensorflow.Example](https://www.tensorflow.org/tutorials/load_data/tf_records) format. Here, we'll preprocess our data into tf.Examples, and also extract the predicted class from our classifier, which is binary." + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "lqZeO9aGtn2s", + "colab": {} + }, + "cell_type": "code", + "source": [ + "unique_id_field = 'ID' #@param\n", + "prediction_field_score = 'predicted_default_payment_next_month_tables_score' #@param\n", + "prediction_field_value = 'predicted_default_payment_next_month_tables_value' #@param\n", + "\n", + "\n", + "def extract_top_class(prediction_tuples):\n", + " # values from Tables show up as a CSV of individual json (prediction, confidence) objects.\n", + " best_score = 0\n", + " best_class = u''\n", + " for val, sco in prediction_tuples:\n", + " if sco > best_score:\n", + " best_score = sco\n", + " best_class = val\n", + " return (best_class, best_score)\n", + "\n", + "def df_to_examples(df, columns=None):\n", + " examples = []\n", + " if columns == None:\n", + " columns = df.columns.values.tolist()\n", + " for id in df[unique_id_field].unique():\n", + " example = tf.train.Example()\n", + " prediction_tuples = zip(df.loc[df[unique_id_field] == id][prediction_field_value], df.loc[df[unique_id_field] == id][prediction_field_score])\n", + " row = df.loc[df[unique_id_field] == id].iloc[0]\n", + " for col in columns:\n", + " if col == prediction_field_score or col == prediction_field_value:\n", + " # Deal with prediction fields separately\n", + " continue\n", + " elif df[col].dtype is np.dtype(np.int64):\n", + " example.features.feature[col].int64_list.value.append(int(row[col]))\n", + " elif df[col].dtype is np.dtype(np.float64):\n", + " example.features.feature[col].float_list.value.append(row[col])\n", + " elif row[col] is None:\n", + " continue\n", + " elif row[col] == row[col]:\n", + " example.features.feature[col].bytes_list.value.append(row[col].encode('utf-8'))\n", + " cla, sco = extract_top_class(prediction_tuples)\n", + " example.features.feature['predicted_class'].int64_list.value.append(cla)\n", + " example.features.feature['predicted_class_score'].float_list.value.append(sco)\n", + " examples.append(example)\n", + " return examples\n", + "\n", + "# Fix up some types so analysis is consistent. This code is specific to the dataset.\n", + "df = df.astype({\"PAY_5\": float, \"PAY_6\": float})\n", + "\n", + "# Converts a dataframe column into a column of 0's and 1's based on the provided test.\n", + "def make_label_column_numeric(df, label_column, test):\n", + " df[label_column] = np.where(test(df[label_column]), 1, 0)\n", + " \n", + "# Convert label types to numeric. This code is specific to the dataset.\n", + "make_label_column_numeric(df, 'predicted_default_payment_next_month_tables_value', lambda val: val == '1')\n", + "make_label_column_numeric(df, 'default_payment_next_month', lambda val: val == '1')\n", + "\n", + "examples = df_to_examples(df)\n", + "print(\"Preprocessing complete!\")" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "XwnOX_orVZEs" + }, + "cell_type": "markdown", + "source": [ + "## What-If Tool\n", + "\n", + "First, we'll explore the data and predictions using the [What-If Tool](https://pair-code.github.io/what-if-tool/). The What-If tool is a powerful visual interface to explore data, models, and predictions. Because we're reading our results from BigQuery, we aren't able to use the features of the What-If Tool that query the model directly. But we can still learn a lot about this dataset from the exploration that the What-If tool enables.\n", + "\n", + "Imagine that you're curious to discover whether there's a discrepancy in the predictive power of your model depending on the marital status of the person whose credit history is being analyzed. You can use the What-If Tool to look at a glance and see the relative sizes of the data samples for each class. In this dataset, the marital statuses are encoded as 1 = married; 2 = single; 3 = divorce; 0=others. You can see using the What-If Tool that there are very few samples for classes other than married or single, which might indicate that performance could be compromised. If this lack of representation concerns you, you could consider collecting more data for underrepresented classes, downsampling overrepresented classes, or upweighting underrepresented data types as you train, depending on your use case and data availability.\n" + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "tjWxGOBkVXQ6", + "colab": {} + }, + "cell_type": "code", + "source": [ + "WitWidget = visualization.WitWidget\n", + "WitConfigBuilder = visualization.WitConfigBuilder\n", + "\n", + "num_datapoints = 2965 #@param {type: \"number\"}\n", + "tool_height_in_px = 700 #@param {type: \"number\"}\n", + "\n", + "# Setup the tool with the test examples and the trained classifier\n", + "config_builder = WitConfigBuilder(examples[:num_datapoints])\n", + "# Need to call this so we have inference_address and model_name initialized\n", + "config_builder = config_builder.set_estimator_and_feature_spec('', '')\n", + "config_builder = config_builder.set_compare_estimator_and_feature_spec('', '')\n", + "wv = WitWidget(config_builder, height=tool_height_in_px)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "YHydLAY991Du" + }, + "cell_type": "markdown", + "source": [ + "## Tensorflow Model Analysis\n", + "\n", + "Then, let's examine some sliced metrics. This section of the tutorial will use [TFMA](https://github.com/tensorflow/model-analysis) model agnostic analysis capabilities. \n", + "\n", + "TFMA generates sliced metrics graphs and confusion matrices. We can use these to dig deeper into the question of how well this model performs on different classes of marital status. The model was built to optimize for AUC ROC metric, and it does fairly well for all of the classes, though there is a small performance gap for the \"divorced\" category. But when we look at the AUC-PR metric slices, we can see that the \"divorced\" and \"other\" classes are very poorly served by the model compared to the more common classes. AUC-PR is the metric that measures how well the tradeoff between precision and recall is being made in the model's predictions. If we're concerned about this gap, we could consider retraining to use AUC-PR as the optimization metric and see whether that model does a better job making equitable predictions. " + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "ZfU11b0797le", + "colab": {} + }, + "cell_type": "code", + "source": [ + "import apache_beam as beam\n", + "import tempfile\n", + "\n", + "from collections import OrderedDict\n", + "from google.protobuf import text_format\n", + "from tensorflow_model_analysis import post_export_metrics\n", + "from tensorflow_model_analysis import types\n", + "from tensorflow_model_analysis.api import model_eval_lib\n", + "from tensorflow_model_analysis.evaluators import aggregate\n", + "from tensorflow_model_analysis.extractors import slice_key_extractor\n", + "from tensorflow_model_analysis.model_agnostic_eval import model_agnostic_evaluate_graph\n", + "from tensorflow_model_analysis.model_agnostic_eval import model_agnostic_extractor\n", + "from tensorflow_model_analysis.model_agnostic_eval import model_agnostic_predict\n", + "from tensorflow_model_analysis.proto import metrics_for_slice_pb2\n", + "from tensorflow_model_analysis.slicer import slicer\n", + "from tensorflow_model_analysis.view.widget_view import render_slicing_metrics\n", + "\n", + "# To set up model agnostic extraction, need to specify features and labels of\n", + "# interest in a feature map.\n", + "feature_map = OrderedDict();\n", + "\n", + "for i, column in enumerate(df.columns):\n", + " type = df.dtypes[i]\n", + " if column == prediction_field_score or column == prediction_field_value:\n", + " continue\n", + " elif (type == np.dtype(np.float64)):\n", + " feature_map[column] = tf.FixedLenFeature([], tf.float32)\n", + " elif (type == np.dtype(np.object)):\n", + " feature_map[column] = tf.FixedLenFeature([], tf.string)\n", + " elif (type == np.dtype(np.int64)):\n", + " feature_map[column] = tf.FixedLenFeature([], tf.int64)\n", + " elif (type == np.dtype(np.bool)):\n", + " feature_map[column] = tf.FixedLenFeature([], tf.bool)\n", + " elif (type == np.dtype(np.datetime64)):\n", + " feature_map[column] = tf.FixedLenFeature([], tf.timestamp)\n", + "\n", + "feature_map['predicted_class'] = tf.FixedLenFeature([], tf.int64)\n", + "feature_map['predicted_class_score'] = tf.FixedLenFeature([], tf.float32)\n", + "\n", + "serialized_examples = [e.SerializeToString() for e in examples]\n", + "\n", + "BASE_DIR = tempfile.gettempdir()\n", + "OUTPUT_DIR = os.path.join(BASE_DIR, 'output')\n", + "\n", + "slice_column = 'MARRIAGE' #@param\n", + "predicted_labels = 'predicted_class' #@param\n", + "actual_labels = 'default_payment_next_month' #@param\n", + "predicted_class_score = 'predicted_class_score' #@param\n", + "\n", + "with beam.Pipeline() as pipeline:\n", + " model_agnostic_config = model_agnostic_predict.ModelAgnosticConfig(\n", + " label_keys=[actual_labels],\n", + " prediction_keys=[predicted_labels],\n", + " feature_spec=feature_map)\n", + " \n", + " extractors = [\n", + " model_agnostic_extractor.ModelAgnosticExtractor(\n", + " model_agnostic_config=model_agnostic_config,\n", + " desired_batch_size=3),\n", + " slice_key_extractor.SliceKeyExtractor([\n", + " slicer.SingleSliceSpec(columns=[slice_column])\n", + " ])\n", + " ]\n", + "\n", + " auc_roc_callback = post_export_metrics.auc(\n", + " labels_key=actual_labels,\n", + " target_prediction_keys=[predicted_labels])\n", + " \n", + " auc_pr_callback = post_export_metrics.auc(\n", + " curve='PR',\n", + " labels_key=actual_labels,\n", + " target_prediction_keys=[predicted_labels])\n", + " \n", + " confusion_matrix_callback = post_export_metrics.confusion_matrix_at_thresholds(\n", + " labels_key=actual_labels,\n", + " target_prediction_keys=[predicted_labels],\n", + " example_weight_key=predicted_class_score,\n", + " thresholds=[0.0, 0.5, 0.8, 1.0])\n", + "\n", + " # Create our model agnostic aggregator.\n", + " eval_shared_model = types.EvalSharedModel(\n", + " construct_fn=model_agnostic_evaluate_graph.make_construct_fn(\n", + " add_metrics_callbacks=[confusion_matrix_callback,\n", + " auc_roc_callback,\n", + " auc_pr_callback,\n", + " post_export_metrics.example_count()],\n", + " fpl_feed_config=model_agnostic_extractor\n", + " .ModelAgnosticGetFPLFeedConfig(model_agnostic_config)))\n", + "\n", + " # Run Model Agnostic Eval.\n", + " _ = (\n", + " pipeline\n", + " | beam.Create(serialized_examples)\n", + " | 'ExtractEvaluateAndWriteResults' >>\n", + " model_eval_lib.ExtractEvaluateAndWriteResults(\n", + " eval_shared_model=eval_shared_model,\n", + " output_path=OUTPUT_DIR,\n", + " extractors=extractors))\n", + " \n", + "\n", + "eval_result = tfma.load_eval_result(output_path=OUTPUT_DIR)\n", + "render_slicing_metrics(eval_result, slicing_column = slice_column)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "code", + "id": "mOotC2D5Onqu", + "colab": {} + }, + "cell_type": "code", + "source": [ + "" + ], + "execution_count": 0, + "outputs": [] + } + ] +} \ No newline at end of file diff --git a/tables/automl/notebooks/retail_product_stockout_prediction/README.md b/tables/automl/notebooks/retail_product_stockout_prediction/README.md new file mode 100644 index 00000000000..32168a4a072 --- /dev/null +++ b/tables/automl/notebooks/retail_product_stockout_prediction/README.md @@ -0,0 +1,387 @@ +---------------------------------------- +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License");you may not use this file except in compliance with the License.You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, softwaredistributed under the License is distributed on an "AS IS" BASIS,WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.See the License for the specific language governing permissions and limitations under the License. + +---------------------------------------- + +# Retail Product Stockouts Prediction using AutoML Tables + +AutoML Tables enables you to build machine learning models based on tables of your own data and host them on Google Cloud for scalability. This solution demonstrates how you can use AutoML Tables to solve a product stockouts problem in the retail industry. This problem is solved using a binary classification approach, which predicts whether a particular product at a certain store will be out-of-stock or not in the next four weeks. Once the solution is built, you can plug this in with your production system and proactively predict stock-outs for your business. + + +Our exercise will + +1. [Walk through the problem of stock-out from a business standpoint](##business-problem) +2. [Explaining the challenges in solving this problem with machine learning](#the-machine-learning-solution) +3. [Demonstrate data preparation for machine learning](#data-preparation) +4. [Step-by-step guide to building the model on AutoML Tables UI](#building-the-model-on-automl-tables-ui) +5. [Step-by-step guide to executing the model through a python script that can be integrated with your production system](#building-the-model-using-automl-tables-python-client-library) +6. [Performance of the model built using AutoML Tables](#evaluation-results-and-business-impact) + + +## Business Problem + +### Problem statement + +A stockout, or out-of-stock (OOS) event is an event that causes inventory to be exhausted. While out-of-stocks can occur along the entire supply chain, the most visible kind are retail out-of-stocks in the fast-moving consumer goods industry (e.g., sweets, diapers, fruits). Stockouts are the opposite of overstocks, where too much inventory is retained. + +### Impact + +According to a study by researchers Thomas Gruen and Daniel Corsten, the global average level of out-of-stocks within retail fast-moving consumer goods sector across developed economies was 8.3% in 2002. This means that shoppers would have a 42% chance of fulfilling a ten-item shopping list without encountering a stockout. Despite the initiatives designed to improve the collaboration of retailers and their suppliers, such as Efficient Consumer Response (ECR), and despite the increasing use of new technologies such as radio-frequency identification (RFID) and point-of-sale data analytics, this situation has improved little over the past decades. + +The biggest impacts being +1. Customer dissatisfaction +2. Loss of revenue + +### Machine Learning Solution + +Using machine learning to solve for stock-outs can help with store operations and thus prevent out-of-stock proactively. + +## The Machine Learning Solution + +There are three big challenges any retailer would face as they try and solve this problem with machine learning: + +1. Data silos: Sales data, supply-chain data, inventory data, etc. may all be in silos. Such disjoint datasets could be a challenge to work with as a machine learning model tries to derive insights from all these data points. +2. Missing Features: Features such as vendor location, weather conditions, etc. could add a lot of value to a machine learning algorithm to learn from. But such features are not always available and when building machine learning solutions we think for collecting features as an iterative approach to improving the machine learning model. +3. Imbalanced dataset: Datasets for classification problems such as retail stock-out are traditionally very imbalanced with fewer cases for stock-out. Designing machine learning solutions by hand for such problems would be time consuming effort when your team should be focusing on collecting features. + +Hence, we recommend using AutoML Tables. With AutoML Tables you only need to work on acquiring all data and features, and AutoML Tables would do the rest. This is a one-click deploy to solving the problem of stock-out with machine learning. + + +## Data Preparation + +### Prerequisite + +To perform this exercise, you need to have a GCP (Google Cloud Platform) account. If you don't have a GCP account, see [Create a GCP project](https://cloud.google.com/resource-manager/docs/creating-managing-projects). + +### Data + +In this solution, you will use two datasets: Training/Evaluation data and Batch Prediction inputs. To access the datasets in BigQuery, you need the following information. + +Training/Evaluation dataset: + +`Project ID: product-stockout` \ +`Dataset ID: product_stockout` \ +`Table ID: stockout` + +Batch Prediction inputs: + +`Project ID: product-stockout` \ +`Dataset ID: product_stockout` \ +`Table ID: batch_prediction_inputs` + +### Data Schema + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          Field name + Datatype + Type + Description +
          Item_Number + STRING + Identifier + This is the product/ item identifier +
          Category + STRING + Identifier + Several items could belong to one category +
          Vendor_Number + STRING + Identifier + Product vendor identifier +
          Store_Number + STRING + Identifier + Store identifier +
          Item_Description + STRING + Text Features + Item Description +
          Category_Name + STRING + Text Features + Category Name +
          Vendor_Name + STRING + Text Features + Vendor Name +
          Store_Name + STRING + Text Features + Store Name +
          Address + STRING + Text Features + Address +
          City + STRING + Categorical Features + City +
          Zip_Code + STRING + Categorical Features + Zip-code +
          Store_Location + STRING + Categorical Features + Store Location +
          County_Number + STRING + Categorical Features + County Number +
          County + STRING + Categorical Features + County Name +
          Weekly Sales Quantity +

          + +

          		INTEGER + Time series data + 52 columns for weekly sales quantity from week 1 to week 52 +
          Weekly Sales Dollars + INTEGER + Time series data + 52 columns for weekly sales dollars from week 1 to week 52 +
          Inventory + FLOAT + Numeric Feature + This inventory is stocked by the retailer looking at past sales and seasonality of the product to meet demand for future sales. +
          Stockout + INTEGER + Label + (1 - Stock-out, 0 - No stock-out) +

          +When the demand for four weeks future sales is not met by the inventory in stock we say we see a stock-out. This is because an early warning sign would help the retailer re-stock inventory with a lead time for the stock to be replenished. +

          + + +To use AutoML Tables with BigQuery you do not need to download this dataset. However, if you would like to use AutoML Tables with GCS you may want to download this dataset and upload it into your GCP Project storage bucket. + +Instructions to download dataset: + +Sample Dataset: Download this dataset which contains sales data. + +1. [Link to training data](https://console.cloud.google.com/bigquery?folder=&organizationId=&project=product-stockout&p=product-stockout&d=product_stockout&t=stockout&page=table): \ +Dataset URI: +2. [Link to data for batch predictions](https://console.cloud.google.com/bigquery?folder=&organizationId=&project=product-stockout&p=product-stockout&d=product_stockout&t=batch_prediction_inputs&page=table): \ +Dataset URI: + +Upload this dataset to GCS or BigQuery (optional). + +You could select either [GCS](https://cloud.google.com/storage/) or [BigQuery](https://cloud.google.com/bigquery/) as the location of your choice to store the data for this challenge. + +1. Storing data on GCS: [Creating storage buckets, Uploading data to storage buckets](https://cloud.google.com/storage/docs/creating-buckets) +2. Storing data on BigQuery: [Create and load data to BigQuery](https://cloud.google.com/bigquery/docs/quickstarts/quickstart-web-ui) (optional) + + +## Building the model on AutoML Tables UI + +1. Enable [AutoML Tables](https://cloud.google.com/automl-tables/docs/quickstart#before_you_begin) on GCP. + +2. Visit the [AutoML Tables UI](https://console.cloud.google.com/automl-tables) to begin the process of creating your dataset and training your model. + +![ ](resources/automl_stockout_img/Image%201%202019-03-13%20at%201.02.53%20PM.png) + +3. Import your dataset or the dataset you downloaded in the last section \ +Click <+New Dataset> → Dataset Name → Click Create Dataset + +![ ](resources/automl_stockout_img/Image%202%202019-03-13%20at%201.05.17%20PM.png) + +4. You can import data from BigQuery or GCS bucket \ + a. For BigQuery enter your GCP project ID, Dataset ID and Table ID \ + After specifying dataset click import dataset + +![ ](resources/automl_stockout_img/Image%203%202019-03-13%20at%201.08.44%20PM.png) + + b. For GCS enter the GCS object location by clicking on BROWSE \ + After specifying dataset click import dataset + +![ ](resources/automl_stockout_img/Image%204%202019-03-13%20at%201.09.56%20PM.png) + + Depending on the size of the dataset this import can take some time. + +5. Once the import is complete you can set the schema of the imported dataset based on your business understanding of the data \ + a. Select Label i.e. Stockout \ + b. Select Variable Type for all features \ + c. Click Continue + +![ ](resources/automl_stockout_img/Image%206%202019-03-13%20at%201.20.57%20PM.png) + +6. The imported dataset is now analyzed \ +This helps you analyze the size of your dataset, dig into missing values if any, calculate correlation, mean and standard deviation. If this data quality looks good to you then we can move on to the next tab i.e. train. + +![ ](resources/automl_stockout_img/Image%20new%201%202019-03-25%20at%2012.43.13%20AM.png) + +7. Train \ + a. Select a model name \ + b. Select the training budget \ + c. Select all features you would like to use for training \ + d. Select optimization objectives. Such as: ROC, Log Loss or PR curve \ + (As our data is imbalances we use PR curve as our optimization metric) \ + e. Click TRAIN \ + f. Training the model can take some time + +![ ](resources/automl_stockout_img/Image%208%202019-03-13%20at%201.34.08%20PM.png) + +![ ](resources/automl_stockout_img/Image%20new%202%202019-03-25%20at%2012.44.18%20AM.png) + +8. Once the model is trained you can click on the evaluate tab \ +This tab gives you stats for model evaluation \ + For example our model shows \ + Area Under Precision Recall Curve: 0.645 \ + Area Under ROC Curve: 0.893 \ + Accuracy: 92.5% \ + Log Loss: 0.217 \ +Selecting the threshold lets you set a desired precision and recall on your predictions. + +![ ](resources/automl_stockout_img/Image%20new%203%202019-03-25%20at%2012.49.40%20AM.png) + +9. Using the model created let's use batch prediction to predict stock-out \ + a. Batch prediction data inputs could come from BigQuery or your GCS bucket. \ + b. Select the GCS bucket to store the results of your batch prediction \ + c. Click Send Batch Predictions + +![ ](resources/automl_stockout_img/Image%2012%202019-03-13%20at%201.56.43%20PM.png) + +![ ](resources/automl_stockout_img/Image%2013%202019-03-13%20at%201.59.18%20PM.png) + + +## Building the model using AutoML Tables Python Client Library + +In this notebook, you will learn how to build the same model as you have done on the AutoML Tables UI using its Python Client Library. + + +## Evaluation results and business impact + +![ ](resources/automl_stockout_img/Image%20new%203%202019-03-25%20at%2012.49.40%20AM.png) + +Thus the evaluation results tell us that the model we built can: + +1. 92.5% Accuracy: That is about 92.5% times you should be confident that the stock-out or no stock-out prediction is accurate. +2. 78.2% Precision: Of the sock-outs identified 78.2% results are expected to actually be stock-outs +3. 44.1% Recall: And of all possible stock-outs 44.1% should be identified by this model +4. 1.5% False Positive Rate: Only 1.5% times an item identified as stock-out may not be out-of-stock + +Thus, with such a machine learning model your business could definitely expect time savings and revenue gain by predicting stock-outs. + +Note: You can always improve this model iteratively by adding business relevant features. diff --git a/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image 1 2019-03-13 at 1.02.53 PM.png b/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image 1 2019-03-13 at 1.02.53 PM.png new file mode 100644 index 00000000000..94f11b28bb6 Binary files /dev/null and b/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image 1 2019-03-13 at 1.02.53 PM.png differ diff --git a/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image 12 2019-03-13 at 1.56.43 PM.png b/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image 12 2019-03-13 at 1.56.43 PM.png new file mode 100644 index 00000000000..f60f3aa5d54 Binary files /dev/null and b/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image 12 2019-03-13 at 1.56.43 PM.png differ diff --git a/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image 13 2019-03-13 at 1.59.18 PM.png b/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image 13 2019-03-13 at 1.59.18 PM.png new file mode 100644 index 00000000000..f80bdfb8555 Binary files /dev/null and b/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image 13 2019-03-13 at 1.59.18 PM.png differ diff --git a/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image 2 2019-03-13 at 1.05.17 PM.png b/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image 2 2019-03-13 at 1.05.17 PM.png new file mode 100644 index 00000000000..daeb7d9661e Binary files /dev/null and b/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image 2 2019-03-13 at 1.05.17 PM.png differ diff --git a/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image 3 2019-03-13 at 1.08.44 PM.png b/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image 3 2019-03-13 at 1.08.44 PM.png new file mode 100644 index 00000000000..2cc3f366c13 Binary files /dev/null and b/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image 3 2019-03-13 at 1.08.44 PM.png differ diff --git a/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image 4 2019-03-13 at 1.09.56 PM.png b/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image 4 2019-03-13 at 1.09.56 PM.png new file mode 100644 index 00000000000..66b1fe57c8a Binary files /dev/null and b/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image 4 2019-03-13 at 1.09.56 PM.png differ diff --git a/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image 5 2019-03-13 at 1.10.11 PM.png b/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image 5 2019-03-13 at 1.10.11 PM.png new file mode 100644 index 00000000000..0d27ed38bfb Binary files /dev/null and b/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image 5 2019-03-13 at 1.10.11 PM.png differ diff --git a/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image 6 2019-03-13 at 1.20.57 PM.png b/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image 6 2019-03-13 at 1.20.57 PM.png new file mode 100644 index 00000000000..02ccd865bc8 Binary files /dev/null and b/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image 6 2019-03-13 at 1.20.57 PM.png differ diff --git a/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image 8 2019-03-13 at 1.34.08 PM.png b/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image 8 2019-03-13 at 1.34.08 PM.png new file mode 100644 index 00000000000..d0e7ddb85af Binary files /dev/null and b/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image 8 2019-03-13 at 1.34.08 PM.png differ diff --git a/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image new 1 2019-03-25 at 12.43.13 AM.png b/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image new 1 2019-03-25 at 12.43.13 AM.png new file mode 100644 index 00000000000..e57b543d0de Binary files /dev/null and b/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image new 1 2019-03-25 at 12.43.13 AM.png differ diff --git a/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image new 2 2019-03-25 at 12.44.18 AM.png b/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image new 2 2019-03-25 at 12.44.18 AM.png new file mode 100644 index 00000000000..20667b2ef4a Binary files /dev/null and b/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image new 2 2019-03-25 at 12.44.18 AM.png differ diff --git a/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image new 3 2019-03-25 at 12.49.40 AM.png b/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image new 3 2019-03-25 at 12.49.40 AM.png new file mode 100644 index 00000000000..776d8d42ae0 Binary files /dev/null and b/tables/automl/notebooks/retail_product_stockout_prediction/resources/automl_stockout_img/Image new 3 2019-03-25 at 12.49.40 AM.png differ diff --git a/tables/automl/notebooks/retail_product_stockout_prediction/retail_product_stockout_prediction.ipynb b/tables/automl/notebooks/retail_product_stockout_prediction/retail_product_stockout_prediction.ipynb new file mode 100644 index 00000000000..f869cdc330b --- /dev/null +++ b/tables/automl/notebooks/retail_product_stockout_prediction/retail_product_stockout_prediction.ipynb @@ -0,0 +1,998 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "retail_product_stockout_prediction.ipynb", + "version": "0.3.2", + "provenance": [], + "collapsed_sections": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "cells": [ + { + "metadata": { + "id": "9V5sA5glWemD", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Copyright 2018 Google LLC \n", + "\n", + "Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "you may not use this file except in compliance with the License.\n", + "You may obtain a copy of the License at\n", + "\n", + "http://www.apache.org/licenses/LICENSE-2.0\n", + "\n", + "Unless required by applicable law or agreed to in writing, software\n", + "distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "See the License for the specific language governing permissions and limitations under the License." + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "m26YhtBMvVWA" + }, + "cell_type": "markdown", + "source": [ + "# Retail Product Stockouts Prediction using AutoML Tables\n", + "\n", + "AutoML Tables enables you to build machine learning models based on tables of your own data and host them on Google Cloud for scalability. This solution demonstrates how you can use AutoML Tables to solve a product stockouts problem in the retail industry. This problem is solved using a binary classification approach, which predicts whether a particular product at a certain store will be out-of-stock or not in the next four weeks. Once the solution is built, you can plug this in with your production system and proactively predict stock-outs for your business. \n", + "\n", + "To use this Colab notebook, copy it to your own Google Drive and open it with [Colaboratory](https://colab.research.google.com/) (or Colab). To run a cell hold the Shift key and press the Enter key (or Return key). Colab automatically displays the return value of the last line in each cell. Refer to [this page](https://colab.research.google.com/notebooks/welcome.ipynb) for more information on Colab.\n", + "\n", + "You can run a Colab notebook on a hosted runtime in the Cloud. The hosted VM times out after 90 minutes of inactivity and you will lose all the data stored in the memory including your authentication data. If your session gets disconnected (for example, because you closed your laptop) for less than the 90 minute inactivity timeout limit, press 'RECONNECT' on the top right corner of your notebook and resume the session. After Colab timeout, you'll need to\n", + "\n", + "1. Re-run the initialization and authentication.\n", + "2. Continue from where you left off. You may need to copy-paste the value of some variables such as the `dataset_name` from the printed output of the previous cells.\n", + "\n", + "Alternatively you can connect your Colab notebook to a [local runtime](https://research.google.com/colaboratory/local-runtimes.html)." + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "b--5FDDwCG9C" + }, + "cell_type": "markdown", + "source": [ + "## 1. Project set up\n", + "\n", + "\n", + "\n" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "AZs0ICgy4jkQ" + }, + "cell_type": "markdown", + "source": [ + "Follow the [AutoML Tables documentation](https://cloud.google.com/automl-tables/docs/) to\n", + "* Create a Google Cloud Platform (GCP) project.\n", + "* Enable billing.\n", + "* Apply to whitelist your project.\n", + "* Enable AutoML API.\n", + "* Enable AutoML Talbes API.\n", + "* Create a service account, grant required permissions, and download the service account private key.\n", + "\n", + "You also need to upload your data into Google Cloud Storage (GCS) or BigQuery. For example, to use GCS as your data source\n", + "* Create a GCS bucket.\n", + "* Upload the training and batch prediction files.\n", + "\n", + "\n", + "**Warning:** Private keys must be kept secret. If you expose your private key it is recommended to revoke it immediately from the Google Cloud Console." + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "xZECt1oL429r" + }, + "cell_type": "markdown", + "source": [ + "\n", + "\n", + "---\n", + "\n" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "rstRPH9SyZj_" + }, + "cell_type": "markdown", + "source": [ + "## 2. Initialize and authenticate\n", + "This section runs intialization and authentication. It creates an authenticated session which is required for running any of the following sections." + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "BR0POq2UzE7e" + }, + "cell_type": "markdown", + "source": [ + "### Install the client library in Colab\n", + "Run the following cell to install the client libary using `pip`.\n", + "\n", + "See [documentations ](https://cloud.google.com/automl-tables/docs/client-libraries) of Google Cloud AutoML Client Library for Python. \n" + ] + }, + { + "metadata": { + "id": "43aXKjDRt_qZ", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title Install AutoML Tables client library { vertical-output: true }\n", + "\n", + "!pip install google-cloud-automl" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "eVFsPPEociwF" + }, + "cell_type": "markdown", + "source": [ + "### Authenticate using service account key\n", + "Run the following cell. Click on the __Choose Files__ button and select the service account private key file. If your Service Account Key file or folder is hidden, you can reveal it in a Mac by pressing the __Command + Shift + .__ combo.\n", + "\n" + ] + }, + { + "metadata": { + "id": "u-kCqysAuaJk", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title Authenticate using service account key and create a client. { vertical-output: true }\n", + "\n", + "from google.cloud import automl_v1beta1\n", + "from google.colab import files\n", + "\n", + "# Upload service account key\n", + "keyfile_upload = files.upload()\n", + "keyfile_name = list(keyfile_upload.keys())[0]\n", + "# Authenticate and create an AutoML client.\n", + "client = automl_v1beta1.AutoMlClient.from_service_account_file(keyfile_name)\n", + "# Authenticate and create a prediction service client.\n", + "prediction_client = automl_v1beta1.PredictionServiceClient.from_service_account_file(keyfile_name)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "s3F2xbEJdDvN" + }, + "cell_type": "markdown", + "source": [ + "### Test" + ] + }, + { + "metadata": { + "id": "0uX4aJYUiXh5", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Enter your GCP project ID." + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "6R4h5HF1Dtds", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title GCP project ID and location\n", + "\n", + "project_id = '' #@param {type:'string'}\n", + "location = 'us-central1'\n", + "location_path = client.location_path(project_id, location)\n", + "location_path" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "rUlBcZ3OfWcJ" + }, + "cell_type": "markdown", + "source": [ + "To test whether your project set up and authentication steps were successful, run the following cell to list your datasets in this project.\n", + "\n", + "If no dataset has previously imported into AutoML Tables, you shall expect an empty return." + ] + }, + { + "metadata": { + "cellView": "both", + "colab_type": "code", + "id": "sf32nKXIqYje", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title List datasets. { vertical-output: true }\n", + "\n", + "list_datasets_response = client.list_datasets(location_path)\n", + "datasets = {dataset.display_name: dataset.name for dataset in list_datasets_response}\n", + "datasets" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "t9uE8MvMkOPd" + }, + "cell_type": "markdown", + "source": [ + "You can also print the list of your models by running the following cell.\n", + "\n", + "If no model has previously trained using AutoML Tables, you shall expect an empty return." + ] + }, + { + "metadata": { + "cellView": "both", + "colab_type": "code", + "id": "j4-bYRSWj7xk", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title List models. { vertical-output: true }\n", + "\n", + "list_models_response = client.list_models(location_path)\n", + "models = {model.display_name: model.name for model in list_models_response}\n", + "models" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "qozQWMnOu48y" + }, + "cell_type": "markdown", + "source": [ + "\n", + "\n", + "---\n", + "\n" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "ODt86YuVDZzm" + }, + "cell_type": "markdown", + "source": [ + "## 3. Import training data" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "XwjZc9Q62Fm5" + }, + "cell_type": "markdown", + "source": [ + "### Create dataset" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "_JfZFGSceyE_" + }, + "cell_type": "markdown", + "source": [ + "Select a dataset display name and pass your table source information to create a new dataset." + ] + }, + { + "metadata": { + "id": "Z_JErW3cw-0J", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title Create dataset { vertical-output: true, output-height: 200 }\n", + "\n", + "dataset_display_name = 'stockout_data' #@param {type: 'string'}\n", + "\n", + "dataset_dict = {\n", + " 'display_name': dataset_display_name, \n", + " 'tables_dataset_metadata': {}\n", + "}\n", + "\n", + "create_dataset_response = client.create_dataset(\n", + " location_path,\n", + " dataset_dict\n", + ")\n", + "create_dataset_response" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "RLRgvqzUdxfL", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + " #@title Get dataset name { vertical-output: true }\n", + "\n", + "dataset_name = create_dataset_response.name\n", + "dataset_name" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "35YZ9dy34VqJ" + }, + "cell_type": "markdown", + "source": [ + "### Import data" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "3c0o15gVREAw" + }, + "cell_type": "markdown", + "source": [ + "You can import your data to AutoML Tables from GCS or BigQuery. For this solution, you will import data from a BigQuery Table. The URI for your table is in the format of `bq://PROJECT_ID.DATASET_ID.TABLE_ID`.\n", + "\n", + "The BigQuery Table used for demonstration purpose can be accessed as `bq://product-stockout.product_stockout.stockout`. \n", + "\n", + "See the table schema and dataset description from the README. " + ] + }, + { + "metadata": { + "id": "bB_GdeqCJW5i", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title ... if data source is BigQuery { vertical-output: true }\n", + "\n", + "dataset_bq_input_uri = 'bq://product-stockout.product_stockout.stockout' #@param {type: 'string'}\n", + "# Define input configuration.\n", + "input_config = {\n", + " 'bigquery_source': {\n", + " 'input_uri': dataset_bq_input_uri\n", + " }\n", + "}" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "FNVYfpoXJsNB", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + " #@title Import data { vertical-output: true }\n", + "\n", + "import_data_response = client.import_data(dataset_name, \n", + " input_config)\n", + "print('Dataset import operation: {}'.format(import_data_response.operation))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "1O7tJ8IlefRC", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + " #@title Check if importing the data is complete { vertical-output: true }\n", + "\n", + "# If returns `False`, you can check back again later.\n", + "# Continue with the rest only if this cell returns a `True`.\n", + "import_data_response.done()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "_WLvyGIDe9ah", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Importing this stockout datasets takes about 10 minutes. \n", + "\n", + "If you re-visit this Colab, uncomment the following cell and run the command to retrieve your dataset. Replace `YOUR_DATASET_NAME` with its actual value obtained in the preceding cells.\n", + "\n", + "`YOUR_DATASET_NAME` is a string in the format of `'projects//locations//datasets/'`." + ] + }, + { + "metadata": { + "id": "P6NkRMyJfAGm", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# dataset_name = '' #@param {type: 'string'}\n", + "# dataset = client.get_dataset(dataset_name) " + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "QdxBI4s44ZRI", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Review the specs" + ] + }, + { + "metadata": { + "id": "RC0PWKqH4jwr", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Run the following command to see table specs such as row count." + ] + }, + { + "metadata": { + "id": "v2Vzq_gwXxo-", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title Table schema { vertical-output: true }\n", + "\n", + "import google.cloud.automl_v1beta1.proto.data_types_pb2 as data_types\n", + "import matplotlib.pyplot as plt\n", + "\n", + "# List table specs\n", + "list_table_specs_response = client.list_table_specs(dataset_name)\n", + "table_specs = [s for s in list_table_specs_response]\n", + "# List column specs\n", + "table_spec_name = table_specs[0].name\n", + "list_column_specs_response = client.list_column_specs(table_spec_name)\n", + "column_specs = {s.display_name: s for s in list_column_specs_response}\n", + "# Table schema pie chart.\n", + "type_counts = {}\n", + "for column_spec in column_specs.values():\n", + " type_name = data_types.TypeCode.Name(column_spec.data_type.type_code)\n", + " type_counts[type_name] = type_counts.get(type_name, 0) + 1\n", + "\n", + "plt.pie(x=type_counts.values(), labels=type_counts.keys(), autopct='%1.1f%%')\n", + "plt.axis('equal')\n", + "plt.show()\n" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "Lqjq4X43v3ON", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "In the pie chart above, you see this dataset contains three variable types: `FLOAT64` (treated as `Numeric`), `CATEGORY` (treated as `Categorical`) and `STRING` (treated as `Text`). " + ] + }, + { + "metadata": { + "id": "FNykW_YOYt6d", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "___" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "kNRVJqVOL8h3" + }, + "cell_type": "markdown", + "source": [ + "## 4. Update dataset: assign a label column and enable nullable columns" + ] + }, + { + "metadata": { + "id": "VsOPwxN9fOIl", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Get column specs" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "-57gehId9PQ5" + }, + "cell_type": "markdown", + "source": [ + "AutoML Tables automatically detects your data column type. \n", + "\n", + "There are a total of 120 columns in this stockout dataset.\n", + "\n", + "Run the following command to check the column data type that automaticallyed detected. If columns contains only numerical values, but they represent categories, change that column data type to caregorical by updating your schema.\n", + "\n", + "In addition, AutoML Tables detects `Stockout` to be categorical that chooses to run a classification model. " + ] + }, + { + "metadata": { + "id": "Pyku3AHEfSp4", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title List table specs { vertical-output: true }\n", + "\n", + "list_table_specs_response = client.list_table_specs(dataset_name)\n", + "table_specs = [s for s in list_table_specs_response]\n", + "table_specs" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "jso_JBI9fgy6", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title Check column data type { vertical-output: true }\n", + "\n", + "# Get column specs.\n", + "table_spec_name = table_specs[0].name\n", + "list_column_specs_response = client.list_column_specs(table_spec_name)\n", + "column_specs = {s.display_name: s for s in list_column_specs_response}\n", + "\n", + "# Print column data types.\n", + "for column in column_specs:\n", + " print(column, '-', column_specs[column].data_type)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "iRqdQ7Xiq04x", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Update columns: make categorical\n", + "\n", + "From the column data type, you noticed `Item_Number`, `Category`, `Vendor_Number`, `Store_Number`, `Zip_Code` and `County_Number` have been autodetected as `FLOAT64` (Numerical) instead of `CATEGORY` (Categorical). \n", + "\n", + "In this solution, the columns `Item_Number`, `Category`, `Vendor_Number` and `Store_Number` are not nullable, but `Zip_Code` and `County_Number` can take null values.\n", + "\n", + "To change the data type, you can update the schema by updating the column spec.\n", + "\n", + "`update_column_response = client.update_column_spec(update_column_spec_dict)`" + ] + }, + { + "metadata": { + "id": "gAPg_ymDf4kL", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def create_update_column_sepc_dict(column_name, type_code, nullable):\n", + " \"\"\"\n", + " Create `update_column_spec_dict` with a given column name and target `type_code`.\n", + " Inputs:\n", + " column_name: string. Represents column name.\n", + " type_code: string. Represents variable type. See details: \\\n", + " https://cloud.google.com/automl-tables/docs/reference/rest/v1beta1/projects.locations.datasets.tableSpecs.columnSpecs#typecode\n", + " nullable: boolean. If true, this DataType can also be null.\n", + " Return:\n", + " update_column_spec_dict: dictionary. Encodes the target column specs.\n", + " \"\"\"\n", + " update_column_spec_dict = {\n", + " 'name': column_specs[column_name].name,\n", + " 'data_type': {\n", + " 'type_code': type_code,\n", + " 'nullable': nullable\n", + " }\n", + " }\n", + " return update_column_spec_dict" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "_xePITEYf5po", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# Update dataset\n", + "categorical_column_names = ['Item_Number',\n", + " 'Category',\n", + " 'Vendor_Number',\n", + " 'Store_Number',\n", + " 'Zip_Code',\n", + " 'County_Number']\n", + "is_nullable = [False, \n", + " False,\n", + " False,\n", + " False,\n", + " True,\n", + " True]\n", + "\n", + "for i in range(len(categorical_column_names)):\n", + " column_name = categorical_column_names[i]\n", + " nullable = is_nullable[i]\n", + " update_column_spec_dict = create_update_column_sepc_dict(column_name, 'CATEGORY', nullable)\n", + " update_column_response = client.update_column_spec(update_column_spec_dict)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "nDMH_chybe4w" + }, + "cell_type": "markdown", + "source": [ + "### Update dataset: assign a label\n", + "\n", + "Select the label column and update the dataset." + ] + }, + { + "metadata": { + "id": "hVIruWg0u33t", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title Update dataset { vertical-output: true }\n", + "\n", + "label_column_name = 'Stockout' #@param {type: 'string'}\n", + "label_column_spec = column_specs[label_column_name]\n", + "label_column_id = label_column_spec.name.rsplit('/', 1)[-1]\n", + "print('Label column ID: {}'.format(label_column_id))\n", + "# Define the values of the fields to be updated.\n", + "update_dataset_dict = {\n", + " 'name': dataset_name,\n", + " 'tables_dataset_metadata': {\n", + " 'target_column_spec_id': label_column_id\n", + " }\n", + "}\n", + "\n", + "update_dataset_response = client.update_dataset(update_dataset_dict)\n", + "update_dataset_response" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "z23NITLrcxmi", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "___" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "FcKgvj1-Tbgj" + }, + "cell_type": "markdown", + "source": [ + "## 5. Creating a model" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "Pnlk8vdQlO_k" + }, + "cell_type": "markdown", + "source": [ + "### Train a model\n", + "Training the model may take one hour or more. To obtain the results with less training time or budget, you can set [`train_budget_milli_node_hours`](https://cloud.google.com/automl-tables/docs/reference/rest/v1beta1/projects.locations.models), which is the train budget of creating this model, expressed in milli node hours i.e. 1,000 value in this field means 1 node hour. \n", + "\n", + "For demonstration purpose, the following command sets the budget as 1 node hour. You can increate that number up to a maximum of 72 hours ('train_budget_milli_node_hours': 72000) for the best model performance. \n", + "\n", + "You can also select the objective to optimize your model training by setting `optimization_objective`. This solution optimizes the model by maximizing the Area Under the Precision-Recall (PR) Curve. \n" + ] + }, + { + "metadata": { + "id": "11izNd6Fu37N", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title Create model { vertical-output: true }\n", + "\n", + "feature_list = list(column_specs.keys())\n", + "feature_list.remove('Stockout')\n", + "\n", + "model_display_name = 'stockout_model' #@param {type:'string'}\n", + "dataset_id = dataset_name.rsplit('/', 1)[-1]\n", + "\n", + "model_dict = {\n", + " 'display_name': model_display_name,\n", + " 'dataset_id': dataset_id, \n", + " 'tables_model_metadata': {\n", + " 'target_column_spec': column_specs['Stockout'],\n", + " 'input_feature_column_specs': [column_specs[f] for f in feature_list],\n", + " 'optimization_objective': 'MAXIMIZE_AU_PRC',\n", + " 'train_budget_milli_node_hours': 1000\n", + " }, \n", + "}\n", + "\n", + "create_model_response = client.create_model(location_path, model_dict)\n", + "print('Dataset import operation: {}'.format(create_model_response.operation))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "wCQdx9VyhKY5", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title Check if model training is complete { vertical-output: true }\n", + "# If returns `False`, you can check back again later.\n", + "# Continue with the rest only if this cell returns a `True`.\n", + "create_model_response.done()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "bPiR8zMwhQYO", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title Retrieve the model name { vertical-output: true }\n", + "create_model_result = create_model_response.result()\n", + "model_name = create_model_result.name\n", + "model_name" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "neYjToB36q9E", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "If your Colab times out, use `client.list_models(location_path)` to check whether your model has been created. \n", + "\n", + "Then uncomment the following cell and run the command to retrieve your model. Replace `YOUR_MODEL_NAME` with its actual value obtained in the preceding cell.\n", + "\n", + "`YOUR_MODEL_NAME` is a string in the format of `'projects//locations//models/'`" + ] + }, + { + "metadata": { + "id": "QptCwUIK7yhU", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# model_name = '' #@param {type: 'string'}\n", + "# model = client.get_model(model_name)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "1wS1is9IY5nK", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "___" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "TarOq84-GXch" + }, + "cell_type": "markdown", + "source": [ + "## 6. Batch prediction" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "Soy5OB8Wbp_R" + }, + "cell_type": "markdown", + "source": [ + "### Initialize prediction" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "39bIGjIlau5a" + }, + "cell_type": "markdown", + "source": [ + "Your data source for batch prediction can be GCS or BigQuery. For this solution, you will use a BigQuery Table as the input source. The URI for your table is in the format of `bq://PROJECT_ID.DATASET_ID.TABLE_ID`.\n", + "\n", + "To write out the predictions, you need to specify a GCS bucket `gs://BUCKET_NAME`.\n", + "\n", + "The AutoML Tables logs the errors in the `errors.csv` file.\n", + "\n", + "**NOTE:** The batch prediction output file(s) will be updated to the GCS bucket that you set in the preceding cells." + ] + }, + { + "metadata": { + "id": "gkF3bH0qu4DU", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title Start batch prediction { vertical-output: true, output-height: 200 }\n", + "\n", + "batch_predict_bq_input_uri = 'bq://product-stockout.product_stockout.batch_prediction_inputs'\n", + "batch_predict_gcs_output_uri_prefix = 'gs://' #@param {type:'string'}\n", + "\n", + "# Define input source.\n", + "batch_prediction_input_source = {\n", + " 'bigquery_source': {\n", + " 'input_uri': batch_predict_bq_input_uri\n", + " }\n", + "}\n", + "# Define output target.\n", + "batch_prediction_output_target = {\n", + " 'gcs_destination': {\n", + " 'output_uri_prefix': batch_predict_gcs_output_uri_prefix\n", + " }\n", + "}\n", + "batch_predict_response = prediction_client.batch_predict(model_name, \n", + " batch_prediction_input_source, \n", + " batch_prediction_output_target)\n", + "print('Batch prediction operation: {}'.format(batch_predict_response.operation))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "AVJhh_k0PfxD", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title Check if batch prediction is complete { vertical-output: true }\n", + "\n", + "# If returns `False`, you can check back again later.\n", + "# Continue with the rest only if this cell returns a `True`.\n", + "batch_predict_response.done()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "8nr5q2M8W2VX", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title Retrieve batch prediction metadata { vertical-output: true }\n", + "\n", + "batch_predict_response.metadata" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "kgwbJwS2iLpc", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#@title Check prediction results { vertical-output: true }\n", + "\n", + "gcs_output_directory = batch_predict_response.metadata.batch_predict_details.output_info.gcs_output_directory\n", + "result_file = gcs_output_directory + '/result.csv'\n", + "print('Batch prediction results are stored as: {}'.format(result_file))" + ], + "execution_count": 0, + "outputs": [] + } + ] +} diff --git a/tasks/README.md b/tasks/README.md new file mode 100644 index 00000000000..00503ccd392 --- /dev/null +++ b/tasks/README.md @@ -0,0 +1,87 @@ +# Google Cloud Tasks Samples + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/flexible/tasks/README.md + +Sample command-line programs for interacting with the Cloud Tasks API +. + +App Engine queues push tasks to an App Engine HTTP target. This directory +contains both the App Engine app to deploy, as well as the snippets to run +locally to push tasks to it, which could also be called on App Engine. + +`create_http_task.py` is a simple command-line program to create +tasks to be pushed to an URL endpoint. + +`create_http_task_with_token.py` is a simple command-line program to create +tasks to be pushed to an URL endpoint with authorization header. + +`main.py` is the main App Engine app. This app serves as an endpoint to receive +App Engine task attempts. + +`app.yaml` configures the App Engine app. + + +## Prerequisites to run locally: + +Please refer to [Setting Up a Python Development Environment](https://cloud.google.com/python/setup). + +## Authentication + +To set up authentication, please refer to our +[authentication getting started guide](https://cloud.google.com/docs/authentication/getting-started). + +## Creating a queue + +To create a queue using the Cloud SDK, use the following gcloud command: + +``` +gcloud beta tasks queues create-app-engine-queue my-appengine-queue +``` + +Note: A newly created queue will route to the default App Engine service and +version unless configured to do otherwise. + +## Run the Sample Using the Command Line + +Set environment variables: + +First, your project ID: + +``` +export PROJECT_ID=my-project-id +``` + +Then the queue ID, as specified at queue creation time. Queue IDs already +created can be listed with `gcloud beta tasks queues list`. + +``` +export QUEUE_ID=my-appengine-queue +``` + +And finally the location ID, which can be discovered with +`gcloud beta tasks queues describe $QUEUE_ID`, with the location embedded in +the "name" value (for instance, if the name is +"projects/my-project/locations/us-central1/queues/my-appengine-queue", then the +location is "us-central1"). + +``` +export LOCATION_ID=us-central1 +``` + +### Using HTTP Push Queues + +Set an environment variable for the endpoint to your task handler. This is an +example url to send requests to the App Engine task handler: +``` +export URL=https://.appspot.com/example_task_handler +``` + +Running the sample will create a task and send the task to the specific URL +endpoint, with a payload specified: + +``` +python create_http_task.py --project=$PROJECT_ID --queue=$QUEUE_ID --location=$LOCATION_ID --url=$URL --payload=hello +``` diff --git a/tasks/create_http_task.py b/tasks/create_http_task.py new file mode 100644 index 00000000000..6ff3a6a2cfa --- /dev/null +++ b/tasks/create_http_task.py @@ -0,0 +1,122 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import argparse +import datetime + + +def create_http_task(project, + queue, + location, + url, + payload=None, + in_seconds=None): + # [START cloud_tasks_create_http_task] + """Create a task for a given queue with an arbitrary payload.""" + + from google.cloud import tasks_v2beta3 + from google.protobuf import timestamp_pb2 + + # Create a client. + client = tasks_v2beta3.CloudTasksClient() + + # TODO(developer): Uncomment these lines and replace with your values. + # project = 'my-project-id' + # queue = 'my-appengine-queue' + # location = 'us-central1' + # url = 'https://.appspot.com/example_task_handler' + # payload = 'hello' + + # Construct the fully qualified queue name. + parent = client.queue_path(project, location, queue) + + # Construct the request body. + task = { + 'http_request': { # Specify the type of request. + 'http_method': 'POST', + 'url': url # The full url path that the task will be sent to. + } + } + if payload is not None: + # The API expects a payload of type bytes. + converted_payload = payload.encode() + + # Add the payload to the request. + task['http_request']['body'] = converted_payload + + if in_seconds is not None: + # Convert "seconds from now" into an rfc3339 datetime string. + d = datetime.datetime.utcnow() + datetime.timedelta(seconds=in_seconds) + + # Create Timestamp protobuf. + timestamp = timestamp_pb2.Timestamp() + timestamp.FromDatetime(d) + + # Add the timestamp to the tasks. + task['schedule_time'] = timestamp + + # Use the client to build and send the task. + response = client.create_task(parent, task) + + print('Created task {}'.format(response.name)) + return response +# [END cloud_tasks_create_http_task] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=create_http_task.__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + + parser.add_argument( + '--project', + help='Project of the queue to add the task to.', + required=True, + ) + + parser.add_argument( + '--queue', + help='ID (short name) of the queue to add the task to.', + required=True, + ) + + parser.add_argument( + '--location', + help='Location of the queue to add the task to.', + required=True, + ) + + parser.add_argument( + '--url', + help='The full url path that the request will be sent to.', + required=True, + ) + + parser.add_argument( + '--payload', + help='Optional payload to attach to the push queue.' + ) + + parser.add_argument( + '--in_seconds', type=int, + help='The number of seconds from now to schedule task attempt.' + ) + + args = parser.parse_args() + + create_http_task( + args.project, args.queue, args.location, args.url, + args.payload, args.in_seconds) diff --git a/tasks/create_http_task_test.py b/tasks/create_http_task_test.py new file mode 100644 index 00000000000..6acfeaacbdb --- /dev/null +++ b/tasks/create_http_task_test.py @@ -0,0 +1,28 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import create_http_task + +TEST_PROJECT_ID = os.getenv('GCLOUD_PROJECT') +TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') +TEST_QUEUE_NAME = os.getenv('TEST_QUEUE_NAME', 'my-appengine-queue') + + +def test_create_http_task(): + url = 'https://example.appspot.com/example_task_handler' + result = create_http_task.create_http_task( + TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION, url) + assert TEST_QUEUE_NAME in result.name diff --git a/tasks/create_http_task_with_token.py b/tasks/create_http_task_with_token.py new file mode 100644 index 00000000000..1b79a9b3fc7 --- /dev/null +++ b/tasks/create_http_task_with_token.py @@ -0,0 +1,80 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import datetime + + +def create_http_task(project, + queue, + location, + url, + service_account_email, + payload=None, + in_seconds=None): + # [START cloud_tasks_create_http_task_with_token] + """Create a task for a given queue with an arbitrary payload.""" + + from google.cloud import tasks_v2beta3 + from google.protobuf import timestamp_pb2 + + # Create a client. + client = tasks_v2beta3.CloudTasksClient() + + # TODO(developer): Uncomment these lines and replace with your values. + # project = 'my-project-id' + # queue = 'my-appengine-queue' + # location = 'us-central1' + # url = 'https://example.com/example_task_handler' + # payload = 'hello' + + # Construct the fully qualified queue name. + parent = client.queue_path(project, location, queue) + + # Construct the request body. + task = { + 'http_request': { # Specify the type of request. + 'http_method': 'POST', + 'url': url, # The full url path that the task will be sent to. + 'oidc_token': { + 'service_account_email': service_account_email + } + } + } + + if payload is not None: + # The API expects a payload of type bytes. + converted_payload = payload.encode() + + # Add the payload to the request. + task['http_request']['body'] = converted_payload + + if in_seconds is not None: + # Convert "seconds from now" into an rfc3339 datetime string. + d = datetime.datetime.utcnow() + datetime.timedelta(seconds=in_seconds) + + # Create Timestamp protobuf. + timestamp = timestamp_pb2.Timestamp() + timestamp.FromDatetime(d) + + # Add the timestamp to the tasks. + task['schedule_time'] = timestamp + + # Use the client to build and send the task. + response = client.create_task(parent, task) + + print('Created task {}'.format(response.name)) + return response +# [END cloud_tasks_create_http_task_with_token] diff --git a/tasks/create_http_task_with_token_test.py b/tasks/create_http_task_with_token_test.py new file mode 100644 index 00000000000..5b98c566e9e --- /dev/null +++ b/tasks/create_http_task_with_token_test.py @@ -0,0 +1,33 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import create_http_task_with_token + +TEST_PROJECT_ID = os.getenv('GCLOUD_PROJECT') +TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') +TEST_QUEUE_NAME = os.getenv('TEST_QUEUE_NAME', 'my-appengine-queue') +TEST_SERVICE_ACCOUNT = ( + 'test-run-invoker@python-docs-samples-tests.iam.gserviceaccount.com') + + +def test_create_http_task_with_token(): + url = 'https://example.com/example_task_handler' + result = create_http_task_with_token.create_http_task(TEST_PROJECT_ID, + TEST_QUEUE_NAME, + TEST_LOCATION, + url, + TEST_SERVICE_ACCOUNT) + assert TEST_QUEUE_NAME in result.name diff --git a/tasks/requirements.txt b/tasks/requirements.txt new file mode 100644 index 00000000000..fe50a5aa3b2 --- /dev/null +++ b/tasks/requirements.txt @@ -0,0 +1,3 @@ +Flask==1.0.2 +gunicorn==19.9.0 +google-cloud-tasks==0.7.0 diff --git a/testing/.gitignore b/testing/.gitignore new file mode 100644 index 00000000000..40f47fa771e --- /dev/null +++ b/testing/.gitignore @@ -0,0 +1,3 @@ +test-env.sh +service-account.json +client-secrets.json diff --git a/testing/requirements.txt b/testing/requirements.txt new file mode 100644 index 00000000000..925321f69d9 --- /dev/null +++ b/testing/requirements.txt @@ -0,0 +1,18 @@ +beautifulsoup4==4.7.1 +coverage==4.5.2 +flaky==3.5.3 +funcsigs==1.0.2 +mock==2.0.0 +mysql-python==1.2.5; python_version < "3.0" +PyCrypto==2.6.1 +pytest-cov==2.6.1 +pytest==4.2.0 +pyyaml==3.13 +responses==0.10.5 +WebTest==2.0.32 +webapp2==2.5.2 +google-api-python-client==1.7.8 +google-cloud-core==0.29.1 +gcp-devrel-py-tools==0.0.15 +flask==1.0.2 +websocket-client==0.54.0 diff --git a/testing/resources/test-env.tmpl.sh b/testing/resources/test-env.tmpl.sh deleted file mode 100644 index 60c0f6f253a..00000000000 --- a/testing/resources/test-env.tmpl.sh +++ /dev/null @@ -1,19 +0,0 @@ -# Environment variables for system tests. -export GCLOUD_PROJECT=your-project-id -export CLOUD_STORAGE_BUCKET=$GCLOUD_PROJECT - -# Environment variables for Managed VMs system tests. -export GA_TRACKING_ID= -export SQLALCHEMY_DATABASE_URI=sqlite:// -export PUBSUB_TOPIC=gae-mvm-pubsub-topic -export PUBSUB_VERIFICATION_TOKEN=1234abc - -# Mailgun, Sendgrid, and Twilio config. -# These aren't current used because tests do not exist for these. -export MAILGUN_DOMAIN_NAME= -export MAILGUN_API_KEY= -export SENDGRID_API_KEY= -export SENDGRID_SENDER= -export TWILIO_ACCOUNT_SID= -export TWILIO_AUTH_TOKEN= -export TWILIO_NUMBER= diff --git a/testing/secrets.tar.enc b/testing/secrets.tar.enc new file mode 100644 index 00000000000..59f9e624f90 Binary files /dev/null and b/testing/secrets.tar.enc differ diff --git a/testing/test-env.tmpl.sh b/testing/test-env.tmpl.sh new file mode 100644 index 00000000000..2c3a396166e --- /dev/null +++ b/testing/test-env.tmpl.sh @@ -0,0 +1,39 @@ +# Environment variables for system tests. +export GCLOUD_PROJECT=your-project-id +export GCP_PROJECT=$GCLOUD_PROJECT +export GOOGLE_CLOUD_PROJECT=$GCLOUD_PROJECT +export FIRESTORE_PROJECT= + +export CLOUD_STORAGE_BUCKET=$GCLOUD_PROJECT +export API_KEY= +export BIGTABLE_CLUSTER=bigtable-test +export BIGTABLE_ZONE=us-central1-c +export SPANNER_INSTANCE= +export COMPOSER_LOCATION=us-central1 +export COMPOSER_ENVIRONMENT= +export CLOUD_KMS_KEY= + +export MYSQL_INSTANCE= +export MYSQL_USER= +export MYSQL_PASSWORD= +export MYSQL_DATABASE= +export POSTGRES_INSTANCE= +export POSTGRES_USER= +export POSTGRES_PASSWORD= +export POSTGRES_DATABASE= + +# Environment variables for App Engine Flexible system tests. +export GA_TRACKING_ID= +export SQLALCHEMY_DATABASE_URI=sqlite:// +export PUBSUB_TOPIC=gae-mvm-pubsub-topic +export PUBSUB_VERIFICATION_TOKEN=1234abc + +# Mailgun, Sendgrid, and Twilio config. +# These aren't current used because tests do not exist for these. +export MAILGUN_DOMAIN_NAME= +export MAILGUN_API_KEY= +export SENDGRID_API_KEY= +export SENDGRID_SENDER= +export TWILIO_ACCOUNT_SID= +export TWILIO_AUTH_TOKEN= +export TWILIO_NUMBER= diff --git a/texttospeech/cloud-client/README.rst b/texttospeech/cloud-client/README.rst new file mode 100644 index 00000000000..7b877be1328 --- /dev/null +++ b/texttospeech/cloud-client/README.rst @@ -0,0 +1,206 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Text-to-Speech API Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=/README.rst + + +This directory contains samples for Google Cloud Text-to-Speech API. The `Google Cloud Text To Speech API`_ enables you to generate and customize synthesized speech from text or SSML. + + + + +.. _Google Cloud Text-to-Speech API: https://cloud.google.com/text-to-speech/docs/ + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Quickstart ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=/quickstart.py,/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python quickstart.py + + +List voices ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=/list_voices.py,/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python list_voices.py + + +Synthesize text ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=/synthesize_text.py,/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python synthesize_text.py + + usage: synthesize_text.py [-h] (--text TEXT | --ssml SSML) + + Google Cloud Text-To-Speech API sample application . + + Example usage: + python synthesize_text.py --text "hello" + python synthesize_text.py --ssml "Hello there." + + optional arguments: + -h, --help show this help message and exit + --text TEXT The text from which to synthesize speech. + --ssml SSML The ssml string from which to synthesize speech. + + + +Synthesize file ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=/synthesize_file.py,/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python synthesize_file.py + + usage: synthesize_file.py [-h] (--text TEXT | --ssml SSML) + + Google Cloud Text-To-Speech API sample application . + + Example usage: + python synthesize_file.py --text resources/hello.txt + python synthesize_file.py --ssml resources/hello.ssml + + optional arguments: + -h, --help show this help message and exit + --text TEXT The text file from which to synthesize speech. + --ssml SSML The ssml file from which to synthesize speech. + + + +Audio profile ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=/audio_profile.py,/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python audio_profile.py + + usage: audio_profile.py [-h] [--output OUTPUT] [--text TEXT] + [--effects_profile_id EFFECTS_PROFILE_ID] + + Google Cloud Text-To-Speech API sample application for audio profile. + + Example usage: + python audio_profile.py --text "hello" --effects_profile_id + "telephony-class-application" + + optional arguments: + -h, --help show this help message and exit + --output OUTPUT The output mp3 file. + --text TEXT The text from which to synthesize speech. + --effects_profile_id EFFECTS_PROFILE_ID + The audio effects profile id to be applied. + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/texttospeech/cloud-client/README.rst.in b/texttospeech/cloud-client/README.rst.in new file mode 100644 index 00000000000..8be6ab46774 --- /dev/null +++ b/texttospeech/cloud-client/README.rst.in @@ -0,0 +1,29 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Text-to-Speech API + short_name: Cloud TTS API + url: https://cloud.google.com/text-to-speech/docs/ + description: > + The `Google Cloud Text To Speech API`_ enables you to generate and customize synthesized speech from text or SSML. + +setup: +- auth +- install_deps + +samples: +- name: Quickstart + file: quickstart.py +- name: List voices + file: list_voices.py +- name: Synthesize text + file: synthesize_text.py + show_help: True +- name: Synthesize file + file: synthesize_file.py + show_help: True +- name: Audio profile + file: audio_profile.py + show_help: True + +cloud_client_library: true diff --git a/texttospeech/cloud-client/audio_profile.py b/texttospeech/cloud-client/audio_profile.py new file mode 100644 index 00000000000..01f2ba884ba --- /dev/null +++ b/texttospeech/cloud-client/audio_profile.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Google Cloud Text-To-Speech API sample application for audio profile. + +Example usage: + python audio_profile.py --text "hello" --effects_profile_id + "telephony-class-application" --output "output.mp3" +""" + +import argparse + + +# [START tts_synthesize_text_audio_profile] +# [START tts_synthesize_text_audio_profile_beta] +def synthesize_text_with_audio_profile(text, output, effects_profile_id): + """Synthesizes speech from the input string of text.""" + from google.cloud import texttospeech + + client = texttospeech.TextToSpeechClient() + + input_text = texttospeech.types.SynthesisInput(text=text) + + # Note: the voice can also be specified by name. + # Names of voices can be retrieved with client.list_voices(). + voice = texttospeech.types.VoiceSelectionParams(language_code='en-US') + + # Note: you can pass in multiple effects_profile_id. They will be applied + # in the same order they are provided. + audio_config = texttospeech.types.AudioConfig( + audio_encoding=texttospeech.enums.AudioEncoding.MP3, + effects_profile_id=[effects_profile_id]) + + response = client.synthesize_speech(input_text, voice, audio_config) + + # The response's audio_content is binary. + with open(output, 'wb') as out: + out.write(response.audio_content) + print('Audio content written to file "%s"' % output) + +# [END tts_synthesize_text_audio_profile_beta] +# [END tts_synthesize_text_audio_profile] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('--output', + help='The output mp3 file.') + parser.add_argument('--text', + help='The text from which to synthesize speech.') + parser.add_argument('--effects_profile_id', + help='The audio effects profile id to be applied.') + + args = parser.parse_args() + + synthesize_text_with_audio_profile(args.text, args.output, + args.effects_profile_id) diff --git a/texttospeech/cloud-client/audio_profile_test.py b/texttospeech/cloud-client/audio_profile_test.py new file mode 100644 index 00000000000..43a30bf06d1 --- /dev/null +++ b/texttospeech/cloud-client/audio_profile_test.py @@ -0,0 +1,34 @@ +# Copyright 2018, Google, LLC. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import os.path + +import audio_profile + +TEXT = 'hello' +OUTPUT = 'output.mp3' +EFFECTS_PROFILE_ID = 'telephony-class-application' + + +def test_audio_profile(capsys): + if os.path.exists(OUTPUT): + os.remove(OUTPUT) + assert not os.path.exists(OUTPUT) + audio_profile.synthesize_text_with_audio_profile(TEXT, OUTPUT, + EFFECTS_PROFILE_ID) + out, err = capsys.readouterr() + + assert ('Audio content written to file "%s"' % OUTPUT) in out + assert os.path.exists(OUTPUT) + os.remove(OUTPUT) diff --git a/texttospeech/cloud-client/list_voices.py b/texttospeech/cloud-client/list_voices.py new file mode 100644 index 00000000000..a2da5a17852 --- /dev/null +++ b/texttospeech/cloud-client/list_voices.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python + +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Google Cloud Text-To-Speech API sample application. + +Example usage: + python list_voices.py +""" + + +# [START tts_list_voices] +def list_voices(): + """Lists the available voices.""" + from google.cloud import texttospeech + from google.cloud.texttospeech import enums + client = texttospeech.TextToSpeechClient() + + # Performs the list voices request + voices = client.list_voices() + + for voice in voices.voices: + # Display the voice's name. Example: tpc-vocoded + print('Name: {}'.format(voice.name)) + + # Display the supported language codes for this voice. Example: "en-US" + for language_code in voice.language_codes: + print('Supported language: {}'.format(language_code)) + + ssml_gender = enums.SsmlVoiceGender(voice.ssml_gender) + + # Display the SSML Voice Gender + print('SSML Voice Gender: {}'.format(ssml_gender.name)) + + # Display the natural sample rate hertz for this voice. Example: 24000 + print('Natural Sample Rate Hertz: {}\n'.format( + voice.natural_sample_rate_hertz)) +# [END tts_list_voices] + + +if __name__ == '__main__': + list_voices() diff --git a/texttospeech/cloud-client/list_voices_test.py b/texttospeech/cloud-client/list_voices_test.py new file mode 100644 index 00000000000..fd325569492 --- /dev/null +++ b/texttospeech/cloud-client/list_voices_test.py @@ -0,0 +1,23 @@ +# Copyright 2018, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import list_voices + + +def test_list_voices(capsys): + list_voices.list_voices() + out, err = capsys.readouterr() + + assert 'en-US' in out + assert 'SSML Voice Gender: MALE' in out + assert 'SSML Voice Gender: FEMALE' in out diff --git a/texttospeech/cloud-client/quickstart.py b/texttospeech/cloud-client/quickstart.py new file mode 100644 index 00000000000..f462139d794 --- /dev/null +++ b/texttospeech/cloud-client/quickstart.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python + +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Google Cloud Text-To-Speech API sample application . + +Example usage: + python quickstart.py +""" + + +def run_quickstart(): + # [START tts_quickstart] + """Synthesizes speech from the input string of text or ssml. + + Note: ssml must be well-formed according to: + https://www.w3.org/TR/speech-synthesis/ + """ + from google.cloud import texttospeech + + # Instantiates a client + client = texttospeech.TextToSpeechClient() + + # Set the text input to be synthesized + synthesis_input = texttospeech.types.SynthesisInput(text="Hello, World!") + + # Build the voice request, select the language code ("en-US") and the ssml + # voice gender ("neutral") + voice = texttospeech.types.VoiceSelectionParams( + language_code='en-US', + ssml_gender=texttospeech.enums.SsmlVoiceGender.NEUTRAL) + + # Select the type of audio file you want returned + audio_config = texttospeech.types.AudioConfig( + audio_encoding=texttospeech.enums.AudioEncoding.MP3) + + # Perform the text-to-speech request on the text input with the selected + # voice parameters and audio file type + response = client.synthesize_speech(synthesis_input, voice, audio_config) + + # The response's audio_content is binary. + with open('output.mp3', 'wb') as out: + # Write the response to the output file. + out.write(response.audio_content) + print('Audio content written to file "output.mp3"') + # [END tts_quickstart] + + +if __name__ == '__main__': + run_quickstart() diff --git a/texttospeech/cloud-client/requirements.txt b/texttospeech/cloud-client/requirements.txt new file mode 100644 index 00000000000..0db01b710b9 --- /dev/null +++ b/texttospeech/cloud-client/requirements.txt @@ -0,0 +1 @@ +google-cloud-texttospeech==0.4.0 diff --git a/texttospeech/cloud-client/resources/hello.ssml b/texttospeech/cloud-client/resources/hello.ssml new file mode 100644 index 00000000000..cd347b71fe5 --- /dev/null +++ b/texttospeech/cloud-client/resources/hello.ssml @@ -0,0 +1 @@ +Hello there. \ No newline at end of file diff --git a/texttospeech/cloud-client/resources/hello.txt b/texttospeech/cloud-client/resources/hello.txt new file mode 100644 index 00000000000..cd773cd131f --- /dev/null +++ b/texttospeech/cloud-client/resources/hello.txt @@ -0,0 +1 @@ +Hello there! \ No newline at end of file diff --git a/texttospeech/cloud-client/synthesize_file.py b/texttospeech/cloud-client/synthesize_file.py new file mode 100644 index 00000000000..f62d6330d7d --- /dev/null +++ b/texttospeech/cloud-client/synthesize_file.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python + +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Google Cloud Text-To-Speech API sample application . + +Example usage: + python synthesize_file.py --text resources/hello.txt + python synthesize_file.py --ssml resources/hello.ssml +""" + +import argparse + + +# [START tts_synthesize_text_file] +def synthesize_text_file(text_file): + """Synthesizes speech from the input file of text.""" + from google.cloud import texttospeech + client = texttospeech.TextToSpeechClient() + + with open(text_file, 'r') as f: + text = f.read() + input_text = texttospeech.types.SynthesisInput(text=text) + + # Note: the voice can also be specified by name. + # Names of voices can be retrieved with client.list_voices(). + voice = texttospeech.types.VoiceSelectionParams( + language_code='en-US', + ssml_gender=texttospeech.enums.SsmlVoiceGender.FEMALE) + + audio_config = texttospeech.types.AudioConfig( + audio_encoding=texttospeech.enums.AudioEncoding.MP3) + + response = client.synthesize_speech(input_text, voice, audio_config) + + # The response's audio_content is binary. + with open('output.mp3', 'wb') as out: + out.write(response.audio_content) + print('Audio content written to file "output.mp3"') +# [END tts_synthesize_text_file] + + +# [START tts_synthesize_ssml_file] +def synthesize_ssml_file(ssml_file): + """Synthesizes speech from the input file of ssml. + + Note: ssml must be well-formed according to: + https://www.w3.org/TR/speech-synthesis/ + """ + from google.cloud import texttospeech + client = texttospeech.TextToSpeechClient() + + with open(ssml_file, 'r') as f: + ssml = f.read() + input_text = texttospeech.types.SynthesisInput(ssml=ssml) + + # Note: the voice can also be specified by name. + # Names of voices can be retrieved with client.list_voices(). + voice = texttospeech.types.VoiceSelectionParams( + language_code='en-US', + ssml_gender=texttospeech.enums.SsmlVoiceGender.FEMALE) + + audio_config = texttospeech.types.AudioConfig( + audio_encoding=texttospeech.enums.AudioEncoding.MP3) + + response = client.synthesize_speech(input_text, voice, audio_config) + + # The response's audio_content is binary. + with open('output.mp3', 'wb') as out: + out.write(response.audio_content) + print('Audio content written to file "output.mp3"') +# [END tts_synthesize_ssml_file] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('--text', + help='The text file from which to synthesize speech.') + group.add_argument('--ssml', + help='The ssml file from which to synthesize speech.') + + args = parser.parse_args() + + if args.text: + synthesize_text_file(args.text) + else: + synthesize_ssml_file(args.ssml) diff --git a/texttospeech/cloud-client/synthesize_file_test.py b/texttospeech/cloud-client/synthesize_file_test.py new file mode 100644 index 00000000000..2652009f98b --- /dev/null +++ b/texttospeech/cloud-client/synthesize_file_test.py @@ -0,0 +1,37 @@ +# Copyright 2018, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import synthesize_file + +TEXT_FILE = 'resources/hello.txt' +SSML_FILE = 'resources/hello.ssml' + + +def test_synthesize_text_file(capsys): + synthesize_file.synthesize_text_file(text_file=TEXT_FILE) + out, err = capsys.readouterr() + + assert 'Audio content written to file' in out + statinfo = os.stat('output.mp3') + assert statinfo.st_size > 0 + + +def test_synthesize_ssml_file(capsys): + synthesize_file.synthesize_ssml_file(ssml_file=SSML_FILE) + out, err = capsys.readouterr() + + assert 'Audio content written to file' in out + statinfo = os.stat('output.mp3') + assert statinfo.st_size > 0 diff --git a/texttospeech/cloud-client/synthesize_text.py b/texttospeech/cloud-client/synthesize_text.py new file mode 100644 index 00000000000..d5886bd16f1 --- /dev/null +++ b/texttospeech/cloud-client/synthesize_text.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python + +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Google Cloud Text-To-Speech API sample application . + +Example usage: + python synthesize_text.py --text "hello" + python synthesize_text.py --ssml "Hello there." +""" + +import argparse + + +# [START tts_synthesize_text] +def synthesize_text(text): + """Synthesizes speech from the input string of text.""" + from google.cloud import texttospeech + client = texttospeech.TextToSpeechClient() + + input_text = texttospeech.types.SynthesisInput(text=text) + + # Note: the voice can also be specified by name. + # Names of voices can be retrieved with client.list_voices(). + voice = texttospeech.types.VoiceSelectionParams( + language_code='en-US', + ssml_gender=texttospeech.enums.SsmlVoiceGender.FEMALE) + + audio_config = texttospeech.types.AudioConfig( + audio_encoding=texttospeech.enums.AudioEncoding.MP3) + + response = client.synthesize_speech(input_text, voice, audio_config) + + # The response's audio_content is binary. + with open('output.mp3', 'wb') as out: + out.write(response.audio_content) + print('Audio content written to file "output.mp3"') +# [END tts_synthesize_text] + + +# [START tts_synthesize_ssml] +def synthesize_ssml(ssml): + """Synthesizes speech from the input string of ssml. + + Note: ssml must be well-formed according to: + https://www.w3.org/TR/speech-synthesis/ + + Example: Hello there. + """ + from google.cloud import texttospeech + client = texttospeech.TextToSpeechClient() + + input_text = texttospeech.types.SynthesisInput(ssml=ssml) + + # Note: the voice can also be specified by name. + # Names of voices can be retrieved with client.list_voices(). + voice = texttospeech.types.VoiceSelectionParams( + language_code='en-US', + ssml_gender=texttospeech.enums.SsmlVoiceGender.FEMALE) + + audio_config = texttospeech.types.AudioConfig( + audio_encoding=texttospeech.enums.AudioEncoding.MP3) + + response = client.synthesize_speech(input_text, voice, audio_config) + + # The response's audio_content is binary. + with open('output.mp3', 'wb') as out: + out.write(response.audio_content) + print('Audio content written to file "output.mp3"') +# [END tts_synthesize_ssml] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('--text', + help='The text from which to synthesize speech.') + group.add_argument('--ssml', + help='The ssml string from which to synthesize speech.') + + args = parser.parse_args() + + if args.text: + synthesize_text(args.text) + else: + synthesize_ssml(args.ssml) diff --git a/texttospeech/cloud-client/synthesize_text_test.py b/texttospeech/cloud-client/synthesize_text_test.py new file mode 100644 index 00000000000..948d58da26d --- /dev/null +++ b/texttospeech/cloud-client/synthesize_text_test.py @@ -0,0 +1,37 @@ +# Copyright 2018, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import synthesize_text + +TEXT = 'Hello there.' +SSML = 'Hello there.' + + +def test_synthesize_text(capsys): + synthesize_text.synthesize_text(text=TEXT) + out, err = capsys.readouterr() + + assert 'Audio content written to file' in out + statinfo = os.stat('output.mp3') + assert statinfo.st_size > 0 + + +def test_synthesize_ssml(capsys): + synthesize_text.synthesize_ssml(ssml=SSML) + out, err = capsys.readouterr() + + assert 'Audio content written to file' in out + statinfo = os.stat('output.mp3') + assert statinfo.st_size > 0 diff --git a/third_party/apache-airflow/LICENSE b/third_party/apache-airflow/LICENSE new file mode 100644 index 00000000000..68ca6b6117c --- /dev/null +++ b/third_party/apache-airflow/LICENSE @@ -0,0 +1,263 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +============================================================================ + APACHE AIRFLOW SUBCOMPONENTS: + + The Apache Airflow project contains subcomponents with separate copyright + notices and license terms. Your use of the source code for the these + subcomponents is subject to the terms and conditions of the following + licenses. + + +======================================================================== +Third party Apache 2.0 licenses +======================================================================== + +The following components are provided under the Apache 2.0 License. +See project link for details. The text of each license is also included +at licenses/LICENSE-[project].txt. + + (ALv2 License) hue (https://github.com/cloudera/hue/) + (ALv2 License) jqclock (https://github.com/JohnRDOrazio/jQuery-Clock-Plugin) + (ALv2 License) bootstrap3-typeahead (https://github.com/bassjobsen/Bootstrap-3-Typeahead) + (ALv2 License) airflow.contrib.auth.backends.github_enterprise_auth + +======================================================================== +MIT licenses +======================================================================== + +The following components are provided under the MIT License. See project link for details. +The text of each license is also included at licenses/LICENSE-[project].txt. + + (MIT License) jquery (https://jquery.org/license/) + (MIT License) dagre-d3 (https://github.com/cpettitt/dagre-d3) + (MIT License) bootstrap (https://github.com/twbs/bootstrap/) + (MIT License) d3-tip (https://github.com/Caged/d3-tip) + (MIT License) dataTables (https://datatables.net) + (MIT License) WebGL-2D (https://github.com/gameclosure/webgl-2d) + (MIT License) Underscorejs (http://underscorejs.org) + (MIT License) Bootstrap Toggle (http://www.bootstraptoggle.com) + (MIT License) normalize.css (http://necolas.github.io/normalize.css/) + (MIT License) ElasticMock (https://github.com/vrcmarcos/elasticmock) + (MIT License) MomentJS (http://momentjs.com/) + +======================================================================== +BSD 2-Clause licenses +======================================================================== +The following components are provided under the BSD 2-Clause license. +See file headers and project links for details. +The text of each license is also included at licenses/LICENSE-[project].txt. + + (BSD 2 License) flask-kerberos (https://github.com/mkomitee/flask-kerberos) + +======================================================================== +BSD 3-Clause licenses +======================================================================== +The following components are provided under the BSD 3-Clause license. See project links for details. +The text of each license is also included at licenses/LICENSE-[project].txt. + + (BSD 3 License) Ace (https://github.com/ajaxorg/ace) + (BSD 3 License) d3js (https://d3js.org) + (BSD 3 License) parallel-coordinates (http://syntagmatic.github.com/parallel-coordinates/) + (BSD 3 License) scikit-learn (https://github.com/scikit-learn/scikit-learn) + diff --git a/third_party/apache-airflow/README.md b/third_party/apache-airflow/README.md new file mode 100644 index 00000000000..8a95cfc2fe8 --- /dev/null +++ b/third_party/apache-airflow/README.md @@ -0,0 +1,22 @@ +## Google Cloud Platform Python Samples - Apache Airflow Third party libraries + +This folder holds modified third party modules from [Apache Airflow] +(https://airflow.apache.org/), including operators and hooks to be deployed as +plugins. + +## Instructions + +1. Copy the required operator or hook module inside +`plugins/$defined_plugin/operators` or `plugins/$defined_plugin/hooks` +respectively within your Airflow/Composer environment + +2. If requires another module as dependency, download it from the Airflow +Github repository, e.g., GCS hook: + ``` + cd plugins/gcs_plugin/hooks + wget https://raw.githubusercontent.com/apache/incubator-airflow/v1-10-stable/airflow/contrib/hooks/gcs_hook.py + ``` + +## Licensing + +* See [LICENSE](LICENSE) diff --git a/third_party/apache-airflow/plugins/__init__.py b/third_party/apache-airflow/plugins/__init__.py new file mode 100644 index 00000000000..bef4caf619c --- /dev/null +++ b/third_party/apache-airflow/plugins/__init__.py @@ -0,0 +1,30 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""GCS Plugin + +This plugin provides an interface to GCS operator from Airflow Master. +""" + +from airflow.plugins_manager import AirflowPlugin + +from gcs_plugin.hooks.gcs_hook import GoogleCloudStorageHook +from gcs_plugin.operators.gcs_to_gcs import \ + GoogleCloudStorageToGoogleCloudStorageOperator + + +class GCSPlugin(AirflowPlugin): + name = "gcs_plugin" + operators = [GoogleCloudStorageToGoogleCloudStorageOperator] + hooks = [GoogleCloudStorageHook] diff --git a/third_party/apache-airflow/plugins/gcs_plugin/__init__.py b/third_party/apache-airflow/plugins/gcs_plugin/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/third_party/apache-airflow/plugins/gcs_plugin/hooks/__init__.py b/third_party/apache-airflow/plugins/gcs_plugin/hooks/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/third_party/apache-airflow/plugins/gcs_plugin/hooks/gcs_hook.py b/third_party/apache-airflow/plugins/gcs_plugin/hooks/gcs_hook.py new file mode 100644 index 00000000000..6cfa1cf5650 --- /dev/null +++ b/third_party/apache-airflow/plugins/gcs_plugin/hooks/gcs_hook.py @@ -0,0 +1,539 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +from apiclient.discovery import build +from apiclient.http import MediaFileUpload +from googleapiclient import errors + +from airflow.contrib.hooks.gcp_api_base_hook import GoogleCloudBaseHook +from airflow.exceptions import AirflowException + +import re + + +class GoogleCloudStorageHook(GoogleCloudBaseHook): + """ + Interact with Google Cloud Storage. This hook uses the Google Cloud Platform + connection. + """ + + def __init__(self, + google_cloud_storage_conn_id='google_cloud_default', + delegate_to=None): + super(GoogleCloudStorageHook, self).__init__(google_cloud_storage_conn_id, + delegate_to) + + def get_conn(self): + """ + Returns a Google Cloud Storage service object. + """ + http_authorized = self._authorize() + return build( + 'storage', 'v1', http=http_authorized, cache_discovery=False) + + # pylint:disable=redefined-builtin + def copy(self, source_bucket, source_object, destination_bucket=None, + destination_object=None): + """ + Copies an object from a bucket to another, with renaming if requested. + + destination_bucket or destination_object can be omitted, in which case + source bucket/object is used, but not both. + + :param source_bucket: The bucket of the object to copy from. + :type source_bucket: str + :param source_object: The object to copy. + :type source_object: str + :param destination_bucket: The destination of the object to copied to. + Can be omitted; then the same bucket is used. + :type destination_bucket: str + :param destination_object: The (renamed) path of the object if given. + Can be omitted; then the same name is used. + :type destination_object: str + """ + destination_bucket = destination_bucket or source_bucket + destination_object = destination_object or source_object + if source_bucket == destination_bucket and \ + source_object == destination_object: + + raise ValueError( + 'Either source/destination bucket or source/destination object ' + 'must be different, not both the same: bucket=%s, object=%s' % + (source_bucket, source_object)) + if not source_bucket or not source_object: + raise ValueError('source_bucket and source_object cannot be empty.') + + service = self.get_conn() + try: + service \ + .objects() \ + .copy(sourceBucket=source_bucket, sourceObject=source_object, + destinationBucket=destination_bucket, + destinationObject=destination_object, body='') \ + .execute() + return True + except errors.HttpError as ex: + if ex.resp['status'] == '404': + return False + raise + + def rewrite(self, source_bucket, source_object, destination_bucket, + destination_object=None): + """ + Has the same functionality as copy, except that will work on files + over 5 TB, as well as when copying between locations and/or storage + classes. + + destination_object can be omitted, in which case source_object is used. + + :param source_bucket: The bucket of the object to copy from. + :type source_bucket: str + :param source_object: The object to copy. + :type source_object: str + :param destination_bucket: The destination of the object to copied to. + :type destination_bucket: str + :param destination_object: The (renamed) path of the object if given. + Can be omitted; then the same name is used. + """ + destination_object = destination_object or source_object + if (source_bucket == destination_bucket and + source_object == destination_object): + raise ValueError( + 'Either source/destination bucket or source/destination object ' + 'must be different, not both the same: bucket=%s, object=%s' % + (source_bucket, source_object)) + if not source_bucket or not source_object: + raise ValueError('source_bucket and source_object cannot be empty.') + + service = self.get_conn() + request_count = 1 + try: + result = service.objects() \ + .rewrite(sourceBucket=source_bucket, sourceObject=source_object, + destinationBucket=destination_bucket, + destinationObject=destination_object, body='') \ + .execute() + self.log.info('Rewrite request #%s: %s', request_count, result) + while not result['done']: + request_count += 1 + result = service.objects() \ + .rewrite(sourceBucket=source_bucket, sourceObject=source_object, + destinationBucket=destination_bucket, + destinationObject=destination_object, + rewriteToken=result['rewriteToken'], body='') \ + .execute() + self.log.info('Rewrite request #%s: %s', request_count, result) + return True + except errors.HttpError as ex: + if ex.resp['status'] == '404': + return False + raise + + # pylint:disable=redefined-builtin + def download(self, bucket, object, filename=None): + """ + Get a file from Google Cloud Storage. + + :param bucket: The bucket to fetch from. + :type bucket: str + :param object: The object to fetch. + :type object: str + :param filename: If set, a local file path where the file should be written to. + :type filename: str + """ + service = self.get_conn() + downloaded_file_bytes = service \ + .objects() \ + .get_media(bucket=bucket, object=object) \ + .execute() + + # Write the file to local file path, if requested. + if filename: + write_argument = 'wb' if isinstance(downloaded_file_bytes, bytes) else 'w' + with open(filename, write_argument) as file_fd: + file_fd.write(downloaded_file_bytes) + + return downloaded_file_bytes + + # pylint:disable=redefined-builtin + def upload(self, bucket, object, filename, mime_type='application/octet-stream'): + """ + Uploads a local file to Google Cloud Storage. + + :param bucket: The bucket to upload to. + :type bucket: str + :param object: The object name to set when uploading the local file. + :type object: str + :param filename: The local file path to the file to be uploaded. + :type filename: str + :param mime_type: The MIME type to set when uploading the file. + :type mime_type: str + """ + service = self.get_conn() + media = MediaFileUpload(filename, mime_type) + try: + service \ + .objects() \ + .insert(bucket=bucket, name=object, media_body=media) \ + .execute() + return True + except errors.HttpError as ex: + if ex.resp['status'] == '404': + return False + raise + + # pylint:disable=redefined-builtin + def exists(self, bucket, object): + """ + Checks for the existence of a file in Google Cloud Storage. + + :param bucket: The Google cloud storage bucket where the object is. + :type bucket: str + :param object: The name of the object to check in the Google cloud + storage bucket. + :type object: str + """ + service = self.get_conn() + try: + service \ + .objects() \ + .get(bucket=bucket, object=object) \ + .execute() + return True + except errors.HttpError as ex: + if ex.resp['status'] == '404': + return False + raise + + # pylint:disable=redefined-builtin + def is_updated_after(self, bucket, object, ts): + """ + Checks if an object is updated in Google Cloud Storage. + + :param bucket: The Google cloud storage bucket where the object is. + :type bucket: str + :param object: The name of the object to check in the Google cloud + storage bucket. + :type object: str + :param ts: The timestamp to check against. + :type ts: datetime + """ + service = self.get_conn() + try: + response = (service + .objects() + .get(bucket=bucket, object=object) + .execute()) + + if 'updated' in response: + import dateutil.parser + import dateutil.tz + + if not ts.tzinfo: + ts = ts.replace(tzinfo=dateutil.tz.tzutc()) + + updated = dateutil.parser.parse(response['updated']) + self.log.info("Verify object date: %s > %s", updated, ts) + + if updated > ts: + return True + + except errors.HttpError as ex: + if ex.resp['status'] != '404': + raise + + return False + + def delete(self, bucket, object, generation=None): + """ + Delete an object if versioning is not enabled for the bucket, or if generation + parameter is used. + + :param bucket: name of the bucket, where the object resides + :type bucket: str + :param object: name of the object to delete + :type object: str + :param generation: if present, permanently delete the object of this generation + :type generation: str + :return: True if succeeded + """ + service = self.get_conn() + + try: + service \ + .objects() \ + .delete(bucket=bucket, object=object, generation=generation) \ + .execute() + return True + except errors.HttpError as ex: + if ex.resp['status'] == '404': + return False + raise + + def list(self, bucket, versions=None, maxResults=None, prefix=None, delimiter=None): + """ + List all objects from the bucket with the give string prefix in name + + :param bucket: bucket name + :type bucket: str + :param versions: if true, list all versions of the objects + :type versions: bool + :param maxResults: max count of items to return in a single page of responses + :type maxResults: int + :param prefix: prefix string which filters objects whose name begin with + this prefix + :type prefix: str + :param delimiter: filters objects based on the delimiter (for e.g '.csv') + :type delimiter: str + :return: a stream of object names matching the filtering criteria + """ + service = self.get_conn() + + ids = list() + pageToken = None + while True: + response = service.objects().list( + bucket=bucket, + versions=versions, + maxResults=maxResults, + pageToken=pageToken, + prefix=prefix, + delimiter=delimiter + ).execute() + + if 'prefixes' not in response: + if 'items' not in response: + self.log.info("No items found for prefix: %s", prefix) + break + + for item in response['items']: + if item and 'name' in item: + ids.append(item['name']) + else: + for item in response['prefixes']: + ids.append(item) + + if 'nextPageToken' not in response: + # no further pages of results, so stop the loop + break + + pageToken = response['nextPageToken'] + if not pageToken: + # empty next page token + break + return ids + + def get_size(self, bucket, object): + """ + Gets the size of a file in Google Cloud Storage. + + :param bucket: The Google cloud storage bucket where the object is. + :type bucket: str + :param object: The name of the object to check in the Google cloud storage bucket. + :type object: str + + """ + self.log.info('Checking the file size of object: %s in bucket: %s', + object, + bucket) + service = self.get_conn() + try: + response = service.objects().get( + bucket=bucket, + object=object + ).execute() + + if 'name' in response and response['name'][-1] != '/': + # Remove Directories & Just check size of files + size = response['size'] + self.log.info('The file size of %s is %s bytes.', object, size) + return size + else: + raise ValueError('Object is not a file') + except errors.HttpError as ex: + if ex.resp['status'] == '404': + raise ValueError('Object Not Found') + + def get_crc32c(self, bucket, object): + """ + Gets the CRC32c checksum of an object in Google Cloud Storage. + + :param bucket: The Google cloud storage bucket where the object is. + :type bucket: str + :param object: The name of the object to check in the Google cloud + storage bucket. + :type object: str + """ + self.log.info('Retrieving the crc32c checksum of ' + 'object: %s in bucket: %s', object, bucket) + service = self.get_conn() + try: + response = service.objects().get( + bucket=bucket, + object=object + ).execute() + + crc32c = response['crc32c'] + self.log.info('The crc32c checksum of %s is %s', object, crc32c) + return crc32c + + except errors.HttpError as ex: + if ex.resp['status'] == '404': + raise ValueError('Object Not Found') + + def get_md5hash(self, bucket, object): + """ + Gets the MD5 hash of an object in Google Cloud Storage. + + :param bucket: The Google cloud storage bucket where the object is. + :type bucket: str + :param object: The name of the object to check in the Google cloud + storage bucket. + :type object: str + """ + self.log.info('Retrieving the MD5 hash of ' + 'object: %s in bucket: %s', object, bucket) + service = self.get_conn() + try: + response = service.objects().get( + bucket=bucket, + object=object + ).execute() + + md5hash = response['md5Hash'] + self.log.info('The md5Hash of %s is %s', object, md5hash) + return md5hash + + except errors.HttpError as ex: + if ex.resp['status'] == '404': + raise ValueError('Object Not Found') + + def create_bucket(self, + bucket_name, + storage_class='MULTI_REGIONAL', + location='US', + project_id=None, + labels=None + ): + """ + Creates a new bucket. Google Cloud Storage uses a flat namespace, so + you can't create a bucket with a name that is already in use. + + .. seealso:: + For more information, see Bucket Naming Guidelines: + https://cloud.google.com/storage/docs/bucketnaming.html#requirements + + :param bucket_name: The name of the bucket. + :type bucket_name: str + :param storage_class: This defines how objects in the bucket are stored + and determines the SLA and the cost of storage. Values include + + - ``MULTI_REGIONAL`` + - ``REGIONAL`` + - ``STANDARD`` + - ``NEARLINE`` + - ``COLDLINE``. + If this value is not specified when the bucket is + created, it will default to STANDARD. + :type storage_class: str + :param location: The location of the bucket. + Object data for objects in the bucket resides in physical storage + within this region. Defaults to US. + + .. seealso:: + https://developers.google.com/storage/docs/bucket-locations + + :type location: str + :param project_id: The ID of the GCP Project. + :type project_id: str + :param labels: User-provided labels, in key/value pairs. + :type labels: dict + :return: If successful, it returns the ``id`` of the bucket. + """ + + project_id = project_id if project_id is not None else self.project_id + storage_classes = [ + 'MULTI_REGIONAL', + 'REGIONAL', + 'NEARLINE', + 'COLDLINE', + 'STANDARD', # alias for MULTI_REGIONAL/REGIONAL, based on location + ] + + self.log.info('Creating Bucket: %s; Location: %s; Storage Class: %s', + bucket_name, location, storage_class) + if storage_class not in storage_classes: + raise ValueError( + 'Invalid value ({}) passed to storage_class. Value should be ' + 'one of {}'.format(storage_class, storage_classes)) + + if not re.match('[a-zA-Z0-9]+', bucket_name[0]): + raise ValueError('Bucket names must start with a number or letter.') + + if not re.match('[a-zA-Z0-9]+', bucket_name[-1]): + raise ValueError('Bucket names must end with a number or letter.') + + service = self.get_conn() + bucket_resource = { + 'name': bucket_name, + 'location': location, + 'storageClass': storage_class + } + + self.log.info('The Default Project ID is %s', self.project_id) + + if labels is not None: + bucket_resource['labels'] = labels + + try: + response = service.buckets().insert( + project=project_id, + body=bucket_resource + ).execute() + + self.log.info('Bucket: %s created successfully.', bucket_name) + + return response['id'] + + except errors.HttpError as ex: + raise AirflowException( + 'Bucket creation failed. Error was: {}'.format(ex.content) + ) + + +def _parse_gcs_url(gsurl): + """ + Given a Google Cloud Storage URL (gs:///), returns a + tuple containing the corresponding bucket and blob. + """ + # Python 3 + try: + from urllib.parse import urlparse + # Python 2 + except ImportError: + from urlparse import urlparse + + parsed_url = urlparse(gsurl) + if not parsed_url.netloc: + raise AirflowException('Please provide a bucket name') + else: + bucket = parsed_url.netloc + # Remove leading '/' but NOT trailing one + blob = parsed_url.path.lstrip('/') + return bucket, blob diff --git a/third_party/apache-airflow/plugins/gcs_plugin/operators/__init__.py b/third_party/apache-airflow/plugins/gcs_plugin/operators/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/third_party/apache-airflow/plugins/gcs_plugin/operators/gcs_to_gcs.py b/third_party/apache-airflow/plugins/gcs_plugin/operators/gcs_to_gcs.py new file mode 100644 index 00000000000..4af1645fbf7 --- /dev/null +++ b/third_party/apache-airflow/plugins/gcs_plugin/operators/gcs_to_gcs.py @@ -0,0 +1,177 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from airflow.models import BaseOperator +from airflow.utils.decorators import apply_defaults +from gcs_plugin.hooks.gcs_hook import GoogleCloudStorageHook + + +class GoogleCloudStorageToGoogleCloudStorageOperator(BaseOperator): + """ + Copies objects from a bucket to another, with renaming if requested. + :param source_bucket: The source Google cloud storage bucket where the + object is. (templated) + :type source_bucket: string + :param source_object: The source name of the object to copy in the + Google cloud storage bucket. (templated) + If wildcards are used in this argument: + You can use only one wildcard for objects (filenames) within your + bucket. The wildcard can appear inside the object name or at the + end of the object name. Appending a wildcard to the bucket name is + unsupported. + :type source_object: string + :param destination_bucket: The destination Google cloud storage bucket + where the object should be. (templated) + :type destination_bucket: string + :param destination_object: The destination name of the object in the + destination Google cloud storage bucket. (templated) + If a wildcard is supplied in the source_object argument, this is the + prefix that will be prepended to the final destination objects' paths. + Note that the source path's part before the wildcard will be removed; + if it needs to be retained it should be appended to destination_object. + For example, with prefix ``foo/*`` and destination_object `'blah/``, + the file ``foo/baz`` will be copied to ``blah/baz``; to retain the + prefix write the destination_object as e.g. ``blah/foo``, in which + case the copied file will be named ``blah/foo/baz``. + :type destination_object: string + :param move_object: When move object is True, the object is moved instead + of copied to the new location. + This is the equivalent of a mv command as opposed to a + cp command. + :type move_object: bool + :param google_cloud_storage_conn_id: The connection ID to use when + connecting to Google cloud storage. + :type google_cloud_storage_conn_id: string + :param delegate_to: The account to impersonate, if any. + For this to work, the service account making the request must have + domain-wide delegation enabled. + :type delegate_to: string + **Examples**: + The following Operator would copy a single file named + ``sales/sales-2017/january.avro`` in the ``data`` bucket to the file + named ``copied_sales/2017/january-backup.avro` in the + ``data_backup`` bucket :: + copy_single_file = GoogleCloudStorageToGoogleCloudStorageOperator( + task_id='copy_single_file', + source_bucket='data', + source_object='sales/sales-2017/january.avro', + destination_bucket='data_backup', + destination_object='copied_sales/2017/january-backup.avro', + google_cloud_storage_conn_id=google_cloud_conn_id + ) + The following Operator would copy all the Avro files from + ``sales/sales-2017`` folder (i.e. with names starting with that + prefix) in ``data`` bucket to the ``copied_sales/2017`` folder in + the ``data_backup`` bucket. :: + copy_files = GoogleCloudStorageToGoogleCloudStorageOperator( + task_id='copy_files', + source_bucket='data', + source_object='sales/sales-2017/*.avro', + destination_bucket='data_backup', + destination_object='copied_sales/2017/', + google_cloud_storage_conn_id=google_cloud_conn_id + ) + The following Operator would move all the Avro files from + ``sales/sales-2017`` folder (i.e. with names starting with that + prefix) in ``data`` bucket to the same folder in the ``data_backup`` + bucket, deleting the original files in the process. :: + move_files = GoogleCloudStorageToGoogleCloudStorageOperator( + task_id='move_files', + source_bucket='data', + source_object='sales/sales-2017/*.avro', + destination_bucket='data_backup', + move_object=True, + google_cloud_storage_conn_id=google_cloud_conn_id + ) + Notes: + 1. Place this module inside plugins/gcs_plugin/operators + 2. The GCS hook must be downloaded from the Airflow repository + cd plugins/gcs_plugin/hook + wget https://raw.githubusercontent.com/apache/incubator-airflow/\ + v1-10-stable/airflow/contrib/hooks/gcs_hook.py + """ + template_fields = ('source_bucket', 'source_object', 'destination_bucket', + 'destination_object',) + ui_color = '#f0eee4' + + @apply_defaults + def __init__(self, + source_bucket, + source_object, + destination_bucket=None, + destination_object=None, + move_object=False, + google_cloud_storage_conn_id='google_cloud_default', + delegate_to=None, + *args, + **kwargs): + super(GoogleCloudStorageToGoogleCloudStorageOperator, + self).__init__(*args, **kwargs) + self.source_bucket = source_bucket + self.source_object = source_object + self.destination_bucket = destination_bucket + self.destination_object = destination_object + self.move_object = move_object + self.google_cloud_storage_conn_id = google_cloud_storage_conn_id + self.delegate_to = delegate_to + self.wildcard = '*' + + def execute(self, context): + + hook = GoogleCloudStorageHook( + google_cloud_storage_conn_id=self.google_cloud_storage_conn_id, + delegate_to=self.delegate_to + ) + log_message = 'Executing copy of gs://{0}/{1} to gs://{2}/{3}' + + if self.wildcard in self.source_object: + prefix, delimiter = self.source_object.split(self.wildcard, 1) + objects = hook.list(self.source_bucket, prefix=prefix, + delimiter=delimiter) + + for source_object in objects: + if self.destination_object is None: + destination_object = source_object + else: + destination_object = source_object\ + .replace(prefix, self.destination_object, 1) + self.log.info( + log_message.format(self.source_bucket, source_object, + self.destination_bucket, + destination_object) + ) + + hook.rewrite(self.source_bucket, source_object, + self.destination_bucket, destination_object) + if self.move_object: + hook.delete(self.source_bucket, source_object) + + else: + self.log.info( + log_message.format(self.source_bucket, self.source_object, + self.destination_bucket or + self.source_bucket, + self.destination_object or + self.source_object) + ) + hook.rewrite(self.source_bucket, self.source_object, + self.destination_bucket, self.destination_object) + + if self.move_object: + hook.delete(self.source_bucket, self.source_object) diff --git a/trace/README.rst b/trace/README.rst new file mode 100644 index 00000000000..f0aa701b038 --- /dev/null +++ b/trace/README.rst @@ -0,0 +1,97 @@ +.. This file is automatically generated. Do not edit this file directly. + +Stackdriver Trace Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=trace/README.rst + + +This directory contains samples for Stackdriver Trace. `Stackdriver Trace`_ collects latency data from applications and displays it in near real time in the Google Cloud Platform Console. + + + + +.. _Stackdriver Trace: https://cloud.google.com/trace/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Web Server ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=trace/main.py,trace/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python main.py + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/trace/README.rst.in b/trace/README.rst.in new file mode 100644 index 00000000000..471ffa008eb --- /dev/null +++ b/trace/README.rst.in @@ -0,0 +1,21 @@ +# This file is used to generate README.rst + +product: + name: Stackdriver Trace + short_name: Stackdriver Trace + url: https://cloud.google.com/trace/docs + description: > + `Stackdriver Trace`_ collects latency data from applications and displays + it in near real time in the Google Cloud Platform Console. + +setup: +- auth +- install_deps + +samples: +- name: Web Server + file: main.py + +cloud_client_library: true + +folder: trace diff --git a/trace/main.py b/trace/main.py new file mode 100644 index 00000000000..bdfdb00e497 --- /dev/null +++ b/trace/main.py @@ -0,0 +1,72 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import random +import time + +from flask import Flask, redirect, url_for + + +# [START trace_setup_python_configure] +from opencensus.trace.exporters import stackdriver_exporter +import opencensus.trace.tracer + + +def initialize_tracer(project_id): + exporter = stackdriver_exporter.StackdriverExporter( + project_id=project_id + ) + tracer = opencensus.trace.tracer.Tracer(exporter=exporter) + + return tracer +# [END trace_setup_python_configure] + + +app = Flask(__name__) + + +@app.route('/', methods=['GET']) +def root(): + return redirect(url_for('index')) + + +# [START trace_setup_python_quickstart] +@app.route('/index.html', methods=['GET']) +def index(): + tracer = app.config['TRACER'] + tracer.start_span(name='index') + + # Add up to 1 sec delay, weighted toward zero + time.sleep(random.random() ** 2) + result = "Tracing requests" + + tracer.end_span() + return result +# [END trace_setup_python_quickstart] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + '--project_id', help='Project ID you want to access.', required=True) + args = parser.parse_args() + + tracer = initialize_tracer(args.project_id) + app.config['TRACER'] = tracer + + app.run() diff --git a/trace/main_test.py b/trace/main_test.py new file mode 100644 index 00000000000..ee590fdcfe3 --- /dev/null +++ b/trace/main_test.py @@ -0,0 +1,38 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import main + + +def test_index(): + project_id = os.environ['GCLOUD_PROJECT'] + main.app.testing = True + main.app.config['TRACER'] = main.initialize_tracer(project_id) + client = main.app.test_client() + + resp = client.get('/index.html') + assert resp.status_code == 200 + assert 'Tracing requests' in resp.data.decode('utf-8') + + +def test_redirect(): + project_id = os.environ['GCLOUD_PROJECT'] + main.app.testing = True + main.app.config['TRACER'] = main.initialize_tracer(project_id) + client = main.app.test_client() + + resp = client.get('/') + assert resp.status_code == 302 + assert '/index.html' in resp.headers.get('location', '') diff --git a/trace/requirements.txt b/trace/requirements.txt new file mode 100644 index 00000000000..10ad502458d --- /dev/null +++ b/trace/requirements.txt @@ -0,0 +1,3 @@ +google-cloud-trace==0.20.2 +opencensus==0.2.0 +Flask==1.0.2 diff --git a/translate/automl/automl_translation_dataset.py b/translate/automl/automl_translation_dataset.py new file mode 100755 index 00000000000..cf3e50ae4c1 --- /dev/null +++ b/translate/automl/automl_translation_dataset.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to perform basic operations on dataset +with the Google AutoML Translation API. + +For more information, see the documentation at +https://cloud.google.com/translate/automl/docs +""" + +import argparse +import os + + +def create_dataset(project_id, compute_region, dataset_name, source, target): + """Create a dataset.""" + # [START automl_translate_create_dataset] + # TODO(developer): Uncomment and set the following variables + # project_id = 'PROJECT_ID_HERE' + # compute_region = 'COMPUTE_REGION_HERE' + # dataset_name = 'DATASET_NAME_HERE' + # source = 'LANGUAGE_CODE_OF_SOURCE_LANGUAGE' + # target = 'LANGUAGE_CODE_OF_TARGET_LANGUAGE' + + from google.cloud import automl_v1beta1 as automl + + client = automl.AutoMlClient() + + # A resource that represents Google Cloud Platform location. + project_location = client.location_path(project_id, compute_region) + + # Specify the source and target language. + dataset_metadata = { + "source_language_code": source, + "target_language_code": target, + } + # Set dataset name and dataset metadata + my_dataset = { + "display_name": dataset_name, + "translation_dataset_metadata": dataset_metadata, + } + + # Create a dataset with the dataset metadata in the region. + dataset = client.create_dataset(project_location, my_dataset) + + # Display the dataset information + print("Dataset name: {}".format(dataset.name)) + print("Dataset id: {}".format(dataset.name.split("/")[-1])) + print("Dataset display name: {}".format(dataset.display_name)) + print("Translation dataset Metadata:") + print( + "\tsource_language_code: {}".format( + dataset.translation_dataset_metadata.source_language_code + ) + ) + print( + "\ttarget_language_code: {}".format( + dataset.translation_dataset_metadata.target_language_code + ) + ) + print("Dataset create time:") + print("\tseconds: {}".format(dataset.create_time.seconds)) + print("\tnanos: {}".format(dataset.create_time.nanos)) + + # [END automl_translate_create_dataset] + + +def list_datasets(project_id, compute_region, filter_): + """List Datasets.""" + # [START automl_translate_list_datasets] + # TODO(developer): Uncomment and set the following variables + # project_id = 'PROJECT_ID_HERE' + # compute_region = 'COMPUTE_REGION_HERE' + # filter_ = 'filter expression here' + + from google.cloud import automl_v1beta1 as automl + + client = automl.AutoMlClient() + + # A resource that represents Google Cloud Platform location. + project_location = client.location_path(project_id, compute_region) + + # List all the datasets available in the region by applying filter. + response = client.list_datasets(project_location, filter_) + + print("List of datasets:") + for dataset in response: + # Display the dataset information + print("Dataset name: {}".format(dataset.name)) + print("Dataset id: {}".format(dataset.name.split("/")[-1])) + print("Dataset display name: {}".format(dataset.display_name)) + print("Translation dataset metadata:") + print( + "\tsource_language_code: {}".format( + dataset.translation_dataset_metadata.source_language_code + ) + ) + print( + "\ttarget_language_code: {}".format( + dataset.translation_dataset_metadata.target_language_code + ) + ) + print("Dataset create time:") + print("\tseconds: {}".format(dataset.create_time.seconds)) + print("\tnanos: {}".format(dataset.create_time.nanos)) + + # [END automl_translate_list_datasets] + + +def get_dataset(project_id, compute_region, dataset_id): + """Get the dataset.""" + # [START automl_translate_get_dataset] + # TODO(developer): Uncomment and set the following variables + # project_id = 'PROJECT_ID_HERE' + # compute_region = 'COMPUTE_REGION_HERE' + # dataset_id = 'DATASET_ID_HERE' + + from google.cloud import automl_v1beta1 as automl + + client = automl.AutoMlClient() + + # Get the full path of the dataset + dataset_full_id = client.dataset_path( + project_id, compute_region, dataset_id + ) + + # Get complete detail of the dataset. + dataset = client.get_dataset(dataset_full_id) + + # Display the dataset information + print("Dataset name: {}".format(dataset.name)) + print("Dataset id: {}".format(dataset.name.split("/")[-1])) + print("Dataset display name: {}".format(dataset.display_name)) + print("Translation dataset metadata:") + print( + "\tsource_language_code: {}".format( + dataset.translation_dataset_metadata.source_language_code + ) + ) + print( + "\ttarget_language_code: {}".format( + dataset.translation_dataset_metadata.target_language_code + ) + ) + print("Dataset create time:") + print("\tseconds: {}".format(dataset.create_time.seconds)) + print("\tnanos: {}".format(dataset.create_time.nanos)) + + # [END automl_translate_get_dataset] + + +def import_data(project_id, compute_region, dataset_id, path): + """Import sentence pairs to the dataset.""" + # [START automl_translate_import_data] + # TODO(developer): Uncomment and set the following variables + # project_id = 'PROJECT_ID_HERE' + # compute_region = 'COMPUTE_REGION_HERE' + # dataset_id = 'DATASET_ID_HERE' + # path = 'gs://path/to/file.csv' + + from google.cloud import automl_v1beta1 as automl + + client = automl.AutoMlClient() + + # Get the full path of the dataset. + dataset_full_id = client.dataset_path( + project_id, compute_region, dataset_id + ) + + # Get the multiple Google Cloud Storage URIs + input_uris = path.split(",") + input_config = {"gcs_source": {"input_uris": input_uris}} + + # Import data from the input URI + response = client.import_data(dataset_full_id, input_config) + + print("Processing import...") + # synchronous check of operation status + print("Data imported. {}".format(response.result())) + + # [END automl_translate_import_data] + + +def delete_dataset(project_id, compute_region, dataset_id): + """Delete a dataset.""" + # [START automl_translate_delete_dataset] + # TODO(developer): Uncomment and set the following variables + # project_id = 'PROJECT_ID_HERE' + # compute_region = 'COMPUTE_REGION_HERE' + # dataset_id = 'DATASET_ID_HERE' + + from google.cloud import automl_v1beta1 as automl + + client = automl.AutoMlClient() + + # Get the full path of the dataset. + dataset_full_id = client.dataset_path( + project_id, compute_region, dataset_id + ) + + # Delete a dataset. + response = client.delete_dataset(dataset_full_id) + + # synchronous check of operation status + print("Dataset deleted. {}".format(response.result())) + + # [END automl_translate_delete_dataset] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + subparsers = parser.add_subparsers(dest="command") + + create_dataset_parser = subparsers.add_parser( + "create_dataset", help=create_dataset.__doc__ + ) + create_dataset_parser.add_argument("dataset_name") + create_dataset_parser.add_argument("source") + create_dataset_parser.add_argument("target") + + list_datasets_parser = subparsers.add_parser( + "list_datasets", help=list_datasets.__doc__ + ) + list_datasets_parser.add_argument("filter", nargs="?", default="") + + import_data_parser = subparsers.add_parser( + "import_data", help=import_data.__doc__ + ) + import_data_parser.add_argument("dataset_id") + import_data_parser.add_argument("path") + + delete_dataset_parser = subparsers.add_parser( + "delete_dataset", help=delete_dataset.__doc__ + ) + delete_dataset_parser.add_argument("dataset_id") + + get_dataset_parser = subparsers.add_parser( + "get_dataset", help=get_dataset.__doc__ + ) + get_dataset_parser.add_argument("dataset_id") + + project_id = os.environ["PROJECT_ID"] + compute_region = os.environ["REGION_NAME"] + + args = parser.parse_args() + + if args.command == "create_dataset": + create_dataset( + project_id, + compute_region, + args.dataset_name, + args.source, + args.target, + ) + if args.command == "list_datasets": + list_datasets(project_id, compute_region, args.filter) + if args.command == "get_dataset": + get_dataset(project_id, compute_region, args.dataset_id) + if args.command == "import_data": + import_data(project_id, compute_region, args.dataset_id, args.path) + if args.command == "delete_dataset": + delete_dataset(project_id, compute_region, args.dataset_id) diff --git a/translate/automl/automl_translation_model.py b/translate/automl/automl_translation_model.py new file mode 100755 index 00000000000..77a4ed73ca9 --- /dev/null +++ b/translate/automl/automl_translation_model.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to perform basic operations on model +with the Google AutoML Translation API. + +For more information, see the documentation at +https://cloud.google.com/translate/automl/docs +""" + +import argparse +import os + + +def create_model(project_id, compute_region, dataset_id, model_name): + """Create a model.""" + # [START automl_translate_create_model] + # TODO(developer): Uncomment and set the following variables + # project_id = 'PROJECT_ID_HERE' + # compute_region = 'COMPUTE_REGION_HERE' + # dataset_id = 'DATASET_ID_HERE' + # model_name = 'MODEL_NAME_HERE' + + from google.cloud import automl_v1beta1 as automl + + client = automl.AutoMlClient() + + # A resource that represents Google Cloud Platform location. + project_location = client.location_path(project_id, compute_region) + + # Set model name and dataset. + my_model = { + "display_name": model_name, + "dataset_id": dataset_id, + "translation_model_metadata": {"base_model": ""}, + } + + # Create a model with the model metadata in the region. + response = client.create_model(project_location, my_model) + + print("Training operation name: {}".format(response.operation.name)) + print("Training started...") + + # [END automl_translate_create_model] + + +def list_models(project_id, compute_region, filter_): + """List all models.""" + # [START automl_translate_list_models] + # TODO(developer): Uncomment and set the following variables + # project_id = 'PROJECT_ID_HERE' + # compute_region = 'COMPUTE_REGION_HERE' + # filter_ = 'DATASET_ID_HERE' + + from google.cloud import automl_v1beta1 as automl + from google.cloud.automl_v1beta1 import enums + + client = automl.AutoMlClient() + + # A resource that represents Google Cloud Platform location. + project_location = client.location_path(project_id, compute_region) + + # List all the models available in the region by applying filter. + response = client.list_models(project_location, filter_) + + print("List of models:") + for model in response: + # Display the model information. + if model.deployment_state == enums.Model.DeploymentState.DEPLOYED: + deployment_state = "deployed" + else: + deployment_state = "undeployed" + + print("Model name: {}".format(model.name)) + print("Model id: {}".format(model.name.split("/")[-1])) + print("Model display name: {}".format(model.display_name)) + print("Model create time:") + print("\tseconds: {}".format(model.create_time.seconds)) + print("\tnanos: {}".format(model.create_time.nanos)) + print("Model deployment state: {}".format(deployment_state)) + + # [END automl_translate_list_models] + + +def get_model(project_id, compute_region, model_id): + """Get model details.""" + # [START automl_translate_get_model] + # TODO(developer): Uncomment and set the following variables + # project_id = 'PROJECT_ID_HERE' + # compute_region = 'COMPUTE_REGION_HERE' + # model_id = 'MODEL_ID_HERE' + + from google.cloud import automl_v1beta1 as automl + from google.cloud.automl_v1beta1 import enums + + client = automl.AutoMlClient() + + # Get the full path of the model. + model_full_id = client.model_path(project_id, compute_region, model_id) + + # Get complete detail of the model. + model = client.get_model(model_full_id) + + # Retrieve deployment state. + if model.deployment_state == enums.Model.DeploymentState.DEPLOYED: + deployment_state = "deployed" + else: + deployment_state = "undeployed" + + # Display the model information. + print("Model name: {}".format(model.name)) + print("Model id: {}".format(model.name.split("/")[-1])) + print("Model display name: {}".format(model.display_name)) + print("Model create time:") + print("\tseconds: {}".format(model.create_time.seconds)) + print("\tnanos: {}".format(model.create_time.nanos)) + print("Model deployment state: {}".format(deployment_state)) + + # [END automl_translate_get_model] + + +def list_model_evaluations(project_id, compute_region, model_id, filter_): + """List model evaluations.""" + # [START automl_translate_list_model_evaluations] + # TODO(developer): Uncomment and set the following variables + # project_id = 'PROJECT_ID_HERE' + # compute_region = 'COMPUTE_REGION_HERE' + # model_id = 'MODEL_ID_HERE' + # filter_ = 'filter expression here' + + from google.cloud import automl_v1beta1 as automl + + client = automl.AutoMlClient() + + # Get the full path of the model. + model_full_id = client.model_path(project_id, compute_region, model_id) + + print("List of model evaluations:") + for element in client.list_model_evaluations(model_full_id, filter_): + print(element) + + # [END automl_translate_list_model_evaluations] + + +def get_model_evaluation( + project_id, compute_region, model_id, model_evaluation_id +): + """Get model evaluation.""" + # [START automl_translate_get_model_evaluation] + # TODO(developer): Uncomment and set the following variables + # project_id = 'PROJECT_ID_HERE' + # compute_region = 'COMPUTE_REGION_HERE' + # model_id = 'MODEL_ID_HERE' + # model_evaluation_id = 'MODEL_EVALUATION_ID_HERE' + + from google.cloud import automl_v1beta1 as automl + + client = automl.AutoMlClient() + + # Get the full path of the model evaluation. + model_evaluation_full_id = client.model_evaluation_path( + project_id, compute_region, model_id, model_evaluation_id + ) + + # Get complete detail of the model evaluation. + response = client.get_model_evaluation(model_evaluation_full_id) + + print(response) + + # [END automl_translate_get_model_evaluation] + + +def delete_model(project_id, compute_region, model_id): + """Delete a model.""" + # [START automl_translate_delete_model] + # TODO(developer): Uncomment and set the following variables + # project_id = 'PROJECT_ID_HERE' + # compute_region = 'COMPUTE_REGION_HERE' + # model_id = 'MODEL_ID_HERE' + + from google.cloud import automl_v1beta1 as automl + + client = automl.AutoMlClient() + + # Get the full path of the model. + model_full_id = client.model_path(project_id, compute_region, model_id) + + # Delete a model. + response = client.delete_model(model_full_id) + + # synchronous check of operation status. + print("Model deleted. {}".format(response.result())) + + # [END automl_translate_delete_model] + + +def get_operation_status(operation_full_id): + """Get operation status.""" + # [START automl_translate_get_operation_status] + # TODO(developer): Uncomment and set the following variables + # operation_full_id = + # 'projects//locations//operations/' + + from google.cloud import automl_v1beta1 as automl + + client = automl.AutoMlClient() + + # Get the latest state of a long-running operation. + response = client.transport._operations_client.get_operation( + operation_full_id + ) + + print("Operation status: {}".format(response)) + + # [END automl_translate_get_operation_status] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + subparsers = parser.add_subparsers(dest="command") + + create_model_parser = subparsers.add_parser( + "create_model", help=create_model.__doc__ + ) + create_model_parser.add_argument("dataset_id") + create_model_parser.add_argument("model_name") + + list_model_evaluations_parser = subparsers.add_parser( + "list_model_evaluations", help=list_model_evaluations.__doc__ + ) + list_model_evaluations_parser.add_argument("model_id") + list_model_evaluations_parser.add_argument("filter", nargs="?", default="") + + get_model_evaluation_parser = subparsers.add_parser( + "get_model_evaluation", help=get_model_evaluation.__doc__ + ) + get_model_evaluation_parser.add_argument("model_id") + get_model_evaluation_parser.add_argument("model_evaluation_id") + + get_model_parser = subparsers.add_parser( + "get_model", help=get_model.__doc__ + ) + get_model_parser.add_argument("model_id") + + get_operation_status_parser = subparsers.add_parser( + "get_operation_status", help=get_operation_status.__doc__ + ) + get_operation_status_parser.add_argument("operation_full_id") + + list_models_parser = subparsers.add_parser( + "list_models", help=list_models.__doc__ + ) + list_models_parser.add_argument("filter", nargs="?", default="") + + delete_model_parser = subparsers.add_parser( + "delete_model", help=delete_model.__doc__ + ) + delete_model_parser.add_argument("model_id") + + project_id = os.environ["PROJECT_ID"] + compute_region = os.environ["REGION_NAME"] + + args = parser.parse_args() + + if args.command == "create_model": + create_model( + project_id, compute_region, args.dataset_id, args.model_name + ) + if args.command == "list_models": + list_models(project_id, compute_region, args.filter) + if args.command == "get_model": + get_model(project_id, compute_region, args.model_id) + if args.command == "list_model_evaluations": + list_model_evaluations( + project_id, compute_region, args.model_id, args.filter + ) + if args.command == "get_model_evaluation": + get_model_evaluation( + project_id, compute_region, args.model_id, args.model_evaluation_id + ) + if args.command == "delete_model": + delete_model(project_id, compute_region, args.model_id) + if args.command == "get_operation_status": + get_operation_status(args.operation_full_id) diff --git a/translate/automl/automl_translation_predict.py b/translate/automl/automl_translation_predict.py new file mode 100644 index 00000000000..b15e0e3096f --- /dev/null +++ b/translate/automl/automl_translation_predict.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to perform basic operations on prediction +with the Google AutoML Translation API. + +For more information, see the documentation at +https://cloud.google.com/translate/automl/docs +""" + +import argparse +import os + + +def predict(project_id, compute_region, model_id, file_path): + """Translate the content.""" + # [START automl_translate_predict] + # project_id = 'PROJECT_ID_HERE' + # compute_region = 'COMPUTE_REGION_HERE' + # model_id = 'MODEL_ID_HERE' + # file_path = '/local/path/to/file' + + from google.cloud import automl_v1beta1 as automl + + automl_client = automl.AutoMlClient() + + # Create client for prediction service. + prediction_client = automl.PredictionServiceClient() + + # Get the full path of the model. + model_full_id = automl_client.model_path( + project_id, compute_region, model_id + ) + + # Read the file content for translation. + with open(file_path, "rb") as content_file: + content = content_file.read() + content.decode("utf-8") + + # Set the payload by giving the content of the file. + payload = {"text_snippet": {"content": content}} + + # params is additional domain-specific parameters. + params = {} + + response = prediction_client.predict(model_full_id, payload, params) + translated_content = response.payload[0].translation.translated_content + + print(u"Translated content: {}".format(translated_content.content)) + + # [END automl_translate_predict] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + subparsers = parser.add_subparsers(dest="command") + + predict_parser = subparsers.add_parser("predict", help=predict.__doc__) + predict_parser.add_argument("model_id") + predict_parser.add_argument("file_path") + + project_id = os.environ["PROJECT_ID"] + compute_region = os.environ["REGION_NAME"] + + args = parser.parse_args() + + if args.command == "predict": + predict(project_id, compute_region, args.model_id, args.file_path) diff --git a/translate/automl/dataset_test.py b/translate/automl/dataset_test.py new file mode 100644 index 00000000000..29e3e5c9fe9 --- /dev/null +++ b/translate/automl/dataset_test.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import os + +import pytest + +import automl_translation_dataset + +project_id = os.environ["GCLOUD_PROJECT"] +compute_region = "us-central1" + + +@pytest.mark.slow +def test_dataset_create_import_delete(capsys): + # create dataset + dataset_name = "test_" + datetime.datetime.now().strftime("%Y%m%d%H%M%S") + automl_translation_dataset.create_dataset( + project_id, compute_region, dataset_name, "en", "ja" + ) + out, _ = capsys.readouterr() + create_dataset_output = out.splitlines() + assert "Dataset id: " in create_dataset_output[1] + + # import data + dataset_id = create_dataset_output[1].split()[2] + data = "gs://{}-vcm/en-ja.csv".format(project_id) + automl_translation_dataset.import_data( + project_id, compute_region, dataset_id, data + ) + out, _ = capsys.readouterr() + assert "Data imported." in out + + # delete dataset + automl_translation_dataset.delete_dataset( + project_id, compute_region, dataset_id + ) + out, _ = capsys.readouterr() + assert "Dataset deleted." in out + + +def test_dataset_list_get(capsys): + # list datasets + automl_translation_dataset.list_datasets(project_id, compute_region, "") + out, _ = capsys.readouterr() + list_dataset_output = out.splitlines() + assert "Dataset id: " in list_dataset_output[2] + + # get dataset + dataset_id = list_dataset_output[2].split()[2] + automl_translation_dataset.get_dataset( + project_id, compute_region, dataset_id + ) + out, _ = capsys.readouterr() + assert "Dataset name: " in out diff --git a/translate/automl/model_test.py b/translate/automl/model_test.py new file mode 100644 index 00000000000..0d37a85c674 --- /dev/null +++ b/translate/automl/model_test.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import os + +from google.cloud import automl_v1beta1 as automl +import pytest + +import automl_translation_model + +project_id = os.environ["GCLOUD_PROJECT"] +compute_region = "us-central1" + + +@pytest.mark.skip(reason="creates too many models") +def test_model_create_status_delete(capsys): + # create model + client = automl.AutoMlClient() + model_name = "test_" + datetime.datetime.now().strftime("%Y%m%d%H%M%S") + project_location = client.location_path(project_id, compute_region) + my_model = { + "display_name": model_name, + "dataset_id": "3876092572857648864", + "translation_model_metadata": {"base_model": ""}, + } + response = client.create_model(project_location, my_model) + operation_name = response.operation.name + assert operation_name + + # get operation status + automl_translation_model.get_operation_status(operation_name) + out, _ = capsys.readouterr() + assert "Operation status: " in out + + # cancel operation + response.cancel() + + +def test_model_list_get_evaluate(capsys): + # list models + automl_translation_model.list_models(project_id, compute_region, "") + out, _ = capsys.readouterr() + list_models_output = out.splitlines() + assert "Model id: " in list_models_output[2] + + # get model + model_id = list_models_output[2].split()[2] + automl_translation_model.get_model(project_id, compute_region, model_id) + out, _ = capsys.readouterr() + assert "Model name: " in out + + # list model evaluations + automl_translation_model.list_model_evaluations( + project_id, compute_region, model_id, "" + ) + out, _ = capsys.readouterr() + list_evals_output = out.splitlines() + assert "name: " in list_evals_output[1] + + # get model evaluation + model_evaluation_id = list_evals_output[1].split("/")[-1][:-1] + automl_translation_model.get_model_evaluation( + project_id, compute_region, model_id, model_evaluation_id + ) + out, _ = capsys.readouterr() + assert "evaluation_metric" in out diff --git a/translate/automl/predict_test.py b/translate/automl/predict_test.py new file mode 100644 index 00000000000..f9d98dfb053 --- /dev/null +++ b/translate/automl/predict_test.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import automl_translation_predict + +project_id = os.environ["GCLOUD_PROJECT"] +compute_region = "us-central1" + + +def test_predict(capsys): + model_id = "TRL3128559826197068699" + automl_translation_predict.predict( + project_id, compute_region, model_id, "resources/input.txt" + ) + out, _ = capsys.readouterr() + assert "Translated content: " in out diff --git a/translate/automl/requirements.txt b/translate/automl/requirements.txt new file mode 100644 index 00000000000..ebc8794cf08 --- /dev/null +++ b/translate/automl/requirements.txt @@ -0,0 +1 @@ +google-cloud-automl==0.2.0 diff --git a/translate/automl/resources/input.txt b/translate/automl/resources/input.txt new file mode 100644 index 00000000000..5aecd6590fc --- /dev/null +++ b/translate/automl/resources/input.txt @@ -0,0 +1 @@ +Tell me how this ends \ No newline at end of file diff --git a/translate/cloud-client/README.rst b/translate/cloud-client/README.rst new file mode 100644 index 00000000000..f5065ed4be2 --- /dev/null +++ b/translate/cloud-client/README.rst @@ -0,0 +1,178 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Translation API Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=translate/cloud-client/README.rst + + +This directory contains samples for Google Translation API. With `Google Translation API`, you can dynamically translate text between thousands of language pairs. + + + + +.. _Google Translation API: https://cloud.google.com/translate/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Quickstart ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=translate/cloud-client/quickstart.py,translate/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python quickstart.py + + +Snippets ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=translate/cloud-client/snippets.py,translate/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python snippets.py + + usage: snippets.py [-h] + {detect-language,list-languages,list-languages-with-target,translate-text} + ... + + This application demonstrates how to perform basic operations with the + Google Cloud Translate API + + For more information, the documentation at + https://cloud.google.com/translate/docs. + + positional arguments: + {detect-language,list-languages,list-languages-with-target,translate-text} + detect-language Detects the text's language. + list-languages Lists all available languages. + list-languages-with-target + Lists all available languages and localizes them to + the target language. Target must be an ISO 639-1 + language code. See https://g.co/cloud/translate/v2 + /translate-reference#supported_languages + translate-text Translates text into the target language. Target must + be an ISO 639-1 language code. See + https://g.co/cloud/translate/v2/translate- + reference#supported_languages + + optional arguments: + -h, --help show this help message and exit + + + +Beta Snippets ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=translate/cloud-client/beta_snippets.py,translate/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python beta_snippets.py + + usage: beta_snippets.py [-h] + {translate-text,batch-translate-text,detect-language,list-languages,list-languages-with-target,create-glossary,get-glossary,list-glossaries,delete-glossary,translate-with-glossary} + ... + + positional arguments: + {translate-text,batch-translate-text,detect-language,list-languages,list-languages-with-target,create-glossary,get-glossary,list-glossaries,delete-glossary,translate-with-glossary} + translate-text + batch-translate-text + detect-language + list-languages + list-languages-with-target + create-glossary + get-glossary + list-glossaries + delete-glossary + translate-with-glossary + + optional arguments: + -h, --help show this help message and exit + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/translate/cloud-client/README.rst.in b/translate/cloud-client/README.rst.in new file mode 100644 index 00000000000..ba804e74de3 --- /dev/null +++ b/translate/cloud-client/README.rst.in @@ -0,0 +1,27 @@ +# This file is used to generate README.rst + +product: + name: Google Translation API + short_name: Translation API + url: https://cloud.google.com/translate/docs + description: > + With `Google Translation API`, you can dynamically translate text between + thousands of language pairs. + +setup: +- auth +- install_deps + +samples: +- name: Quickstart + file: quickstart.py +- name: Snippets + file: snippets.py + show_help: true +- name: Beta Snippets + file: beta_snippets.py + show_help: true + +cloud_client_library: true + +folder: translate/cloud-client \ No newline at end of file diff --git a/translate/cloud-client/beta_snippets.py b/translate/cloud-client/beta_snippets.py new file mode 100644 index 00000000000..7cd94aed59a --- /dev/null +++ b/translate/cloud-client/beta_snippets.py @@ -0,0 +1,357 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + + +def translate_text(project_id, text): + # [START translate_translate_text_beta] + from google.cloud import translate_v3beta1 as translate + client = translate.TranslationServiceClient() + + # project_id = YOUR_PROJECT_ID + # text = 'Text you wish to translate' + location = 'global' + + parent = client.location_path(project_id, location) + + response = client.translate_text( + parent=parent, + contents=[text], + mime_type='text/plain', # mime types: text/plain, text/html + source_language_code='en-US', + target_language_code='sr-Latn') + + for translation in response.translations: + print('Translated Text: {}'.format(translation)) + # [END translate_translate_text_beta] + + +def batch_translate_text(project_id, input_uri, output_uri): + # [START translate_batch_translate_text_beta] + from google.cloud import translate_v3beta1 as translate + client = translate.TranslationServiceClient() + + # project_id = YOUR_PROJECT_ID + # input_uri = 'gs://cloud-samples-data/translation/text.txt' + # output_uri = 'gs://YOUR_BUCKET_ID/path_to_store_results/' + location = 'us-central1' + + parent = client.location_path(project_id, location) + + gcs_source = translate.types.GcsSource(input_uri=input_uri) + + input_config = translate.types.InputConfig( + mime_type='text/plain', # mime types: text/plain, text/html + gcs_source=gcs_source) + + gcs_destination = translate.types.GcsDestination( + output_uri_prefix=output_uri) + + output_config = translate.types.OutputConfig( + gcs_destination=gcs_destination) + + operation = client.batch_translate_text( + parent=parent, + source_language_code='en-US', + target_language_codes=['sr-Latn'], + input_configs=[input_config], + output_config=output_config) + + result = operation.result(90) + + print('Total Characters: {}'.format(result.total_characters)) + print('Translated Characters: {}'.format(result.translated_characters)) + # [END translate_batch_translate_text_beta] + + +def detect_language(project_id, text): + # [START translate_detect_language_beta] + from google.cloud import translate_v3beta1 as translate + client = translate.TranslationServiceClient() + + # project_id = YOUR_PROJECT_ID + # text = 'Text you wish to translate' + location = 'global' + + parent = client.location_path(project_id, location) + + response = client.detect_language( + parent=parent, + content=text, + mime_type='text/plain') # mime types: text/plain, text/html + + for language in response.languages: + print('Language Code: {} (Confidence: {})'.format( + language.language_code, + language.confidence)) + # [END translate_detect_language_beta] + + +def list_languages(project_id): + # [START translate_list_codes_beta] + from google.cloud import translate_v3beta1 as translate + client = translate.TranslationServiceClient() + + # project_id = YOUR_PROJECT_ID + location = 'global' + + parent = client.location_path(project_id, location) + + response = client.get_supported_languages(parent) + + print('Supported Languages:') + for language in response.languages: + print('Language Code: {}'.format(language.language_code)) + # [END translate_list_codes_beta] + + +def list_languages_with_target(project_id, display_language_code): + # [START translate_list_language_names_beta] + from google.cloud import translate_v3beta1 as translate + client = translate.TranslationServiceClient() + + # project_id = YOUR_PROJECT_ID + # display_language_code = 'is' + location = 'global' + + parent = client.location_path(project_id, location) + + response = client.get_supported_languages( + parent=parent, + display_language_code=display_language_code) + + print('Supported Languages:') + for language in response.languages: + print('Language Code: {}'.format(language.language_code)) + print('Display Name: {}\n'.format(language.display_name)) + # [END translate_list_language_names_beta] + + +def create_glossary(project_id, glossary_id): + # [START translate_create_glossary_beta] + from google.cloud import translate_v3beta1 as translate + client = translate.TranslationServiceClient() + + # project_id = 'YOUR_PROJECT_ID' + # glossary_id = 'glossary-id' + location = 'us-central1' # The location of the glossary + + name = client.glossary_path( + project_id, + location, + glossary_id) + + language_codes_set = translate.types.Glossary.LanguageCodesSet( + language_codes=['en', 'es']) + + gcs_source = translate.types.GcsSource( + input_uri='gs://cloud-samples-data/translation/glossary.csv') + + input_config = translate.types.GlossaryInputConfig( + gcs_source=gcs_source) + + glossary = translate.types.Glossary( + name=name, + language_codes_set=language_codes_set, + input_config=input_config) + + parent = client.location_path(project_id, location) + + operation = client.create_glossary(parent=parent, glossary=glossary) + + result = operation.result(timeout=90) + print('Created: {}'.format(result.name)) + print('Input Uri: {}'.format(result.input_config.gcs_source.input_uri)) + # [END translate_create_glossary_beta] + + +def list_glossaries(project_id): + # [START translate_list_glossary_beta] + from google.cloud import translate_v3beta1 as translate + client = translate.TranslationServiceClient() + + # project_id = 'YOUR_PROJECT_ID' + location = 'us-central1' # The location of the glossary + + parent = client.location_path(project_id, location) + + for glossary in client.list_glossaries(parent): + print('Name: {}'.format(glossary.name)) + print('Entry count: {}'.format(glossary.entry_count)) + print('Input uri: {}'.format( + glossary.input_config.gcs_source.input_uri)) + for language_code in glossary.language_codes_set.language_codes: + print('Language code: {}'.format(language_code)) + # [END translate_list_glossary_beta] + + +def get_glossary(project_id, glossary_id): + # [START translate_get_glossary_beta] + from google.cloud import translate_v3beta1 as translate + client = translate.TranslationServiceClient() + + # project_id = 'YOUR_PROJECT_ID' + # glossary_id = 'GLOSSARY_ID' + + parent = client.glossary_path( + project_id, + 'us-central1', # The location of the glossary + glossary_id) + + response = client.get_glossary(parent) + print('Name: {}'.format(response.name)) + print('Language Pair:') + print('\tSource Language Code: {}'.format( + response.language_pair.source_language_code)) + print('\tTarget Language Code: {}'.format( + response.language_pair.target_language_code)) + print('Input Uri: {}'.format( + response.input_config.gcs_source.input_uri)) + # [END translate_get_glossary_beta] + + +def delete_glossary(project_id, glossary_id): + # [START translate_delete_glossary_beta] + from google.cloud import translate_v3beta1 as translate + client = translate.TranslationServiceClient() + + # project_id = 'YOUR_PROJECT_ID' + # glossary_id = 'GLOSSARY_ID' + + parent = client.glossary_path( + project_id, + 'us-central1', # The location of the glossary + glossary_id) + + operation = client.delete_glossary(parent) + result = operation.result(timeout=90) + print('Deleted: {}'.format(result.name)) + # [END translate_delete_glossary_beta] + + +def translate_text_with_glossary(project_id, glossary_id, text): + # [START translate_translate_text_with_glossary_beta] + from google.cloud import translate_v3beta1 as translate + client = translate.TranslationServiceClient() + + # project_id = 'YOUR_PROJECT_ID' + # glossary_id = 'GLOSSARY_ID' + # text = 'Text you wish to translate' + location = 'us-central1' # The location of the glossary + + glossary = client.glossary_path( + project_id, + 'us-central1', # The location of the glossary + glossary_id) + + glossary_config = translate.types.TranslateTextGlossaryConfig( + glossary=glossary) + + parent = client.location_path(project_id, location) + + result = client.translate_text( + parent=parent, + contents=[text], + mime_type='text/plain', # mime types: text/plain, text/html + source_language_code='en', + target_language_code='es', + glossary_config=glossary_config) + + for translation in result.translations: + print(translation) + # [END translate_translate_text_with_glossary_beta] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + + subparsers = parser.add_subparsers(dest='command') + + translate_text_parser = subparsers.add_parser( + 'translate-text', help=translate_text.__doc__) + translate_text_parser.add_argument('project_id') + translate_text_parser.add_argument('text') + + batch_translate_text_parser = subparsers.add_parser( + 'batch-translate-text', help=translate_text.__doc__) + batch_translate_text_parser.add_argument('project_id') + batch_translate_text_parser.add_argument('gcs_source') + batch_translate_text_parser.add_argument('gcs_destination') + + detect_langage_parser = subparsers.add_parser( + 'detect-language', help=detect_language.__doc__) + detect_langage_parser.add_argument('project_id') + detect_langage_parser.add_argument('text') + + list_languages_parser = subparsers.add_parser( + 'list-languages', help=list_languages.__doc__) + list_languages_parser.add_argument('project_id') + + list_languages_with_target_parser = subparsers.add_parser( + 'list-languages-with-target', help=list_languages_with_target.__doc__) + list_languages_with_target_parser.add_argument('project_id') + list_languages_with_target_parser.add_argument('display_language_code') + + create_glossary_parser = subparsers.add_parser( + 'create-glossary', help=create_glossary.__doc__) + create_glossary_parser.add_argument('project_id') + create_glossary_parser.add_argument('glossary_id') + + get_glossary_parser = subparsers.add_parser( + 'get-glossary', help=get_glossary.__doc__) + get_glossary_parser.add_argument('project_id') + get_glossary_parser.add_argument('glossary_id') + + list_glossary_parser = subparsers.add_parser( + 'list-glossaries', help=list_glossaries.__doc__) + list_glossary_parser.add_argument('project_id') + + delete_glossary_parser = subparsers.add_parser( + 'delete-glossary', help=delete_glossary.__doc__) + delete_glossary_parser.add_argument('project_id') + delete_glossary_parser.add_argument('glossary_id') + + translate_with_glossary_parser = subparsers.add_parser( + 'translate-with-glossary', help=translate_text_with_glossary.__doc__) + translate_with_glossary_parser.add_argument('project_id') + translate_with_glossary_parser.add_argument('glossary_id') + translate_with_glossary_parser.add_argument('text') + + args = parser.parse_args() + + if args.command == 'translate-text': + translate_text(args.project_id, args.text) + elif args.command == 'batch-translate-text': + batch_translate_text( + args.project_id, args.gcs_source, args.gcs_destination) + elif args.command == 'detect-language': + detect_language(args.project_id, args.text) + elif args.command == 'list-languages': + list_languages(args.project_id) + elif args.command == 'list-languages-with-target': + list_languages_with_target(args.project_id, args.display_language_code) + elif args.command == 'create-glossary': + create_glossary(args.project_id, args.glossary_id) + elif args.command == 'get-glossary': + get_glossary(args.project_id, args.glossary_id) + elif args.command == 'list-glossaries': + list_glossaries(args.project_id) + elif args.command == 'delete-glossary': + delete_glossary(args.project_id, args.glossary_id) + elif args.command == 'translate-with-glossary': + translate_text_with_glossary( + args.project_id, args.glossary_id, args.text) diff --git a/translate/cloud-client/beta_snippets_test.py b/translate/cloud-client/beta_snippets_test.py new file mode 100644 index 00000000000..f7099a27cae --- /dev/null +++ b/translate/cloud-client/beta_snippets_test.py @@ -0,0 +1,134 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pytest +import uuid +import beta_snippets +from google.cloud import storage + +PROJECT_ID = os.environ['GCLOUD_PROJECT'] + + +@pytest.fixture(scope='function') +def bucket(): + """Create a temporary bucket to store annotation output.""" + bucket_name = str(uuid.uuid1()) + storage_client = storage.Client() + bucket = storage_client.create_bucket(bucket_name) + + yield bucket + + bucket.delete(force=True) + + +@pytest.fixture(scope='session') +def glossary(): + """Get the ID of a glossary available to session (do not mutate/delete).""" + glossary_id = 'must-start-with-letters-' + str(uuid.uuid1()) + beta_snippets.create_glossary(PROJECT_ID, glossary_id) + + yield glossary_id + + try: + beta_snippets.delete_glossary(PROJECT_ID, glossary_id) + except Exception: + pass + + +@pytest.fixture(scope='function') +def unique_glossary_id(): + """Get a unique ID. Attempts to delete glossary with this ID after test.""" + glossary_id = 'must-start-with-letters-' + str(uuid.uuid1()) + + yield glossary_id + + try: + beta_snippets.delete_glossary(PROJECT_ID, glossary_id) + except Exception: + pass + + +def test_translate_text(capsys): + beta_snippets.translate_text(PROJECT_ID, 'Hello world') + out, _ = capsys.readouterr() + assert 'Zdravo svet' in out + + +def test_batch_translate_text(capsys, bucket): + beta_snippets.batch_translate_text( + PROJECT_ID, + 'gs://cloud-samples-data/translation/text.txt', + 'gs://{}/translation/BATCH_TRANSLATION_OUTPUT/'.format(bucket.name)) + out, _ = capsys.readouterr() + assert 'Total Characters: 13' in out + assert 'Translated Characters: 13' in out + + +def test_detect_language(capsys): + beta_snippets.detect_language(PROJECT_ID, 'Hæ sæta') + out, _ = capsys.readouterr() + assert 'is' in out + + +def test_list_languages(capsys): + beta_snippets.list_languages(PROJECT_ID) + out, _ = capsys.readouterr() + assert 'zh-CN' in out + + +def test_list_languages_with_target(capsys): + beta_snippets.list_languages_with_target(PROJECT_ID, 'is') + out, _ = capsys.readouterr() + assert u'Language Code: sq' in out + assert u'Display Name: albanska' in out + + +def test_create_glossary(capsys, unique_glossary_id): + beta_snippets.create_glossary(PROJECT_ID, unique_glossary_id) + out, _ = capsys.readouterr() + assert 'Created' in out + assert PROJECT_ID in out + assert unique_glossary_id in out + assert 'gs://cloud-samples-data/translation/glossary.csv' in out + + +def test_get_glossary(capsys, glossary): + beta_snippets.get_glossary(PROJECT_ID, glossary) + out, _ = capsys.readouterr() + assert glossary in out + assert 'gs://cloud-samples-data/translation/glossary.csv' in out + + +def test_list_glossary(capsys, glossary): + beta_snippets.list_glossaries(PROJECT_ID) + out, _ = capsys.readouterr() + assert glossary in out + assert 'gs://cloud-samples-data/translation/glossary.csv' in out + + +def test_translate_text_with_glossary(capsys, glossary): + beta_snippets.translate_text_with_glossary( + PROJECT_ID, glossary, 'directions') + out, _ = capsys.readouterr() + assert 'direcciones' in out + + +def test_delete_glossary(capsys, unique_glossary_id): + beta_snippets.create_glossary(PROJECT_ID, unique_glossary_id) + beta_snippets.delete_glossary(PROJECT_ID, unique_glossary_id) + out, _ = capsys.readouterr() + assert PROJECT_ID in out + assert 'us-central1' in out + assert unique_glossary_id in out diff --git a/translate/cloud-client/quickstart.py b/translate/cloud-client/quickstart.py new file mode 100644 index 00000000000..c84ea892eda --- /dev/null +++ b/translate/cloud-client/quickstart.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def run_quickstart(): + # [START translate_quickstart] + # Imports the Google Cloud client library + from google.cloud import translate + + # Instantiates a client + translate_client = translate.Client() + + # The text to translate + text = u'Hello, world!' + # The target language + target = 'ru' + + # Translates some text into Russian + translation = translate_client.translate( + text, + target_language=target) + + print(u'Text: {}'.format(text)) + print(u'Translation: {}'.format(translation['translatedText'])) + # [END translate_quickstart] + + +if __name__ == '__main__': + run_quickstart() diff --git a/translate/cloud-client/quickstart_test.py b/translate/cloud-client/quickstart_test.py new file mode 100644 index 00000000000..4018daa060f --- /dev/null +++ b/translate/cloud-client/quickstart_test.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import quickstart + + +def test_quickstart(capsys): + quickstart.run_quickstart() + out, _ = capsys.readouterr() + assert u'Привет, мир!' in out diff --git a/translate/cloud-client/requirements.txt b/translate/cloud-client/requirements.txt new file mode 100644 index 00000000000..318e3485aa0 --- /dev/null +++ b/translate/cloud-client/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-translate==1.4.0 +google-cloud-storage==1.14.0 diff --git a/translate/cloud-client/snippets.py b/translate/cloud-client/snippets.py new file mode 100644 index 00000000000..b5719d23212 --- /dev/null +++ b/translate/cloud-client/snippets.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python + +# Copyright 2016 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to perform basic operations with the +Google Cloud Translate API + +For more information, the documentation at +https://cloud.google.com/translate/docs. +""" + +import argparse + +from google.cloud import translate +import six + + +def detect_language(text): + # [START translate_detect_language] + """Detects the text's language.""" + translate_client = translate.Client() + + # Text can also be a sequence of strings, in which case this method + # will return a sequence of results for each text. + result = translate_client.detect_language(text) + + print('Text: {}'.format(text)) + print('Confidence: {}'.format(result['confidence'])) + print('Language: {}'.format(result['language'])) + # [END translate_detect_language] + + +def list_languages(): + # [START translate_list_codes] + """Lists all available languages.""" + translate_client = translate.Client() + + results = translate_client.get_languages() + + for language in results: + print(u'{name} ({language})'.format(**language)) + # [END translate_list_codes] + + +def list_languages_with_target(target): + # [START translate_list_language_names] + """Lists all available languages and localizes them to the target language. + + Target must be an ISO 639-1 language code. + See https://g.co/cloud/translate/v2/translate-reference#supported_languages + """ + translate_client = translate.Client() + + results = translate_client.get_languages(target_language=target) + + for language in results: + print(u'{name} ({language})'.format(**language)) + # [END translate_list_language_names] + + +def translate_text_with_model(target, text, model=translate.NMT): + # [START translate_text_with_model] + """Translates text into the target language. + + Make sure your project is whitelisted. + + Target must be an ISO 639-1 language code. + See https://g.co/cloud/translate/v2/translate-reference#supported_languages + """ + translate_client = translate.Client() + + if isinstance(text, six.binary_type): + text = text.decode('utf-8') + + # Text can also be a sequence of strings, in which case this method + # will return a sequence of results for each text. + result = translate_client.translate( + text, target_language=target, model=model) + + print(u'Text: {}'.format(result['input'])) + print(u'Translation: {}'.format(result['translatedText'])) + print(u'Detected source language: {}'.format( + result['detectedSourceLanguage'])) + # [END translate_text_with_model] + + +def translate_text(target, text): + # [START translate_translate_text] + """Translates text into the target language. + + Target must be an ISO 639-1 language code. + See https://g.co/cloud/translate/v2/translate-reference#supported_languages + """ + translate_client = translate.Client() + + if isinstance(text, six.binary_type): + text = text.decode('utf-8') + + # Text can also be a sequence of strings, in which case this method + # will return a sequence of results for each text. + result = translate_client.translate( + text, target_language=target) + + print(u'Text: {}'.format(result['input'])) + print(u'Translation: {}'.format(result['translatedText'])) + print(u'Detected source language: {}'.format( + result['detectedSourceLanguage'])) + # [END translate_translate_text] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + subparsers = parser.add_subparsers(dest='command') + + detect_langage_parser = subparsers.add_parser( + 'detect-language', help=detect_language.__doc__) + detect_langage_parser.add_argument('text') + + list_languages_parser = subparsers.add_parser( + 'list-languages', help=list_languages.__doc__) + + list_languages_with_target_parser = subparsers.add_parser( + 'list-languages-with-target', help=list_languages_with_target.__doc__) + list_languages_with_target_parser.add_argument('target') + + translate_text_parser = subparsers.add_parser( + 'translate-text', help=translate_text.__doc__) + translate_text_parser.add_argument('target') + translate_text_parser.add_argument('text') + + args = parser.parse_args() + + if args.command == 'detect-language': + detect_language(args.text) + elif args.command == 'list-languages': + list_languages() + elif args.command == 'list-languages-with-target': + list_languages_with_target(args.target) + elif args.command == 'translate-text': + translate_text(args.target, args.text) diff --git a/translate/cloud-client/snippets_test.py b/translate/cloud-client/snippets_test.py new file mode 100644 index 00000000000..5123576698a --- /dev/null +++ b/translate/cloud-client/snippets_test.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import snippets + + +def test_detect_language(capsys): + snippets.detect_language('Hæ sæta') + out, _ = capsys.readouterr() + assert 'is' in out + + +def test_list_languages(capsys): + snippets.list_languages() + out, _ = capsys.readouterr() + assert 'Icelandic (is)' in out + + +def test_list_languages_with_target(capsys): + snippets.list_languages_with_target('is') + out, _ = capsys.readouterr() + assert u'íslenska (is)' in out + + +def test_translate_text(capsys): + snippets.translate_text('is', 'Hello world') + out, _ = capsys.readouterr() + assert u'Halló heimur' in out + + +def test_translate_utf8(capsys): + text = u'나는 파인애플을 좋아한다.' + snippets.translate_text('en', text) + out, _ = capsys.readouterr() + assert u'I like pineapples.' in out diff --git a/video/cloud-client/analyze/README.rst b/video/cloud-client/analyze/README.rst new file mode 100644 index 00000000000..82b8cee3642 --- /dev/null +++ b/video/cloud-client/analyze/README.rst @@ -0,0 +1,160 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Video Intelligence API Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=video/cloud-client/analyze/README.rst + + +This directory contains samples for Google Cloud Video Intelligence API. `Google Cloud Video Intelligence API`_ allows developers to easily integrate feature detection in video. + + + + +.. _Google Cloud Video Intelligence API: https://cloud.google.com/video-intelligence/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +analyze ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=video/cloud-client/analyze/analyze.py,video/cloud-client/analyze/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python analyze.py + + usage: analyze.py [-h] {labels,labels_file,explicit_content,shots} ... + + This application demonstrates label detection, + explicit content, and shot change detection using the Google Cloud API. + + Usage Examples: + + python analyze.py labels gs://cloud-ml-sandbox/video/chicago.mp4 + python analyze.py labels_file resources/cat.mp4 + python analyze.py shots gs://demomaker/gbikes_dinosaur.mp4 + python analyze.py explicit_content gs://demomaker/gbikes_dinosaur.mp4 + + positional arguments: + {labels,labels_file,explicit_content,shots} + labels Detects labels given a GCS path. + labels_file Detect labels given a file path. + explicit_content Detects explicit content from the GCS path to a video. + shots Detects camera shot changes. + + optional arguments: + -h, --help show this help message and exit + + + +beta samples ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=video/cloud-client/analyze/beta_snippets.py,video/cloud-client/analyze/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python beta_snippets.py + + usage: beta_snippets.py [-h] + {transcription,video-text-gcs,video-text,track-objects-gcs,track-objects} + ... + + This application demonstrates speech transcription using the + Google Cloud API. + + Usage Examples: + python beta_snippets.py transcription gs://python-docs-samples-tests/video/googlework_short.mp4 + python beta_snippets.py video-text-gcs gs://python-docs-samples-tests/video/googlework_short.mp4 + python beta_snippets.py track-objects /resources/cat.mp4 + + positional arguments: + {transcription,video-text-gcs,video-text,track-objects-gcs,track-objects} + transcription Transcribe speech from a video stored on GCS. + video-text-gcs Detect text in a video stored on GCS. + video-text Detect text in a local video. + track-objects-gcs Object Tracking. + track-objects Object Tracking. + + optional arguments: + -h, --help show this help message and exit + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/video/cloud-client/analyze/README.rst.in b/video/cloud-client/analyze/README.rst.in new file mode 100644 index 00000000000..a01d163f930 --- /dev/null +++ b/video/cloud-client/analyze/README.rst.in @@ -0,0 +1,25 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Video Intelligence API + short_name: Cloud Video Intelligence API + url: https://cloud.google.com/video-intelligence/docs + description: > + `Google Cloud Video Intelligence API`_ allows developers to easily + integrate feature detection in video. + +setup: +- auth +- install_deps + +samples: +- name: analyze + file: analyze.py + show_help: True +- name: beta samples + file: beta_snippets.py + show_help: True + +cloud_client_library: true + +folder: video/cloud-client/analyze \ No newline at end of file diff --git a/video/cloud-client/analyze/analyze.py b/video/cloud-client/analyze/analyze.py new file mode 100644 index 00000000000..e86193c10e4 --- /dev/null +++ b/video/cloud-client/analyze/analyze.py @@ -0,0 +1,531 @@ +#!/usr/bin/env python + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates label detection, +explicit content, and shot change detection using the Google Cloud API. + +Usage Examples: + + python analyze.py labels gs://cloud-samples-data/video/chicago.mp4 + python analyze.py labels_file resources/cat.mp4 + python analyze.py shots gs://cloud-samples-data/video/gbikes_dinosaur.mp4 + python analyze.py explicit_content \ + gs://cloud-samples-data/video/gbikes_dinosaur.mp4 + python analyze.py text_gcs \ + gs://cloud-samples-data/video/googlework_short.mp4 + python analyze.py text_file resources/googlework_short.mp4 + python analyze.py objects_gcs gs://cloud-samples-data/video/cat.mp4 + python analyze.py objects_file resources/cat.mp4 +""" + +import argparse +import io + +from google.cloud import videointelligence +from google.cloud.videointelligence import enums + + +def analyze_explicit_content(path): + # [START video_analyze_explicit_content] + """ Detects explicit content from the GCS path to a video. """ + video_client = videointelligence.VideoIntelligenceServiceClient() + features = [videointelligence.enums.Feature.EXPLICIT_CONTENT_DETECTION] + + operation = video_client.annotate_video(path, features=features) + print('\nProcessing video for explicit content annotations:') + + result = operation.result(timeout=90) + print('\nFinished processing.') + + # first result is retrieved because a single video was processed + for frame in result.annotation_results[0].explicit_annotation.frames: + likelihood = enums.Likelihood(frame.pornography_likelihood) + frame_time = frame.time_offset.seconds + frame.time_offset.nanos / 1e9 + print('Time: {}s'.format(frame_time)) + print('\tpornography: {}'.format(likelihood.name)) + # [END video_analyze_explicit_content] + + +def analyze_labels(path): + # [START video_analyze_labels_gcs] + """ Detects labels given a GCS path. """ + video_client = videointelligence.VideoIntelligenceServiceClient() + features = [videointelligence.enums.Feature.LABEL_DETECTION] + + mode = videointelligence.enums.LabelDetectionMode.SHOT_AND_FRAME_MODE + config = videointelligence.types.LabelDetectionConfig( + label_detection_mode=mode) + context = videointelligence.types.VideoContext( + label_detection_config=config) + + operation = video_client.annotate_video( + path, features=features, video_context=context) + print('\nProcessing video for label annotations:') + + result = operation.result(timeout=180) + print('\nFinished processing.') + + # Process video/segment level label annotations + segment_labels = result.annotation_results[0].segment_label_annotations + for i, segment_label in enumerate(segment_labels): + print('Video label description: {}'.format( + segment_label.entity.description)) + for category_entity in segment_label.category_entities: + print('\tLabel category description: {}'.format( + category_entity.description)) + + for i, segment in enumerate(segment_label.segments): + start_time = (segment.segment.start_time_offset.seconds + + segment.segment.start_time_offset.nanos / 1e9) + end_time = (segment.segment.end_time_offset.seconds + + segment.segment.end_time_offset.nanos / 1e9) + positions = '{}s to {}s'.format(start_time, end_time) + confidence = segment.confidence + print('\tSegment {}: {}'.format(i, positions)) + print('\tConfidence: {}'.format(confidence)) + print('\n') + + # Process shot level label annotations + shot_labels = result.annotation_results[0].shot_label_annotations + for i, shot_label in enumerate(shot_labels): + print('Shot label description: {}'.format( + shot_label.entity.description)) + for category_entity in shot_label.category_entities: + print('\tLabel category description: {}'.format( + category_entity.description)) + + for i, shot in enumerate(shot_label.segments): + start_time = (shot.segment.start_time_offset.seconds + + shot.segment.start_time_offset.nanos / 1e9) + end_time = (shot.segment.end_time_offset.seconds + + shot.segment.end_time_offset.nanos / 1e9) + positions = '{}s to {}s'.format(start_time, end_time) + confidence = shot.confidence + print('\tSegment {}: {}'.format(i, positions)) + print('\tConfidence: {}'.format(confidence)) + print('\n') + + # Process frame level label annotations + frame_labels = result.annotation_results[0].frame_label_annotations + for i, frame_label in enumerate(frame_labels): + print('Frame label description: {}'.format( + frame_label.entity.description)) + for category_entity in frame_label.category_entities: + print('\tLabel category description: {}'.format( + category_entity.description)) + + # Each frame_label_annotation has many frames, + # here we print information only about the first frame. + frame = frame_label.frames[0] + time_offset = (frame.time_offset.seconds + + frame.time_offset.nanos / 1e9) + print('\tFirst frame time offset: {}s'.format(time_offset)) + print('\tFirst frame confidence: {}'.format(frame.confidence)) + print('\n') + # [END video_analyze_labels_gcs] + + +def analyze_labels_file(path): + # [START video_analyze_labels] + """Detect labels given a file path.""" + video_client = videointelligence.VideoIntelligenceServiceClient() + features = [videointelligence.enums.Feature.LABEL_DETECTION] + + with io.open(path, 'rb') as movie: + input_content = movie.read() + + operation = video_client.annotate_video( + features=features, input_content=input_content) + print('\nProcessing video for label annotations:') + + result = operation.result(timeout=90) + print('\nFinished processing.') + + # Process video/segment level label annotations + segment_labels = result.annotation_results[0].segment_label_annotations + for i, segment_label in enumerate(segment_labels): + print('Video label description: {}'.format( + segment_label.entity.description)) + for category_entity in segment_label.category_entities: + print('\tLabel category description: {}'.format( + category_entity.description)) + + for i, segment in enumerate(segment_label.segments): + start_time = (segment.segment.start_time_offset.seconds + + segment.segment.start_time_offset.nanos / 1e9) + end_time = (segment.segment.end_time_offset.seconds + + segment.segment.end_time_offset.nanos / 1e9) + positions = '{}s to {}s'.format(start_time, end_time) + confidence = segment.confidence + print('\tSegment {}: {}'.format(i, positions)) + print('\tConfidence: {}'.format(confidence)) + print('\n') + + # Process shot level label annotations + shot_labels = result.annotation_results[0].shot_label_annotations + for i, shot_label in enumerate(shot_labels): + print('Shot label description: {}'.format( + shot_label.entity.description)) + for category_entity in shot_label.category_entities: + print('\tLabel category description: {}'.format( + category_entity.description)) + + for i, shot in enumerate(shot_label.segments): + start_time = (shot.segment.start_time_offset.seconds + + shot.segment.start_time_offset.nanos / 1e9) + end_time = (shot.segment.end_time_offset.seconds + + shot.segment.end_time_offset.nanos / 1e9) + positions = '{}s to {}s'.format(start_time, end_time) + confidence = shot.confidence + print('\tSegment {}: {}'.format(i, positions)) + print('\tConfidence: {}'.format(confidence)) + print('\n') + + # Process frame level label annotations + frame_labels = result.annotation_results[0].frame_label_annotations + for i, frame_label in enumerate(frame_labels): + print('Frame label description: {}'.format( + frame_label.entity.description)) + for category_entity in frame_label.category_entities: + print('\tLabel category description: {}'.format( + category_entity.description)) + + # Each frame_label_annotation has many frames, + # here we print information only about the first frame. + frame = frame_label.frames[0] + time_offset = frame.time_offset.seconds + frame.time_offset.nanos / 1e9 + print('\tFirst frame time offset: {}s'.format(time_offset)) + print('\tFirst frame confidence: {}'.format(frame.confidence)) + print('\n') + # [END video_analyze_labels] + + +def analyze_shots(path): + # [START video_analyze_shots] + """ Detects camera shot changes. """ + video_client = videointelligence.VideoIntelligenceServiceClient() + features = [videointelligence.enums.Feature.SHOT_CHANGE_DETECTION] + operation = video_client.annotate_video(path, features=features) + print('\nProcessing video for shot change annotations:') + + result = operation.result(timeout=90) + print('\nFinished processing.') + + # first result is retrieved because a single video was processed + for i, shot in enumerate(result.annotation_results[0].shot_annotations): + start_time = (shot.start_time_offset.seconds + + shot.start_time_offset.nanos / 1e9) + end_time = (shot.end_time_offset.seconds + + shot.end_time_offset.nanos / 1e9) + print('\tShot {}: {} to {}'.format(i, start_time, end_time)) + # [END video_analyze_shots] + + +def speech_transcription(path): + # [START video_speech_transcription_gcs] + """Transcribe speech from a video stored on GCS.""" + from google.cloud import videointelligence + + video_client = videointelligence.VideoIntelligenceServiceClient() + features = [videointelligence.enums.Feature.SPEECH_TRANSCRIPTION] + + config = videointelligence.types.SpeechTranscriptionConfig( + language_code='en-US', + enable_automatic_punctuation=True) + video_context = videointelligence.types.VideoContext( + speech_transcription_config=config) + + operation = video_client.annotate_video( + path, features=features, + video_context=video_context) + + print('\nProcessing video for speech transcription.') + + result = operation.result(timeout=600) + + # There is only one annotation_result since only + # one video is processed. + annotation_results = result.annotation_results[0] + for speech_transcription in annotation_results.speech_transcriptions: + + # The number of alternatives for each transcription is limited by + # SpeechTranscriptionConfig.max_alternatives. + # Each alternative is a different possible transcription + # and has its own confidence score. + for alternative in speech_transcription.alternatives: + print('Alternative level information:') + + print('Transcript: {}'.format(alternative.transcript)) + print('Confidence: {}\n'.format(alternative.confidence)) + + print('Word level information:') + for word_info in alternative.words: + word = word_info.word + start_time = word_info.start_time + end_time = word_info.end_time + print('\t{}s - {}s: {}'.format( + start_time.seconds + start_time.nanos * 1e-9, + end_time.seconds + end_time.nanos * 1e-9, + word)) + # [END video_speech_transcription_gcs] + + +def video_detect_text_gcs(input_uri): + # [START video_detect_text_gcs] + """Detect text in a video stored on GCS.""" + from google.cloud import videointelligence + + video_client = videointelligence.VideoIntelligenceServiceClient() + features = [videointelligence.enums.Feature.TEXT_DETECTION] + + operation = video_client.annotate_video( + input_uri=input_uri, + features=features) + + print('\nProcessing video for text detection.') + result = operation.result(timeout=300) + + # The first result is retrieved because a single video was processed. + annotation_result = result.annotation_results[0] + + for text_annotation in annotation_result.text_annotations: + print('\nText: {}'.format(text_annotation.text)) + + # Get the first text segment + text_segment = text_annotation.segments[0] + start_time = text_segment.segment.start_time_offset + end_time = text_segment.segment.end_time_offset + print('start_time: {}, end_time: {}'.format( + start_time.seconds + start_time.nanos * 1e-9, + end_time.seconds + end_time.nanos * 1e-9)) + + print('Confidence: {}'.format(text_segment.confidence)) + + # Show the result for the first frame in this segment. + frame = text_segment.frames[0] + time_offset = frame.time_offset + print('Time offset for the first frame: {}'.format( + time_offset.seconds + time_offset.nanos * 1e-9)) + print('Rotated Bounding Box Vertices:') + for vertex in frame.rotated_bounding_box.vertices: + print('\tVertex.x: {}, Vertex.y: {}'.format(vertex.x, vertex.y)) + # [END video_detect_text_gcs] + + +def video_detect_text(path): + # [START video_detect_text] + """Detect text in a local video.""" + from google.cloud import videointelligence + + video_client = videointelligence.VideoIntelligenceServiceClient() + features = [videointelligence.enums.Feature.TEXT_DETECTION] + video_context = videointelligence.types.VideoContext() + + with io.open(path, 'rb') as file: + input_content = file.read() + + operation = video_client.annotate_video( + input_content=input_content, # the bytes of the video file + features=features, + video_context=video_context) + + print('\nProcessing video for text detection.') + result = operation.result(timeout=300) + + # The first result is retrieved because a single video was processed. + annotation_result = result.annotation_results[0] + + for text_annotation in annotation_result.text_annotations: + print('\nText: {}'.format(text_annotation.text)) + + # Get the first text segment + text_segment = text_annotation.segments[0] + start_time = text_segment.segment.start_time_offset + end_time = text_segment.segment.end_time_offset + print('start_time: {}, end_time: {}'.format( + start_time.seconds + start_time.nanos * 1e-9, + end_time.seconds + end_time.nanos * 1e-9)) + + print('Confidence: {}'.format(text_segment.confidence)) + + # Show the result for the first frame in this segment. + frame = text_segment.frames[0] + time_offset = frame.time_offset + print('Time offset for the first frame: {}'.format( + time_offset.seconds + time_offset.nanos * 1e-9)) + print('Rotated Bounding Box Vertices:') + for vertex in frame.rotated_bounding_box.vertices: + print('\tVertex.x: {}, Vertex.y: {}'.format(vertex.x, vertex.y)) + # [END video_detect_text] + + +def track_objects_gcs(gcs_uri): + # [START video_object_tracking_gcs] + """Object tracking in a video stored on GCS.""" + from google.cloud import videointelligence + + video_client = videointelligence.VideoIntelligenceServiceClient() + features = [videointelligence.enums.Feature.OBJECT_TRACKING] + operation = video_client.annotate_video( + input_uri=gcs_uri, features=features) + print('\nProcessing video for object annotations.') + + result = operation.result(timeout=300) + print('\nFinished processing.\n') + + # The first result is retrieved because a single video was processed. + object_annotations = result.annotation_results[0].object_annotations + + for object_annotation in object_annotations: + print('Entity description: {}'.format( + object_annotation.entity.description)) + if object_annotation.entity.entity_id: + print('Entity id: {}'.format(object_annotation.entity.entity_id)) + + print('Segment: {}s to {}s'.format( + object_annotation.segment.start_time_offset.seconds + + object_annotation.segment.start_time_offset.nanos / 1e9, + object_annotation.segment.end_time_offset.seconds + + object_annotation.segment.end_time_offset.nanos / 1e9)) + + print('Confidence: {}'.format(object_annotation.confidence)) + + # Here we print only the bounding box of the first frame in the segment + frame = object_annotation.frames[0] + box = frame.normalized_bounding_box + print('Time offset of the first frame: {}s'.format( + frame.time_offset.seconds + frame.time_offset.nanos / 1e9)) + print('Bounding box position:') + print('\tleft : {}'.format(box.left)) + print('\ttop : {}'.format(box.top)) + print('\tright : {}'.format(box.right)) + print('\tbottom: {}'.format(box.bottom)) + print('\n') + # [END video_object_tracking_gcs] + + +def track_objects(path): + # [START video_object_tracking] + """Object tracking in a local video.""" + from google.cloud import videointelligence + + video_client = videointelligence.VideoIntelligenceServiceClient() + features = [videointelligence.enums.Feature.OBJECT_TRACKING] + + with io.open(path, 'rb') as file: + input_content = file.read() + + operation = video_client.annotate_video( + input_content=input_content, features=features) + print('\nProcessing video for object annotations.') + + result = operation.result(timeout=300) + print('\nFinished processing.\n') + + # The first result is retrieved because a single video was processed. + object_annotations = result.annotation_results[0].object_annotations + + # Get only the first annotation for demo purposes. + object_annotation = object_annotations[0] + print('Entity description: {}'.format( + object_annotation.entity.description)) + if object_annotation.entity.entity_id: + print('Entity id: {}'.format(object_annotation.entity.entity_id)) + + print('Segment: {}s to {}s'.format( + object_annotation.segment.start_time_offset.seconds + + object_annotation.segment.start_time_offset.nanos / 1e9, + object_annotation.segment.end_time_offset.seconds + + object_annotation.segment.end_time_offset.nanos / 1e9)) + + print('Confidence: {}'.format(object_annotation.confidence)) + + # Here we print only the bounding box of the first frame in this segment + frame = object_annotation.frames[0] + box = frame.normalized_bounding_box + print('Time offset of the first frame: {}s'.format( + frame.time_offset.seconds + frame.time_offset.nanos / 1e9)) + print('Bounding box position:') + print('\tleft : {}'.format(box.left)) + print('\ttop : {}'.format(box.top)) + print('\tright : {}'.format(box.right)) + print('\tbottom: {}'.format(box.bottom)) + print('\n') + # [END video_object_tracking] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + subparsers = parser.add_subparsers(dest='command') + + analyze_labels_parser = subparsers.add_parser( + 'labels', help=analyze_labels.__doc__) + analyze_labels_parser.add_argument('path') + + analyze_labels_file_parser = subparsers.add_parser( + 'labels_file', help=analyze_labels_file.__doc__) + analyze_labels_file_parser.add_argument('path') + + analyze_explicit_content_parser = subparsers.add_parser( + 'explicit_content', help=analyze_explicit_content.__doc__) + analyze_explicit_content_parser.add_argument('path') + + analyze_shots_parser = subparsers.add_parser( + 'shots', help=analyze_shots.__doc__) + analyze_shots_parser.add_argument('path') + + transcribe_speech_parser = subparsers.add_parser( + 'transcribe', help=speech_transcription.__doc__) + transcribe_speech_parser.add_argument('path') + + detect_text_parser = subparsers.add_parser( + 'text_gcs', help=video_detect_text_gcs.__doc__) + detect_text_parser.add_argument('path') + + detect_text_file_parser = subparsers.add_parser( + 'text_file', help=video_detect_text.__doc__) + detect_text_file_parser.add_argument('path') + + tack_objects_parser = subparsers.add_parser( + 'objects_gcs', help=track_objects_gcs.__doc__) + tack_objects_parser.add_argument('path') + + tack_objects_file_parser = subparsers.add_parser( + 'objects_file', help=track_objects.__doc__) + tack_objects_file_parser.add_argument('path') + + args = parser.parse_args() + + if args.command == 'labels': + analyze_labels(args.path) + if args.command == 'labels_file': + analyze_labels_file(args.path) + if args.command == 'shots': + analyze_shots(args.path) + if args.command == 'explicit_content': + analyze_explicit_content(args.path) + if args.command == 'transcribe': + speech_transcription(args.path) + if args.command == 'text_gcs': + video_detect_text_gcs(args.path) + if args.command == 'text_file': + video_detect_text(args.path) + if args.command == 'objects_gcs': + track_objects_gcs(args.path) + if args.command == 'objects_file': + track_objects(args.path) diff --git a/video/cloud-client/analyze/analyze_test.py b/video/cloud-client/analyze/analyze_test.py new file mode 100644 index 00000000000..36288e8e9db --- /dev/null +++ b/video/cloud-client/analyze/analyze_test.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python + +# Copyright 2017 Google, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +import analyze + + +@pytest.mark.slow +def test_analyze_shots(capsys): + analyze.analyze_shots('gs://cloud-samples-data/video/gbikes_dinosaur.mp4') + out, _ = capsys.readouterr() + assert 'Shot 1:' in out + + +@pytest.mark.slow +def test_analyze_labels(capsys): + analyze.analyze_labels('gs://cloud-samples-data/video/cat.mp4') + out, _ = capsys.readouterr() + assert 'label description: cat' in out + + +@pytest.mark.slow +def test_analyze_explicit_content(capsys): + analyze.analyze_explicit_content('gs://cloud-samples-data/video/cat.mp4') + out, _ = capsys.readouterr() + assert 'pornography' in out + + +@pytest.mark.slow +def test_speech_transcription(capsys): + analyze.speech_transcription( + 'gs://cloud-samples-data/video/googlework_short.mp4') + out, _ = capsys.readouterr() + assert 'cultural' in out + + +@pytest.mark.slow +def test_detect_text_gcs(capsys): + analyze.video_detect_text_gcs( + 'gs://cloud-samples-data/video/googlework_short.mp4') + out, _ = capsys.readouterr() + assert 'GOOGLE' in out + + +@pytest.mark.slow +def test_track_objects_gcs(capsys): + analyze.track_objects_gcs( + 'gs://cloud-samples-data/video/cat.mp4') + out, _ = capsys.readouterr() + assert 'cat' in out diff --git a/video/cloud-client/analyze/beta_snippets.py b/video/cloud-client/analyze/beta_snippets.py new file mode 100644 index 00000000000..356b9ec114d --- /dev/null +++ b/video/cloud-client/analyze/beta_snippets.py @@ -0,0 +1,670 @@ +#!/usr/bin/env python + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates speech transcription using the +Google Cloud API. + +Usage Examples: + python beta_snippets.py transcription \ + gs://python-docs-samples-tests/video/googlework_short.mp4 + + python beta_snippets.py video-text-gcs \ + gs://python-docs-samples-tests/video/googlework_short.mp4 + + python beta_snippets.py track-objects resources/cat.mp4 + + python beta_snippets.py streaming-labels resources/cat.mp4 + + python beta_snippets.py streaming-shot-change resources/cat.mp4 + + python beta_snippets.py streaming-objects resources/cat.mp4 + + python beta_snippets.py streaming-explicit-content resources/cat.mp4 + + python beta_snippets.py streaming-annotation-storage resources/cat.mp4 \ + gs://mybucket/myfolder +""" + +import argparse +import io + + +def speech_transcription(input_uri): + # [START video_speech_transcription_gcs_beta] + """Transcribe speech from a video stored on GCS.""" + from google.cloud import videointelligence_v1p1beta1 as videointelligence + + video_client = videointelligence.VideoIntelligenceServiceClient() + + features = [videointelligence.enums.Feature.SPEECH_TRANSCRIPTION] + + config = videointelligence.types.SpeechTranscriptionConfig( + language_code='en-US', + enable_automatic_punctuation=True) + video_context = videointelligence.types.VideoContext( + speech_transcription_config=config) + + operation = video_client.annotate_video( + input_uri, features=features, + video_context=video_context) + + print('\nProcessing video for speech transcription.') + + result = operation.result(timeout=180) + + # There is only one annotation_result since only + # one video is processed. + annotation_results = result.annotation_results[0] + for speech_transcription in annotation_results.speech_transcriptions: + + # The number of alternatives for each transcription is limited by + # SpeechTranscriptionConfig.max_alternatives. + # Each alternative is a different possible transcription + # and has its own confidence score. + for alternative in speech_transcription.alternatives: + print('Alternative level information:') + + print('Transcript: {}'.format(alternative.transcript)) + print('Confidence: {}\n'.format(alternative.confidence)) + + print('Word level information:') + for word_info in alternative.words: + word = word_info.word + start_time = word_info.start_time + end_time = word_info.end_time + print('\t{}s - {}s: {}'.format( + start_time.seconds + start_time.nanos * 1e-9, + end_time.seconds + end_time.nanos * 1e-9, + word)) + # [END video_speech_transcription_gcs_beta] + + +def video_detect_text_gcs(input_uri): + # [START video_detect_text_gcs_beta] + """Detect text in a video stored on GCS.""" + from google.cloud import videointelligence_v1p2beta1 as videointelligence + + video_client = videointelligence.VideoIntelligenceServiceClient() + features = [videointelligence.enums.Feature.TEXT_DETECTION] + + operation = video_client.annotate_video( + input_uri=input_uri, + features=features) + + print('\nProcessing video for text detection.') + result = operation.result(timeout=300) + + # The first result is retrieved because a single video was processed. + annotation_result = result.annotation_results[0] + + # Get only the first result + text_annotation = annotation_result.text_annotations[0] + print('\nText: {}'.format(text_annotation.text)) + + # Get the first text segment + text_segment = text_annotation.segments[0] + start_time = text_segment.segment.start_time_offset + end_time = text_segment.segment.end_time_offset + print('start_time: {}, end_time: {}'.format( + start_time.seconds + start_time.nanos * 1e-9, + end_time.seconds + end_time.nanos * 1e-9)) + + print('Confidence: {}'.format(text_segment.confidence)) + + # Show the result for the first frame in this segment. + frame = text_segment.frames[0] + time_offset = frame.time_offset + print('Time offset for the first frame: {}'.format( + time_offset.seconds + time_offset.nanos * 1e-9)) + print('Rotated Bounding Box Vertices:') + for vertex in frame.rotated_bounding_box.vertices: + print('\tVertex.x: {}, Vertex.y: {}'.format(vertex.x, vertex.y)) + # [END video_detect_text_gcs_beta] + return annotation_result.text_annotations + + +def video_detect_text(path): + # [START video_detect_text_beta] + """Detect text in a local video.""" + from google.cloud import videointelligence_v1p2beta1 as videointelligence + + video_client = videointelligence.VideoIntelligenceServiceClient() + features = [videointelligence.enums.Feature.TEXT_DETECTION] + video_context = videointelligence.types.VideoContext() + + with io.open(path, 'rb') as file: + input_content = file.read() + + operation = video_client.annotate_video( + input_content=input_content, # the bytes of the video file + features=features, + video_context=video_context) + + print('\nProcessing video for text detection.') + result = operation.result(timeout=300) + + # The first result is retrieved because a single video was processed. + annotation_result = result.annotation_results[0] + + # Get only the first result + text_annotation = annotation_result.text_annotations[0] + print('\nText: {}'.format(text_annotation.text)) + + # Get the first text segment + text_segment = text_annotation.segments[0] + start_time = text_segment.segment.start_time_offset + end_time = text_segment.segment.end_time_offset + print('start_time: {}, end_time: {}'.format( + start_time.seconds + start_time.nanos * 1e-9, + end_time.seconds + end_time.nanos * 1e-9)) + + print('Confidence: {}'.format(text_segment.confidence)) + + # Show the result for the first frame in this segment. + frame = text_segment.frames[0] + time_offset = frame.time_offset + print('Time offset for the first frame: {}'.format( + time_offset.seconds + time_offset.nanos * 1e-9)) + print('Rotated Bounding Box Vertices:') + for vertex in frame.rotated_bounding_box.vertices: + print('\tVertex.x: {}, Vertex.y: {}'.format(vertex.x, vertex.y)) + # [END video_detect_text_beta] + return annotation_result.text_annotations + + +def track_objects_gcs(gcs_uri): + # [START video_object_tracking_gcs_beta] + """Object Tracking.""" + from google.cloud import videointelligence_v1p2beta1 as videointelligence + + # It is recommended to use location_id as 'us-east1' for the best latency + # due to different types of processors used in this region and others. + video_client = videointelligence.VideoIntelligenceServiceClient() + features = [videointelligence.enums.Feature.OBJECT_TRACKING] + operation = video_client.annotate_video( + input_uri=gcs_uri, features=features, location_id='us-east1') + print('\nProcessing video for object annotations.') + + result = operation.result(timeout=300) + print('\nFinished processing.\n') + + # The first result is retrieved because a single video was processed. + object_annotations = result.annotation_results[0].object_annotations + + # Get only the first annotation for demo purposes. + object_annotation = object_annotations[0] + print('Entity description: {}'.format( + object_annotation.entity.description)) + if object_annotation.entity.entity_id: + print('Entity id: {}'.format(object_annotation.entity.entity_id)) + + print('Segment: {}s to {}s'.format( + object_annotation.segment.start_time_offset.seconds + + object_annotation.segment.start_time_offset.nanos / 1e9, + object_annotation.segment.end_time_offset.seconds + + object_annotation.segment.end_time_offset.nanos / 1e9)) + + print('Confidence: {}'.format(object_annotation.confidence)) + + # Here we print only the bounding box of the first frame in this segment + frame = object_annotation.frames[0] + box = frame.normalized_bounding_box + print('Time offset of the first frame: {}s'.format( + frame.time_offset.seconds + frame.time_offset.nanos / 1e9)) + print('Bounding box position:') + print('\tleft : {}'.format(box.left)) + print('\ttop : {}'.format(box.top)) + print('\tright : {}'.format(box.right)) + print('\tbottom: {}'.format(box.bottom)) + print('\n') + # [END video_object_tracking_gcs_beta] + return object_annotations + + +def track_objects(path): + # [START video_object_tracking_beta] + """Object Tracking.""" + from google.cloud import videointelligence_v1p2beta1 as videointelligence + + video_client = videointelligence.VideoIntelligenceServiceClient() + features = [videointelligence.enums.Feature.OBJECT_TRACKING] + + with io.open(path, 'rb') as file: + input_content = file.read() + + # It is recommended to use location_id as 'us-east1' for the best latency + # due to different types of processors used in this region and others. + operation = video_client.annotate_video( + input_content=input_content, features=features, location_id='us-east1') + print('\nProcessing video for object annotations.') + + result = operation.result(timeout=300) + print('\nFinished processing.\n') + + # The first result is retrieved because a single video was processed. + object_annotations = result.annotation_results[0].object_annotations + + # Get only the first annotation for demo purposes. + object_annotation = object_annotations[0] + print('Entity description: {}'.format( + object_annotation.entity.description)) + if object_annotation.entity.entity_id: + print('Entity id: {}'.format(object_annotation.entity.entity_id)) + + print('Segment: {}s to {}s'.format( + object_annotation.segment.start_time_offset.seconds + + object_annotation.segment.start_time_offset.nanos / 1e9, + object_annotation.segment.end_time_offset.seconds + + object_annotation.segment.end_time_offset.nanos / 1e9)) + + print('Confidence: {}'.format(object_annotation.confidence)) + + # Here we print only the bounding box of the first frame in this segment + frame = object_annotation.frames[0] + box = frame.normalized_bounding_box + print('Time offset of the first frame: {}s'.format( + frame.time_offset.seconds + frame.time_offset.nanos / 1e9)) + print('Bounding box position:') + print('\tleft : {}'.format(box.left)) + print('\ttop : {}'.format(box.top)) + print('\tright : {}'.format(box.right)) + print('\tbottom: {}'.format(box.bottom)) + print('\n') + # [END video_object_tracking_beta] + return object_annotations + + +def detect_labels_streaming(path): + # [START video_streaming_label_detection_beta] + from google.cloud import videointelligence_v1p3beta1 as videointelligence + + # path = 'path_to_file' + + client = videointelligence.StreamingVideoIntelligenceServiceClient() + + # Set streaming config. + config = videointelligence.types.StreamingVideoConfig( + feature=(videointelligence.enums. + StreamingFeature.STREAMING_LABEL_DETECTION)) + + # config_request should be the first in the stream of requests. + config_request = videointelligence.types.StreamingAnnotateVideoRequest( + video_config=config) + + # Set the chunk size to 5MB (recommended less than 10MB). + chunk_size = 5 * 1024 * 1024 + + # Load file content. + stream = [] + with io.open(path, 'rb') as video_file: + while True: + data = video_file.read(chunk_size) + if not data: + break + stream.append(data) + + def stream_generator(): + yield config_request + for chunk in stream: + yield videointelligence.types.StreamingAnnotateVideoRequest( + input_content=chunk) + + requests = stream_generator() + + # streaming_annotate_video returns a generator. + responses = client.streaming_annotate_video(requests) + + # Each response corresponds to about 1 second of video. + for response in responses: + # Check for errors. + if response.error.message: + print(response.error.message) + break + + # Get the time offset of the response. + frame = response.annotation_results.label_annotations[0].frames[0] + time_offset = frame.time_offset.seconds + frame.time_offset.nanos / 1e9 + print('{}s:'.format(time_offset)) + + for annotation in response.annotation_results.label_annotations: + description = annotation.entity.description + # Every annotation has only one frame + confidence = annotation.frames[0].confidence + print('\t{} (confidence: {})'.format(description, confidence)) + # [END video_streaming_label_detection_beta] + + +def detect_shot_change_streaming(path): + # [START video_streaming_shot_change_detection_beta] + from google.cloud import videointelligence_v1p3beta1 as videointelligence + + # path = 'path_to_file' + + client = videointelligence.StreamingVideoIntelligenceServiceClient() + + # Set streaming config. + config = videointelligence.types.StreamingVideoConfig( + feature=(videointelligence.enums.StreamingFeature. + STREAMING_SHOT_CHANGE_DETECTION)) + + # config_request should be the first in the stream of requests. + config_request = videointelligence.types.StreamingAnnotateVideoRequest( + video_config=config) + + # Set the chunk size to 5MB (recommended less than 10MB). + chunk_size = 5 * 1024 * 1024 + + # Load file content. + stream = [] + with io.open(path, 'rb') as video_file: + while True: + data = video_file.read(chunk_size) + if not data: + break + stream.append(data) + + def stream_generator(): + yield config_request + for chunk in stream: + yield videointelligence.types.StreamingAnnotateVideoRequest( + input_content=chunk) + + requests = stream_generator() + + # streaming_annotate_video returns a generator. + responses = client.streaming_annotate_video(requests) + + # Each response corresponds to about 1 second of video. + for response in responses: + # Check for errors. + if response.error.message: + print(response.error.message) + break + + for annotation in response.annotation_results.shot_annotations: + start = (annotation.start_time_offset.seconds + + annotation.start_time_offset.nanos / 1e9) + end = (annotation.end_time_offset.seconds + + annotation.end_time_offset.nanos / 1e9) + + print('Shot: {}s to {}s'.format(start, end)) + # [END video_streaming_shot_change_detection_beta] + + +def track_objects_streaming(path): + # [START video_streaming_object_tracking_beta] + from google.cloud import videointelligence_v1p3beta1 as videointelligence + + # path = 'path_to_file' + + client = videointelligence.StreamingVideoIntelligenceServiceClient() + + # Set streaming config. + config = videointelligence.types.StreamingVideoConfig( + feature=(videointelligence.enums. + StreamingFeature.STREAMING_OBJECT_TRACKING)) + + # config_request should be the first in the stream of requests. + config_request = videointelligence.types.StreamingAnnotateVideoRequest( + video_config=config) + + # Set the chunk size to 5MB (recommended less than 10MB). + chunk_size = 5 * 1024 * 1024 + + # Load file content. + stream = [] + with io.open(path, 'rb') as video_file: + while True: + data = video_file.read(chunk_size) + if not data: + break + stream.append(data) + + def stream_generator(): + yield config_request + for chunk in stream: + yield videointelligence.types.StreamingAnnotateVideoRequest( + input_content=chunk) + + requests = stream_generator() + + # streaming_annotate_video returns a generator. + responses = client.streaming_annotate_video(requests) + + # Each response corresponds to about 1 second of video. + for response in responses: + # Check for errors. + if response.error.message: + print(response.error.message) + break + + # Get the time offset of the response. + frame = response.annotation_results.object_annotations[0].frames[0] + time_offset = frame.time_offset.seconds + frame.time_offset.nanos / 1e9 + print('{}s:'.format(time_offset)) + + for annotation in response.annotation_results.object_annotations: + description = annotation.entity.description + confidence = annotation.confidence + + # track_id tracks the same object in the video. + track_id = annotation.track_id + + print('\tEntity description: {}'.format(description)) + print('\tTrack Id: {}'.format(track_id)) + if annotation.entity.entity_id: + print('\tEntity id: {}'.format(annotation.entity.entity_id)) + + print('\tConfidence: {}'.format(confidence)) + + # Every annotation has only one frame + frame = annotation.frames[0] + box = frame.normalized_bounding_box + print('\tBounding box position:') + print('\tleft : {}'.format(box.left)) + print('\ttop : {}'.format(box.top)) + print('\tright : {}'.format(box.right)) + print('\tbottom: {}\n'.format(box.bottom)) + # [END video_streaming_object_tracking_beta] + + +def detect_explicit_content_streaming(path): + # [START video_streaming_explicit_content_detection_beta] + from google.cloud import videointelligence_v1p3beta1 as videointelligence + + # path = 'path_to_file' + + client = videointelligence.StreamingVideoIntelligenceServiceClient() + + # Set streaming config. + config = videointelligence.types.StreamingVideoConfig( + feature=(videointelligence.enums.StreamingFeature. + STREAMING_EXPLICIT_CONTENT_DETECTION)) + + # config_request should be the first in the stream of requests. + config_request = videointelligence.types.StreamingAnnotateVideoRequest( + video_config=config) + + # Set the chunk size to 5MB (recommended less than 10MB). + chunk_size = 5 * 1024 * 1024 + + # Load file content. + stream = [] + with io.open(path, 'rb') as video_file: + while True: + data = video_file.read(chunk_size) + if not data: + break + stream.append(data) + + def stream_generator(): + yield config_request + for chunk in stream: + yield videointelligence.types.StreamingAnnotateVideoRequest( + input_content=chunk) + + requests = stream_generator() + + # streaming_annotate_video returns a generator. + responses = client.streaming_annotate_video(requests) + + # Each response corresponds to about 1 second of video. + for response in responses: + # Check for errors. + if response.error.message: + print(response.error.message) + break + + for frame in response.annotation_results.explicit_annotation.frames: + time_offset = (frame.time_offset.seconds + + frame.time_offset.nanos / 1e9) + pornography_likelihood = videointelligence.enums.Likelihood( + frame.pornography_likelihood) + + print('Time: {}s'.format(time_offset)) + print('\tpornogaphy: {}'.format(pornography_likelihood.name)) + # [END video_streaming_explicit_content_detection_beta] + + +def annotation_to_storage_streaming(path, output_uri): + # [START video_streaming_annotation_to_storage_beta] + from google.cloud import videointelligence_v1p3beta1 as videointelligence + + # path = 'path_to_file' + # output_uri = 'gs://path_to_output' + + client = videointelligence.StreamingVideoIntelligenceServiceClient() + + # Set streaming config specifying the output_uri. + # The output_uri is the prefix of the actual output files. + storage_config = videointelligence.types.StreamingStorageConfig( + enable_storage_annotation_result=True, + annotation_result_storage_directory=output_uri) + # Here we use label detection as an example. + # All features support output to GCS. + config = videointelligence.types.StreamingVideoConfig( + feature=(videointelligence.enums. + StreamingFeature.STREAMING_LABEL_DETECTION), + storage_config=storage_config) + + # config_request should be the first in the stream of requests. + config_request = videointelligence.types.StreamingAnnotateVideoRequest( + video_config=config) + + # Set the chunk size to 5MB (recommended less than 10MB). + chunk_size = 5 * 1024 * 1024 + + # Load file content. + stream = [] + with io.open(path, 'rb') as video_file: + while True: + data = video_file.read(chunk_size) + if not data: + break + stream.append(data) + + def stream_generator(): + yield config_request + for chunk in stream: + yield videointelligence.types.StreamingAnnotateVideoRequest( + input_content=chunk) + + requests = stream_generator() + + # streaming_annotate_video returns a generator. + responses = client.streaming_annotate_video(requests) + + for response in responses: + # Check for errors. + if response.error.message: + print(response.error.message) + break + + print('Storage URI: {}'.format(response.annotation_results_uri)) + # [END video_streaming_annotation_to_storage_beta] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + subparsers = parser.add_subparsers(dest='command') + + speech_transcription_parser = subparsers.add_parser( + 'transcription', help=speech_transcription.__doc__) + speech_transcription_parser.add_argument('gcs_uri') + + video_text_gcs_parser = subparsers.add_parser( + 'video-text-gcs', help=video_detect_text_gcs.__doc__) + video_text_gcs_parser.add_argument('gcs_uri') + + video_text_parser = subparsers.add_parser( + 'video-text', help=video_detect_text.__doc__) + video_text_parser.add_argument('path') + + video_object_tracking_gcs_parser = subparsers.add_parser( + 'track-objects-gcs', help=track_objects_gcs.__doc__) + video_object_tracking_gcs_parser.add_argument('gcs_uri') + + video_object_tracking_parser = subparsers.add_parser( + 'track-objects', help=track_objects.__doc__) + video_object_tracking_parser.add_argument('path') + + video_streaming_labels_parser = subparsers.add_parser( + 'streaming-labels', help=detect_labels_streaming.__doc__) + video_streaming_labels_parser.add_argument('path') + + video_streaming_shot_change_parser = subparsers.add_parser( + 'streaming-shot-change', help=detect_shot_change_streaming.__doc__) + video_streaming_shot_change_parser.add_argument('path') + + video_streaming_objects_parser = subparsers.add_parser( + 'streaming-objects', help=track_objects_streaming.__doc__) + video_streaming_objects_parser.add_argument('path') + + video_streaming_explicit_content_parser = subparsers.add_parser( + 'streaming-explicit-content', + help=detect_explicit_content_streaming.__doc__) + video_streaming_explicit_content_parser.add_argument('path') + + video_streaming_annotation_to_storage_parser = subparsers.add_parser( + 'streaming-annotation-storage', + help=annotation_to_storage_streaming.__doc__) + video_streaming_annotation_to_storage_parser.add_argument('path') + video_streaming_annotation_to_storage_parser.add_argument('output_uri') + + args = parser.parse_args() + + if args.command == 'transcription': + speech_transcription(args.gcs_uri) + elif args.command == 'video-text-gcs': + video_detect_text_gcs(args.gcs_uri) + elif args.command == 'video-text': + video_detect_text(args.path) + elif args.command == 'track-objects-gcs': + track_objects_gcs(args.gcs_uri) + elif args.command == 'track-objects': + track_objects(args.path) + elif args.command == 'streaming-labels': + detect_labels_streaming(args.path) + elif args.command == 'streaming-shot-change': + detect_shot_change_streaming(args.path) + elif args.command == 'streaming-objects': + track_objects_streaming(args.path) + elif args.command == 'streaming-explicit-content': + detect_explicit_content_streaming(args.path) + elif args.command == 'streaming-annotation-storage': + annotation_to_storage_streaming(args.path, args.output_uri) diff --git a/video/cloud-client/analyze/beta_snippets_test.py b/video/cloud-client/analyze/beta_snippets_test.py new file mode 100644 index 00000000000..7402b014c95 --- /dev/null +++ b/video/cloud-client/analyze/beta_snippets_test.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python + +# Copyright 2017 Google, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from six.moves.urllib.request import urlopen +import time +import uuid + +import beta_snippets +from google.cloud import storage +import pytest + + +POSSIBLE_TEXTS = ['Google', 'SUR', 'SUR', 'ROTO', 'Vice President', '58oo9', + 'LONDRES', 'OMAR', 'PARIS', 'METRO', 'RUE', 'CARLO'] + + +@pytest.fixture(scope='session') +def video_path(tmpdir_factory): + file = urlopen( + 'http://storage.googleapis.com/cloud-samples-data/video/cat.mp4') + path = tmpdir_factory.mktemp('video').join('file.mp4') + with open(str(path), 'wb') as f: + f.write(file.read()) + + return str(path) + + +@pytest.fixture(scope='function') +def bucket(): + # Create a temporaty bucket to store annotation output. + bucket_name = str(uuid.uuid1()) + storage_client = storage.Client() + bucket = storage_client.create_bucket(bucket_name) + + yield bucket + + # Teardown. + bucket.delete(force=True) + + +@pytest.mark.slow +def test_speech_transcription(capsys): + beta_snippets.speech_transcription( + 'gs://python-docs-samples-tests/video/googlework_short.mp4') + out, _ = capsys.readouterr() + assert 'cultural' in out + + +@pytest.mark.slow +def test_detect_labels_streaming(capsys, video_path): + beta_snippets.detect_labels_streaming(video_path) + + out, _ = capsys.readouterr() + assert 'cat' in out + + +@pytest.mark.slow +def test_detect_shot_change_streaming(capsys, video_path): + beta_snippets.detect_shot_change_streaming(video_path) + + out, _ = capsys.readouterr() + assert 'Shot' in out + + +@pytest.mark.slow +def test_track_objects_streaming(capsys, video_path): + beta_snippets.track_objects_streaming(video_path) + + out, _ = capsys.readouterr() + assert 'cat' in out + + +@pytest.mark.slow +def test_detect_explicit_content_streaming(capsys, video_path): + beta_snippets.detect_explicit_content_streaming(video_path) + + out, _ = capsys.readouterr() + assert 'Time' in out + + +@pytest.mark.slow +def test_annotation_to_storage_streaming(capsys, video_path, bucket): + output_uri = 'gs://{}'.format(bucket.name) + beta_snippets.annotation_to_storage_streaming(video_path, output_uri) + + out, _ = capsys.readouterr() + assert 'Storage' in out + + # It takes a few seconds before the results show up on GCS. + time.sleep(3) + + # Confirm that one output blob had been written to GCS. + blobs_iterator = bucket.list_blobs() + blobs = [blob for blob in blobs_iterator] + assert len(blobs) == 1 + + +@pytest.mark.slow +def test_detect_text(): + in_file = './resources/googlework_short.mp4' + text_annotations = beta_snippets.video_detect_text(in_file) + + text_exists = False + for text_annotation in text_annotations: + for possible_text in POSSIBLE_TEXTS: + if possible_text.upper() in text_annotation.text.upper(): + text_exists = True + assert text_exists + + +@pytest.mark.slow +def test_detect_text_gcs(): + in_file = 'gs://python-docs-samples-tests/video/googlework_short.mp4' + text_annotations = beta_snippets.video_detect_text_gcs(in_file) + + text_exists = False + for text_annotation in text_annotations: + for possible_text in POSSIBLE_TEXTS: + if possible_text.upper() in text_annotation.text.upper(): + text_exists = True + assert text_exists + + +@pytest.mark.slow +def test_track_objects(): + in_file = './resources/cat.mp4' + object_annotations = beta_snippets.track_objects(in_file) + + text_exists = False + for object_annotation in object_annotations: + if 'CAT' in object_annotation.entity.description.upper(): + text_exists = True + assert text_exists + assert object_annotations[0].frames[0].normalized_bounding_box.left >= 0.0 + assert object_annotations[0].frames[0].normalized_bounding_box.left <= 1.0 + + +@pytest.mark.slow +def test_track_objects_gcs(): + in_file = 'gs://demomaker/cat.mp4' + object_annotations = beta_snippets.track_objects_gcs(in_file) + + text_exists = False + for object_annotation in object_annotations: + if 'CAT' in object_annotation.entity.description.upper(): + text_exists = True + assert text_exists + assert object_annotations[0].frames[0].normalized_bounding_box.left >= 0.0 + assert object_annotations[0].frames[0].normalized_bounding_box.left <= 1.0 diff --git a/video/cloud-client/analyze/requirements.txt b/video/cloud-client/analyze/requirements.txt new file mode 100644 index 00000000000..ba28944b652 --- /dev/null +++ b/video/cloud-client/analyze/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-videointelligence==1.8.0 +google-cloud-storage==1.14.0 diff --git a/video/cloud-client/analyze/resources/README.md b/video/cloud-client/analyze/resources/README.md new file mode 100644 index 00000000000..e33d7137c4f --- /dev/null +++ b/video/cloud-client/analyze/resources/README.md @@ -0,0 +1,17 @@ +# Resources folder for local files + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=video/cloud-client/analyze/resources/README.md + +Copy from Google Cloud Storage to this folder for testing video analysis +of local files. For `cat.mp4` used in the usage example, run the following +`gcloud` command. + + gsutil cp gs://demomaker/cat.mp4 . + +Now, when you run the following command, the video used for label detection +will be passed from here: + + python analyze.py labels_file resources/cat.mp4 diff --git a/video/cloud-client/analyze/resources/cat.mp4 b/video/cloud-client/analyze/resources/cat.mp4 new file mode 100644 index 00000000000..0e071b9ec67 Binary files /dev/null and b/video/cloud-client/analyze/resources/cat.mp4 differ diff --git a/video/cloud-client/analyze/resources/googlework_short.mp4 b/video/cloud-client/analyze/resources/googlework_short.mp4 new file mode 100644 index 00000000000..be0f40f8ad6 Binary files /dev/null and b/video/cloud-client/analyze/resources/googlework_short.mp4 differ diff --git a/video/cloud-client/labels/README.rst b/video/cloud-client/labels/README.rst new file mode 100644 index 00000000000..ab919e7fdc8 --- /dev/null +++ b/video/cloud-client/labels/README.rst @@ -0,0 +1,117 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Video Intelligence API Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=video/cloud-client/labels/README.rst + + +This directory contains samples for Google Cloud Video Intelligence API. `Google Cloud Video Intelligence API`_ allows developers to easily integrate feature detection in video. + + + + +.. _Google Cloud Video Intelligence API: https://cloud.google.com/video-intelligence/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +labels ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=video/cloud-client/labels/labels.py,video/cloud-client/labels/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python labels.py + + usage: labels.py [-h] path + + This application demonstrates how to detect labels from a video + based on the image content with the Google Cloud Video Intelligence + API. + + For more information, check out the documentation at + https://cloud.google.com/videointelligence/docs. + + Usage Example: + + python labels.py gs://cloud-ml-sandbox/video/chicago.mp4 + + positional arguments: + path GCS file path for label detection. + + optional arguments: + -h, --help show this help message and exit + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/video/cloud-client/labels/README.rst.in b/video/cloud-client/labels/README.rst.in new file mode 100644 index 00000000000..2d6b97cf6e6 --- /dev/null +++ b/video/cloud-client/labels/README.rst.in @@ -0,0 +1,22 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Video Intelligence API + short_name: Cloud Video Intelligence API + url: https://cloud.google.com/video-intelligence/docs + description: > + `Google Cloud Video Intelligence API`_ allows developers to easily + integrate feature detection in video. + +setup: +- auth +- install_deps + +samples: +- name: labels + file: labels.py + show_help: True + +cloud_client_library: true + +folder: video/cloud-client/labels \ No newline at end of file diff --git a/video/cloud-client/labels/labels.py b/video/cloud-client/labels/labels.py new file mode 100644 index 00000000000..4a1cf7679ec --- /dev/null +++ b/video/cloud-client/labels/labels.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to detect labels from a video +based on the image content with the Google Cloud Video Intelligence +API. + +For more information, check out the documentation at +https://cloud.google.com/videointelligence/docs. + +Usage Example: + + python labels.py gs://cloud-ml-sandbox/video/chicago.mp4 + +""" + +# [START video_label_tutorial] +# [START video_label_tutorial_imports] +import argparse + +from google.cloud import videointelligence +# [END video_label_tutorial_imports] + + +def analyze_labels(path): + """ Detects labels given a GCS path. """ + # [START video_label_tutorial_construct_request] + video_client = videointelligence.VideoIntelligenceServiceClient() + features = [videointelligence.enums.Feature.LABEL_DETECTION] + operation = video_client.annotate_video(path, features=features) + # [END video_label_tutorial_construct_request] + print('\nProcessing video for label annotations:') + + # [START video_label_tutorial_check_operation] + result = operation.result(timeout=90) + print('\nFinished processing.') + # [END video_label_tutorial_check_operation] + + # [START video_label_tutorial_parse_response] + segment_labels = result.annotation_results[0].segment_label_annotations + for i, segment_label in enumerate(segment_labels): + print('Video label description: {}'.format( + segment_label.entity.description)) + for category_entity in segment_label.category_entities: + print('\tLabel category description: {}'.format( + category_entity.description)) + + for i, segment in enumerate(segment_label.segments): + start_time = (segment.segment.start_time_offset.seconds + + segment.segment.start_time_offset.nanos / 1e9) + end_time = (segment.segment.end_time_offset.seconds + + segment.segment.end_time_offset.nanos / 1e9) + positions = '{}s to {}s'.format(start_time, end_time) + confidence = segment.confidence + print('\tSegment {}: {}'.format(i, positions)) + print('\tConfidence: {}'.format(confidence)) + print('\n') + # [END video_label_tutorial_parse_response] + + +if __name__ == '__main__': + # [START video_label_tutorial_run_application] + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('path', help='GCS file path for label detection.') + args = parser.parse_args() + + analyze_labels(args.path) + # [END video_label_tutorial_run_application] +# [END video_label_tutorial] diff --git a/video/cloud-client/labels/labels_test.py b/video/cloud-client/labels/labels_test.py new file mode 100644 index 00000000000..ae455b90c64 --- /dev/null +++ b/video/cloud-client/labels/labels_test.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python + +# Copyright 2017 Google, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +import labels + + +@pytest.mark.slow +def test_feline_video_labels(capsys): + labels.analyze_labels('gs://demomaker/cat.mp4') + out, _ = capsys.readouterr() + assert 'Video label description: cat' in out diff --git a/video/cloud-client/labels/requirements.txt b/video/cloud-client/labels/requirements.txt new file mode 100644 index 00000000000..0a5c79b12c9 --- /dev/null +++ b/video/cloud-client/labels/requirements.txt @@ -0,0 +1 @@ +google-cloud-videointelligence==1.6.1 diff --git a/video/cloud-client/quickstart/README.rst b/video/cloud-client/quickstart/README.rst new file mode 100644 index 00000000000..26bc921b8ab --- /dev/null +++ b/video/cloud-client/quickstart/README.rst @@ -0,0 +1,97 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Video Intelligence API Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=video/cloud-client/quickstart/README.rst + + +This directory contains samples for Google Cloud Video Intelligence API. `Google Cloud Video Intelligence API`_ allows developers to easily integrate feature detection in video. + + + + +.. _Google Cloud Video Intelligence API: https://cloud.google.com/video-intelligence/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +quickstart ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=video/cloud-client/quickstart/quickstart.py,video/cloud-client/quickstart/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python quickstart.py + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/video/cloud-client/quickstart/README.rst.in b/video/cloud-client/quickstart/README.rst.in new file mode 100644 index 00000000000..9763ec6334a --- /dev/null +++ b/video/cloud-client/quickstart/README.rst.in @@ -0,0 +1,21 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Video Intelligence API + short_name: Cloud Video Intelligence API + url: https://cloud.google.com/video-intelligence/docs + description: > + `Google Cloud Video Intelligence API`_ allows developers to easily + integrate feature detection in video. + +setup: +- auth +- install_deps + +samples: +- name: quickstart + file: quickstart.py + +cloud_client_library: true + +folder: video/cloud-client/quickstart \ No newline at end of file diff --git a/video/cloud-client/quickstart/quickstart.py b/video/cloud-client/quickstart/quickstart.py new file mode 100644 index 00000000000..1300e774707 --- /dev/null +++ b/video/cloud-client/quickstart/quickstart.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates label detection on a demo video using +the Google Cloud API. + +Usage: + python quickstart.py + +""" + + +def run_quickstart(): + # [START video_quickstart] + from google.cloud import videointelligence + + video_client = videointelligence.VideoIntelligenceServiceClient() + features = [videointelligence.enums.Feature.LABEL_DETECTION] + operation = video_client.annotate_video( + 'gs://demomaker/cat.mp4', features=features) + print('\nProcessing video for label annotations:') + + result = operation.result(timeout=120) + print('\nFinished processing.') + + # first result is retrieved because a single video was processed + segment_labels = result.annotation_results[0].segment_label_annotations + for i, segment_label in enumerate(segment_labels): + print('Video label description: {}'.format( + segment_label.entity.description)) + for category_entity in segment_label.category_entities: + print('\tLabel category description: {}'.format( + category_entity.description)) + + for i, segment in enumerate(segment_label.segments): + start_time = (segment.segment.start_time_offset.seconds + + segment.segment.start_time_offset.nanos / 1e9) + end_time = (segment.segment.end_time_offset.seconds + + segment.segment.end_time_offset.nanos / 1e9) + positions = '{}s to {}s'.format(start_time, end_time) + confidence = segment.confidence + print('\tSegment {}: {}'.format(i, positions)) + print('\tConfidence: {}'.format(confidence)) + print('\n') + # [END video_quickstart] + + +if __name__ == '__main__': + run_quickstart() diff --git a/video/cloud-client/quickstart/quickstart_test.py b/video/cloud-client/quickstart/quickstart_test.py new file mode 100644 index 00000000000..1d1534c46cf --- /dev/null +++ b/video/cloud-client/quickstart/quickstart_test.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python + +# Copyright 2017 Google, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +import quickstart + + +@pytest.mark.slow +def test_quickstart(capsys): + quickstart.run_quickstart() + out, _ = capsys.readouterr() + assert 'Video label description: cat' in out diff --git a/video/cloud-client/quickstart/requirements.txt b/video/cloud-client/quickstart/requirements.txt new file mode 100644 index 00000000000..0a5c79b12c9 --- /dev/null +++ b/video/cloud-client/quickstart/requirements.txt @@ -0,0 +1 @@ +google-cloud-videointelligence==1.6.1 diff --git a/video/cloud-client/shotchange/README.rst b/video/cloud-client/shotchange/README.rst new file mode 100644 index 00000000000..3d5bf2e7bcd --- /dev/null +++ b/video/cloud-client/shotchange/README.rst @@ -0,0 +1,116 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Video Intelligence API Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=video/cloud-client/shotchange/README.rst + + +This directory contains samples for Google Cloud Video Intelligence API. `Google Cloud Video Intelligence API`_ allows developers to easily integrate feature detection in video. + + + + +.. _Google Cloud Video Intelligence API: https://cloud.google.com/video-intelligence/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Shot Change Detection ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=video/cloud-client/shotchange/shotchange.py,video/cloud-client/shotchange/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python shotchange.py + + usage: shotchange.py [-h] path + + This application demonstrates how to identify all different shots + in a video using the Google Cloud Video Intelligence API. + + For more information, check out the documentation at + https://cloud.google.com/videointelligence/docs. + + Example Usage: + + python shotchange.py gs://demomaker/gbikes_dinosaur.mp4 + + positional arguments: + path GCS path for shot change detection. + + optional arguments: + -h, --help show this help message and exit + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/video/cloud-client/shotchange/README.rst.in b/video/cloud-client/shotchange/README.rst.in new file mode 100644 index 00000000000..6463d192f72 --- /dev/null +++ b/video/cloud-client/shotchange/README.rst.in @@ -0,0 +1,22 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Video Intelligence API + short_name: Cloud Video Intelligence API + url: https://cloud.google.com/video-intelligence/docs + description: > + `Google Cloud Video Intelligence API`_ allows developers to easily + integrate feature detection in video. + +setup: +- auth +- install_deps + +samples: +- name: Shot Change Detection + file: shotchange.py + show_help: True + +cloud_client_library: true + +folder: video/cloud-client/shotchange \ No newline at end of file diff --git a/video/cloud-client/shotchange/requirements.txt b/video/cloud-client/shotchange/requirements.txt new file mode 100644 index 00000000000..0a5c79b12c9 --- /dev/null +++ b/video/cloud-client/shotchange/requirements.txt @@ -0,0 +1 @@ +google-cloud-videointelligence==1.6.1 diff --git a/video/cloud-client/shotchange/shotchange.py b/video/cloud-client/shotchange/shotchange.py new file mode 100644 index 00000000000..2db0e832d49 --- /dev/null +++ b/video/cloud-client/shotchange/shotchange.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to identify all different shots +in a video using the Google Cloud Video Intelligence API. + +For more information, check out the documentation at +https://cloud.google.com/videointelligence/docs. + +Example Usage: + + python shotchange.py gs://demomaker/gbikes_dinosaur.mp4 + +""" + +# [START video_shot_tutorial] +# [START video_shot_tutorial_imports] +import argparse + +from google.cloud import videointelligence +# [END video_shot_tutorial_imports] + + +def analyze_shots(path): + """ Detects camera shot changes. """ + # [START video_shot_tutorial_construct_request] + video_client = videointelligence.VideoIntelligenceServiceClient() + features = [videointelligence.enums.Feature.SHOT_CHANGE_DETECTION] + operation = video_client.annotate_video(path, features=features) + # [END video_shot_tutorial_construct_request] + print('\nProcessing video for shot change annotations:') + + # [START video_shot_tutorial_check_operation] + result = operation.result(timeout=120) + print('\nFinished processing.') + # [END video_shot_tutorial_check_operation] + + # [START video_shot_tutorial_parse_response] + for i, shot in enumerate(result.annotation_results[0].shot_annotations): + start_time = (shot.start_time_offset.seconds + + shot.start_time_offset.nanos / 1e9) + end_time = (shot.end_time_offset.seconds + + shot.end_time_offset.nanos / 1e9) + print('\tShot {}: {} to {}'.format(i, start_time, end_time)) + # [END video_shot_tutorial_parse_response] + + +if __name__ == '__main__': + # [START video_shot_tutorial_run_application] + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('path', help='GCS path for shot change detection.') + args = parser.parse_args() + + analyze_shots(args.path) + # [END video_shot_tutorial_run_application] +# [END video_shot_tutorial] diff --git a/video/cloud-client/shotchange/shotchange_test.py b/video/cloud-client/shotchange/shotchange_test.py new file mode 100644 index 00000000000..752e595cd7d --- /dev/null +++ b/video/cloud-client/shotchange/shotchange_test.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python + +# Copyright 2017 Google, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +import shotchange + + +@pytest.mark.slow +def test_shots_dino(capsys): + shotchange.analyze_shots('gs://demomaker/gbikes_dinosaur.mp4') + out, _ = capsys.readouterr() + assert 'Shot 1:' in out diff --git a/vision/automl/automl_vision_dataset.py b/vision/automl/automl_vision_dataset.py new file mode 100755 index 00000000000..1af60cd46f0 --- /dev/null +++ b/vision/automl/automl_vision_dataset.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to perform basic operations on dataset +with the Google AutoML Vision API. + +For more information, the documentation at +https://cloud.google.com/vision/automl/docs. +""" + +import argparse +import os + + +def create_dataset(project_id, compute_region, dataset_name, multilabel=False): + """Create a dataset.""" + # [START automl_vision_create_dataset] + # TODO(developer): Uncomment and set the following variables + # project_id = 'PROJECT_ID_HERE' + # compute_region = 'COMPUTE_REGION_HERE' + # dataset_name = 'DATASET_NAME_HERE' + # multilabel = True for multilabel or False for multiclass + + from google.cloud import automl_v1beta1 as automl + + client = automl.AutoMlClient() + + # A resource that represents Google Cloud Platform location. + project_location = client.location_path(project_id, compute_region) + + # Classification type is assigned based on multilabel value. + classification_type = "MULTICLASS" + if multilabel: + classification_type = "MULTILABEL" + + # Specify the image classification type for the dataset. + dataset_metadata = {"classification_type": classification_type} + # Set dataset name and metadata of the dataset. + my_dataset = { + "display_name": dataset_name, + "image_classification_dataset_metadata": dataset_metadata, + } + + # Create a dataset with the dataset metadata in the region. + dataset = client.create_dataset(project_location, my_dataset) + + # Display the dataset information. + print("Dataset name: {}".format(dataset.name)) + print("Dataset id: {}".format(dataset.name.split("/")[-1])) + print("Dataset display name: {}".format(dataset.display_name)) + print("Image classification dataset metadata:") + print("\t{}".format(dataset.image_classification_dataset_metadata)) + print("Dataset example count: {}".format(dataset.example_count)) + print("Dataset create time:") + print("\tseconds: {}".format(dataset.create_time.seconds)) + print("\tnanos: {}".format(dataset.create_time.nanos)) + + # [END automl_vision_create_dataset] + + +def list_datasets(project_id, compute_region, filter_): + """List all datasets.""" + # [START automl_vision_list_datasets] + # TODO(developer): Uncomment and set the following variables + # project_id = 'PROJECT_ID_HERE' + # compute_region = 'COMPUTE_REGION_HERE' + # filter_ = 'filter expression here' + + from google.cloud import automl_v1beta1 as automl + + client = automl.AutoMlClient() + + # A resource that represents Google Cloud Platform location. + project_location = client.location_path(project_id, compute_region) + + # List all the datasets available in the region by applying filter. + response = client.list_datasets(project_location, filter_) + + print("List of datasets:") + for dataset in response: + # Display the dataset information. + print("Dataset name: {}".format(dataset.name)) + print("Dataset id: {}".format(dataset.name.split("/")[-1])) + print("Dataset display name: {}".format(dataset.display_name)) + print("Image classification dataset metadata:") + print("\t{}".format(dataset.image_classification_dataset_metadata)) + print("Dataset example count: {}".format(dataset.example_count)) + print("Dataset create time:") + print("\tseconds: {}".format(dataset.create_time.seconds)) + print("\tnanos: {}".format(dataset.create_time.nanos)) + + # [END automl_vision_list_datasets] + + +def get_dataset(project_id, compute_region, dataset_id): + """Get the dataset.""" + # [START automl_vision_get_dataset] + # TODO(developer): Uncomment and set the following variables + # project_id = 'PROJECT_ID_HERE' + # compute_region = 'COMPUTE_REGION_HERE' + # dataset_id = 'DATASET_ID_HERE' + + from google.cloud import automl_v1beta1 as automl + + client = automl.AutoMlClient() + + # Get the full path of the dataset. + dataset_full_id = client.dataset_path( + project_id, compute_region, dataset_id + ) + + # Get complete detail of the dataset. + dataset = client.get_dataset(dataset_full_id) + + # Display the dataset information. + print("Dataset name: {}".format(dataset.name)) + print("Dataset id: {}".format(dataset.name.split("/")[-1])) + print("Dataset display name: {}".format(dataset.display_name)) + print("Image classification dataset metadata:") + print("\t{}".format(dataset.image_classification_dataset_metadata)) + print("Dataset example count: {}".format(dataset.example_count)) + print("Dataset create time:") + print("\tseconds: {}".format(dataset.create_time.seconds)) + print("\tnanos: {}".format(dataset.create_time.nanos)) + + # [START automl_vision_get_dataset] + + +def import_data(project_id, compute_region, dataset_id, path): + """Import labeled images.""" + # [START automl_vision_import_data] + # TODO(developer): Uncomment and set the following variables + # project_id = 'PROJECT_ID_HERE' + # compute_region = 'COMPUTE_REGION_HERE' + # dataset_id = 'DATASET_ID_HERE' + # path = 'gs://path/to/file.csv' + + from google.cloud import automl_v1beta1 as automl + + client = automl.AutoMlClient() + + # Get the full path of the dataset. + dataset_full_id = client.dataset_path( + project_id, compute_region, dataset_id + ) + + # Get the multiple Google Cloud Storage URIs. + input_uris = path.split(",") + input_config = {"gcs_source": {"input_uris": input_uris}} + + # Import data from the input URI. + response = client.import_data(dataset_full_id, input_config) + + print("Processing import...") + # synchronous check of operation status. + print("Data imported. {}".format(response.result())) + + # [END automl_vision_import_data] + + +def export_data(project_id, compute_region, dataset_id, gcs_uri): + """Export a dataset to a Google Cloud Storage bucket.""" + # [START automl_vision_export_data] + # TODO(developer): Uncomment and set the following variables + # project_id = 'PROJECT_ID_HERE' + # compute_region = 'COMPUTE_REGION_HERE' + # dataset_id = 'DATASET_ID_HERE' + # output_uri: 'gs://location/to/export/data' + + from google.cloud import automl_v1beta1 as automl + + client = automl.AutoMlClient() + + # Get the full path of the dataset. + dataset_full_id = client.dataset_path( + project_id, compute_region, dataset_id + ) + + # Set the output URI + output_config = {"gcs_destination": {"output_uri_prefix": gcs_uri}} + + # Export the dataset to the output URI. + response = client.export_data(dataset_full_id, output_config) + + print("Processing export...") + # synchronous check of operation status. + print("Data exported. {}".format(response.result())) + + # [END automl_vision_export_data] + + +def delete_dataset(project_id, compute_region, dataset_id): + """Delete a dataset""" + # [START automl_vision_delete_dataset] + # TODO(developer): Uncomment and set the following variables + # project_id = 'PROJECT_ID_HERE' + # compute_region = 'COMPUTE_REGION_HERE' + # dataset_id = 'DATASET_ID_HERE' + + from google.cloud import automl_v1beta1 as automl + + client = automl.AutoMlClient() + + # Get the full path of the dataset. + dataset_full_id = client.dataset_path( + project_id, compute_region, dataset_id + ) + + # Delete a dataset. + response = client.delete_dataset(dataset_full_id) + + # synchronous check of operation status. + print("Dataset deleted. {}".format(response.result())) + # [END automl_vision_delete_dataset] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + subparsers = parser.add_subparsers(dest="command") + + create_dataset_parser = subparsers.add_parser( + "create_dataset", help=create_dataset.__doc__ + ) + create_dataset_parser.add_argument("dataset_name") + create_dataset_parser.add_argument( + "multilabel", nargs="?", choices=["False", "True"], default="False" + ) + + list_datasets_parser = subparsers.add_parser( + "list_datasets", help=list_datasets.__doc__ + ) + list_datasets_parser.add_argument("filter_") + + get_dataset_parser = subparsers.add_parser( + "get_dataset", help=get_dataset.__doc__ + ) + get_dataset_parser.add_argument("dataset_id") + + import_data_parser = subparsers.add_parser( + "import_data", help=import_data.__doc__ + ) + import_data_parser.add_argument("dataset_id") + import_data_parser.add_argument("path") + + export_data_parser = subparsers.add_parser( + "export_data", help=export_data.__doc__ + ) + export_data_parser.add_argument("dataset_id") + export_data_parser.add_argument("gcs_uri") + + delete_dataset_parser = subparsers.add_parser( + "delete_dataset", help=delete_dataset.__doc__ + ) + delete_dataset_parser.add_argument("dataset_id") + + project_id = os.environ["PROJECT_ID"] + compute_region = os.environ["REGION_NAME"] + + args = parser.parse_args() + + if args.command == "create_dataset": + multilabel = True if args.multilabel == "True" else False + create_dataset( + project_id, compute_region, args.dataset_name, multilabel + ) + if args.command == "list_datasets": + list_datasets(project_id, compute_region, args.filter_) + if args.command == "get_dataset": + get_dataset(project_id, compute_region, args.dataset_id) + if args.command == "import_data": + import_data(project_id, compute_region, args.dataset_id, args.path) + if args.command == "export_data": + export_data(project_id, compute_region, args.dataset_id, args.gcs_uri) + if args.command == "delete_dataset": + delete_dataset(project_id, compute_region, args.dataset_id) diff --git a/vision/automl/automl_vision_model.py b/vision/automl/automl_vision_model.py new file mode 100755 index 00000000000..0c1f621d8b3 --- /dev/null +++ b/vision/automl/automl_vision_model.py @@ -0,0 +1,445 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to perform basic operations on model +with the Google AutoML Vision API. + +For more information, the documentation at +https://cloud.google.com/vision/automl/docs. +""" + +import argparse +import os + + +def create_model( + project_id, compute_region, dataset_id, model_name, train_budget=24 +): + """Create a model.""" + # [START automl_vision_create_model] + # TODO(developer): Uncomment and set the following variables + # project_id = 'PROJECT_ID_HERE' + # compute_region = 'COMPUTE_REGION_HERE' + # dataset_id = 'DATASET_ID_HERE' + # model_name = 'MODEL_NAME_HERE' + # train_budget = integer amount for maximum cost of model + + from google.cloud import automl_v1beta1 as automl + + client = automl.AutoMlClient() + + # A resource that represents Google Cloud Platform location. + project_location = client.location_path(project_id, compute_region) + + # Set model name and model metadata for the image dataset. + my_model = { + "display_name": model_name, + "dataset_id": dataset_id, + "image_classification_model_metadata": {"train_budget": train_budget} + if train_budget + else {}, + } + + # Create a model with the model metadata in the region. + response = client.create_model(project_location, my_model) + + print("Training operation name: {}".format(response.operation.name)) + print("Training started...") + + # [END automl_vision_create_model] + + +def get_operation_status(operation_full_id): + """Get operation status.""" + # [START automl_vision_get_operation_status] + # TODO(developer): Uncomment and set the following variables + # operation_full_id = + # 'projects//locations//operations/' + + from google.cloud import automl_v1beta1 as automl + + client = automl.AutoMlClient() + + # Get the latest state of a long-running operation. + response = client.transport._operations_client.get_operation( + operation_full_id + ) + + print("Operation status: {}".format(response)) + + # [END automl_vision_get_operation_status] + + +def list_models(project_id, compute_region, filter_): + """List all models.""" + # [START automl_vision_list_models] + # TODO(developer): Uncomment and set the following variables + # project_id = 'PROJECT_ID_HERE' + # compute_region = 'COMPUTE_REGION_HERE' + # filter_ = 'DATASET_ID_HERE' + + from google.cloud import automl_v1beta1 as automl + from google.cloud.automl_v1beta1 import enums + + client = automl.AutoMlClient() + + # A resource that represents Google Cloud Platform location. + project_location = client.location_path(project_id, compute_region) + + # List all the models available in the region by applying filter. + response = client.list_models(project_location, filter_) + + print("List of models:") + for model in response: + # Retrieve deployment state. + if model.deployment_state == enums.Model.DeploymentState.DEPLOYED: + deployment_state = "deployed" + else: + deployment_state = "undeployed" + + # Display the model information. + print("Model name: {}".format(model.name)) + print("Model id: {}".format(model.name.split("/")[-1])) + print("Model display name: {}".format(model.display_name)) + print("Image classification model metadata:") + print( + "Training budget: {}".format( + model.image_classification_model_metadata.train_budget + ) + ) + print( + "Training cost: {}".format( + model.image_classification_model_metadata.train_cost + ) + ) + print( + "Stop reason: {}".format( + model.image_classification_model_metadata.stop_reason + ) + ) + print( + "Base model id: {}".format( + model.image_classification_model_metadata.base_model_id + ) + ) + print("Model create time:") + print("\tseconds: {}".format(model.create_time.seconds)) + print("\tnanos: {}".format(model.create_time.nanos)) + print("Model deployment state: {}".format(deployment_state)) + + # [END automl_vision_list_models] + + +def get_model(project_id, compute_region, model_id): + """Get model details.""" + # [START automl_vision_get_model] + # TODO(developer): Uncomment and set the following variables + # project_id = 'PROJECT_ID_HERE' + # compute_region = 'COMPUTE_REGION_HERE' + # model_id = 'MODEL_ID_HERE' + + from google.cloud import automl_v1beta1 as automl + from google.cloud.automl_v1beta1 import enums + + client = automl.AutoMlClient() + + # Get the full path of the model. + model_full_id = client.model_path(project_id, compute_region, model_id) + + # Get complete detail of the model. + model = client.get_model(model_full_id) + + # Retrieve deployment state. + if model.deployment_state == enums.Model.DeploymentState.DEPLOYED: + deployment_state = "deployed" + else: + deployment_state = "undeployed" + + # Display the model information. + print("Model name: {}".format(model.name)) + print("Model id: {}".format(model.name.split("/")[-1])) + print("Model display name: {}".format(model.display_name)) + print("Image classification model metadata:") + print( + "Training budget: {}".format( + model.image_classification_model_metadata.train_budget + ) + ) + print( + "Training cost: {}".format( + model.image_classification_model_metadata.train_cost + ) + ) + print( + "Stop reason: {}".format( + model.image_classification_model_metadata.stop_reason + ) + ) + print( + "Base model id: {}".format( + model.image_classification_model_metadata.base_model_id + ) + ) + print("Model create time:") + print("\tseconds: {}".format(model.create_time.seconds)) + print("\tnanos: {}".format(model.create_time.nanos)) + print("Model deployment state: {}".format(deployment_state)) + + # [END automl_vision_get_model] + + +def list_model_evaluations(project_id, compute_region, model_id, filter_): + """List model evaluations.""" + # [START automl_vision_list_model_evaluations] + # TODO(developer): Uncomment and set the following variables + # project_id = 'PROJECT_ID_HERE' + # compute_region = 'COMPUTE_REGION_HERE' + # model_id = 'MODEL_ID_HERE' + # filter_ = 'filter expression here' + + from google.cloud import automl_v1beta1 as automl + + client = automl.AutoMlClient() + + # Get the full path of the model. + model_full_id = client.model_path(project_id, compute_region, model_id) + + # List all the model evaluations in the model by applying filter. + response = client.list_model_evaluations(model_full_id, filter_) + + print("List of model evaluations:") + for element in response: + print(element) + + # [END automl_vision_list_model_evaluations] + + +def get_model_evaluation( + project_id, compute_region, model_id, model_evaluation_id +): + """Get model evaluation.""" + # [START automl_vision_get_model_evaluation] + # TODO(developer): Uncomment and set the following variables + # project_id = 'PROJECT_ID_HERE' + # compute_region = 'COMPUTE_REGION_HERE' + # model_id = 'MODEL_ID_HERE' + # model_evaluation_id = 'MODEL_EVALUATION_ID_HERE' + + from google.cloud import automl_v1beta1 as automl + + client = automl.AutoMlClient() + + # Get the full path of the model evaluation. + model_evaluation_full_id = client.model_evaluation_path( + project_id, compute_region, model_id, model_evaluation_id + ) + + # Get complete detail of the model evaluation. + response = client.get_model_evaluation(model_evaluation_full_id) + + print(response) + + # [END automl_vision_get_model_evaluation] + + +def display_evaluation(project_id, compute_region, model_id, filter_): + """Display evaluation.""" + # [START automl_vision_display_evaluation] + # TODO(developer): Uncomment and set the following variables + # project_id = 'PROJECT_ID_HERE' + # compute_region = 'COMPUTE_REGION_HERE' + # model_id = 'MODEL_ID_HERE' + # filter_ = 'filter expression here' + + from google.cloud import automl_v1beta1 as automl + + client = automl.AutoMlClient() + + # Get the full path of the model. + model_full_id = client.model_path(project_id, compute_region, model_id) + + # List all the model evaluations in the model by applying filter. + response = client.list_model_evaluations(model_full_id, filter_) + + # Iterate through the results. + for element in response: + # There is evaluation for each class in a model and for overall model. + # Get only the evaluation of overall model. + if not element.annotation_spec_id: + model_evaluation_id = element.name.split("/")[-1] + + # Resource name for the model evaluation. + model_evaluation_full_id = client.model_evaluation_path( + project_id, compute_region, model_id, model_evaluation_id + ) + + # Get a model evaluation. + model_evaluation = client.get_model_evaluation(model_evaluation_full_id) + + class_metrics = model_evaluation.classification_evaluation_metrics + confidence_metrics_entries = class_metrics.confidence_metrics_entry + + # Showing model score based on threshold of 0.5 + for confidence_metrics_entry in confidence_metrics_entries: + if confidence_metrics_entry.confidence_threshold == 0.5: + print("Precision and recall are based on a score threshold of 0.5") + print( + "Model Precision: {}%".format( + round(confidence_metrics_entry.precision * 100, 2) + ) + ) + print( + "Model Recall: {}%".format( + round(confidence_metrics_entry.recall * 100, 2) + ) + ) + print( + "Model F1 score: {}%".format( + round(confidence_metrics_entry.f1_score * 100, 2) + ) + ) + print( + "Model Precision@1: {}%".format( + round(confidence_metrics_entry.precision_at1 * 100, 2) + ) + ) + print( + "Model Recall@1: {}%".format( + round(confidence_metrics_entry.recall_at1 * 100, 2) + ) + ) + print( + "Model F1 score@1: {}%".format( + round(confidence_metrics_entry.f1_score_at1 * 100, 2) + ) + ) + + # [END automl_vision_display_evaluation] + + +def delete_model(project_id, compute_region, model_id): + """Delete a model.""" + # [START automl_vision_delete_model] + # TODO(developer): Uncomment and set the following variables + # project_id = 'PROJECT_ID_HERE' + # compute_region = 'COMPUTE_REGION_HERE' + # model_id = 'MODEL_ID_HERE' + + from google.cloud import automl_v1beta1 as automl + + client = automl.AutoMlClient() + + # Get the full path of the model. + model_full_id = client.model_path(project_id, compute_region, model_id) + + # Delete a model. + response = client.delete_model(model_full_id) + + # synchronous check of operation status. + print("Model deleted. {}".format(response.result())) + + # [END automl_vision_delete_model] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + subparsers = parser.add_subparsers(dest="command") + + create_model_parser = subparsers.add_parser( + "create_model", help=create_model.__doc__ + ) + create_model_parser.add_argument("dataset_id") + create_model_parser.add_argument("model_name") + create_model_parser.add_argument( + "train_budget", type=int, nargs="?", default=0 + ) + + get_operation_status_parser = subparsers.add_parser( + "get_operation_status", help=get_operation_status.__doc__ + ) + get_operation_status_parser.add_argument("operation_full_id") + + list_models_parser = subparsers.add_parser( + "list_models", help=list_models.__doc__ + ) + list_models_parser.add_argument("filter_") + + get_model_parser = subparsers.add_parser( + "get_model", help=get_model.__doc__ + ) + get_model_parser.add_argument("model_id") + + list_model_evaluations_parser = subparsers.add_parser( + "list_model_evaluations", help=list_model_evaluations.__doc__ + ) + list_model_evaluations_parser.add_argument("model_id") + list_model_evaluations_parser.add_argument( + "filter_", nargs="?", default="" + ) + + get_model_evaluation_parser = subparsers.add_parser( + "get_model_evaluation", help=get_model_evaluation.__doc__ + ) + get_model_evaluation_parser.add_argument("model_id") + get_model_evaluation_parser.add_argument("model_evaluation_id") + + display_evaluation_parser = subparsers.add_parser( + "display_evaluation", help=display_evaluation.__doc__ + ) + display_evaluation_parser.add_argument("model_id") + display_evaluation_parser.add_argument("filter_", nargs="?", default="") + + delete_model_parser = subparsers.add_parser( + "delete_model", help=delete_model.__doc__ + ) + delete_model_parser.add_argument("model_id") + + project_id = os.environ["PROJECT_ID"] + compute_region = os.environ["REGION_NAME"] + + args = parser.parse_args() + + if args.command == "create_model": + create_model( + project_id, + compute_region, + args.dataset_id, + args.model_name, + args.train_budget, + ) + if args.command == "get_operation_status": + get_operation_status(args.operation_full_id) + if args.command == "list_models": + list_models(project_id, compute_region, args.filter_) + if args.command == "get_model": + get_model(project_id, compute_region, args.model_id) + if args.command == "list_model_evaluations": + list_model_evaluations( + project_id, compute_region, args.model_id, args.filter_ + ) + if args.command == "get_model_evaluation": + get_model_evaluation( + project_id, compute_region, args.model_id, args.model_evaluation_id + ) + if args.command == "display_evaluation": + display_evaluation( + project_id, compute_region, args.model_id, args.filter_ + ) + if args.command == "delete_model": + delete_model(project_id, compute_region, args.model_id) diff --git a/vision/automl/automl_vision_predict.py b/vision/automl/automl_vision_predict.py new file mode 100755 index 00000000000..0478c5c2feb --- /dev/null +++ b/vision/automl/automl_vision_predict.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to perform basic operations on prediction +with the Google AutoML Vision API. + +For more information, the documentation at +https://cloud.google.com/vision/automl/docs. +""" + +import argparse +import os + + +def predict( + project_id, compute_region, model_id, file_path, score_threshold="" +): + """Make a prediction for an image.""" + # [START automl_vision_predict] + # TODO(developer): Uncomment and set the following variables + # project_id = 'PROJECT_ID_HERE' + # compute_region = 'COMPUTE_REGION_HERE' + # model_id = 'MODEL_ID_HERE' + # file_path = '/local/path/to/file' + # score_threshold = 'value from 0.0 to 0.5' + + from google.cloud import automl_v1beta1 as automl + + automl_client = automl.AutoMlClient() + + # Get the full path of the model. + model_full_id = automl_client.model_path( + project_id, compute_region, model_id + ) + + # Create client for prediction service. + prediction_client = automl.PredictionServiceClient() + + # Read the image and assign to payload. + with open(file_path, "rb") as image_file: + content = image_file.read() + payload = {"image": {"image_bytes": content}} + + # params is additional domain-specific parameters. + # score_threshold is used to filter the result + # Initialize params + params = {} + if score_threshold: + params = {"score_threshold": score_threshold} + + response = prediction_client.predict(model_full_id, payload, params) + print("Prediction results:") + for result in response.payload: + print("Predicted class name: {}".format(result.display_name)) + print("Predicted class score: {}".format(result.classification.score)) + + # [END automl_vision_predict] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + subparsers = parser.add_subparsers(dest="command") + + predict_parser = subparsers.add_parser("predict", help=predict.__doc__) + predict_parser.add_argument("model_id") + predict_parser.add_argument("file_path") + predict_parser.add_argument("score_threshold", nargs="?", default="") + + project_id = os.environ["PROJECT_ID"] + compute_region = os.environ["REGION_NAME"] + + args = parser.parse_args() + + if args.command == "predict": + predict( + project_id, + compute_region, + args.model_id, + args.file_path, + args.score_threshold, + ) diff --git a/vision/automl/dataset_test.py b/vision/automl/dataset_test.py new file mode 100644 index 00000000000..1da0433b53a --- /dev/null +++ b/vision/automl/dataset_test.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import os + +import pytest + +import automl_vision_dataset + +project_id = os.environ["GCLOUD_PROJECT"] +compute_region = "us-central1" + + +@pytest.mark.slow +def test_dataset_create_import_delete(capsys): + # create dataset + dataset_name = "test_" + datetime.datetime.now().strftime("%Y%m%d%H%M%S") + automl_vision_dataset.create_dataset( + project_id, compute_region, dataset_name + ) + out, _ = capsys.readouterr() + create_dataset_output = out.splitlines() + assert "Dataset id: " in create_dataset_output[1] + + # import data + dataset_id = create_dataset_output[1].split()[2] + data = "gs://{}-vcm/flower_traindata.csv".format(project_id) + automl_vision_dataset.import_data( + project_id, compute_region, dataset_id, data + ) + out, _ = capsys.readouterr() + assert "Data imported." in out + + # delete dataset + automl_vision_dataset.delete_dataset( + project_id, compute_region, dataset_id + ) + out, _ = capsys.readouterr() + assert "Dataset deleted." in out + + +def test_dataset_list_get(capsys): + # list datasets + automl_vision_dataset.list_datasets(project_id, compute_region, "") + out, _ = capsys.readouterr() + list_dataset_output = out.splitlines() + assert "Dataset id: " in list_dataset_output[2] + + # get dataset + dataset_id = list_dataset_output[2].split()[2] + automl_vision_dataset.get_dataset(project_id, compute_region, dataset_id) + out, _ = capsys.readouterr() + assert "Dataset name: " in out diff --git a/vision/automl/model_test.py b/vision/automl/model_test.py new file mode 100644 index 00000000000..9792c341682 --- /dev/null +++ b/vision/automl/model_test.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import os + +from google.cloud import automl_v1beta1 as automl +import pytest + +import automl_vision_model + +project_id = os.environ["GCLOUD_PROJECT"] +compute_region = "us-central1" + + +@pytest.mark.skip(reason="creates too many models") +def test_model_create_status_delete(capsys): + # create model + client = automl.AutoMlClient() + model_name = "test_" + datetime.datetime.now().strftime("%Y%m%d%H%M%S") + project_location = client.location_path(project_id, compute_region) + my_model = { + "display_name": model_name, + "dataset_id": "3946265060617537378", + "image_classification_model_metadata": {"train_budget": 24}, + } + response = client.create_model(project_location, my_model) + operation_name = response.operation.name + assert operation_name + + # get operation status + automl_vision_model.get_operation_status(operation_name) + out, _ = capsys.readouterr() + assert "Operation status: " in out + + # cancel operation + response.cancel() + + +def test_model_list_get_evaluate(capsys): + # list models + automl_vision_model.list_models(project_id, compute_region, "") + out, _ = capsys.readouterr() + list_models_output = out.splitlines() + assert "Model id: " in list_models_output[2] + + # get model + model_id = list_models_output[2].split()[2] + automl_vision_model.get_model(project_id, compute_region, model_id) + out, _ = capsys.readouterr() + assert "Model name: " in out + + # list model evaluations + automl_vision_model.list_model_evaluations( + project_id, compute_region, model_id, "" + ) + out, _ = capsys.readouterr() + list_evals_output = out.splitlines() + assert "name: " in list_evals_output[1] + + # get model evaluation + model_evaluation_id = list_evals_output[1].split("/")[-1][:-1] + automl_vision_model.get_model_evaluation( + project_id, compute_region, model_id, model_evaluation_id + ) + out, _ = capsys.readouterr() + assert "evaluation_metric" in out diff --git a/vision/automl/predict_test.py b/vision/automl/predict_test.py new file mode 100644 index 00000000000..a0e3eba833b --- /dev/null +++ b/vision/automl/predict_test.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import automl_vision_predict + +project_id = os.environ["GCLOUD_PROJECT"] +compute_region = "us-central1" + + +def test_predict(capsys): + model_id = "ICN7383667271543079510" + automl_vision_predict.predict( + project_id, compute_region, model_id, "resources/test.png" + ) + out, _ = capsys.readouterr() + assert "maize" in out diff --git a/vision/automl/requirements.txt b/vision/automl/requirements.txt new file mode 100644 index 00000000000..db96c59966c --- /dev/null +++ b/vision/automl/requirements.txt @@ -0,0 +1 @@ +google-cloud-automl==0.1.2 diff --git a/vision/automl/resources/test.png b/vision/automl/resources/test.png new file mode 100644 index 00000000000..653342a46e5 Binary files /dev/null and b/vision/automl/resources/test.png differ diff --git a/vision/cloud-client/crop_hints/.gitignore b/vision/cloud-client/crop_hints/.gitignore new file mode 100644 index 00000000000..69e003866fb --- /dev/null +++ b/vision/cloud-client/crop_hints/.gitignore @@ -0,0 +1,2 @@ +output-crop.jpg +output-hint.jpg diff --git a/vision/cloud-client/crop_hints/README.rst b/vision/cloud-client/crop_hints/README.rst new file mode 100644 index 00000000000..4ca8652f5a2 --- /dev/null +++ b/vision/cloud-client/crop_hints/README.rst @@ -0,0 +1,111 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Vision API Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=vision/cloud-client/crop_hints/README.rst + + +This directory contains samples for Google Cloud Vision API. `Google Cloud Vision API`_ allows developers to easily integrate vision detection features within applications, including image labeling, face and landmark detection, optical character recognition (OCR), and tagging of explicit content. + +- See the `migration guide`_ for information about migrating to Python client library v0.25.1. + +.. _migration guide: https://cloud.google.com/vision/docs/python-client-migration + + + + +.. _Google Cloud Vision API: https://cloud.google.com/vision/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Crop Hints Tutorial ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=vision/cloud-client/crop_hints/crop_hints.py,vision/cloud-client/crop_hints/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python crop_hints.py + + usage: crop_hints.py [-h] image_file mode + + positional arguments: + image_file The image you'd like to crop. + mode Set to "crop" or "draw". + + optional arguments: + -h, --help show this help message and exit + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/vision/cloud-client/crop_hints/README.rst.in b/vision/cloud-client/crop_hints/README.rst.in new file mode 100644 index 00000000000..113d2771044 --- /dev/null +++ b/vision/cloud-client/crop_hints/README.rst.in @@ -0,0 +1,30 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Vision API + short_name: Cloud Vision API + url: https://cloud.google.com/vision/docs + description: > + `Google Cloud Vision API`_ allows developers to easily integrate vision + detection features within applications, including image labeling, face and + landmark detection, optical character recognition (OCR), and tagging of + explicit content. + + + - See the `migration guide`_ for information about migrating to Python client library v0.25.1. + + + .. _migration guide: https://cloud.google.com/vision/docs/python-client-migration + +setup: +- auth +- install_deps + +samples: +- name: Crop Hints Tutorial + file: crop_hints.py + show_help: True + +cloud_client_library: true + +folder: vision/cloud-client/crop_hints \ No newline at end of file diff --git a/vision/cloud-client/crop_hints/crop_hints.py b/vision/cloud-client/crop_hints/crop_hints.py new file mode 100644 index 00000000000..882cf5499ed --- /dev/null +++ b/vision/cloud-client/crop_hints/crop_hints.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Outputs a cropped image or an image highlighting crop regions on an image. + +Examples: + python crop_hints.py resources/cropme.jpg draw + python crop_hints.py resources/cropme.jpg crop +""" +# [START vision_crop_hints_tutorial] +# [START vision_crop_hints_tutorial_imports] +import argparse +import io + +from google.cloud import vision +from google.cloud.vision import types +from PIL import Image, ImageDraw +# [END vision_crop_hints_tutorial_imports] + + +def get_crop_hint(path): + # [START vision_crop_hints_tutorial_get_crop_hints] + """Detect crop hints on a single image and return the first result.""" + client = vision.ImageAnnotatorClient() + + with io.open(path, 'rb') as image_file: + content = image_file.read() + + image = types.Image(content=content) + + crop_hints_params = types.CropHintsParams(aspect_ratios=[1.77]) + image_context = types.ImageContext(crop_hints_params=crop_hints_params) + + response = client.crop_hints(image=image, image_context=image_context) + hints = response.crop_hints_annotation.crop_hints + + # Get bounds for the first crop hint using an aspect ratio of 1.77. + vertices = hints[0].bounding_poly.vertices + # [END vision_crop_hints_tutorial_get_crop_hints] + + return vertices + + +def draw_hint(image_file): + """Draw a border around the image using the hints in the vector list.""" + # [START vision_crop_hints_tutorial_draw_crop_hints] + vects = get_crop_hint(image_file) + + im = Image.open(image_file) + draw = ImageDraw.Draw(im) + draw.polygon([ + vects[0].x, vects[0].y, + vects[1].x, vects[1].y, + vects[2].x, vects[2].y, + vects[3].x, vects[3].y], None, 'red') + im.save('output-hint.jpg', 'JPEG') + print('Saved new image to output-crop.jpg') + # [END vision_crop_hints_tutorial_draw_crop_hints] + + +def crop_to_hint(image_file): + """Crop the image using the hints in the vector list.""" + # [START vision_crop_hints_tutorial_crop_to_hints] + vects = get_crop_hint(image_file) + + im = Image.open(image_file) + im2 = im.crop([vects[0].x, vects[0].y, + vects[2].x - 1, vects[2].y - 1]) + im2.save('output-crop.jpg', 'JPEG') + print('Saved new image to output-crop.jpg') + # [END vision_crop_hints_tutorial_crop_to_hints] + + +if __name__ == '__main__': + # [START vision_crop_hints_tutorial_run_application] + parser = argparse.ArgumentParser() + parser.add_argument('image_file', help='The image you\'d like to crop.') + parser.add_argument('mode', help='Set to "crop" or "draw".') + args = parser.parse_args() + + parser = argparse.ArgumentParser() + + if args.mode == 'crop': + crop_to_hint(args.image_file) + elif args.mode == 'draw': + draw_hint(args.image_file) + # [END vision_crop_hints_tutorial_run_application] +# [END vision_crop_hints_tutorial] diff --git a/vision/cloud-client/crop_hints/crop_hints_test.py b/vision/cloud-client/crop_hints/crop_hints_test.py new file mode 100644 index 00000000000..2ba900f48b1 --- /dev/null +++ b/vision/cloud-client/crop_hints/crop_hints_test.py @@ -0,0 +1,37 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import crop_hints + + +def test_crop(capsys): + """Checks the output image for cropping the image is created.""" + file_name = os.path.join( + os.path.dirname(__file__), + 'resources/cropme.jpg') + crop_hints.crop_to_hint(file_name) + out, _ = capsys.readouterr() + assert os.path.isfile('output-crop.jpg') + + +def test_draw(capsys): + """Checks the output image for drawing the crop hint is created.""" + file_name = os.path.join( + os.path.dirname(__file__), + 'resources/cropme.jpg') + crop_hints.draw_hint(file_name) + out, _ = capsys.readouterr() + assert os.path.isfile('output-hint.jpg') diff --git a/vision/cloud-client/crop_hints/requirements.txt b/vision/cloud-client/crop_hints/requirements.txt new file mode 100644 index 00000000000..044a582a532 --- /dev/null +++ b/vision/cloud-client/crop_hints/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-vision==0.35.2 +pillow==5.4.1 diff --git a/vision/cloud-client/crop_hints/resources/cropme.jpg b/vision/cloud-client/crop_hints/resources/cropme.jpg new file mode 100644 index 00000000000..50145895863 Binary files /dev/null and b/vision/cloud-client/crop_hints/resources/cropme.jpg differ diff --git a/vision/cloud-client/detect/README.rst b/vision/cloud-client/detect/README.rst new file mode 100644 index 00000000000..5e5f4f57ba3 --- /dev/null +++ b/vision/cloud-client/detect/README.rst @@ -0,0 +1,239 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Vision API Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=vision/cloud-client/detect/README.rst + + +This directory contains samples for Google Cloud Vision API. `Google Cloud Vision API`_ allows developers to easily integrate vision detection features within applications, including image labeling, face and landmark detection, optical character recognition (OCR), and tagging of explicit content. + +- See the `migration guide`_ for information about migrating to Python client library v0.25.1. + +.. _migration guide: https://cloud.google.com/vision/docs/python-client-migration + + + + +.. _Google Cloud Vision API: https://cloud.google.com/vision/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Detect ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=vision/cloud-client/detect/detect.py,vision/cloud-client/detect/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python detect.py + + usage: detect.py [-h] + {faces,faces-uri,labels,labels-uri,landmarks,landmarks-uri,text,text-uri,logos,logos-uri,safe-search,safe-search-uri,properties,properties-uri,web,web-uri,web-geo,web-geo-uri,crophints,crophints-uri,document,document-uri,ocr-uri,object-localization,object-localization-uri} + ... + + This application demonstrates how to perform basic operations with the + Google Cloud Vision API. + + Example Usage: + python detect.py text ./resources/wakeupcat.jpg + python detect.py labels ./resources/landmark.jpg + python detect.py web ./resources/landmark.jpg + python detect.py web-uri http://wheresgus.com/dog.JPG + python detect.py web-geo ./resources/city.jpg + python detect.py faces-uri gs://your-bucket/file.jpg + python detect.py ocr-uri gs://python-docs-samples-tests/HodgeConj.pdf gs://BUCKET_NAME/PREFIX/ + python detect.py object-localization ./resources/puppies.jpg + python detect.py object-localization-uri gs://... + + For more information, the documentation at + https://cloud.google.com/vision/docs. + + positional arguments: + {faces,faces-uri,labels,labels-uri,landmarks,landmarks-uri,text,text-uri,logos,logos-uri,safe-search,safe-search-uri,properties,properties-uri,web,web-uri,web-geo,web-geo-uri,crophints,crophints-uri,document,document-uri,ocr-uri,object-localization,object-localization-uri} + faces Detects faces in an image. + faces-uri Detects faces in the file located in Google Cloud + Storage or the web. + labels Detects labels in the file. + labels-uri Detects labels in the file located in Google Cloud + Storage or on the Web. + landmarks Detects landmarks in the file. + landmarks-uri Detects landmarks in the file located in Google Cloud + Storage or on the Web. + text Detects text in the file. + text-uri Detects text in the file located in Google Cloud + Storage or on the Web. + logos Detects logos in the file. + logos-uri Detects logos in the file located in Google Cloud + Storage or on the Web. + safe-search Detects unsafe features in the file. + safe-search-uri Detects unsafe features in the file located in Google + Cloud Storage or on the Web. + properties Detects image properties in the file. + properties-uri Detects image properties in the file located in Google + Cloud Storage or on the Web. + web Detects web annotations given an image. + web-uri Detects web annotations in the file located in Google + Cloud Storage. + web-geo Detects web annotations given an image, using the + geotag metadata in the image to detect web entities. + web-geo-uri Detects web annotations given an image in the file + located in Google Cloud Storage., using the geotag + metadata in the image to detect web entities. + crophints Detects crop hints in an image. + crophints-uri Detects crop hints in the file located in Google Cloud + Storage. + document Detects document features in an image. + document-uri Detects document features in the file located in + Google Cloud Storage. + ocr-uri OCR with PDF/TIFF as source files on GCS + object-localization + OCR with PDF/TIFF as source files on GCS + object-localization-uri + OCR with PDF/TIFF as source files on GCS + + optional arguments: + -h, --help show this help message and exit + + + +Beta Detect ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=vision/cloud-client/detect/beta_snippets.py,vision/cloud-client/detect/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python beta_snippets.py + + usage: beta_snippets.py [-h] + {object-localization,object-localization-uri,handwritten-ocr,handwritten-ocr-uri,batch-annotate-files,batch-annotate-files-uri,batch-annotate-images-uri} + ... + + Google Cloud Vision API Python Beta Snippets + + Example Usage: + python beta_snippets.py -h + python beta_snippets.py object-localization INPUT_IMAGE + python beta_snippets.py object-localization-uri gs://... + python beta_snippets.py handwritten-ocr INPUT_IMAGE + python beta_snippets.py handwritten-ocr-uri gs://... + python beta_snippets.py batch-annotate-files INPUT_PDF + python beta_snippets.py batch-annotate-files-uri gs://... + python beta_snippets.py batch-annotate-images-uri gs://... gs://... + + For more information, the documentation at + https://cloud.google.com/vision/docs. + + positional arguments: + {object-localization,object-localization-uri,handwritten-ocr,handwritten-ocr-uri,batch-annotate-files,batch-annotate-files-uri,batch-annotate-images-uri} + object-localization + Localize objects in the local image. Args: path: The + path to the local file. + object-localization-uri + Localize objects in the image on Google Cloud Storage + Args: uri: The path to the file in Google Cloud + Storage (gs://...) + handwritten-ocr Detects handwritten characters in a local image. Args: + path: The path to the local file. + handwritten-ocr-uri + Detects handwritten characters in the file located in + Google Cloud Storage. Args: uri: The path to the file + in Google Cloud Storage (gs://...) + batch-annotate-files + Detects document features in a PDF/TIFF/GIF file. + While your PDF file may have several pages, this API + can process up to 5 pages only. Args: path: The path + to the local file. + batch-annotate-files-uri + Detects document features in a PDF/TIFF/GIF file. + While your PDF file may have several pages, this API + can process up to 5 pages only. Args: uri: The path to + the file in Google Cloud Storage (gs://...) + batch-annotate-images-uri + Batch annotation of images on Google Cloud Storage + asynchronously. Args: image_uri: The path to the image + in Google Cloud Storage (gs://...) gcs_uri: The path + to the output path in Google Cloud Storage (gs://...) + + optional arguments: + -h, --help show this help message and exit + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/vision/cloud-client/detect/README.rst.in b/vision/cloud-client/detect/README.rst.in new file mode 100644 index 00000000000..0d105411cff --- /dev/null +++ b/vision/cloud-client/detect/README.rst.in @@ -0,0 +1,33 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Vision API + short_name: Cloud Vision API + url: https://cloud.google.com/vision/docs + description: > + `Google Cloud Vision API`_ allows developers to easily integrate vision + detection features within applications, including image labeling, face and + landmark detection, optical character recognition (OCR), and tagging of + explicit content. + + + - See the `migration guide`_ for information about migrating to Python client library v0.25.1. + + + .. _migration guide: https://cloud.google.com/vision/docs/python-client-migration + +setup: +- auth +- install_deps + +samples: +- name: Detect + file: detect.py + show_help: True +- name: Beta Detect + file: beta_snippets.py + show_help: True + +cloud_client_library: true + +folder: vision/cloud-client/detect \ No newline at end of file diff --git a/vision/cloud-client/detect/beta_snippets.py b/vision/cloud-client/detect/beta_snippets.py new file mode 100644 index 00000000000..5d792b9138b --- /dev/null +++ b/vision/cloud-client/detect/beta_snippets.py @@ -0,0 +1,401 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Google Cloud Vision API Python Beta Snippets + +Example Usage: +python beta_snippets.py -h +python beta_snippets.py object-localization INPUT_IMAGE +python beta_snippets.py object-localization-uri gs://... +python beta_snippets.py handwritten-ocr INPUT_IMAGE +python beta_snippets.py handwritten-ocr-uri gs://... +python beta_snippets.py batch-annotate-files INPUT_PDF +python beta_snippets.py batch-annotate-files-uri gs://... +python beta_snippets.py batch-annotate-images-uri gs://... gs://... + + +For more information, the documentation at +https://cloud.google.com/vision/docs. +""" + +import argparse +import io + + +# [START vision_localize_objects_beta] +def localize_objects(path): + """Localize objects in the local image. + + Args: + path: The path to the local file. + """ + from google.cloud import vision_v1p3beta1 as vision + client = vision.ImageAnnotatorClient() + + with open(path, 'rb') as image_file: + content = image_file.read() + image = vision.types.Image(content=content) + + objects = client.object_localization( + image=image).localized_object_annotations + + print('Number of objects found: {}'.format(len(objects))) + for object_ in objects: + print('\n{} (confidence: {})'.format(object_.name, object_.score)) + print('Normalized bounding polygon vertices: ') + for vertex in object_.bounding_poly.normalized_vertices: + print(' - ({}, {})'.format(vertex.x, vertex.y)) +# [END vision_localize_objects_beta] + + +# [START vision_localize_objects_gcs_beta] +def localize_objects_uri(uri): + """Localize objects in the image on Google Cloud Storage + + Args: + uri: The path to the file in Google Cloud Storage (gs://...) + """ + from google.cloud import vision_v1p3beta1 as vision + client = vision.ImageAnnotatorClient() + + image = vision.types.Image() + image.source.image_uri = uri + + objects = client.object_localization( + image=image).localized_object_annotations + + print('Number of objects found: {}'.format(len(objects))) + for object_ in objects: + print('\n{} (confidence: {})'.format(object_.name, object_.score)) + print('Normalized bounding polygon vertices: ') + for vertex in object_.bounding_poly.normalized_vertices: + print(' - ({}, {})'.format(vertex.x, vertex.y)) +# [END vision_localize_objects_gcs_beta] + + +# [START vision_handwritten_ocr_beta] +def detect_handwritten_ocr(path): + """Detects handwritten characters in a local image. + + Args: + path: The path to the local file. + """ + from google.cloud import vision_v1p3beta1 as vision + client = vision.ImageAnnotatorClient() + + with io.open(path, 'rb') as image_file: + content = image_file.read() + + image = vision.types.Image(content=content) + + # Language hint codes for handwritten OCR: + # en-t-i0-handwrit, mul-Latn-t-i0-handwrit + # Note: Use only one language hint code per request for handwritten OCR. + image_context = vision.types.ImageContext( + language_hints=['en-t-i0-handwrit']) + + response = client.document_text_detection(image=image, + image_context=image_context) + + print('Full Text: {}'.format(response.full_text_annotation.text)) + for page in response.full_text_annotation.pages: + for block in page.blocks: + print('\nBlock confidence: {}\n'.format(block.confidence)) + + for paragraph in block.paragraphs: + print('Paragraph confidence: {}'.format( + paragraph.confidence)) + + for word in paragraph.words: + word_text = ''.join([ + symbol.text for symbol in word.symbols + ]) + print('Word text: {} (confidence: {})'.format( + word_text, word.confidence)) + + for symbol in word.symbols: + print('\tSymbol: {} (confidence: {})'.format( + symbol.text, symbol.confidence)) +# [END vision_handwritten_ocr_beta] + + +# [START vision_handwritten_ocr_gcs_beta] +def detect_handwritten_ocr_uri(uri): + """Detects handwritten characters in the file located in Google Cloud + Storage. + + Args: + uri: The path to the file in Google Cloud Storage (gs://...) + """ + from google.cloud import vision_v1p3beta1 as vision + client = vision.ImageAnnotatorClient() + image = vision.types.Image() + image.source.image_uri = uri + + # Language hint codes for handwritten OCR: + # en-t-i0-handwrit, mul-Latn-t-i0-handwrit + # Note: Use only one language hint code per request for handwritten OCR. + image_context = vision.types.ImageContext( + language_hints=['en-t-i0-handwrit']) + + response = client.document_text_detection(image=image, + image_context=image_context) + + print('Full Text: {}'.format(response.full_text_annotation.text)) + for page in response.full_text_annotation.pages: + for block in page.blocks: + print('\nBlock confidence: {}\n'.format(block.confidence)) + + for paragraph in block.paragraphs: + print('Paragraph confidence: {}'.format( + paragraph.confidence)) + + for word in paragraph.words: + word_text = ''.join([ + symbol.text for symbol in word.symbols + ]) + print('Word text: {} (confidence: {})'.format( + word_text, word.confidence)) + + for symbol in word.symbols: + print('\tSymbol: {} (confidence: {})'.format( + symbol.text, symbol.confidence)) +# [END vision_handwritten_ocr_gcs_beta] + + +# [START vision_batch_annotate_files_beta] +def detect_batch_annotate_files(path): + """Detects document features in a PDF/TIFF/GIF file. + + While your PDF file may have several pages, + this API can process up to 5 pages only. + + Args: + path: The path to the local file. + """ + from google.cloud import vision_v1p4beta1 as vision + client = vision.ImageAnnotatorClient() + + with open(path, 'rb') as pdf_file: + content = pdf_file.read() + + # Other supported mime_types: image/tiff' or 'image/gif' + mime_type = 'application/pdf' + input_config = vision.types.InputConfig( + content=content, mime_type=mime_type) + + feature = vision.types.Feature( + type=vision.enums.Feature.Type.DOCUMENT_TEXT_DETECTION) + # Annotate the first two pages and the last one (max 5 pages) + # First page starts at 1, and not 0. Last page is -1. + pages = [1, 2, -1] + + request = vision.types.AnnotateFileRequest( + input_config=input_config, + features=[feature], + pages=pages) + + response = client.batch_annotate_files(requests=[request]) + + for image_response in response.responses[0].responses: + for page in image_response.full_text_annotation.pages: + for block in page.blocks: + print('\nBlock confidence: {}\n'.format(block.confidence)) + for par in block.paragraphs: + print('\tParagraph confidence: {}'.format(par.confidence)) + for word in par.words: + symbol_texts = [symbol.text for symbol in word.symbols] + word_text = ''.join(symbol_texts) + print('\t\tWord text: {} (confidence: {})'.format( + word_text, word.confidence)) + for symbol in word.symbols: + print('\t\t\tSymbol: {} (confidence: {})'.format( + symbol.text, symbol.confidence)) +# [END vision_batch_annotate_files_beta] + + +# [START vision_batch_annotate_files_gcs_beta] +def detect_batch_annotate_files_uri(gcs_uri): + """Detects document features in a PDF/TIFF/GIF file. + + While your PDF file may have several pages, + this API can process up to 5 pages only. + + Args: + uri: The path to the file in Google Cloud Storage (gs://...) + """ + from google.cloud import vision_v1p4beta1 as vision + client = vision.ImageAnnotatorClient() + + # Other supported mime_types: image/tiff' or 'image/gif' + mime_type = 'application/pdf' + input_config = vision.types.InputConfig( + gcs_source=vision.types.GcsSource(uri=gcs_uri), mime_type=mime_type) + + feature = vision.types.Feature( + type=vision.enums.Feature.Type.DOCUMENT_TEXT_DETECTION) + # Annotate the first two pages and the last one (max 5 pages) + # First page starts at 1, and not 0. Last page is -1. + pages = [1, 2, -1] + + request = vision.types.AnnotateFileRequest( + input_config=input_config, + features=[feature], + pages=pages) + + response = client.batch_annotate_files(requests=[request]) + + for image_response in response.responses[0].responses: + for page in image_response.full_text_annotation.pages: + for block in page.blocks: + print('\nBlock confidence: {}\n'.format(block.confidence)) + for par in block.paragraphs: + print('\tParagraph confidence: {}'.format(par.confidence)) + for word in par.words: + symbol_texts = [symbol.text for symbol in word.symbols] + word_text = ''.join(symbol_texts) + print('\t\tWord text: {} (confidence: {})'.format( + word_text, word.confidence)) + for symbol in word.symbols: + print('\t\t\tSymbol: {} (confidence: {})'.format( + symbol.text, symbol.confidence)) +# [END vision_batch_annotate_files_gcs_beta] + + +# [START vision_async_batch_annotate_images_beta] +def async_batch_annotate_images_uri(input_image_uri, output_uri): + """Batch annotation of images on Google Cloud Storage asynchronously. + + Args: + input_image_uri: The path to the image in Google Cloud Storage (gs://...) + output_uri: The path to the output path in Google Cloud Storage (gs://...) + """ + import re + + from google.cloud import storage + from google.protobuf import json_format + from google.cloud import vision_v1p4beta1 as vision + client = vision.ImageAnnotatorClient() + + # Construct the request for the image(s) to be annotated: + image_source = vision.types.ImageSource(image_uri=input_image_uri) + image = vision.types.Image(source=image_source) + features = [ + vision.types.Feature(type=vision.enums.Feature.Type.LABEL_DETECTION), + vision.types.Feature(type=vision.enums.Feature.Type.TEXT_DETECTION), + vision.types.Feature(type=vision.enums.Feature.Type.IMAGE_PROPERTIES), + ] + requests = [ + vision.types.AnnotateImageRequest(image=image, features=features), + ] + + gcs_destination = vision.types.GcsDestination(uri=output_uri) + output_config = vision.types.OutputConfig( + gcs_destination=gcs_destination, batch_size=2) + + operation = client.async_batch_annotate_images( + requests=requests, output_config=output_config) + + print('Waiting for the operation to finish.') + operation.result(timeout=10000) + + # Once the request has completed and the output has been + # written to Google Cloud Storage, we can list all the output files. + storage_client = storage.Client() + + match = re.match(r'gs://([^/]+)/(.+)', output_uri) + bucket_name = match.group(1) + prefix = match.group(2) + + bucket = storage_client.get_bucket(bucket_name=bucket_name) + + # Lists objects with the given prefix. + blob_list = list(bucket.list_blobs(prefix=prefix)) + print('Output files:') + for blob in blob_list: + print(blob.name) + + # Processes the first output file from Google Cloud Storage. + # Since we specified batch_size=2, the first response contains + # annotations for the first two annotate image requests. + output = blob_list[0] + + json_string = output.download_as_string() + response = json_format.Parse(json_string, + vision.types.BatchAnnotateImagesResponse()) + + # Prints the actual response for the first annotate image request. + print(u'The annotation response for the first request: {}'.format( + response.responses[0])) +# [END vision_async_batch_annotate_images_beta] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + subparsers = parser.add_subparsers(dest='command') + + object_parser = subparsers.add_parser( + 'object-localization', help=localize_objects.__doc__) + object_parser.add_argument('path') + + object_uri_parser = subparsers.add_parser( + 'object-localization-uri', help=localize_objects_uri.__doc__) + object_uri_parser.add_argument('uri') + + handwritten_parser = subparsers.add_parser( + 'handwritten-ocr', help=detect_handwritten_ocr.__doc__) + handwritten_parser.add_argument('path') + + handwritten_uri_parser = subparsers.add_parser( + 'handwritten-ocr-uri', help=detect_handwritten_ocr_uri.__doc__) + handwritten_uri_parser.add_argument('uri') + + batch_annotate_parser = subparsers.add_parser( + 'batch-annotate-files', help=detect_batch_annotate_files.__doc__) + batch_annotate_parser.add_argument('path') + + batch_annotate_uri_parser = subparsers.add_parser( + 'batch-annotate-files-uri', + help=detect_batch_annotate_files_uri.__doc__) + batch_annotate_uri_parser.add_argument('uri') + + batch_annotate__image_uri_parser = subparsers.add_parser( + 'batch-annotate-images-uri', + help=async_batch_annotate_images_uri.__doc__) + batch_annotate__image_uri_parser.add_argument('uri') + batch_annotate__image_uri_parser.add_argument('output') + + args = parser.parse_args() + + if 'uri' in args.command: + if 'object-localization-uri' in args.command: + localize_objects_uri(args.uri) + elif 'handwritten-ocr-uri' in args.command: + detect_handwritten_ocr_uri(args.uri) + elif 'batch-annotate-files' in args.command: + detect_batch_annotate_files_uri(args.uri) + elif 'batch-annotate-images' in args.command: + async_batch_annotate_images_uri(args.uri, args.output) + else: + if 'object-localization' in args.command: + localize_objects(args.path) + elif 'handwritten-ocr' in args.command: + detect_handwritten_ocr(args.path) + elif 'batch-annotate-files' in args.command: + detect_batch_annotate_files(args.path) diff --git a/vision/cloud-client/detect/beta_snippets_test.py b/vision/cloud-client/detect/beta_snippets_test.py new file mode 100644 index 00000000000..acbb5b7015e --- /dev/null +++ b/vision/cloud-client/detect/beta_snippets_test.py @@ -0,0 +1,83 @@ +# Copyright 2018 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import beta_snippets + +RESOURCES = os.path.join(os.path.dirname(__file__), 'resources') +GCS_ROOT = 'gs://cloud-samples-data/vision/' + +BUCKET = os.environ['CLOUD_STORAGE_BUCKET'] +OUTPUT_PREFIX = 'OCR_PDF_TEST_OUTPUT' +GCS_DESTINATION_URI = 'gs://{}/{}/'.format(BUCKET, OUTPUT_PREFIX) + + +def test_localize_objects(capsys): + path = os.path.join(RESOURCES, 'puppies.jpg') + + beta_snippets.localize_objects(path) + + out, _ = capsys.readouterr() + assert 'Dog' in out + + +def test_localize_objects_uri(capsys): + uri = GCS_ROOT + 'puppies.jpg' + + beta_snippets.localize_objects_uri(uri) + + out, _ = capsys.readouterr() + assert 'Dog' in out + + +def test_handwritten_ocr(capsys): + path = os.path.join(RESOURCES, 'handwritten.jpg') + + beta_snippets.detect_handwritten_ocr(path) + + out, _ = capsys.readouterr() + assert 'Cloud Vision API' in out + + +def test_handwritten_ocr_uri(capsys): + uri = GCS_ROOT + 'handwritten.jpg' + + beta_snippets.detect_handwritten_ocr_uri(uri) + + out, _ = capsys.readouterr() + assert 'Cloud Vision API' in out + + +def test_detect_batch_annotate_files(capsys): + file_name = os.path.join(RESOURCES, 'kafka.pdf') + beta_snippets.detect_batch_annotate_files(file_name) + out, _ = capsys.readouterr() + assert 'Symbol: a' in out + assert 'Word text: evenings' in out + + +def test_detect_batch_annotate_files_uri(capsys): + gcs_uri = GCS_ROOT + 'document_understanding/kafka.pdf' + beta_snippets.detect_batch_annotate_files_uri(gcs_uri) + out, _ = capsys.readouterr() + assert 'Symbol' in out + assert 'Word text' in out + + +def test_async_batch_annotate_images(capsys): + gcs_uri = GCS_ROOT + 'landmark/eiffel_tower.jpg' + beta_snippets.async_batch_annotate_images_uri(gcs_uri, GCS_DESTINATION_URI) + out, _ = capsys.readouterr() + assert 'language_code: "en"' in out + assert 'description: "Tower"' in out diff --git a/vision/cloud-client/detect/detect.py b/vision/cloud-client/detect/detect.py new file mode 100644 index 00000000000..8a285b78558 --- /dev/null +++ b/vision/cloud-client/detect/detect.py @@ -0,0 +1,961 @@ +#!/usr/bin/env python + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to perform basic operations with the +Google Cloud Vision API. + +Example Usage: +python detect.py text ./resources/wakeupcat.jpg +python detect.py labels ./resources/landmark.jpg +python detect.py web ./resources/landmark.jpg +python detect.py web-uri http://wheresgus.com/dog.JPG +python detect.py web-geo ./resources/city.jpg +python detect.py faces-uri gs://your-bucket/file.jpg +python detect.py ocr-uri gs://python-docs-samples-tests/HodgeConj.pdf \ +gs://BUCKET_NAME/PREFIX/ +python detect.py object-localization ./resources/puppies.jpg +python detect.py object-localization-uri gs://... + +For more information, the documentation at +https://cloud.google.com/vision/docs. +""" + +import argparse +import io +import re + + +# [START vision_face_detection] +def detect_faces(path): + """Detects faces in an image.""" + from google.cloud import vision + client = vision.ImageAnnotatorClient() + + # [START vision_python_migration_face_detection] + # [START vision_python_migration_image_file] + with io.open(path, 'rb') as image_file: + content = image_file.read() + + image = vision.types.Image(content=content) + # [END vision_python_migration_image_file] + + response = client.face_detection(image=image) + faces = response.face_annotations + + # Names of likelihood from google.cloud.vision.enums + likelihood_name = ('UNKNOWN', 'VERY_UNLIKELY', 'UNLIKELY', 'POSSIBLE', + 'LIKELY', 'VERY_LIKELY') + print('Faces:') + + for face in faces: + print('anger: {}'.format(likelihood_name[face.anger_likelihood])) + print('joy: {}'.format(likelihood_name[face.joy_likelihood])) + print('surprise: {}'.format(likelihood_name[face.surprise_likelihood])) + + vertices = (['({},{})'.format(vertex.x, vertex.y) + for vertex in face.bounding_poly.vertices]) + + print('face bounds: {}'.format(','.join(vertices))) + # [END vision_python_migration_face_detection] +# [END vision_face_detection] + + +# [START vision_face_detection_gcs] +def detect_faces_uri(uri): + """Detects faces in the file located in Google Cloud Storage or the web.""" + from google.cloud import vision + client = vision.ImageAnnotatorClient() + # [START vision_python_migration_image_uri] + image = vision.types.Image() + image.source.image_uri = uri + # [END vision_python_migration_image_uri] + + response = client.face_detection(image=image) + faces = response.face_annotations + + # Names of likelihood from google.cloud.vision.enums + likelihood_name = ('UNKNOWN', 'VERY_UNLIKELY', 'UNLIKELY', 'POSSIBLE', + 'LIKELY', 'VERY_LIKELY') + print('Faces:') + + for face in faces: + print('anger: {}'.format(likelihood_name[face.anger_likelihood])) + print('joy: {}'.format(likelihood_name[face.joy_likelihood])) + print('surprise: {}'.format(likelihood_name[face.surprise_likelihood])) + + vertices = (['({},{})'.format(vertex.x, vertex.y) + for vertex in face.bounding_poly.vertices]) + + print('face bounds: {}'.format(','.join(vertices))) +# [END vision_face_detection_gcs] + + +# [START vision_label_detection] +def detect_labels(path): + """Detects labels in the file.""" + from google.cloud import vision + client = vision.ImageAnnotatorClient() + + # [START vision_python_migration_label_detection] + with io.open(path, 'rb') as image_file: + content = image_file.read() + + image = vision.types.Image(content=content) + + response = client.label_detection(image=image) + labels = response.label_annotations + print('Labels:') + + for label in labels: + print(label.description) + # [END vision_python_migration_label_detection] +# [END vision_label_detection] + + +# [START vision_label_detection_gcs] +def detect_labels_uri(uri): + """Detects labels in the file located in Google Cloud Storage or on the + Web.""" + from google.cloud import vision + client = vision.ImageAnnotatorClient() + image = vision.types.Image() + image.source.image_uri = uri + + response = client.label_detection(image=image) + labels = response.label_annotations + print('Labels:') + + for label in labels: + print(label.description) +# [END vision_label_detection_gcs] + + +# [START vision_landmark_detection] +def detect_landmarks(path): + """Detects landmarks in the file.""" + from google.cloud import vision + client = vision.ImageAnnotatorClient() + + # [START vision_python_migration_landmark_detection] + with io.open(path, 'rb') as image_file: + content = image_file.read() + + image = vision.types.Image(content=content) + + response = client.landmark_detection(image=image) + landmarks = response.landmark_annotations + print('Landmarks:') + + for landmark in landmarks: + print(landmark.description) + for location in landmark.locations: + lat_lng = location.lat_lng + print('Latitude {}'.format(lat_lng.latitude)) + print('Longitude {}'.format(lat_lng.longitude)) + # [END vision_python_migration_landmark_detection] +# [END vision_landmark_detection] + + +# [START vision_landmark_detection_gcs] +def detect_landmarks_uri(uri): + """Detects landmarks in the file located in Google Cloud Storage or on the + Web.""" + from google.cloud import vision + client = vision.ImageAnnotatorClient() + image = vision.types.Image() + image.source.image_uri = uri + + response = client.landmark_detection(image=image) + landmarks = response.landmark_annotations + print('Landmarks:') + + for landmark in landmarks: + print(landmark.description) +# [END vision_landmark_detection_gcs] + + +# [START vision_logo_detection] +def detect_logos(path): + """Detects logos in the file.""" + from google.cloud import vision + client = vision.ImageAnnotatorClient() + + # [START vision_python_migration_logo_detection] + with io.open(path, 'rb') as image_file: + content = image_file.read() + + image = vision.types.Image(content=content) + + response = client.logo_detection(image=image) + logos = response.logo_annotations + print('Logos:') + + for logo in logos: + print(logo.description) + # [END vision_python_migration_logo_detection] +# [END vision_logo_detection] + + +# [START vision_logo_detection_gcs] +def detect_logos_uri(uri): + """Detects logos in the file located in Google Cloud Storage or on the Web. + """ + from google.cloud import vision + client = vision.ImageAnnotatorClient() + image = vision.types.Image() + image.source.image_uri = uri + + response = client.logo_detection(image=image) + logos = response.logo_annotations + print('Logos:') + + for logo in logos: + print(logo.description) +# [END vision_logo_detection_gcs] + + +# [START vision_safe_search_detection] +def detect_safe_search(path): + """Detects unsafe features in the file.""" + from google.cloud import vision + client = vision.ImageAnnotatorClient() + + # [START vision_python_migration_safe_search_detection] + with io.open(path, 'rb') as image_file: + content = image_file.read() + + image = vision.types.Image(content=content) + + response = client.safe_search_detection(image=image) + safe = response.safe_search_annotation + + # Names of likelihood from google.cloud.vision.enums + likelihood_name = ('UNKNOWN', 'VERY_UNLIKELY', 'UNLIKELY', 'POSSIBLE', + 'LIKELY', 'VERY_LIKELY') + print('Safe search:') + + print('adult: {}'.format(likelihood_name[safe.adult])) + print('medical: {}'.format(likelihood_name[safe.medical])) + print('spoofed: {}'.format(likelihood_name[safe.spoof])) + print('violence: {}'.format(likelihood_name[safe.violence])) + print('racy: {}'.format(likelihood_name[safe.racy])) + # [END vision_python_migration_safe_search_detection] +# [END vision_safe_search_detection] + + +# [START vision_safe_search_detection_gcs] +def detect_safe_search_uri(uri): + """Detects unsafe features in the file located in Google Cloud Storage or + on the Web.""" + from google.cloud import vision + client = vision.ImageAnnotatorClient() + image = vision.types.Image() + image.source.image_uri = uri + + response = client.safe_search_detection(image=image) + safe = response.safe_search_annotation + + # Names of likelihood from google.cloud.vision.enums + likelihood_name = ('UNKNOWN', 'VERY_UNLIKELY', 'UNLIKELY', 'POSSIBLE', + 'LIKELY', 'VERY_LIKELY') + print('Safe search:') + + print('adult: {}'.format(likelihood_name[safe.adult])) + print('medical: {}'.format(likelihood_name[safe.medical])) + print('spoofed: {}'.format(likelihood_name[safe.spoof])) + print('violence: {}'.format(likelihood_name[safe.violence])) + print('racy: {}'.format(likelihood_name[safe.racy])) +# [END vision_safe_search_detection_gcs] + + +# [START vision_text_detection] +def detect_text(path): + """Detects text in the file.""" + from google.cloud import vision + client = vision.ImageAnnotatorClient() + + # [START vision_python_migration_text_detection] + with io.open(path, 'rb') as image_file: + content = image_file.read() + + image = vision.types.Image(content=content) + + response = client.text_detection(image=image) + texts = response.text_annotations + print('Texts:') + + for text in texts: + print('\n"{}"'.format(text.description)) + + vertices = (['({},{})'.format(vertex.x, vertex.y) + for vertex in text.bounding_poly.vertices]) + + print('bounds: {}'.format(','.join(vertices))) + # [END vision_python_migration_text_detection] +# [END vision_text_detection] + + +# [START vision_text_detection_gcs] +def detect_text_uri(uri): + """Detects text in the file located in Google Cloud Storage or on the Web. + """ + from google.cloud import vision + client = vision.ImageAnnotatorClient() + image = vision.types.Image() + image.source.image_uri = uri + + response = client.text_detection(image=image) + texts = response.text_annotations + print('Texts:') + + for text in texts: + print('\n"{}"'.format(text.description)) + + vertices = (['({},{})'.format(vertex.x, vertex.y) + for vertex in text.bounding_poly.vertices]) + + print('bounds: {}'.format(','.join(vertices))) +# [END vision_text_detection_gcs] + + +# [START vision_image_property_detection] +def detect_properties(path): + """Detects image properties in the file.""" + from google.cloud import vision + client = vision.ImageAnnotatorClient() + + # [START vision_python_migration_image_properties] + with io.open(path, 'rb') as image_file: + content = image_file.read() + + image = vision.types.Image(content=content) + + response = client.image_properties(image=image) + props = response.image_properties_annotation + print('Properties:') + + for color in props.dominant_colors.colors: + print('fraction: {}'.format(color.pixel_fraction)) + print('\tr: {}'.format(color.color.red)) + print('\tg: {}'.format(color.color.green)) + print('\tb: {}'.format(color.color.blue)) + print('\ta: {}'.format(color.color.alpha)) + # [END vision_python_migration_image_properties] +# [END vision_image_property_detection] + + +# [START vision_image_property_detection_gcs] +def detect_properties_uri(uri): + """Detects image properties in the file located in Google Cloud Storage or + on the Web.""" + from google.cloud import vision + client = vision.ImageAnnotatorClient() + image = vision.types.Image() + image.source.image_uri = uri + + response = client.image_properties(image=image) + props = response.image_properties_annotation + print('Properties:') + + for color in props.dominant_colors.colors: + print('frac: {}'.format(color.pixel_fraction)) + print('\tr: {}'.format(color.color.red)) + print('\tg: {}'.format(color.color.green)) + print('\tb: {}'.format(color.color.blue)) + print('\ta: {}'.format(color.color.alpha)) +# [END vision_image_property_detection_gcs] + + +# [START vision_web_detection] +def detect_web(path): + """Detects web annotations given an image.""" + from google.cloud import vision + client = vision.ImageAnnotatorClient() + + # [START vision_python_migration_web_detection] + with io.open(path, 'rb') as image_file: + content = image_file.read() + + image = vision.types.Image(content=content) + + response = client.web_detection(image=image) + annotations = response.web_detection + + if annotations.best_guess_labels: + for label in annotations.best_guess_labels: + print('\nBest guess label: {}'.format(label.label)) + + if annotations.pages_with_matching_images: + print('\n{} Pages with matching images found:'.format( + len(annotations.pages_with_matching_images))) + + for page in annotations.pages_with_matching_images: + print('\n\tPage url : {}'.format(page.url)) + + if page.full_matching_images: + print('\t{} Full Matches found: '.format( + len(page.full_matching_images))) + + for image in page.full_matching_images: + print('\t\tImage url : {}'.format(image.url)) + + if page.partial_matching_images: + print('\t{} Partial Matches found: '.format( + len(page.partial_matching_images))) + + for image in page.partial_matching_images: + print('\t\tImage url : {}'.format(image.url)) + + if annotations.web_entities: + print('\n{} Web entities found: '.format( + len(annotations.web_entities))) + + for entity in annotations.web_entities: + print('\n\tScore : {}'.format(entity.score)) + print(u'\tDescription: {}'.format(entity.description)) + + if annotations.visually_similar_images: + print('\n{} visually similar images found:\n'.format( + len(annotations.visually_similar_images))) + + for image in annotations.visually_similar_images: + print('\tImage url : {}'.format(image.url)) + # [END vision_python_migration_web_detection] +# [END vision_web_detection] + + +# [START vision_web_detection_gcs] +def detect_web_uri(uri): + """Detects web annotations in the file located in Google Cloud Storage.""" + from google.cloud import vision + client = vision.ImageAnnotatorClient() + image = vision.types.Image() + image.source.image_uri = uri + + response = client.web_detection(image=image) + annotations = response.web_detection + + if annotations.best_guess_labels: + for label in annotations.best_guess_labels: + print('\nBest guess label: {}'.format(label.label)) + + if annotations.pages_with_matching_images: + print('\n{} Pages with matching images found:'.format( + len(annotations.pages_with_matching_images))) + + for page in annotations.pages_with_matching_images: + print('\n\tPage url : {}'.format(page.url)) + + if page.full_matching_images: + print('\t{} Full Matches found: '.format( + len(page.full_matching_images))) + + for image in page.full_matching_images: + print('\t\tImage url : {}'.format(image.url)) + + if page.partial_matching_images: + print('\t{} Partial Matches found: '.format( + len(page.partial_matching_images))) + + for image in page.partial_matching_images: + print('\t\tImage url : {}'.format(image.url)) + + if annotations.web_entities: + print('\n{} Web entities found: '.format( + len(annotations.web_entities))) + + for entity in annotations.web_entities: + print('\n\tScore : {}'.format(entity.score)) + print(u'\tDescription: {}'.format(entity.description)) + + if annotations.visually_similar_images: + print('\n{} visually similar images found:\n'.format( + len(annotations.visually_similar_images))) + + for image in annotations.visually_similar_images: + print('\tImage url : {}'.format(image.url)) +# [END vision_web_detection_gcs] + + +# [START vision_web_detection_include_geo] +def web_entities_include_geo_results(path): + """Detects web annotations given an image, using the geotag metadata + in the image to detect web entities.""" + from google.cloud import vision + client = vision.ImageAnnotatorClient() + + with io.open(path, 'rb') as image_file: + content = image_file.read() + + image = vision.types.Image(content=content) + + web_detection_params = vision.types.WebDetectionParams( + include_geo_results=True) + image_context = vision.types.ImageContext( + web_detection_params=web_detection_params) + + response = client.web_detection(image=image, image_context=image_context) + + for entity in response.web_detection.web_entities: + print('\n\tScore : {}'.format(entity.score)) + print(u'\tDescription: {}'.format(entity.description)) +# [END vision_web_detection_include_geo] + + +# [START vision_web_detection_include_geo_gcs] +def web_entities_include_geo_results_uri(uri): + """Detects web annotations given an image in the file located in + Google Cloud Storage., using the geotag metadata in the image to + detect web entities.""" + from google.cloud import vision + client = vision.ImageAnnotatorClient() + + image = vision.types.Image() + image.source.image_uri = uri + + web_detection_params = vision.types.WebDetectionParams( + include_geo_results=True) + image_context = vision.types.ImageContext( + web_detection_params=web_detection_params) + + response = client.web_detection(image=image, image_context=image_context) + + for entity in response.web_detection.web_entities: + print('\n\tScore : {}'.format(entity.score)) + print(u'\tDescription: {}'.format(entity.description)) +# [END vision_web_detection_include_geo_gcs] + + +# [START vision_crop_hint_detection] +def detect_crop_hints(path): + """Detects crop hints in an image.""" + from google.cloud import vision + client = vision.ImageAnnotatorClient() + + # [START vision_python_migration_crop_hints] + with io.open(path, 'rb') as image_file: + content = image_file.read() + image = vision.types.Image(content=content) + + crop_hints_params = vision.types.CropHintsParams(aspect_ratios=[1.77]) + image_context = vision.types.ImageContext( + crop_hints_params=crop_hints_params) + + response = client.crop_hints(image=image, image_context=image_context) + hints = response.crop_hints_annotation.crop_hints + + for n, hint in enumerate(hints): + print('\nCrop Hint: {}'.format(n)) + + vertices = (['({},{})'.format(vertex.x, vertex.y) + for vertex in hint.bounding_poly.vertices]) + + print('bounds: {}'.format(','.join(vertices))) + # [END vision_python_migration_crop_hints] +# [END vision_crop_hint_detection] + + +# [START vision_crop_hint_detection_gcs] +def detect_crop_hints_uri(uri): + """Detects crop hints in the file located in Google Cloud Storage.""" + from google.cloud import vision + client = vision.ImageAnnotatorClient() + image = vision.types.Image() + image.source.image_uri = uri + + crop_hints_params = vision.types.CropHintsParams(aspect_ratios=[1.77]) + image_context = vision.types.ImageContext( + crop_hints_params=crop_hints_params) + + response = client.crop_hints(image=image, image_context=image_context) + hints = response.crop_hints_annotation.crop_hints + + for n, hint in enumerate(hints): + print('\nCrop Hint: {}'.format(n)) + + vertices = (['({},{})'.format(vertex.x, vertex.y) + for vertex in hint.bounding_poly.vertices]) + + print('bounds: {}'.format(','.join(vertices))) +# [END vision_crop_hint_detection_gcs] + + +# [START vision_fulltext_detection] +def detect_document(path): + """Detects document features in an image.""" + from google.cloud import vision + client = vision.ImageAnnotatorClient() + + # [START vision_python_migration_document_text_detection] + with io.open(path, 'rb') as image_file: + content = image_file.read() + + image = vision.types.Image(content=content) + + response = client.document_text_detection(image=image) + + for page in response.full_text_annotation.pages: + for block in page.blocks: + print('\nBlock confidence: {}\n'.format(block.confidence)) + + for paragraph in block.paragraphs: + print('Paragraph confidence: {}'.format( + paragraph.confidence)) + + for word in paragraph.words: + word_text = ''.join([ + symbol.text for symbol in word.symbols + ]) + print('Word text: {} (confidence: {})'.format( + word_text, word.confidence)) + + for symbol in word.symbols: + print('\tSymbol: {} (confidence: {})'.format( + symbol.text, symbol.confidence)) + # [END vision_python_migration_document_text_detection] +# [END vision_fulltext_detection] + + +# [START vision_fulltext_detection_gcs] +def detect_document_uri(uri): + """Detects document features in the file located in Google Cloud + Storage.""" + from google.cloud import vision + client = vision.ImageAnnotatorClient() + image = vision.types.Image() + image.source.image_uri = uri + + response = client.document_text_detection(image=image) + + for page in response.full_text_annotation.pages: + for block in page.blocks: + print('\nBlock confidence: {}\n'.format(block.confidence)) + + for paragraph in block.paragraphs: + print('Paragraph confidence: {}'.format( + paragraph.confidence)) + + for word in paragraph.words: + word_text = ''.join([ + symbol.text for symbol in word.symbols + ]) + print('Word text: {} (confidence: {})'.format( + word_text, word.confidence)) + + for symbol in word.symbols: + print('\tSymbol: {} (confidence: {})'.format( + symbol.text, symbol.confidence)) +# [END vision_fulltext_detection_gcs] + + +# [START vision_text_detection_pdf_gcs] +def async_detect_document(gcs_source_uri, gcs_destination_uri): + """OCR with PDF/TIFF as source files on GCS""" + from google.cloud import vision + from google.cloud import storage + from google.protobuf import json_format + # Supported mime_types are: 'application/pdf' and 'image/tiff' + mime_type = 'application/pdf' + + # How many pages should be grouped into each json output file. + batch_size = 2 + + client = vision.ImageAnnotatorClient() + + feature = vision.types.Feature( + type=vision.enums.Feature.Type.DOCUMENT_TEXT_DETECTION) + + gcs_source = vision.types.GcsSource(uri=gcs_source_uri) + input_config = vision.types.InputConfig( + gcs_source=gcs_source, mime_type=mime_type) + + gcs_destination = vision.types.GcsDestination(uri=gcs_destination_uri) + output_config = vision.types.OutputConfig( + gcs_destination=gcs_destination, batch_size=batch_size) + + async_request = vision.types.AsyncAnnotateFileRequest( + features=[feature], input_config=input_config, + output_config=output_config) + + operation = client.async_batch_annotate_files( + requests=[async_request]) + + print('Waiting for the operation to finish.') + operation.result(timeout=180) + + # Once the request has completed and the output has been + # written to GCS, we can list all the output files. + storage_client = storage.Client() + + match = re.match(r'gs://([^/]+)/(.+)', gcs_destination_uri) + bucket_name = match.group(1) + prefix = match.group(2) + + bucket = storage_client.get_bucket(bucket_name=bucket_name) + + # List objects with the given prefix. + blob_list = list(bucket.list_blobs(prefix=prefix)) + print('Output files:') + for blob in blob_list: + print(blob.name) + + # Process the first output file from GCS. + # Since we specified batch_size=2, the first response contains + # the first two pages of the input file. + output = blob_list[0] + + json_string = output.download_as_string() + response = json_format.Parse( + json_string, vision.types.AnnotateFileResponse()) + + # The actual response for the first page of the input file. + first_page_response = response.responses[0] + annotation = first_page_response.full_text_annotation + + # Here we print the full text from the first page. + # The response contains more information: + # annotation/pages/blocks/paragraphs/words/symbols + # including confidence scores and bounding boxes + print(u'Full text:\n{}'.format( + annotation.text)) +# [END vision_text_detection_pdf_gcs] + + +# [START vision_localize_objects] +def localize_objects(path): + """Localize objects in the local image. + + Args: + path: The path to the local file. + """ + from google.cloud import vision + client = vision.ImageAnnotatorClient() + + with open(path, 'rb') as image_file: + content = image_file.read() + image = vision.types.Image(content=content) + + objects = client.object_localization( + image=image).localized_object_annotations + + print('Number of objects found: {}'.format(len(objects))) + for object_ in objects: + print('\n{} (confidence: {})'.format(object_.name, object_.score)) + print('Normalized bounding polygon vertices: ') + for vertex in object_.bounding_poly.normalized_vertices: + print(' - ({}, {})'.format(vertex.x, vertex.y)) +# [END vision_localize_objects] + + +# [START vision_localize_objects_gcs] +def localize_objects_uri(uri): + """Localize objects in the image on Google Cloud Storage + + Args: + uri: The path to the file in Google Cloud Storage (gs://...) + """ + from google.cloud import vision + client = vision.ImageAnnotatorClient() + + image = vision.types.Image() + image.source.image_uri = uri + + objects = client.object_localization( + image=image).localized_object_annotations + + print('Number of objects found: {}'.format(len(objects))) + for object_ in objects: + print('\n{} (confidence: {})'.format(object_.name, object_.score)) + print('Normalized bounding polygon vertices: ') + for vertex in object_.bounding_poly.normalized_vertices: + print(' - ({}, {})'.format(vertex.x, vertex.y)) +# [END vision_localize_objects_gcs] + + +def run_local(args): + if args.command == 'faces': + detect_faces(args.path) + elif args.command == 'labels': + detect_labels(args.path) + elif args.command == 'landmarks': + detect_landmarks(args.path) + elif args.command == 'text': + detect_text(args.path) + elif args.command == 'logos': + detect_logos(args.path) + elif args.command == 'safe-search': + detect_safe_search(args.path) + elif args.command == 'properties': + detect_properties(args.path) + elif args.command == 'web': + detect_web(args.path) + elif args.command == 'crophints': + detect_crop_hints(args.path) + elif args.command == 'document': + detect_document(args.path) + elif args.command == 'web-geo': + web_entities_include_geo_results(args.path) + elif args.command == 'object-localization': + localize_objects(args.path) + + +def run_uri(args): + if args.command == 'text-uri': + detect_text_uri(args.uri) + elif args.command == 'faces-uri': + detect_faces_uri(args.uri) + elif args.command == 'labels-uri': + detect_labels_uri(args.uri) + elif args.command == 'landmarks-uri': + detect_landmarks_uri(args.uri) + elif args.command == 'logos-uri': + detect_logos_uri(args.uri) + elif args.command == 'safe-search-uri': + detect_safe_search_uri(args.uri) + elif args.command == 'properties-uri': + detect_properties_uri(args.uri) + elif args.command == 'web-uri': + detect_web_uri(args.uri) + elif args.command == 'crophints-uri': + detect_crop_hints_uri(args.uri) + elif args.command == 'document-uri': + detect_document_uri(args.uri) + elif args.command == 'web-geo-uri': + web_entities_include_geo_results_uri(args.uri) + elif args.command == 'ocr-uri': + async_detect_document(args.uri, args.destination_uri) + elif args.command == 'object-localization-uri': + localize_objects_uri(args.uri) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + subparsers = parser.add_subparsers(dest='command') + + detect_faces_parser = subparsers.add_parser( + 'faces', help=detect_faces.__doc__) + detect_faces_parser.add_argument('path') + + faces_file_parser = subparsers.add_parser( + 'faces-uri', help=detect_faces_uri.__doc__) + faces_file_parser.add_argument('uri') + + detect_labels_parser = subparsers.add_parser( + 'labels', help=detect_labels.__doc__) + detect_labels_parser.add_argument('path') + + labels_file_parser = subparsers.add_parser( + 'labels-uri', help=detect_labels_uri.__doc__) + labels_file_parser.add_argument('uri') + + detect_landmarks_parser = subparsers.add_parser( + 'landmarks', help=detect_landmarks.__doc__) + detect_landmarks_parser.add_argument('path') + + landmark_file_parser = subparsers.add_parser( + 'landmarks-uri', help=detect_landmarks_uri.__doc__) + landmark_file_parser.add_argument('uri') + + detect_text_parser = subparsers.add_parser( + 'text', help=detect_text.__doc__) + detect_text_parser.add_argument('path') + + text_file_parser = subparsers.add_parser( + 'text-uri', help=detect_text_uri.__doc__) + text_file_parser.add_argument('uri') + + detect_logos_parser = subparsers.add_parser( + 'logos', help=detect_logos.__doc__) + detect_logos_parser.add_argument('path') + + logos_file_parser = subparsers.add_parser( + 'logos-uri', help=detect_logos_uri.__doc__) + logos_file_parser.add_argument('uri') + + safe_search_parser = subparsers.add_parser( + 'safe-search', help=detect_safe_search.__doc__) + safe_search_parser.add_argument('path') + + safe_search_file_parser = subparsers.add_parser( + 'safe-search-uri', + help=detect_safe_search_uri.__doc__) + safe_search_file_parser.add_argument('uri') + + properties_parser = subparsers.add_parser( + 'properties', help=detect_properties.__doc__) + properties_parser.add_argument('path') + + properties_file_parser = subparsers.add_parser( + 'properties-uri', + help=detect_properties_uri.__doc__) + properties_file_parser.add_argument('uri') + + # 1.1 Vision features + web_parser = subparsers.add_parser( + 'web', help=detect_web.__doc__) + web_parser.add_argument('path') + + web_uri_parser = subparsers.add_parser( + 'web-uri', + help=detect_web_uri.__doc__) + web_uri_parser.add_argument('uri') + + web_geo_parser = subparsers.add_parser( + 'web-geo', help=web_entities_include_geo_results.__doc__) + web_geo_parser.add_argument('path') + + web_geo_uri_parser = subparsers.add_parser( + 'web-geo-uri', + help=web_entities_include_geo_results_uri.__doc__) + web_geo_uri_parser.add_argument('uri') + + crop_hints_parser = subparsers.add_parser( + 'crophints', help=detect_crop_hints.__doc__) + crop_hints_parser.add_argument('path') + + crop_hints_uri_parser = subparsers.add_parser( + 'crophints-uri', help=detect_crop_hints_uri.__doc__) + crop_hints_uri_parser.add_argument('uri') + + document_parser = subparsers.add_parser( + 'document', help=detect_document.__doc__) + document_parser.add_argument('path') + + document_uri_parser = subparsers.add_parser( + 'document-uri', help=detect_document_uri.__doc__) + document_uri_parser.add_argument('uri') + + ocr_uri_parser = subparsers.add_parser( + 'ocr-uri', help=async_detect_document.__doc__) + ocr_uri_parser.add_argument('uri') + ocr_uri_parser.add_argument('destination_uri') + + object_localization_parser = subparsers.add_parser( + 'object-localization', help=async_detect_document.__doc__) + object_localization_parser.add_argument('path') + + object_localization_uri_parser = subparsers.add_parser( + 'object-localization-uri', help=async_detect_document.__doc__) + object_localization_uri_parser.add_argument('uri') + + args = parser.parse_args() + + if 'uri' in args.command: + run_uri(args) + else: + run_local(args) diff --git a/vision/cloud-client/detect/detect_test.py b/vision/cloud-client/detect/detect_test.py new file mode 100644 index 00000000000..9f8c9f4f3cb --- /dev/null +++ b/vision/cloud-client/detect/detect_test.py @@ -0,0 +1,323 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from google.cloud import storage + +import detect + +ASSET_BUCKET = "cloud-samples-data" + +BUCKET = os.environ['CLOUD_STORAGE_BUCKET'] +OUTPUT_PREFIX = 'OCR_PDF_TEST_OUTPUT' +GCS_SOURCE_URI = 'gs://{}/HodgeConj.pdf'.format(BUCKET) +GCS_DESTINATION_URI = 'gs://{}/{}/'.format(BUCKET, OUTPUT_PREFIX) + + +def test_labels(capsys): + file_name = os.path.join( + os.path.dirname(__file__), + 'resources/wakeupcat.jpg') + detect.detect_labels(file_name) + out, _ = capsys.readouterr() + assert 'Labels' in out + + +def test_labels_uri(capsys): + file_name = 'gs://{}/vision/wakeupcat.jpg'.format(ASSET_BUCKET) + detect.detect_labels_uri(file_name) + out, _ = capsys.readouterr() + assert 'Labels' in out + + +def test_labels_http(capsys): + uri = 'https://storage-download.googleapis.com/{}' \ + '/vision/label/wakeupcat.jpg' + detect.detect_labels_uri(uri.format(ASSET_BUCKET)) + out, _ = capsys.readouterr() + assert 'Labels' in out + + +def test_landmarks(capsys): + file_name = os.path.join( + os.path.dirname(__file__), + 'resources/landmark.jpg') + detect.detect_landmarks(file_name) + out, _ = capsys.readouterr() + assert 'palace' in out.lower() + + +def test_landmarks_uri(capsys): + file_name = 'gs://{}/vision/landmark/pofa.jpg'.format(ASSET_BUCKET) + detect.detect_landmarks_uri(file_name) + out, _ = capsys.readouterr() + assert 'palace' in out.lower() + + +def test_landmarks_http(capsys): + uri = 'https://storage-download.googleapis.com/{}/vision/landmark/pofa.jpg' + detect.detect_landmarks_uri(uri.format(ASSET_BUCKET)) + out, _ = capsys.readouterr() + assert 'palace' in out.lower() + + +def test_faces(capsys): + file_name = os.path.join( + os.path.dirname(__file__), + 'resources/face_no_surprise.jpg') + detect.detect_faces(file_name) + out, _ = capsys.readouterr() + assert 'POSSIBLE' in out + + +def test_faces_uri(capsys): + file_name = 'gs://{}/vision/face/face_no_surprise.jpg'.format(ASSET_BUCKET) + detect.detect_faces_uri(file_name) + out, _ = capsys.readouterr() + assert 'POSSIBLE' in out + + +def test_faces_http(capsys): + uri = ('https://storage-download.googleapis.com/{}/vision/' + + 'face/face_no_surprise.jpg') + detect.detect_faces_uri(uri.format(ASSET_BUCKET)) + out, _ = capsys.readouterr() + assert 'POSSIBLE' in out + + +def test_logos(capsys): + file_name = os.path.join( + os.path.dirname(__file__), + 'resources/logos.png') + detect.detect_logos(file_name) + out, _ = capsys.readouterr() + assert 'google' in out.lower() + + +def test_logos_uri(capsys): + file_name = 'gs://{}/vision/logo/logo_google.png'.format(ASSET_BUCKET) + detect.detect_logos_uri(file_name) + out, _ = capsys.readouterr() + assert 'google' in out.lower() + + +def test_logos_http(capsys): + uri = 'https://storage-download.googleapis.com/{}' \ + '/vision/logo/logo_google.png' + detect.detect_logos_uri(uri.format(ASSET_BUCKET)) + out, _ = capsys.readouterr() + assert 'google' in out.lower() + + +def test_safe_search(capsys): + file_name = os.path.join( + os.path.dirname(__file__), + 'resources/wakeupcat.jpg') + detect.detect_safe_search(file_name) + out, _ = capsys.readouterr() + assert 'VERY_LIKELY' in out + assert 'racy: ' in out + + +def test_safe_search_uri(capsys): + file_name = 'gs://{}/vision/label/wakeupcat.jpg'.format(ASSET_BUCKET) + detect.detect_safe_search_uri(file_name) + out, _ = capsys.readouterr() + assert 'VERY_LIKELY' in out + assert 'racy: ' in out + + +def test_safe_search_http(capsys): + uri = 'https://storage-download.googleapis.com/{}' \ + '/vision/label/wakeupcat.jpg' + detect.detect_safe_search_uri(uri.format(ASSET_BUCKET)) + out, _ = capsys.readouterr() + assert 'VERY_LIKELY' in out + assert 'racy: ' in out + + +def test_detect_text(capsys): + file_name = os.path.join( + os.path.dirname(__file__), + 'resources/text.jpg') + detect.detect_text(file_name) + out, _ = capsys.readouterr() + assert '37%' in out + + +def test_detect_text_uri(capsys): + file_name = 'gs://{}/vision/text/screen.jpg'.format(ASSET_BUCKET) + detect.detect_text_uri(file_name) + out, _ = capsys.readouterr() + assert '37%' in out + + +def test_detect_text_http(capsys): + uri = 'https://storage-download.googleapis.com/{}/vision/text/screen.jpg' + detect.detect_text_uri(uri.format(ASSET_BUCKET)) + out, _ = capsys.readouterr() + assert '37%' in out + + +def test_detect_properties(capsys): + file_name = os.path.join( + os.path.dirname(__file__), + 'resources/landmark.jpg') + detect.detect_properties(file_name) + out, _ = capsys.readouterr() + assert 'frac' in out + + +def test_detect_properties_uri(capsys): + file_name = 'gs://{}/vision/landmark/pofa.jpg'.format(ASSET_BUCKET) + detect.detect_properties_uri(file_name) + out, _ = capsys.readouterr() + assert 'frac' in out + + +def test_detect_properties_http(capsys): + uri = 'https://storage-download.googleapis.com/{}/vision/landmark/pofa.jpg' + detect.detect_properties_uri(uri.format(ASSET_BUCKET)) + out, _ = capsys.readouterr() + assert 'frac' in out + + +# Vision 1.1 tests +def test_detect_web(capsys): + file_name = os.path.join( + os.path.dirname(__file__), + 'resources/landmark.jpg') + detect.detect_web(file_name) + out, _ = capsys.readouterr() + assert 'best guess label: palace of fine arts' in out.lower() + + +def test_detect_web_uri(capsys): + file_name = 'gs://{}/vision/landmark/pofa.jpg'.format(ASSET_BUCKET) + detect.detect_web_uri(file_name) + out, _ = capsys.readouterr() + assert 'best guess label: palace of fine arts' in out.lower() + + +def test_detect_web_http(capsys): + uri = 'https://storage-download.googleapis.com/{}/vision/landmark/pofa.jpg' + detect.detect_web_uri(uri.format(ASSET_BUCKET)) + out, _ = capsys.readouterr() + assert 'best guess label: palace of fine arts' in out.lower() + + +def test_detect_web_with_geo(capsys): + file_name = os.path.join( + os.path.dirname(__file__), + 'resources/city.jpg') + detect.web_entities_include_geo_results(file_name) + out, _ = capsys.readouterr() + out = out.lower() + assert 'zepra' in out or 'electra tower' in out + + +def test_detect_web_with_geo_uri(capsys): + file_name = 'gs://{}/vision/web/city.jpg'.format(ASSET_BUCKET) + detect.web_entities_include_geo_results_uri(file_name) + out, _ = capsys.readouterr() + out = out.lower() + assert 'zepra' in out or 'electra tower' in out + + +def test_detect_document(capsys): + file_name = os.path.join( + os.path.dirname(__file__), + 'resources/text.jpg') + detect.detect_document(file_name) + out, _ = capsys.readouterr() + assert 'class' in out + + +def test_detect_document_uri(capsys): + file_name = 'gs://{}/vision/text/screen.jpg'.format(ASSET_BUCKET) + detect.detect_document_uri(file_name) + out, _ = capsys.readouterr() + assert 'class' in out + + +def test_detect_document_http(capsys): + uri = 'https://storage-download.googleapis.com/{}/vision/text/screen.jpg' + detect.detect_document_uri(uri.format(ASSET_BUCKET)) + out, _ = capsys.readouterr() + assert 'class' in out + + +def test_detect_crop_hints(capsys): + file_name = os.path.join( + os.path.dirname(__file__), + 'resources/wakeupcat.jpg') + detect.detect_crop_hints(file_name) + out, _ = capsys.readouterr() + assert 'bounds: (0,0)' in out + + +def test_detect_crop_hints_uri(capsys): + file_name = 'gs://{}/vision/label/wakeupcat.jpg'.format(ASSET_BUCKET) + detect.detect_crop_hints_uri(file_name) + out, _ = capsys.readouterr() + assert 'bounds: (0,0)' in out + + +def test_detect_crop_hints_http(capsys): + uri = 'https://storage-download.googleapis.com/{}' \ + '/vision/label/wakeupcat.jpg' + detect.detect_crop_hints_uri(uri.format(ASSET_BUCKET)) + out, _ = capsys.readouterr() + assert 'bounds: (0,0)' in out + + +def test_async_detect_document(capsys): + storage_client = storage.Client() + bucket = storage_client.get_bucket(BUCKET) + if len(list(bucket.list_blobs(prefix=OUTPUT_PREFIX))) > 0: + for blob in bucket.list_blobs(prefix=OUTPUT_PREFIX): + blob.delete() + + assert len(list(bucket.list_blobs(prefix=OUTPUT_PREFIX))) == 0 + + uri = 'gs://{}/vision/document/custom_0773375000.pdf'.format(ASSET_BUCKET) + detect.async_detect_document( + gcs_source_uri=uri, + gcs_destination_uri=GCS_DESTINATION_URI) + out, _ = capsys.readouterr() + + assert 'OIL, GAS AND MINERAL LEASE' in out + assert len(list(bucket.list_blobs(prefix=OUTPUT_PREFIX))) > 0 + + for blob in bucket.list_blobs(prefix=OUTPUT_PREFIX): + blob.delete() + + assert len(list(bucket.list_blobs(prefix=OUTPUT_PREFIX))) == 0 + + +def test_localize_objects(capsys): + detect.localize_objects('resources/puppies.jpg') + + out, _ = capsys.readouterr() + assert 'dog' in out.lower() + + +def test_localize_objects_uri(capsys): + uri = 'gs://cloud-samples-data/vision/puppies.jpg' + + detect.localize_objects_uri(uri) + + out, _ = capsys.readouterr() + assert 'dog' in out.lower() diff --git a/vision/cloud-client/detect/requirements.txt b/vision/cloud-client/detect/requirements.txt new file mode 100644 index 00000000000..7dff97ac93b --- /dev/null +++ b/vision/cloud-client/detect/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-vision==0.36.0 +google-cloud-storage==1.13.2 diff --git a/vision/cloud-client/detect/resources/city.jpg b/vision/cloud-client/detect/resources/city.jpg new file mode 100644 index 00000000000..b14282e7539 Binary files /dev/null and b/vision/cloud-client/detect/resources/city.jpg differ diff --git a/vision/cloud-client/detect/resources/duck_and_truck.jpg b/vision/cloud-client/detect/resources/duck_and_truck.jpg new file mode 100644 index 00000000000..5c560fe774f Binary files /dev/null and b/vision/cloud-client/detect/resources/duck_and_truck.jpg differ diff --git a/vision/cloud-client/detect/resources/face_no_surprise.jpg b/vision/cloud-client/detect/resources/face_no_surprise.jpg new file mode 100644 index 00000000000..0e2894adb83 Binary files /dev/null and b/vision/cloud-client/detect/resources/face_no_surprise.jpg differ diff --git a/vision/cloud-client/detect/resources/handwritten.jpg b/vision/cloud-client/detect/resources/handwritten.jpg new file mode 100644 index 00000000000..50a9575b5ad Binary files /dev/null and b/vision/cloud-client/detect/resources/handwritten.jpg differ diff --git a/vision/cloud-client/detect/resources/kafka.pdf b/vision/cloud-client/detect/resources/kafka.pdf new file mode 100644 index 00000000000..ffa2e2fac2f Binary files /dev/null and b/vision/cloud-client/detect/resources/kafka.pdf differ diff --git a/vision/cloud-client/detect/resources/landmark.jpg b/vision/cloud-client/detect/resources/landmark.jpg new file mode 100644 index 00000000000..41c3d0fc935 Binary files /dev/null and b/vision/cloud-client/detect/resources/landmark.jpg differ diff --git a/vision/cloud-client/detect/resources/logos.png b/vision/cloud-client/detect/resources/logos.png new file mode 100644 index 00000000000..5538eaed2bd Binary files /dev/null and b/vision/cloud-client/detect/resources/logos.png differ diff --git a/vision/cloud-client/detect/resources/puppies.jpg b/vision/cloud-client/detect/resources/puppies.jpg new file mode 100644 index 00000000000..1bfbbc9c5e4 Binary files /dev/null and b/vision/cloud-client/detect/resources/puppies.jpg differ diff --git a/vision/cloud-client/detect/resources/text.jpg b/vision/cloud-client/detect/resources/text.jpg new file mode 100644 index 00000000000..3b17d55de0e Binary files /dev/null and b/vision/cloud-client/detect/resources/text.jpg differ diff --git a/vision/cloud-client/detect/resources/wakeupcat.jpg b/vision/cloud-client/detect/resources/wakeupcat.jpg new file mode 100644 index 00000000000..139cf461eca Binary files /dev/null and b/vision/cloud-client/detect/resources/wakeupcat.jpg differ diff --git a/vision/cloud-client/document_text/.gitignore b/vision/cloud-client/document_text/.gitignore new file mode 100644 index 00000000000..a4c44706caf --- /dev/null +++ b/vision/cloud-client/document_text/.gitignore @@ -0,0 +1 @@ +output-text.jpg diff --git a/vision/cloud-client/document_text/README.rst b/vision/cloud-client/document_text/README.rst new file mode 100644 index 00000000000..a38564a27d0 --- /dev/null +++ b/vision/cloud-client/document_text/README.rst @@ -0,0 +1,111 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Vision API Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=vision/cloud-client/document_text/README.rst + + +This directory contains samples for Google Cloud Vision API. `Google Cloud Vision API`_ allows developers to easily integrate vision detection features within applications, including image labeling, face and landmark detection, optical character recognition (OCR), and tagging of explicit content. + +- See the `migration guide`_ for information about migrating to Python client library v0.25.1. + +.. _migration guide: https://cloud.google.com/vision/docs/python-client-migration + + + + +.. _Google Cloud Vision API: https://cloud.google.com/vision/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Document Text tutorial ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=vision/cloud-client/document_text/doctext.py,vision/cloud-client/document_text/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python doctext.py + + usage: doctext.py [-h] [-out_file OUT_FILE] detect_file + + positional arguments: + detect_file The image for text detection. + + optional arguments: + -h, --help show this help message and exit + -out_file OUT_FILE Optional output file + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/vision/cloud-client/document_text/README.rst.in b/vision/cloud-client/document_text/README.rst.in new file mode 100644 index 00000000000..4746e327eca --- /dev/null +++ b/vision/cloud-client/document_text/README.rst.in @@ -0,0 +1,30 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Vision API + short_name: Cloud Vision API + url: https://cloud.google.com/vision/docs + description: > + `Google Cloud Vision API`_ allows developers to easily integrate vision + detection features within applications, including image labeling, face and + landmark detection, optical character recognition (OCR), and tagging of + explicit content. + + + - See the `migration guide`_ for information about migrating to Python client library v0.25.1. + + + .. _migration guide: https://cloud.google.com/vision/docs/python-client-migration + +setup: +- auth +- install_deps + +samples: +- name: Document Text tutorial + file: doctext.py + show_help: True + +cloud_client_library: true + +folder: vision/cloud-client/document_text \ No newline at end of file diff --git a/vision/cloud-client/document_text/doctext.py b/vision/cloud-client/document_text/doctext.py new file mode 100644 index 00000000000..7ad5b1019ae --- /dev/null +++ b/vision/cloud-client/document_text/doctext.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Outlines document text given an image. + +Example: + python doctext.py resources/text_menu.jpg +""" +# [START vision_document_text_tutorial] +# [START vision_document_text_tutorial_imports] +import argparse +from enum import Enum +import io + +from google.cloud import vision +from google.cloud.vision import types +from PIL import Image, ImageDraw +# [END vision_document_text_tutorial_imports] + + +class FeatureType(Enum): + PAGE = 1 + BLOCK = 2 + PARA = 3 + WORD = 4 + SYMBOL = 5 + + +def draw_boxes(image, bounds, color): + """Draw a border around the image using the hints in the vector list.""" + draw = ImageDraw.Draw(image) + + for bound in bounds: + draw.polygon([ + bound.vertices[0].x, bound.vertices[0].y, + bound.vertices[1].x, bound.vertices[1].y, + bound.vertices[2].x, bound.vertices[2].y, + bound.vertices[3].x, bound.vertices[3].y], None, color) + return image + + +def get_document_bounds(image_file, feature): + # [START vision_document_text_tutorial_detect_bounds] + """Returns document bounds given an image.""" + client = vision.ImageAnnotatorClient() + + bounds = [] + + with io.open(image_file, 'rb') as image_file: + content = image_file.read() + + image = types.Image(content=content) + + response = client.document_text_detection(image=image) + document = response.full_text_annotation + + # Collect specified feature bounds by enumerating all document features + for page in document.pages: + for block in page.blocks: + for paragraph in block.paragraphs: + for word in paragraph.words: + for symbol in word.symbols: + if (feature == FeatureType.SYMBOL): + bounds.append(symbol.bounding_box) + + if (feature == FeatureType.WORD): + bounds.append(word.bounding_box) + + if (feature == FeatureType.PARA): + bounds.append(paragraph.bounding_box) + + if (feature == FeatureType.BLOCK): + bounds.append(block.bounding_box) + + if (feature == FeatureType.PAGE): + bounds.append(block.bounding_box) + + # The list `bounds` contains the coordinates of the bounding boxes. + # [END vision_document_text_tutorial_detect_bounds] + return bounds + + +def render_doc_text(filein, fileout): + image = Image.open(filein) + bounds = get_document_bounds(filein, FeatureType.PAGE) + draw_boxes(image, bounds, 'blue') + bounds = get_document_bounds(filein, FeatureType.PARA) + draw_boxes(image, bounds, 'red') + bounds = get_document_bounds(filein, FeatureType.WORD) + draw_boxes(image, bounds, 'yellow') + + if fileout is not 0: + image.save(fileout) + else: + image.show() + + +if __name__ == '__main__': + # [START vision_document_text_tutorial_run_application] + parser = argparse.ArgumentParser() + parser.add_argument('detect_file', help='The image for text detection.') + parser.add_argument('-out_file', help='Optional output file', default=0) + args = parser.parse_args() + + render_doc_text(args.detect_file, args.out_file) + # [END vision_document_text_tutorial_run_application] +# [END vision_document_text_tutorial] diff --git a/vision/cloud-client/document_text/doctext_test.py b/vision/cloud-client/document_text/doctext_test.py new file mode 100644 index 00000000000..cb881e31967 --- /dev/null +++ b/vision/cloud-client/document_text/doctext_test.py @@ -0,0 +1,24 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import doctext + + +def test_text(capsys): + """Checks the output image for drawing the crop hint is created.""" + doctext.render_doc_text('resources/text_menu.jpg', 'output-text.jpg') + out, _ = capsys.readouterr() + assert os.path.isfile('output-text.jpg') diff --git a/vision/cloud-client/document_text/requirements.txt b/vision/cloud-client/document_text/requirements.txt new file mode 100644 index 00000000000..044a582a532 --- /dev/null +++ b/vision/cloud-client/document_text/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-vision==0.35.2 +pillow==5.4.1 diff --git a/vision/cloud-client/document_text/resources/text_menu.jpg b/vision/cloud-client/document_text/resources/text_menu.jpg new file mode 100644 index 00000000000..caa678b3e7c Binary files /dev/null and b/vision/cloud-client/document_text/resources/text_menu.jpg differ diff --git a/vision/cloud-client/face_detection/.gitignore b/vision/cloud-client/face_detection/.gitignore new file mode 100644 index 00000000000..01f02dff9a7 --- /dev/null +++ b/vision/cloud-client/face_detection/.gitignore @@ -0,0 +1 @@ +out.jpg diff --git a/vision/cloud-client/face_detection/README.rst b/vision/cloud-client/face_detection/README.rst new file mode 100644 index 00000000000..b04a344ecfb --- /dev/null +++ b/vision/cloud-client/face_detection/README.rst @@ -0,0 +1,101 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Vision API Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=vision/cloud-client/face_detection/README.rst + + +This directory contains samples for Google Cloud Vision API. `Google Cloud Vision API`_ allows developers to easily integrate vision detection features within applications, including image labeling, face and landmark detection, optical character recognition (OCR), and tagging of explicit content. + +- See the `migration guide`_ for information about migrating to Python client library v0.25.1. + +.. _migration guide: https://cloud.google.com/vision/docs/python-client-migration + + +This sample demonstrates how to use the Cloud Vision API to do face detection. + + +.. _Google Cloud Vision API: https://cloud.google.com/vision/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Face detection ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=vision/cloud-client/face_detection/faces.py,vision/cloud-client/face_detection/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python faces.py + + usage: faces.py [-h] [--out OUTPUT] [--max-results MAX_RESULTS] input_image + + Detects faces in the given image. + + positional arguments: + input_image the image you'd like to detect faces in. + + optional arguments: + -h, --help show this help message and exit + --out OUTPUT the name of the output file. + --max-results MAX_RESULTS + the max results of face detection. + + + + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/vision/cloud-client/face_detection/README.rst.in b/vision/cloud-client/face_detection/README.rst.in new file mode 100644 index 00000000000..422cec1d11d --- /dev/null +++ b/vision/cloud-client/face_detection/README.rst.in @@ -0,0 +1,31 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Vision API + short_name: Cloud Vision API + url: https://cloud.google.com/vision/docs + description: > + `Google Cloud Vision API`_ allows developers to easily integrate vision + detection features within applications, including image labeling, face and + landmark detection, optical character recognition (OCR), and tagging of + explicit content. + + + - See the `migration guide`_ for information about migrating to Python client library v0.25.1. + + + .. _migration guide: https://cloud.google.com/vision/docs/python-client-migration + +description: > + This sample demonstrates how to use the Cloud Vision API to do face detection. + +setup: +- auth +- install_deps + +samples: +- name: Face detection + file: faces.py + show_help: true + +folder: vision/cloud-client/face_detection \ No newline at end of file diff --git a/vision/cloud-client/face_detection/faces.py b/vision/cloud-client/face_detection/faces.py new file mode 100755 index 00000000000..317057db391 --- /dev/null +++ b/vision/cloud-client/face_detection/faces.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python + +# Copyright 2015 Google, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Draws squares around detected faces in the given image.""" + +import argparse + +# [START vision_face_detection_tutorial_imports] +from google.cloud import vision +from google.cloud.vision import types +from PIL import Image, ImageDraw +# [END vision_face_detection_tutorial_imports] + + +# [START vision_face_detection_tutorial_send_request] +def detect_face(face_file, max_results=4): + """Uses the Vision API to detect faces in the given file. + + Args: + face_file: A file-like object containing an image with faces. + + Returns: + An array of Face objects with information about the picture. + """ + # [START vision_face_detection_tutorial_client] + client = vision.ImageAnnotatorClient() + # [END vision_face_detection_tutorial_client] + + content = face_file.read() + image = types.Image(content=content) + + return client.face_detection(image=image, max_results=max_results).face_annotations +# [END vision_face_detection_tutorial_send_request] + + +# [START vision_face_detection_tutorial_process_response] +def highlight_faces(image, faces, output_filename): + """Draws a polygon around the faces, then saves to output_filename. + + Args: + image: a file containing the image with the faces. + faces: a list of faces found in the file. This should be in the format + returned by the Vision API. + output_filename: the name of the image file to be created, where the + faces have polygons drawn around them. + """ + im = Image.open(image) + draw = ImageDraw.Draw(im) + # Sepecify the font-family and the font-size + for face in faces: + box = [(vertex.x, vertex.y) + for vertex in face.bounding_poly.vertices] + draw.line(box + [box[0]], width=5, fill='#00ff00') + # Place the confidence value/score of the detected faces above the + # detection box in the output image + draw.text(((face.bounding_poly.vertices)[0].x, + (face.bounding_poly.vertices)[0].y - 30), + str(format(face.detection_confidence, '.3f')) + '%', + fill='#FF0000') + im.save(output_filename) +# [END vision_face_detection_tutorial_process_response] + + +# [START vision_face_detection_tutorial_run_application] +def main(input_filename, output_filename, max_results): + with open(input_filename, 'rb') as image: + faces = detect_face(image, max_results) + print('Found {} face{}'.format( + len(faces), '' if len(faces) == 1 else 's')) + + print('Writing to file {}'.format(output_filename)) + # Reset the file pointer, so we can read the file again + image.seek(0) + highlight_faces(image, faces, output_filename) +# [END vision_face_detection_tutorial_run_application] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Detects faces in the given image.') + parser.add_argument( + 'input_image', help='the image you\'d like to detect faces in.') + parser.add_argument( + '--out', dest='output', default='out.jpg', + help='the name of the output file.') + parser.add_argument( + '--max-results', dest='max_results', default=4, + help='the max results of face detection.') + args = parser.parse_args() + + main(args.input_image, args.output, args.max_results) diff --git a/vision/cloud-client/face_detection/faces_test.py b/vision/cloud-client/face_detection/faces_test.py new file mode 100644 index 00000000000..cca63c20e5b --- /dev/null +++ b/vision/cloud-client/face_detection/faces_test.py @@ -0,0 +1,39 @@ +# Copyright 2016, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from PIL import Image + +from faces import main + +RESOURCES = os.path.join(os.path.dirname(__file__), 'resources') + + +def test_main(tmpdir): + out_file = os.path.join(tmpdir.dirname, 'face-output.jpg') + in_file = os.path.join(RESOURCES, 'face-input.jpg') + + # Make sure there isn't already a green box + im = Image.open(in_file) + pixels = im.getdata() + greens = sum(1 for (r, g, b) in pixels if r == 0 and g == 255 and b == 0) + assert greens < 1 + + main(in_file, out_file, 10) + + # Make sure there now is some green drawn + im = Image.open(out_file) + pixels = im.getdata() + greens = sum(1 for (r, g, b) in pixels if r == 0 and g == 255 and b == 0) + assert greens > 10 diff --git a/vision/cloud-client/face_detection/requirements.txt b/vision/cloud-client/face_detection/requirements.txt new file mode 100644 index 00000000000..6a4025f01bb --- /dev/null +++ b/vision/cloud-client/face_detection/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-vision==0.35.2 +Pillow==5.4.1 diff --git a/vision/cloud-client/face_detection/resources/face-input.jpg b/vision/cloud-client/face_detection/resources/face-input.jpg new file mode 100644 index 00000000000..c0ee5580b37 Binary files /dev/null and b/vision/cloud-client/face_detection/resources/face-input.jpg differ diff --git a/vision/cloud-client/product_search/import_product_sets.py b/vision/cloud-client/product_search/import_product_sets.py new file mode 100755 index 00000000000..e2d037143a6 --- /dev/null +++ b/vision/cloud-client/product_search/import_product_sets.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python + +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to perform import product sets operations +on Product set in Cloud Vision Product Search. + +For more information, see the tutorial page at +https://cloud.google.com/vision/product-search/docs/ +""" + +import argparse + +# [START vision_product_search_tutorial_import] +from google.cloud import vision +# [END vision_product_search_tutorial_import] + + +# [START vision_product_search_import_product_images] +def import_product_sets(project_id, location, gcs_uri): + """Import images of different products in the product set. + Args: + project_id: Id of the project. + location: A compute region name. + gcs_uri: Google Cloud Storage URI. + Target files must be in Product Search CSV format. + """ + client = vision.ProductSearchClient() + + # A resource that represents Google Cloud Platform location. + location_path = client.location_path( + project=project_id, location=location) + + # Set the input configuration along with Google Cloud Storage URI + gcs_source = vision.types.ImportProductSetsGcsSource( + csv_file_uri=gcs_uri) + input_config = vision.types.ImportProductSetsInputConfig( + gcs_source=gcs_source) + + # Import the product sets from the input URI. + response = client.import_product_sets( + parent=location_path, input_config=input_config) + + print('Processing operation name: {}'.format(response.operation.name)) + # synchronous check of operation status + result = response.result() + print('Processing done.') + + for i, status in enumerate(result.statuses): + print('Status of processing line {} of the csv: {}'.format( + i, status)) + # Check the status of reference image + # `0` is the code for OK in google.rpc.Code. + if status.code == 0: + reference_image = result.reference_images[i] + print(reference_image) + else: + print('Status code not OK: {}'.format(status.message)) +# [END vision_product_search_import_product_images] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + subparsers = parser.add_subparsers(dest='command') + parser.add_argument( + '--project_id', + help='Project id. Required', + required=True) + parser.add_argument( + '--location', + help='Compute region name', + default='us-west1') + + import_product_sets_parser = subparsers.add_parser( + 'import_product_sets', help=import_product_sets.__doc__) + import_product_sets_parser.add_argument('gcs_uri') + + args = parser.parse_args() + + if args.command == 'import_product_sets': + import_product_sets(args.project_id, args.location, args.gcs_uri) diff --git a/vision/cloud-client/product_search/import_product_sets_test.py b/vision/cloud-client/product_search/import_product_sets_test.py new file mode 100644 index 00000000000..ece29054528 --- /dev/null +++ b/vision/cloud-client/product_search/import_product_sets_test.py @@ -0,0 +1,93 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytest + +from import_product_sets import import_product_sets +from product_in_product_set_management import list_products_in_product_set +from product_management import delete_product, list_products +from product_set_management import delete_product_set, list_product_sets +from reference_image_management import list_reference_images + + +PROJECT_ID = os.getenv('GCLOUD_PROJECT') +LOCATION = 'us-west1' + +GCS_URI = 'gs://cloud-samples-data/vision/product_search/product_sets.csv' +PRODUCT_SET_DISPLAY_NAME = 'fake_product_set_display_name_for_testing' +PRODUCT_SET_ID = 'fake_product_set_id_for_testing' +PRODUCT_ID_1 = 'fake_product_id_for_testing_1' +PRODUCT_ID_2 = 'fake_product_id_for_testing_2' +IMAGE_URI_1 = 'shoes_1.jpg' +IMAGE_URI_2 = 'shoes_2.jpg' + + +@pytest.fixture +def teardown(): + # no set up, tear down only + yield None + + delete_product(PROJECT_ID, LOCATION, PRODUCT_ID_1) + delete_product(PROJECT_ID, LOCATION, PRODUCT_ID_2) + delete_product_set(PROJECT_ID, LOCATION, PRODUCT_SET_ID) + + +def test_import_product_sets(capsys, teardown): + list_product_sets(PROJECT_ID, LOCATION) + out, _ = capsys.readouterr() + assert PRODUCT_SET_ID not in out + + list_products(PROJECT_ID, LOCATION) + out, _ = capsys.readouterr() + assert PRODUCT_ID_1 not in out + assert PRODUCT_ID_2 not in out + + list_products_in_product_set(PROJECT_ID, LOCATION, PRODUCT_SET_ID) + out, _ = capsys.readouterr() + assert PRODUCT_ID_1 not in out + assert PRODUCT_ID_2 not in out + + list_reference_images(PROJECT_ID, LOCATION, PRODUCT_ID_1) + out, _ = capsys.readouterr() + assert IMAGE_URI_1 not in out + + list_reference_images(PROJECT_ID, LOCATION, PRODUCT_ID_2) + out, _ = capsys.readouterr() + assert IMAGE_URI_2 not in out + + import_product_sets(PROJECT_ID, LOCATION, GCS_URI) + + list_product_sets(PROJECT_ID, LOCATION) + out, _ = capsys.readouterr() + assert PRODUCT_SET_ID in out + + list_products(PROJECT_ID, LOCATION) + out, _ = capsys.readouterr() + assert PRODUCT_ID_1 in out + assert PRODUCT_ID_2 in out + + list_products_in_product_set(PROJECT_ID, LOCATION, PRODUCT_SET_ID) + out, _ = capsys.readouterr() + assert PRODUCT_ID_1 in out + assert PRODUCT_ID_2 in out + + list_reference_images(PROJECT_ID, LOCATION, PRODUCT_ID_1) + out, _ = capsys.readouterr() + assert IMAGE_URI_1 in out + + list_reference_images(PROJECT_ID, LOCATION, PRODUCT_ID_2) + out, _ = capsys.readouterr() + assert IMAGE_URI_2 in out diff --git a/vision/cloud-client/product_search/product_in_product_set_management.py b/vision/cloud-client/product_search/product_in_product_set_management.py new file mode 100755 index 00000000000..8c08c8a05e7 --- /dev/null +++ b/vision/cloud-client/product_search/product_in_product_set_management.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python + +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to perform create operations +on Product set in Cloud Vision Product Search. + +For more information, see the tutorial page at +https://cloud.google.com/vision/product-search/docs/ +""" + +import argparse + +# [START vision_product_search_add_product_to_product_set] +# [START vision_product_search_remove_product_from_product_set] +from google.cloud import vision + +# [END vision_product_search_add_product_to_product_set] +# [END vision_product_search_remove_product_from_product_set] + + +# [START vision_product_search_add_product_to_product_set] +def add_product_to_product_set( + project_id, location, product_id, product_set_id): + """Add a product to a product set. + Args: + project_id: Id of the project. + location: A compute region name. + product_id: Id of the product. + product_set_id: Id of the product set. + """ + client = vision.ProductSearchClient() + + # Get the full path of the product set. + product_set_path = client.product_set_path( + project=project_id, location=location, + product_set=product_set_id) + + # Get the full path of the product. + product_path = client.product_path( + project=project_id, location=location, product=product_id) + + # Add the product to the product set. + client.add_product_to_product_set( + name=product_set_path, product=product_path) + print('Product added to product set.') +# [END vision_product_search_add_product_to_product_set] + + +# [START vision_product_search_list_products_in_product_set] +def list_products_in_product_set( + project_id, location, product_set_id): + """List all products in a product set. + Args: + project_id: Id of the project. + location: A compute region name. + product_set_id: Id of the product set. + """ + client = vision.ProductSearchClient() + + # Get the full path of the product set. + product_set_path = client.product_set_path( + project=project_id, location=location, + product_set=product_set_id) + + # List all the products available in the product set. + products = client.list_products_in_product_set(name=product_set_path) + + # Display the product information. + for product in products: + print('Product name: {}'.format(product.name)) + print('Product id: {}'.format(product.name.split('/')[-1])) + print('Product display name: {}'.format(product.display_name)) + print('Product description: {}'.format(product.description)) + print('Product category: {}'.format(product.product_category)) + print('Product labels: {}'.format(product.product_labels)) +# [END vision_product_search_list_products_in_product_set] + + +# [START vision_product_search_remove_product_from_product_set] +def remove_product_from_product_set( + project_id, location, product_id, product_set_id): + """Remove a product from a product set. + Args: + project_id: Id of the project. + location: A compute region name. + product_id: Id of the product. + product_set_id: Id of the product set. + """ + client = vision.ProductSearchClient() + + # Get the full path of the product set. + product_set_path = client.product_set_path( + project=project_id, location=location, + product_set=product_set_id) + + # Get the full path of the product. + product_path = client.product_path( + project=project_id, location=location, product=product_id) + + # Remove the product from the product set. + client.remove_product_from_product_set( + name=product_set_path, product=product_path) + print('Product removed from product set.') +# [END vision_product_search_remove_product_from_product_set] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + subparsers = parser.add_subparsers(dest='command') + parser.add_argument( + '--project_id', + help='Project id. Required', + required=True) + parser.add_argument( + '--location', + help='Compute region name', + default='us-west1') + + add_product_to_product_set_parser = subparsers.add_parser( + 'add_product_to_product_set', help=add_product_to_product_set.__doc__) + add_product_to_product_set_parser.add_argument('product_id') + add_product_to_product_set_parser.add_argument('product_set_id') + + list_products_in_product_set_parser = subparsers.add_parser( + 'list_products_in_product_set', + help=list_products_in_product_set.__doc__) + list_products_in_product_set_parser.add_argument('product_set_id') + + remove_product_from_product_set_parser = subparsers.add_parser( + 'remove_product_from_product_set', + help=remove_product_from_product_set.__doc__) + remove_product_from_product_set_parser.add_argument('product_id') + remove_product_from_product_set_parser.add_argument('product_set_id') + + args = parser.parse_args() + + if args.command == 'add_product_to_product_set': + add_product_to_product_set( + args.project_id, args.location, args.product_id, + args.product_set_id) + elif args.command == 'list_products_in_product_set': + list_products_in_product_set( + args.project_id, args.location, args.product_set_id) + elif args.command == 'remove_product_from_product_set': + remove_product_from_product_set( + args.project_id, args.location, args.product_id, + args.product_set_id) diff --git a/vision/cloud-client/product_search/product_in_product_set_management_test.py b/vision/cloud-client/product_search/product_in_product_set_management_test.py new file mode 100644 index 00000000000..21a23bdb774 --- /dev/null +++ b/vision/cloud-client/product_search/product_in_product_set_management_test.py @@ -0,0 +1,77 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytest + +from product_in_product_set_management import ( + add_product_to_product_set, list_products_in_product_set, + remove_product_from_product_set) +from product_management import create_product, delete_product +from product_set_management import ( + create_product_set, delete_product_set) + + +PROJECT_ID = os.getenv('GCLOUD_PROJECT') +LOCATION = 'us-west1' + +PRODUCT_SET_DISPLAY_NAME = 'fake_product_set_display_name_for_testing' +PRODUCT_SET_ID = 'fake_product_set_id_for_testing' + +PRODUCT_DISPLAY_NAME = 'fake_product_display_name_for_testing' +PRODUCT_CATEGORY = 'homegoods' +PRODUCT_ID = 'fake_product_id_for_testing' + + +@pytest.fixture +def product_and_product_set(): + # set up + create_product_set( + PROJECT_ID, LOCATION, PRODUCT_SET_ID, PRODUCT_SET_DISPLAY_NAME) + create_product( + PROJECT_ID, LOCATION, PRODUCT_ID, + PRODUCT_DISPLAY_NAME, PRODUCT_CATEGORY) + + yield None + + # tear down + delete_product(PROJECT_ID, LOCATION, PRODUCT_ID) + delete_product_set(PROJECT_ID, LOCATION, PRODUCT_SET_ID) + + +def test_add_product_to_product_set(capsys, product_and_product_set): + list_products_in_product_set(PROJECT_ID, LOCATION, PRODUCT_SET_ID) + out, _ = capsys.readouterr() + assert 'Product id: {}'.format(PRODUCT_ID) not in out + + add_product_to_product_set( + PROJECT_ID, LOCATION, PRODUCT_ID, PRODUCT_SET_ID) + list_products_in_product_set(PROJECT_ID, LOCATION, PRODUCT_SET_ID) + out, _ = capsys.readouterr() + assert 'Product id: {}'.format(PRODUCT_ID) in out + + +def test_remove_product_from_product_set(capsys, product_and_product_set): + add_product_to_product_set( + PROJECT_ID, LOCATION, PRODUCT_ID, PRODUCT_SET_ID) + list_products_in_product_set(PROJECT_ID, LOCATION, PRODUCT_SET_ID) + out, _ = capsys.readouterr() + assert 'Product id: {}'.format(PRODUCT_ID) in out + + remove_product_from_product_set( + PROJECT_ID, LOCATION, PRODUCT_ID, PRODUCT_SET_ID) + list_products_in_product_set(PROJECT_ID, LOCATION, PRODUCT_SET_ID) + out, _ = capsys.readouterr() + assert 'Product id: {}'.format(PRODUCT_ID) not in out diff --git a/vision/cloud-client/product_search/product_management.py b/vision/cloud-client/product_search/product_management.py new file mode 100755 index 00000000000..62e95dbabb0 --- /dev/null +++ b/vision/cloud-client/product_search/product_management.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python + +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to perform basic operations on Product +in Cloud Vision Product Search. + +For more information, see the tutorial page at +https://cloud.google.com/vision/product-search/docs/ +""" + +import argparse + +# [START vision_product_search_create_product] +# [START vision_product_search_delete_product] +# [START vision_product_search_list_products] +# [START vision_product_search_get_product] +# [START vision_product_search_update_product_labels] +from google.cloud import vision + +# [END vision_product_search_create_product] +# [END vision_product_search_delete_product] +# [END vision_product_search_list_products] +# [END vision_product_search_get_product] +# [END vision_product_search_update_product_labels] + + +# [START vision_product_search_create_product] +def create_product( + project_id, location, product_id, product_display_name, + product_category): + """Create one product. + Args: + project_id: Id of the project. + location: A compute region name. + product_id: Id of the product. + product_display_name: Display name of the product. + product_category: Category of the product. + """ + client = vision.ProductSearchClient() + + # A resource that represents Google Cloud Platform location. + location_path = client.location_path(project=project_id, location=location) + + # Create a product with the product specification in the region. + # Set product display name and product category. + product = vision.types.Product( + display_name=product_display_name, + product_category=product_category) + + # The response is the product with the `name` field populated. + response = client.create_product( + parent=location_path, + product=product, + product_id=product_id) + + # Display the product information. + print('Product name: {}'.format(response.name)) +# [END vision_product_search_create_product] + + +# [START vision_product_search_list_products] +def list_products(project_id, location): + """List all products. + Args: + project_id: Id of the project. + location: A compute region name. + """ + client = vision.ProductSearchClient() + + # A resource that represents Google Cloud Platform location. + location_path = client.location_path(project=project_id, location=location) + + # List all the products available in the region. + products = client.list_products(parent=location_path) + + # Display the product information. + for product in products: + print('Product name: {}'.format(product.name)) + print('Product id: {}'.format(product.name.split('/')[-1])) + print('Product display name: {}'.format(product.display_name)) + print('Product description: {}'.format(product.description)) + print('Product category: {}'.format(product.product_category)) + print('Product labels: {}\n'.format(product.product_labels)) +# [END vision_product_search_list_products] + + +# [START vision_product_search_get_product] +def get_product(project_id, location, product_id): + """Get information about a product. + Args: + project_id: Id of the project. + location: A compute region name. + product_id: Id of the product. + """ + client = vision.ProductSearchClient() + + # Get the full path of the product. + product_path = client.product_path( + project=project_id, location=location, product=product_id) + + # Get complete detail of the product. + product = client.get_product(name=product_path) + + # Display the product information. + print('Product name: {}'.format(product.name)) + print('Product id: {}'.format(product.name.split('/')[-1])) + print('Product display name: {}'.format(product.display_name)) + print('Product description: {}'.format(product.description)) + print('Product category: {}'.format(product.product_category)) + print('Product labels: {}'.format(product.product_labels)) +# [END vision_product_search_get_product] + + +# [START vision_product_search_update_product_labels] +def update_product_labels( + project_id, location, product_id, key, value): + """Update the product labels. + Args: + project_id: Id of the project. + location: A compute region name. + product_id: Id of the product. + key: The key of the label. + value: The value of the label. + """ + client = vision.ProductSearchClient() + + # Get the name of the product. + product_path = client.product_path( + project=project_id, location=location, product=product_id) + + # Set product name, product label and product display name. + # Multiple labels are also supported. + key_value = vision.types.Product.KeyValue(key=key, value=value) + product = vision.types.Product( + name=product_path, + product_labels=[key_value]) + + # Updating only the product_labels field here. + update_mask = vision.types.FieldMask(paths=['product_labels']) + + # This overwrites the product_labels. + updated_product = client.update_product( + product=product, update_mask=update_mask) + + # Display the updated product information. + print('Product name: {}'.format(updated_product.name)) + print('Updated product labels: {}'.format(product.product_labels)) +# [END vision_product_search_update_product_labels] + + +# [START vision_product_search_delete_product] +def delete_product(project_id, location, product_id): + """Delete the product and all its reference images. + Args: + project_id: Id of the project. + location: A compute region name. + product_id: Id of the product. + """ + client = vision.ProductSearchClient() + + # Get the full path of the product. + product_path = client.product_path( + project=project_id, location=location, product=product_id) + + # Delete a product. + client.delete_product(name=product_path) + print('Product deleted.') +# [END vision_product_search_delete_product] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--project_id', + help='Project id. Required', + required=True) + parser.add_argument( + '--location', + help='Compute region name', + default='us-west1') + + subparsers = parser.add_subparsers(dest='command') + + create_product_parser = subparsers.add_parser( + 'create_product', help=create_product.__doc__) + create_product_parser.add_argument('product_id') + create_product_parser.add_argument('product_display_name') + create_product_parser.add_argument('product_category') + + list_products_parser = subparsers.add_parser( + 'list_products', help=list_products.__doc__) + + get_product_parser = subparsers.add_parser( + 'get_product', help=get_product.__doc__) + get_product_parser.add_argument('product_id') + + update_product_labels_parser = subparsers.add_parser( + 'update_product_labels', help=update_product_labels.__doc__) + update_product_labels_parser.add_argument('product_id') + update_product_labels_parser.add_argument('key') + update_product_labels_parser.add_argument('value') + + delete_product_parser = subparsers.add_parser( + 'delete_product', help=delete_product.__doc__) + delete_product_parser.add_argument('product_id') + + args = parser.parse_args() + + if args.command == 'create_product': + create_product( + args.project_id, args.location, args.product_id, + args.product_display_name, args.product_category) + elif args.command == 'list_products': + list_products(args.project_id, args.location) + elif args.command == 'get_product': + get_product(args.project_id, args.location, args.product_id) + elif args.command == 'update_product_labels': + update_product_labels( + args.project_id, args.location, args.product_id, + args.key, args.value) + elif args.command == 'delete_product': + delete_product(args.project_id, args.location, args.product_id) diff --git a/vision/cloud-client/product_search/product_management_test.py b/vision/cloud-client/product_search/product_management_test.py new file mode 100644 index 00000000000..fd1ad24aeb6 --- /dev/null +++ b/vision/cloud-client/product_search/product_management_test.py @@ -0,0 +1,85 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytest + +from product_management import ( + create_product, delete_product, get_product, list_products, + update_product_labels) + + +PROJECT_ID = os.getenv('GCLOUD_PROJECT') +LOCATION = 'us-west1' + +PRODUCT_DISPLAY_NAME = 'fake_product_display_name_for_testing' +PRODUCT_CATEGORY = 'homegoods' +PRODUCT_ID = 'fake_product_id_for_testing' +KEY = 'fake_key_for_testing' +VALUE = 'fake_value_for_testing' + + +@pytest.fixture +def product(): + # set up + create_product( + PROJECT_ID, LOCATION, PRODUCT_ID, + PRODUCT_DISPLAY_NAME, PRODUCT_CATEGORY) + + yield None + + # tear down + delete_product(PROJECT_ID, LOCATION, PRODUCT_ID) + + +def test_create_product(capsys): + list_products(PROJECT_ID, LOCATION) + out, _ = capsys.readouterr() + assert PRODUCT_ID not in out + + create_product( + PROJECT_ID, LOCATION, PRODUCT_ID, + PRODUCT_DISPLAY_NAME, PRODUCT_CATEGORY) + list_products(PROJECT_ID, LOCATION) + out, _ = capsys.readouterr() + assert PRODUCT_ID in out + + delete_product(PROJECT_ID, LOCATION, PRODUCT_ID) + + +def test_delete_product(capsys, product): + list_products(PROJECT_ID, LOCATION) + out, _ = capsys.readouterr() + assert PRODUCT_ID in out + + delete_product(PROJECT_ID, LOCATION, PRODUCT_ID) + + list_products(PROJECT_ID, LOCATION) + out, _ = capsys.readouterr() + assert PRODUCT_ID not in out + + +def test_update_product_labels(capsys, product): + get_product(PROJECT_ID, LOCATION, PRODUCT_ID) + out, _ = capsys.readouterr() + assert KEY not in out + assert VALUE not in out + + update_product_labels(PROJECT_ID, LOCATION, PRODUCT_ID, KEY, VALUE) + out, _ = capsys.readouterr() + assert KEY in out + assert VALUE in out + + delete_product(PROJECT_ID, LOCATION, PRODUCT_ID) diff --git a/vision/cloud-client/product_search/product_search.py b/vision/cloud-client/product_search/product_search.py new file mode 100755 index 00000000000..343bb6d0173 --- /dev/null +++ b/vision/cloud-client/product_search/product_search.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python + +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This tutorial demonstrates how users query the product set with their +own images and find the products similer to the image using the Cloud +Vision Product Search API. + +For more information, see the tutorial page at +https://cloud.google.com/vision/product-search/docs/ +""" + +import argparse + +# [START vision_product_search_get_similar_products] +# [START vision_product_search_get_similar_products_gcs] +from google.cloud import vision + +# [END vision_product_search_get_similar_products] +# [END vision_product_search_get_similar_products_gcs] + + +# [START vision_product_search_get_similar_products] +def get_similar_products_file( + project_id, location, product_set_id, product_category, + file_path, filter): + """Search similar products to image. + Args: + project_id: Id of the project. + location: A compute region name. + product_set_id: Id of the product set. + product_category: Category of the product. + file_path: Local file path of the image to be searched. + filter: Condition to be applied on the labels. + Example for filter: (color = red OR color = blue) AND style = kids + It will search on all products with the following labels: + color:red AND style:kids + color:blue AND style:kids + """ + # product_search_client is needed only for its helper methods. + product_search_client = vision.ProductSearchClient() + image_annotator_client = vision.ImageAnnotatorClient() + + # Read the image as a stream of bytes. + with open(file_path, 'rb') as image_file: + content = image_file.read() + + # Create annotate image request along with product search feature. + image = vision.types.Image(content=content) + + # product search specific parameters + product_set_path = product_search_client.product_set_path( + project=project_id, location=location, + product_set=product_set_id) + product_search_params = vision.types.ProductSearchParams( + product_set=product_set_path, + product_categories=[product_category], + filter=filter) + image_context = vision.types.ImageContext( + product_search_params=product_search_params) + + # Search products similar to the image. + response = image_annotator_client.product_search( + image, image_context=image_context) + + index_time = response.product_search_results.index_time + print('Product set index time:') + print(' seconds: {}'.format(index_time.seconds)) + print(' nanos: {}\n'.format(index_time.nanos)) + + results = response.product_search_results.results + + print('Search results:') + for result in results: + product = result.product + + print('Score(Confidence): {}'.format(result.score)) + print('Image name: {}'.format(result.image)) + + print('Product name: {}'.format(product.name)) + print('Product display name: {}'.format( + product.display_name)) + print('Product description: {}\n'.format(product.description)) + print('Product labels: {}\n'.format(product.product_labels)) +# [END vision_product_search_get_similar_products] + + +# [START vision_product_search_get_similar_products_gcs] +def get_similar_products_uri( + project_id, location, product_set_id, product_category, + image_uri, filter): + """Search similar products to image. + Args: + project_id: Id of the project. + location: A compute region name. + product_set_id: Id of the product set. + product_category: Category of the product. + file_path: Local file path of the image to be searched. + filter: Condition to be applied on the labels. + Example for filter: (color = red OR color = blue) AND style = kids + It will search on all products with the following labels: + color:red AND style:kids + color:blue AND style:kids + """ + # product_search_client is needed only for its helper methods. + product_search_client = vision.ProductSearchClient() + image_annotator_client = vision.ImageAnnotatorClient() + + # Create annotate image request along with product search feature. + image_source = vision.types.ImageSource(image_uri=image_uri) + image = vision.types.Image(source=image_source) + + # product search specific parameters + product_set_path = product_search_client.product_set_path( + project=project_id, location=location, + product_set=product_set_id) + product_search_params = vision.types.ProductSearchParams( + product_set=product_set_path, + product_categories=[product_category], + filter=filter) + image_context = vision.types.ImageContext( + product_search_params=product_search_params) + + # Search products similar to the image. + response = image_annotator_client.product_search( + image, image_context=image_context) + + index_time = response.product_search_results.index_time + print('Product set index time:') + print(' seconds: {}'.format(index_time.seconds)) + print(' nanos: {}\n'.format(index_time.nanos)) + + results = response.product_search_results.results + + print('Search results:') + for result in results: + product = result.product + + print('Score(Confidence): {}'.format(result.score)) + print('Image name: {}'.format(result.image)) + + print('Product name: {}'.format(product.name)) + print('Product display name: {}'.format( + product.display_name)) + print('Product description: {}\n'.format(product.description)) + print('Product labels: {}\n'.format(product.product_labels)) +# [END vision_product_search_get_similar_products_gcs] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + subparsers = parser.add_subparsers(dest='command') + parser.add_argument( + '--project_id', + help='Project id. Required', + required=True) + parser.add_argument( + '--location', + help='Compute region name', + default='us-west1') + parser.add_argument('--product_set_id') + parser.add_argument('--product_category') + parser.add_argument('--filter', default='') + + get_similar_products_file_parser = subparsers.add_parser( + 'get_similar_products_file', help=get_similar_products_file.__doc__) + get_similar_products_file_parser.add_argument('--file_path') + + get_similar_products_uri_parser = subparsers.add_parser( + 'get_similar_products_uri', help=get_similar_products_uri.__doc__) + get_similar_products_uri_parser.add_argument('--image_uri') + + args = parser.parse_args() + + if args.command == 'get_similar_products_file': + get_similar_products_file( + args.project_id, args.location, args.product_set_id, + args.product_category, args.file_path, args.filter) + elif args.command == 'get_similar_products_uri': + get_similar_products_uri( + args.project_id, args.location, args.product_set_id, + args.product_category, args.image_uri, args.filter) diff --git a/vision/cloud-client/product_search/product_search_test.py b/vision/cloud-client/product_search/product_search_test.py new file mode 100644 index 00000000000..ae518cd24ed --- /dev/null +++ b/vision/cloud-client/product_search/product_search_test.py @@ -0,0 +1,66 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from product_search import get_similar_products_file, get_similar_products_uri + + +PROJECT_ID = os.getenv('GCLOUD_PROJECT') +LOCATION = 'us-west1' + +PRODUCT_SET_ID = 'indexed_product_set_id_for_testing' +PRODUCT_CATEGORY = 'apparel' +PRODUCT_ID_1 = 'indexed_product_id_for_testing_1' +PRODUCT_ID_2 = 'indexed_product_id_for_testing_2' + +FILE_PATH_1 = 'resources/shoes_1.jpg' +IMAGE_URI_1 = 'gs://cloud-samples-data/vision/product_search/shoes_1.jpg' +FILTER = 'style=womens' + + +def test_get_similar_products_file(capsys): + get_similar_products_file( + PROJECT_ID, LOCATION, PRODUCT_SET_ID, PRODUCT_CATEGORY, FILE_PATH_1, + '') + out, _ = capsys.readouterr() + assert PRODUCT_ID_1 in out + assert PRODUCT_ID_2 in out + + +def test_get_similar_products_uri(capsys): + get_similar_products_uri( + PROJECT_ID, LOCATION, PRODUCT_SET_ID, PRODUCT_CATEGORY, IMAGE_URI_1, + '') + out, _ = capsys.readouterr() + assert PRODUCT_ID_1 in out + assert PRODUCT_ID_2 in out + + +def test_get_similar_products_file_with_filter(capsys): + get_similar_products_file( + PROJECT_ID, LOCATION, PRODUCT_SET_ID, PRODUCT_CATEGORY, FILE_PATH_1, + FILTER) + out, _ = capsys.readouterr() + assert PRODUCT_ID_1 in out + assert PRODUCT_ID_2 not in out + + +def test_get_similar_products_uri_with_filter(capsys): + get_similar_products_uri( + PROJECT_ID, LOCATION, PRODUCT_SET_ID, PRODUCT_CATEGORY, IMAGE_URI_1, + FILTER) + out, _ = capsys.readouterr() + assert PRODUCT_ID_1 in out + assert PRODUCT_ID_2 not in out diff --git a/vision/cloud-client/product_search/product_set_management.py b/vision/cloud-client/product_search/product_set_management.py new file mode 100755 index 00000000000..7964bc27b30 --- /dev/null +++ b/vision/cloud-client/product_search/product_set_management.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python + +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to perform operations +on Product set in Cloud Vision Product Search. + +For more information, see the tutorial page at +https://cloud.google.com/vision/product-search/docs/ +""" + +import argparse + +# [START vision_product_search_delete_product_set] +# [START vision_product_search_list_product_sets] +# [START vision_product_search_get_product_set] +# [START vision_product_search_create_product_set] +from google.cloud import vision + +# [END vision_product_search_delete_product_set] +# [END vision_product_search_list_product_sets] +# [END vision_product_search_get_product_set] +# [END vision_product_search_create_product_set] + + +# [START vision_product_search_create_product_set] +def create_product_set( + project_id, location, product_set_id, product_set_display_name): + """Create a product set. + Args: + project_id: Id of the project. + location: A compute region name. + product_set_id: Id of the product set. + product_set_display_name: Display name of the product set. + """ + client = vision.ProductSearchClient() + + # A resource that represents Google Cloud Platform location. + location_path = client.location_path( + project=project_id, location=location) + + # Create a product set with the product set specification in the region. + product_set = vision.types.ProductSet( + display_name=product_set_display_name) + + # The response is the product set with `name` populated. + response = client.create_product_set( + parent=location_path, + product_set=product_set, + product_set_id=product_set_id) + + # Display the product set information. + print('Product set name: {}'.format(response.name)) +# [END vision_product_search_create_product_set] + + +# [START vision_product_search_list_product_sets] +def list_product_sets(project_id, location): + """List all product sets. + Args: + project_id: Id of the project. + location: A compute region name. + """ + client = vision.ProductSearchClient() + + # A resource that represents Google Cloud Platform location. + location_path = client.location_path( + project=project_id, location=location) + + # List all the product sets available in the region. + product_sets = client.list_product_sets(parent=location_path) + + # Display the product set information. + for product_set in product_sets: + print('Product set name: {}'.format(product_set.name)) + print('Product set id: {}'.format(product_set.name.split('/')[-1])) + print('Product set display name: {}'.format(product_set.display_name)) + print('Product set index time:') + print(' seconds: {}'.format(product_set.index_time.seconds)) + print(' nanos: {}\n'.format(product_set.index_time.nanos)) +# [END vision_product_search_list_product_sets] + + +# [START vision_product_search_get_product_set] +def get_product_set(project_id, location, product_set_id): + """Get info about the product set. + Args: + project_id: Id of the project. + location: A compute region name. + product_set_id: Id of the product set. + """ + client = vision.ProductSearchClient() + + # Get the full path of the product set. + product_set_path = client.product_set_path( + project=project_id, location=location, + product_set=product_set_id) + + # Get complete detail of the product set. + product_set = client.get_product_set(name=product_set_path) + + # Display the product set information. + print('Product set name: {}'.format(product_set.name)) + print('Product set id: {}'.format(product_set.name.split('/')[-1])) + print('Product set display name: {}'.format(product_set.display_name)) + print('Product set index time:') + print(' seconds: {}'.format(product_set.index_time.seconds)) + print(' nanos: {}'.format(product_set.index_time.nanos)) +# [END vision_product_search_get_product_set] + + +# [START vision_product_search_delete_product_set] +def delete_product_set(project_id, location, product_set_id): + """Delete a product set. + Args: + project_id: Id of the project. + location: A compute region name. + product_set_id: Id of the product set. + """ + client = vision.ProductSearchClient() + + # Get the full path of the product set. + product_set_path = client.product_set_path( + project=project_id, location=location, + product_set=product_set_id) + + # Delete the product set. + client.delete_product_set(name=product_set_path) + print('Product set deleted.') +# [END vision_product_search_delete_product_set] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + subparsers = parser.add_subparsers(dest='command') + parser.add_argument( + '--project_id', + help='Project id. Required', + required=True) + parser.add_argument( + '--location', + help='Compute region name', + default='us-west1') + + create_product_set_parser = subparsers.add_parser( + 'create_product_set', help=create_product_set.__doc__) + create_product_set_parser.add_argument('product_set_id') + create_product_set_parser.add_argument('product_set_display_name') + + list_product_sets_parser = subparsers.add_parser( + 'list_product_sets', help=list_product_sets.__doc__) + + get_product_set_parser = subparsers.add_parser( + 'get_product_set', help=get_product_set.__doc__) + get_product_set_parser.add_argument('product_set_id') + + delete_product_set_parser = subparsers.add_parser( + 'delete_product_set', help=delete_product_set.__doc__) + delete_product_set_parser.add_argument('product_set_id') + + args = parser.parse_args() + + if args.command == 'create_product_set': + create_product_set( + args.project_id, args.location, args.product_set_id, + args.product_set_display_name) + elif args.command == 'list_product_sets': + list_product_sets(args.project_id, args.location) + elif args.command == 'get_product_set': + get_product_set(args.project_id, args.location, args.product_set_id) + elif args.command == 'delete_product_set': + delete_product_set( + args.project_id, args.location, args.product_set_id) diff --git a/vision/cloud-client/product_search/product_set_management_test.py b/vision/cloud-client/product_search/product_set_management_test.py new file mode 100644 index 00000000000..2148f29e345 --- /dev/null +++ b/vision/cloud-client/product_search/product_set_management_test.py @@ -0,0 +1,66 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytest + +from product_set_management import ( + create_product_set, delete_product_set, list_product_sets) + + +PROJECT_ID = os.getenv('GCLOUD_PROJECT') +LOCATION = 'us-west1' + +PRODUCT_SET_DISPLAY_NAME = 'fake_product_set_display_name_for_testing' +PRODUCT_SET_ID = 'fake_product_set_id_for_testing' + + +@pytest.fixture +def product_set(): + # set up + create_product_set( + PROJECT_ID, LOCATION, PRODUCT_SET_ID, PRODUCT_SET_DISPLAY_NAME) + + yield None + + # tear down + delete_product_set(PROJECT_ID, LOCATION, PRODUCT_SET_ID) + + +def test_create_product_set(capsys): + list_product_sets(PROJECT_ID, LOCATION) + out, _ = capsys.readouterr() + assert PRODUCT_SET_ID not in out + + create_product_set( + PROJECT_ID, LOCATION, PRODUCT_SET_ID, + PRODUCT_SET_DISPLAY_NAME) + list_product_sets(PROJECT_ID, LOCATION) + out, _ = capsys.readouterr() + assert PRODUCT_SET_ID in out + + delete_product_set(PROJECT_ID, LOCATION, PRODUCT_SET_ID) + + +def test_delete_product_set(capsys, product_set): + list_product_sets(PROJECT_ID, LOCATION) + out, _ = capsys.readouterr() + assert PRODUCT_SET_ID in out + + delete_product_set(PROJECT_ID, LOCATION, PRODUCT_SET_ID) + + list_product_sets(PROJECT_ID, LOCATION) + out, _ = capsys.readouterr() + assert PRODUCT_SET_ID not in out diff --git a/vision/cloud-client/product_search/reference_image_management.py b/vision/cloud-client/product_search/reference_image_management.py new file mode 100755 index 00000000000..7e546b7e74a --- /dev/null +++ b/vision/cloud-client/product_search/reference_image_management.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python + +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to perform basic operations on reference +images in Cloud Vision Product Search. + +For more information, see the tutorial page at +https://cloud.google.com/vision/product-search/docs/ +""" + +import argparse + +# [START vision_product_search_create_reference_image] +# [START vision_product_search_delete_reference_image] +# [START vision_product_search_list_reference_images] +# [START vision_product_search_get_reference_image] +from google.cloud import vision + +# [END vision_product_search_create_reference_image] +# [END vision_product_search_delete_reference_image] +# [END vision_product_search_list_reference_images] +# [END vision_product_search_get_reference_image] + + +# [START vision_product_search_create_reference_image] +def create_reference_image( + project_id, location, product_id, reference_image_id, gcs_uri): + """Create a reference image. + Args: + project_id: Id of the project. + location: A compute region name. + product_id: Id of the product. + reference_image_id: Id of the reference image. + gcs_uri: Google Cloud Storage path of the input image. + """ + client = vision.ProductSearchClient() + + # Get the full path of the product. + product_path = client.product_path( + project=project_id, location=location, product=product_id) + + # Create a reference image. + reference_image = vision.types.ReferenceImage(uri=gcs_uri) + + # The response is the reference image with `name` populated. + image = client.create_reference_image( + parent=product_path, + reference_image=reference_image, + reference_image_id=reference_image_id) + + # Display the reference image information. + print('Reference image name: {}'.format(image.name)) + print('Reference image uri: {}'.format(image.uri)) +# [END vision_product_search_create_reference_image] + + +# [START vision_product_search_list_reference_images] +def list_reference_images( + project_id, location, product_id): + """List all images in a product. + Args: + project_id: Id of the project. + location: A compute region name. + product_id: Id of the product. + """ + client = vision.ProductSearchClient() + + # Get the full path of the product. + product_path = client.product_path( + project=project_id, location=location, product=product_id) + + # List all the reference images available in the product. + reference_images = client.list_reference_images(parent=product_path) + + # Display the reference image information. + for image in reference_images: + print('Reference image name: {}'.format(image.name)) + print('Reference image id: {}'.format(image.name.split('/')[-1])) + print('Reference image uri: {}'.format(image.uri)) + print('Reference image bounding polygons: {}'.format( + image.bounding_polys)) +# [END vision_product_search_list_reference_images] + + +# [START vision_product_search_get_reference_image] +def get_reference_image( + project_id, location, product_id, reference_image_id): + """Get info about a reference image. + Args: + project_id: Id of the project. + location: A compute region name. + product_id: Id of the product. + reference_image_id: Id of the reference image. + """ + client = vision.ProductSearchClient() + + # Get the full path of the reference image. + reference_image_path = client.reference_image_path( + project=project_id, location=location, product=product_id, + reference_image=reference_image_id) + + # Get complete detail of the reference image. + image = client.get_reference_image(name=reference_image_path) + + # Display the reference image information. + print('Reference image name: {}'.format(image.name)) + print('Reference image id: {}'.format(image.name.split('/')[-1])) + print('Reference image uri: {}'.format(image.uri)) + print('Reference image bounding polygons: {}'.format(image.bounding_polys)) +# [END vision_product_search_get_reference_image] + + +# [START vision_product_search_delete_reference_image] +def delete_reference_image( + project_id, location, product_id, reference_image_id): + """Delete a reference image. + Args: + project_id: Id of the project. + location: A compute region name. + product_id: Id of the product. + reference_image_id: Id of the reference image. + """ + client = vision.ProductSearchClient() + + # Get the full path of the reference image. + reference_image_path = client.reference_image_path( + project=project_id, location=location, product=product_id, + reference_image=reference_image_id) + + # Delete the reference image. + client.delete_reference_image(name=reference_image_path) + print('Reference image deleted from product.') +# [END vision_product_search_delete_reference_image] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + subparsers = parser.add_subparsers(dest='command') + parser.add_argument( + '--project_id', + help='Project id. Required', + required=True) + parser.add_argument( + '--location', + help='Compute region name', + default='us-west1') + + create_reference_image_parser = subparsers.add_parser( + 'create_reference_image', help=create_reference_image.__doc__) + create_reference_image_parser.add_argument('product_id') + create_reference_image_parser.add_argument('reference_image_id') + create_reference_image_parser.add_argument('gcs_uri') + + list_reference_images_parser = subparsers.add_parser( + 'list_reference_images', + help=list_reference_images.__doc__) + list_reference_images_parser.add_argument('product_id') + + get_reference_image_parser = subparsers.add_parser( + 'get_reference_image', help=get_reference_image.__doc__) + get_reference_image_parser.add_argument('product_id') + get_reference_image_parser.add_argument('reference_image_id') + + delete_reference_image_parser = subparsers.add_parser( + 'delete_reference_image', help=delete_reference_image.__doc__) + delete_reference_image_parser.add_argument('product_id') + delete_reference_image_parser.add_argument('reference_image_id') + + args = parser.parse_args() + + if args.command == 'create_reference_image': + create_reference_image( + args.project_id, args.location, args.product_id, + args.reference_image_id, args.gcs_uri) + elif args.command == 'list_reference_images': + list_reference_images( + args.project_id, args.location, args.product_id) + elif args.command == 'get_reference_image': + get_reference_image( + args.project_id, args.location, args.product_id, + args.reference_image_id) + elif args.command == 'delete_reference_image': + delete_reference_image( + args.project_id, args.location, args.product_id, + args.reference_image_id) diff --git a/vision/cloud-client/product_search/reference_image_management_test.py b/vision/cloud-client/product_search/reference_image_management_test.py new file mode 100644 index 00000000000..335bf4195aa --- /dev/null +++ b/vision/cloud-client/product_search/reference_image_management_test.py @@ -0,0 +1,77 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytest + +from product_management import create_product, delete_product +from reference_image_management import ( + create_reference_image, delete_reference_image, list_reference_images) + + +PROJECT_ID = os.getenv('GCLOUD_PROJECT') +LOCATION = 'us-west1' + +PRODUCT_DISPLAY_NAME = 'fake_product_display_name_for_testing' +PRODUCT_CATEGORY = 'homegoods' +PRODUCT_ID = 'fake_product_id_for_testing' + +REFERENCE_IMAGE_ID = 'fake_reference_image_id_for_testing' +GCS_URI = 'gs://cloud-samples-data/vision/product_search/shoes_1.jpg' + + +@pytest.fixture +def product(): + # set up + create_product( + PROJECT_ID, LOCATION, PRODUCT_ID, + PRODUCT_DISPLAY_NAME, PRODUCT_CATEGORY) + + yield None + + # tear down + delete_product(PROJECT_ID, LOCATION, PRODUCT_ID) + + +def test_create_reference_image(capsys, product): + list_reference_images(PROJECT_ID, LOCATION, PRODUCT_ID) + out, _ = capsys.readouterr() + assert REFERENCE_IMAGE_ID not in out + + create_reference_image( + PROJECT_ID, LOCATION, PRODUCT_ID, REFERENCE_IMAGE_ID, + GCS_URI) + list_reference_images(PROJECT_ID, LOCATION, PRODUCT_ID) + out, _ = capsys.readouterr() + assert REFERENCE_IMAGE_ID in out + + delete_product(PROJECT_ID, LOCATION, PRODUCT_ID) + + +def test_delete_reference_image(capsys, product): + create_reference_image( + PROJECT_ID, LOCATION, PRODUCT_ID, REFERENCE_IMAGE_ID, + GCS_URI) + list_reference_images(PROJECT_ID, LOCATION, PRODUCT_ID) + out, _ = capsys.readouterr() + assert REFERENCE_IMAGE_ID in out + + delete_reference_image( + PROJECT_ID, LOCATION, PRODUCT_ID, REFERENCE_IMAGE_ID) + list_reference_images(PROJECT_ID, LOCATION, PRODUCT_ID) + out, _ = capsys.readouterr() + assert REFERENCE_IMAGE_ID not in out + + delete_product(PROJECT_ID, LOCATION, PRODUCT_ID) diff --git a/vision/cloud-client/product_search/requirements.txt b/vision/cloud-client/product_search/requirements.txt new file mode 100644 index 00000000000..dad4a99e19c --- /dev/null +++ b/vision/cloud-client/product_search/requirements.txt @@ -0,0 +1 @@ +google-cloud-vision==0.35.2 diff --git a/vision/cloud-client/product_search/resources/indexed_product_sets.csv b/vision/cloud-client/product_search/resources/indexed_product_sets.csv new file mode 100644 index 00000000000..329ac2167c7 --- /dev/null +++ b/vision/cloud-client/product_search/resources/indexed_product_sets.csv @@ -0,0 +1,2 @@ +"gs://cloud-samples-data/vision/product_search/shoes_1.jpg","indexed_product_set_id_for_testing","indexed_product_id_for_testing_1","apparel","style=womens","0.1,0.1,0.9,0.1,0.9,0.9,0.1,0.9" +"gs://cloud-samples-data/vision/product_search/shoes_2.jpg","indexed_product_set_id_for_testing","indexed_product_id_for_testing_2","apparel",, \ No newline at end of file diff --git a/vision/cloud-client/product_search/resources/product_sets.csv b/vision/cloud-client/product_search/resources/product_sets.csv new file mode 100644 index 00000000000..68657eed631 --- /dev/null +++ b/vision/cloud-client/product_search/resources/product_sets.csv @@ -0,0 +1,2 @@ +"gs://cloud-samples-data/vision/product_search/shoes_1.jpg","fake_product_set_id_for_testing","fake_product_id_for_testing_1","apparel","style=womens","0.1,0.1,0.9,0.1,0.9,0.9,0.1,0.9" +"gs://cloud-samples-data/vision/product_search/shoes_2.jpg","fake_product_set_id_for_testing","fake_product_id_for_testing_2","apparel",, \ No newline at end of file diff --git a/vision/cloud-client/product_search/resources/shoes_1.jpg b/vision/cloud-client/product_search/resources/shoes_1.jpg new file mode 100644 index 00000000000..78318eeff66 Binary files /dev/null and b/vision/cloud-client/product_search/resources/shoes_1.jpg differ diff --git a/vision/cloud-client/product_search/resources/shoes_2.jpg b/vision/cloud-client/product_search/resources/shoes_2.jpg new file mode 100644 index 00000000000..cdfa80dd899 Binary files /dev/null and b/vision/cloud-client/product_search/resources/shoes_2.jpg differ diff --git a/vision/cloud-client/quickstart/README.rst b/vision/cloud-client/quickstart/README.rst new file mode 100644 index 00000000000..aa4be034e4c --- /dev/null +++ b/vision/cloud-client/quickstart/README.rst @@ -0,0 +1,101 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Vision API Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=vision/cloud-client/quickstart/README.rst + + +This directory contains samples for Google Cloud Vision API. `Google Cloud Vision API`_ allows developers to easily integrate vision detection features within applications, including image labeling, face and landmark detection, optical character recognition (OCR), and tagging of explicit content. + +- See the `migration guide`_ for information about migrating to Python client library v0.25.1. + +.. _migration guide: https://cloud.google.com/vision/docs/python-client-migration + + + + +.. _Google Cloud Vision API: https://cloud.google.com/vision/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Quickstart ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=vision/cloud-client/quickstart/quickstart.py,vision/cloud-client/quickstart/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python quickstart.py + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/vision/cloud-client/quickstart/README.rst.in b/vision/cloud-client/quickstart/README.rst.in new file mode 100644 index 00000000000..bd650a6cb6f --- /dev/null +++ b/vision/cloud-client/quickstart/README.rst.in @@ -0,0 +1,29 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Vision API + short_name: Cloud Vision API + url: https://cloud.google.com/vision/docs + description: > + `Google Cloud Vision API`_ allows developers to easily integrate vision + detection features within applications, including image labeling, face and + landmark detection, optical character recognition (OCR), and tagging of + explicit content. + + + - See the `migration guide`_ for information about migrating to Python client library v0.25.1. + + + .. _migration guide: https://cloud.google.com/vision/docs/python-client-migration + +setup: +- auth +- install_deps + +samples: +- name: Quickstart + file: quickstart.py + +cloud_client_library: true + +folder: vision/cloud-client/quickstart \ No newline at end of file diff --git a/vision/cloud-client/quickstart/quickstart.py b/vision/cloud-client/quickstart/quickstart.py new file mode 100644 index 00000000000..8bb674eb118 --- /dev/null +++ b/vision/cloud-client/quickstart/quickstart.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def run_quickstart(): + # [START vision_quickstart] + import io + import os + + # Imports the Google Cloud client library + # [START vision_python_migration_import] + from google.cloud import vision + from google.cloud.vision import types + # [END vision_python_migration_import] + + # Instantiates a client + # [START vision_python_migration_client] + client = vision.ImageAnnotatorClient() + # [END vision_python_migration_client] + + # The name of the image file to annotate + file_name = os.path.join( + os.path.dirname(__file__), + 'resources/wakeupcat.jpg') + + # Loads the image into memory + with io.open(file_name, 'rb') as image_file: + content = image_file.read() + + image = types.Image(content=content) + + # Performs label detection on the image file + response = client.label_detection(image=image) + labels = response.label_annotations + + print('Labels:') + for label in labels: + print(label.description) + # [END vision_quickstart] + + +if __name__ == '__main__': + run_quickstart() diff --git a/vision/cloud-client/quickstart/quickstart_test.py b/vision/cloud-client/quickstart/quickstart_test.py new file mode 100644 index 00000000000..d483d4131f7 --- /dev/null +++ b/vision/cloud-client/quickstart/quickstart_test.py @@ -0,0 +1,21 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import quickstart + + +def test_quickstart(capsys): + quickstart.run_quickstart() + out, _ = capsys.readouterr() + assert 'Labels' in out diff --git a/vision/cloud-client/quickstart/requirements.txt b/vision/cloud-client/quickstart/requirements.txt new file mode 100644 index 00000000000..dad4a99e19c --- /dev/null +++ b/vision/cloud-client/quickstart/requirements.txt @@ -0,0 +1 @@ +google-cloud-vision==0.35.2 diff --git a/vision/cloud-client/quickstart/resources/wakeupcat.jpg b/vision/cloud-client/quickstart/resources/wakeupcat.jpg new file mode 100644 index 00000000000..139cf461eca Binary files /dev/null and b/vision/cloud-client/quickstart/resources/wakeupcat.jpg differ diff --git a/vision/cloud-client/web/README.rst b/vision/cloud-client/web/README.rst new file mode 100644 index 00000000000..9763ef3a099 --- /dev/null +++ b/vision/cloud-client/web/README.rst @@ -0,0 +1,118 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Vision API Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=vision/cloud-client/web/README.rst + + +This directory contains samples for Google Cloud Vision API. `Google Cloud Vision API`_ allows developers to easily integrate vision detection features within applications, including image labeling, face and landmark detection, optical character recognition (OCR), and tagging of explicit content. + +- See the `migration guide`_ for information about migrating to Python client library v0.25.1. + +.. _migration guide: https://cloud.google.com/vision/docs/python-client-migration + + + + +.. _Google Cloud Vision API: https://cloud.google.com/vision/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Web ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=vision/cloud-client/web/web_detect.py,vision/cloud-client/web/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python web_detect.py + + usage: web_detect.py [-h] image_url + + Demonstrates web detection using the Google Cloud Vision API. + + Example usage: + python web_detect.py https://goo.gl/X4qcB6 + python web_detect.py ../detect/resources/landmark.jpg + python web_detect.py gs://your-bucket/image.png + + positional arguments: + image_url The image to detect, can be web URI, Google Cloud Storage, or + path to local file. + + optional arguments: + -h, --help show this help message and exit + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/vision/cloud-client/web/README.rst.in b/vision/cloud-client/web/README.rst.in new file mode 100644 index 00000000000..8b8533b5261 --- /dev/null +++ b/vision/cloud-client/web/README.rst.in @@ -0,0 +1,30 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Vision API + short_name: Cloud Vision API + url: https://cloud.google.com/vision/docs + description: > + `Google Cloud Vision API`_ allows developers to easily integrate vision + detection features within applications, including image labeling, face and + landmark detection, optical character recognition (OCR), and tagging of + explicit content. + + + - See the `migration guide`_ for information about migrating to Python client library v0.25.1. + + + .. _migration guide: https://cloud.google.com/vision/docs/python-client-migration + +setup: +- auth +- install_deps + +samples: +- name: Web + file: web_detect.py + show_help: True + +cloud_client_library: true + +folder: vision/cloud-client/web \ No newline at end of file diff --git a/vision/cloud-client/web/requirements.txt b/vision/cloud-client/web/requirements.txt new file mode 100644 index 00000000000..dad4a99e19c --- /dev/null +++ b/vision/cloud-client/web/requirements.txt @@ -0,0 +1 @@ +google-cloud-vision==0.35.2 diff --git a/vision/cloud-client/web/web_detect.py b/vision/cloud-client/web/web_detect.py new file mode 100644 index 00000000000..6cdfa25643e --- /dev/null +++ b/vision/cloud-client/web/web_detect.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python + +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Demonstrates web detection using the Google Cloud Vision API. + +Example usage: + python web_detect.py https://goo.gl/X4qcB6 + python web_detect.py ../detect/resources/landmark.jpg + python web_detect.py gs://your-bucket/image.png +""" +# [START vision_web_detection_tutorial] +# [START vision_web_detection_tutorial_imports] +import argparse +import io + +from google.cloud import vision +from google.cloud.vision import types +# [END vision_web_detection_tutorial_imports] + + +def annotate(path): + """Returns web annotations given the path to an image.""" + # [START vision_web_detection_tutorial_annotate] + client = vision.ImageAnnotatorClient() + + if path.startswith('http') or path.startswith('gs:'): + image = types.Image() + image.source.image_uri = path + + else: + with io.open(path, 'rb') as image_file: + content = image_file.read() + + image = types.Image(content=content) + + web_detection = client.web_detection(image=image).web_detection + # [END vision_web_detection_tutorial_annotate] + + return web_detection + + +def report(annotations): + """Prints detected features in the provided web annotations.""" + # [START vision_web_detection_tutorial_print_annotations] + if annotations.pages_with_matching_images: + print('\n{} Pages with matching images retrieved'.format( + len(annotations.pages_with_matching_images))) + + for page in annotations.pages_with_matching_images: + print('Url : {}'.format(page.url)) + + if annotations.full_matching_images: + print('\n{} Full Matches found: '.format( + len(annotations.full_matching_images))) + + for image in annotations.full_matching_images: + print('Url : {}'.format(image.url)) + + if annotations.partial_matching_images: + print('\n{} Partial Matches found: '.format( + len(annotations.partial_matching_images))) + + for image in annotations.partial_matching_images: + print('Url : {}'.format(image.url)) + + if annotations.web_entities: + print('\n{} Web entities found: '.format( + len(annotations.web_entities))) + + for entity in annotations.web_entities: + print('Score : {}'.format(entity.score)) + print('Description: {}'.format(entity.description)) + # [END vision_web_detection_tutorial_print_annotations] + + +if __name__ == '__main__': + # [START vision_web_detection_tutorial_run_application] + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + path_help = str('The image to detect, can be web URI, ' + 'Google Cloud Storage, or path to local file.') + parser.add_argument('image_url', help=path_help) + args = parser.parse_args() + + report(annotate(args.image_url)) + # [END vision_web_detection_tutorial_run_application] +# [END vision_web_detection_tutorial] diff --git a/vision/cloud-client/web/web_detect_test.py b/vision/cloud-client/web/web_detect_test.py new file mode 100644 index 00000000000..3a0ea54840f --- /dev/null +++ b/vision/cloud-client/web/web_detect_test.py @@ -0,0 +1,40 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import web_detect + +ASSET_BUCKET = "cloud-samples-data" + + +def test_detect_file(capsys): + file_name = ('../detect/resources/landmark.jpg') + web_detect.report(web_detect.annotate(file_name)) + out, _ = capsys.readouterr() + print(out) + assert 'description: palace of fine arts' in out.lower() + + +def test_detect_web_gsuri(capsys): + file_name = ('gs://{}/vision/landmark/pofa.jpg'.format( + ASSET_BUCKET)) + web_detect.report(web_detect.annotate(file_name)) + out, _ = capsys.readouterr() + assert 'description: palace of fine arts' in out.lower() + + +def test_detect_web_http(capsys): + web_detect.report(web_detect.annotate( + 'https://cloud.google.com/images/products/vision/extract-text.png')) + out, _ = capsys.readouterr() + assert 'https://cloud.google.com/vision/' in out