diff --git a/nox.py b/nox.py index fc848339616..083fafa8042 100644 --- a/nox.py +++ b/nox.py @@ -320,3 +320,12 @@ def mark_if_necessary(requirement): for requirement in sorted(requirements, key=lambda s: s.lower()): if not requirement.startswith('#'): f.write(requirement) + + +def session_readmegen(session): + session.install('-r', 'testing/requirements-dev.txt') + + in_files = list(list_files('.', 'README.rst.in')) + + for in_file in in_files: + session.run('python', 'scripts/readme-gen/readme_gen.py', in_file) diff --git a/scripts/readme-gen/readme_gen.py b/scripts/readme-gen/readme_gen.py new file mode 100644 index 00000000000..469c53ff52c --- /dev/null +++ b/scripts/readme-gen/readme_gen.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generates READMEs using configuration defined in yaml.""" + +import argparse +import io +import os +import subprocess + +import jinja2 +import yaml + + +jinja_env = jinja2.Environment( + trim_blocks=True, + loader=jinja2.FileSystemLoader( + os.path.abspath(os.path.join(os.path.dirname(__file__), 'templates')))) + +README_TMPL = jinja_env.get_template('README.tmpl.rst') + + +def get_help(file): + return subprocess.check_output(['python', file, '--help']) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('source') + parser.add_argument('--destination', default='README.rst') + + args = parser.parse_args() + + source = os.path.abspath(args.source) + root = os.path.dirname(source) + destination = os.path.join(root, args.destination) + + jinja_env.globals['get_help'] = get_help + + with io.open(source, 'r') as f: + config = yaml.load(f) + + # This allows get_help to execute in the right directory. + os.chdir(root) + + output = README_TMPL.render(config) + + with io.open(destination, 'w') as f: + f.write(output) + +if __name__ == '__main__': + main() diff --git a/scripts/readme-gen/templates/README.tmpl.rst b/scripts/readme-gen/templates/README.tmpl.rst new file mode 100644 index 00000000000..1d960a72404 --- /dev/null +++ b/scripts/readme-gen/templates/README.tmpl.rst @@ -0,0 +1,62 @@ +{# The following line is a lie. BUT! Once jinja2 is done with it, it will + become truth! #} +.. This file is automatically generated. Do not edit this file directly. + +{{product.name}} Python Samples +=============================================================================== + +This directory contains samples for {{product.name}}. {{product.description}} + +.. _{{product.name}}: {{product.url}} + +{% if setup %} +Setup +------------------------------------------------------------------------------- + +{% for section in setup %} + +{% include section + '.tmpl.rst' %} + +{% endfor %} +{% endif %} + +{% if samples %} +Samples +------------------------------------------------------------------------------- + +{% for sample in samples %} +{{sample.name}} ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +{{sample.description}} + +To run this sample: + +.. code-block:: bash + + $ python {{sample.file}} +{% if sample.show_help %} + + {{get_help(sample.file)|indent}} +{% endif %} + + +{% endfor %} +{% endif %} + +{% if cloud_client_library %} + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python `_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source `_ and `report issues `_. + +.. ccl-docs: https://googlecloudplatform.github.io/google-cloud-python/ +.. ccl-source: https://github.com/GoogleCloudPlatform/google-cloud-python +.. ccl-issues: https://github.com/GoogleCloudPlatform/google-cloud-python/issues + +{% endif %} + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ diff --git a/scripts/readme-gen/templates/auth.tmpl.rst b/scripts/readme-gen/templates/auth.tmpl.rst new file mode 100644 index 00000000000..3a6f1abaa0d --- /dev/null +++ b/scripts/readme-gen/templates/auth.tmpl.rst @@ -0,0 +1,33 @@ +Authentication +++++++++++++++ + +Authentication is typically done through `Application Default Credentials`_, +which means you do not have to change the code to authenticate as long as +your environment has credentials. You have a few options for setting up +authentication: + +{% if not no_sdk_credentials %} +#. When running locally, use the `Google Cloud SDK`_ + + .. code-block:: bash + + gcloud beta auth application-default login + +{% endif %} + +#. When running on App Engine or Compute Engine, credentials are already + set-up. However, you may need to configure your Compute Engine instance + with `additional scopes `_. + +#. You can create a `Service Account key file`_. This file can be used to + authenticate to Google Cloud Platform services from any environment. To use + the file, set the ``GOOGLE_APPLICATION_CREDENTIALS`` environment variable to + the path to the key file, for example: + + .. code-block:: bash + + export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service_account.json + +.. _Application Default Credentials: https://cloud.google.com/docs/authentication#getting_credentials_for_server-centric_flow +.. _gce-auth: https://cloud.google.com/compute/docs/authentication#using +.. _Service Account key file: https://developers.google.com/identity/protocols/OAuth2ServiceAccount#creatinganaccount diff --git a/scripts/readme-gen/templates/install_deps.tmpl.rst b/scripts/readme-gen/templates/install_deps.tmpl.rst new file mode 100644 index 00000000000..51b346fdc08 --- /dev/null +++ b/scripts/readme-gen/templates/install_deps.tmpl.rst @@ -0,0 +1,20 @@ +Install Dependencies +++++++++++++++++++++ + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ diff --git a/storage/api/README.md b/storage/api/README.md deleted file mode 100644 index 1125ac17b51..00000000000 --- a/storage/api/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Cloud Storage API Samples - - -These samples are used on the following documentation pages: - -> -* https://cloud.google.com/storage/docs/encryption -* https://cloud.google.com/storage/docs/json_api/v1/json-api-python-samples -* https://cloud.google.com/storage/docs/authentication -* https://cloud.google.com/docs/authentication - - diff --git a/storage/api/README.rst b/storage/api/README.rst new file mode 100644 index 00000000000..a045e1d96f9 --- /dev/null +++ b/storage/api/README.rst @@ -0,0 +1,204 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Storage Python Samples +=============================================================================== + +This directory contains samples for Google Cloud Storage. `Google Cloud Storage`_ allows world-wide storage and retrieval of any amount of data at any time. + + +.. _Google Cloud Storage: https://cloud.google.com/storage/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +Authentication is typically done through `Application Default Credentials`_, +this means you do not have to change the code to authenticate as long as +your environment has credentials. You have a few options for setting up +authentication: + +#. When running locally, use the `Google Cloud SDK`_ + + .. code-block:: bash + + gcloud beta auth application-default login + + +#. When running on App Engine or Compute Engine, credentials are already + set-up. However, you may need to configure your Compute Engine instance + with `additional scopes `_. + +#. You can create a `Service Account key file`_. This file can be used to + authenticate to Google Cloud Platform services from any environment. To use + the file, set the ``GOOGLE_APPLICATION_CREDENTIALS`` environment variable to + the path to the key file, for example: + + .. code-block:: bash + + export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service_account.json + +.. _Application Default Credentials: https://cloud.google.com/docs/authentication#getting_credentials_for_server-centric_flow +.. _gce-auth: https://cloud.google.com/compute/docs/authentication#using +.. _Service Account key file: https://developers.google.com/identity/protocols/OAuth2ServiceAccount#creatinganaccount + +Install Dependencies +++++++++++++++++++++ + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +List Objects ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + + +To run this sample: + +.. code-block:: bash + + $ python list_objects.py + + usage: list_objects.py [-h] bucket + + Command-line sample application for listing all objects in a bucket using + the Cloud Storage API. + + This sample is used on this page: + + https://cloud.google.com/storage/docs/json_api/v1/json-api-python-samples + + For more information, see the README.md under /storage. + + positional arguments: + bucket Your Cloud Storage bucket. + + optional arguments: + -h, --help show this help message and exit + + +CRUD Objects ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + + +To run this sample: + +.. code-block:: bash + + $ python crud_object.py + + usage: crud_object.py [-h] [--reader READER] [--owner OWNER] filename bucket + + Application for uploading an object using the Cloud Storage API. + + This sample is used on this page: + + https://cloud.google.com/storage/docs/json_api/v1/json-api-python-samples + + For more information, see the README.md under /storage. + + positional arguments: + filename The name of the file to upload + bucket Your Cloud Storage bucket. + + optional arguments: + -h, --help show this help message and exit + --reader READER Your Cloud Storage bucket. + --owner OWNER Your Cloud Storage bucket. + + +Compose objects ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + + +To run this sample: + +.. code-block:: bash + + $ python compose_objects.py + + usage: compose_objects.py [-h] bucket destination sources [sources ...] + + Command-line sample application for composing objects using the Cloud + Storage API. + + This sample is used on this page: + + https://cloud.google.com/storage/docs/json_api/v1/json-api-python-samples + + For more information, see the README.md under /storage. + + To run, create a least two sample files: + $ echo "File 1" > file1.txt + $ echo "File 2" > file2.txt + + Example invocation: + $ python compose_objects.py my-bucket destination.txt file1.txt file2.txt + + positional arguments: + bucket Your Cloud Storage bucket. + destination Destination file name. + sources Source files to compose. + + optional arguments: + -h, --help show this help message and exit + + +Customer-Supplied Encryption ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + + +To run this sample: + +.. code-block:: bash + + $ python customer_supplied_keys.py + + usage: customer_supplied_keys.py [-h] bucket filename + + Command-line sample app demonstrating customer-supplied encryption keys. + + This sample demonstrates uploading an object while supplying an encryption key, + retrieving that object's contents, and finally rotating that key to a new + value. + + This sample is used on this page: + + https://cloud.google.com/storage/docs/json_api/v1/json-api-python-samples + + For more information, see the README.md under /storage. + + positional arguments: + bucket Your Cloud Storage bucket. + filename A file to upload and download. + + optional arguments: + -h, --help show this help message and exit + + + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/storage/api/README.rst.in b/storage/api/README.rst.in new file mode 100644 index 00000000000..fafd9f1d8e0 --- /dev/null +++ b/storage/api/README.rst.in @@ -0,0 +1,27 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Storage + short_name: Cloud Storage + url: https://cloud.google.com/storage/docs + description: > + `Google Cloud Storage`_ allows world-wide storage and retrieval of any + amount of data at any time. + +setup: +- auth +- install_deps + +samples: +- name: List Objects + file: list_objects.py + show_help: true +- name: CRUD Objects + file: crud_object.py + show_help: true +- name: Compose objects + file: compose_objects.py + show_help: true +- name: Customer-Supplied Encryption + file: customer_supplied_keys.py + show_help: true diff --git a/storage/cloud-client/README.rst b/storage/cloud-client/README.rst new file mode 100644 index 00000000000..ea49a4af3ef --- /dev/null +++ b/storage/cloud-client/README.rst @@ -0,0 +1,235 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Storage Python Samples +=============================================================================== + +This directory contains samples for Google Cloud Storage. `Google Cloud Storage`_ allows world-wide storage and retrieval of any amount of data at any time. + + +.. _Google Cloud Storage: https://cloud.google.com/storage/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +Authentication is typically done through `Application Default Credentials`_, +this means you do not have to change the code to authenticate as long as +your environment has credentials. You have a few options for setting up +authentication: + +#. When running locally, use the `Google Cloud SDK`_ + + .. code-block:: bash + + gcloud beta auth application-default login + + +#. When running on App Engine or Compute Engine, credentials are already + set-up. However, you may need to configure your Compute Engine instance + with `additional scopes `_. + +#. You can create a `Service Account key file`_. This file can be used to + authenticate to Google Cloud Platform services from any environment. To use + the file, set the ``GOOGLE_APPLICATION_CREDENTIALS`` environment variable to + the path to the key file, for example: + + .. code-block:: bash + + export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service_account.json + +.. _Application Default Credentials: https://cloud.google.com/docs/authentication#getting_credentials_for_server-centric_flow +.. _gce-auth: https://cloud.google.com/compute/docs/authentication#using +.. _Service Account key file: https://developers.google.com/identity/protocols/OAuth2ServiceAccount#creatinganaccount + +Install Dependencies +++++++++++++++++++++ + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Snippets ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + + +To run this sample: + +.. code-block:: bash + + $ python snippets.py + + usage: snippets.py [-h] + bucket_name + {create-bucket,delete-bucket,list,list-with-prefix,upload,download,delete,metadata,make-public,signed-url,rename,copy} + ... + + This application demonstrates how to perform basic operations on blobs + (objects) in a Google Cloud Storage bucket. + + For more information, see the README.md under /storage and the documentation + at https://cloud.google.com/storage/docs. + + positional arguments: + bucket_name Your cloud storage bucket. + {create-bucket,delete-bucket,list,list-with-prefix,upload,download,delete,metadata,make-public,signed-url,rename,copy} + create-bucket Creates a new bucket. + delete-bucket Deletes a bucket. The bucket must be empty. + list Lists all the blobs in the bucket. + list-with-prefix Lists all the blobs in the bucket that begin with the + prefix. This can be used to list all blobs in a + "folder", e.g. "public/". The delimiter argument can + be used to restrict the results to only the "files" in + the given "folder". Without the delimiter, the entire + tree under the prefix is returned. For example, given + these blobs: /a/1.txt /a/b/2.txt If you just specify + prefix = '/a', you'll get back: /a/1.txt /a/b/2.txt + However, if you specify prefix='/a' and delimiter='/', + you'll get back: /a/1.txt + upload Uploads a file to the bucket. + download Downloads a blob from the bucket. + delete Deletes a blob from the bucket. + metadata Prints out a blob's metadata. + make-public Makes a blob publicly accessible. + signed-url Generates a signed URL for a blob. Note that this + method requires a service account key file. You can + not use this if you are using Application Default + Credentials from Google Compute Engine or from the + Google Cloud SDK. + rename Renames a blob. + copy Renames a blob. + + optional arguments: + -h, --help show this help message and exit + + +Access Control Lists ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + + +To run this sample: + +.. code-block:: bash + + $ python acl.py + + usage: acl.py [-h] + {print-bucket-acl,print-bucket-acl-for-user,add-bucket-owner,remove-bucket-owner,add-bucket-default-owner,remove-bucket-default-owner,print-blob-acl,print-blob-acl-for-user,add-blob-owner,remove-blob-owner} + ... + + This application demonstrates how to manage access control lists (acls) in + Google Cloud Storage. + + For more information, see the README.md under /storage and the documentation + at https://cloud.google.com/storage/docs/encryption. + + positional arguments: + {print-bucket-acl,print-bucket-acl-for-user,add-bucket-owner,remove-bucket-owner,add-bucket-default-owner,remove-bucket-default-owner,print-blob-acl,print-blob-acl-for-user,add-blob-owner,remove-blob-owner} + print-bucket-acl Prints out a bucket's access control list. + print-bucket-acl-for-user + Prints out a bucket's access control list. + add-bucket-owner Adds a user as an owner on the given bucket. + remove-bucket-owner + Removes a user from the access control list of the + given bucket. + add-bucket-default-owner + Adds a user as an owner in the given bucket's default + object access control list. + remove-bucket-default-owner + Removes a user from the access control list of the + given bucket's default object access control list. + print-blob-acl Prints out a blob's access control list. + print-blob-acl-for-user + Prints out a blob's access control list for a given + user. + add-blob-owner Adds a user as an owner on the given blob. + remove-blob-owner Removes a user from the access control list of the + given blob in the given bucket. + + optional arguments: + -h, --help show this help message and exit + + +Customer-Supplied Encryption ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + + +To run this sample: + +.. code-block:: bash + + $ python encryption.py + + usage: encryption.py [-h] {generate-encryption-key,upload,download,rotate} ... + + This application demonstrates how to upload and download encrypted blobs + (objects) in Google Cloud Storage. + + Use `generate-encryption-key` to generate an example key: + + python encryption.py generate-encryption-key + + Then use the key to upload and download files encrypted with a custom key. + + For more information, see the README.md under /storage and the documentation + at https://cloud.google.com/storage/docs/encryption. + + positional arguments: + {generate-encryption-key,upload,download,rotate} + generate-encryption-key + Generates a 256 bit (32 byte) AES encryption key and + prints the base64 representation. This is included for + demonstration purposes. You should generate your own + key. Please remember that encryption keys should be + handled with a comprehensive security policy. + upload Uploads a file to a Google Cloud Storage bucket using + a custom encryption key. The file will be encrypted by + Google Cloud Storage and only retrievable using the + provided encryption key. + download Downloads a previously-encrypted blob from Google + Cloud Storage. The encryption key provided must be the + same key provided when uploading the blob. + rotate Performs a key rotation by re-writing an encrypted + blob with a new encryption key. + + optional arguments: + -h, --help show this help message and exit + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python `_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source `_ and `report issues `_. + +.. ccl-docs: https://googlecloudplatform.github.io/google-cloud-python/ +.. ccl-source: https://github.com/GoogleCloudPlatform/google-cloud-python +.. ccl-issues: https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/storage/cloud-client/README.rst.in b/storage/cloud-client/README.rst.in new file mode 100644 index 00000000000..83cf9a5e328 --- /dev/null +++ b/storage/cloud-client/README.rst.in @@ -0,0 +1,26 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Storage + short_name: Cloud Storage + url: https://cloud.google.com/storage/docs + description: > + `Google Cloud Storage`_ allows world-wide storage and retrieval of any + amount of data at any time. + +setup: +- auth +- install_deps + +samples: +- name: Snippets + file: snippets.py + show_help: true +- name: Access Control Lists + file: acl.py + show_help: true +- name: Customer-Supplied Encryption + file: encryption.py + show_help: true + +cloud_client_library: true