diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000000..8a533f7e4e5 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,8 @@ +[run] +include = + datastore/* + localtesting/* + bigquery/* +[report] +exclude_lines = + pragma: NO COVER diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000000..c366151b338 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ + +*.pyc +.coverage +.tox +coverage.xml +nosetests.xml +python-docs-samples.json diff --git a/.travis.yml b/.travis.yml index a7ddccb8660..74d589a6008 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,12 @@ sudo: false #add language, etc. here +language: python cache: directories: - $HOME/gcloud/ env: - - PATH=$PATH:$HOME/gcloud/google-cloud-sdk/bin GOOGLE_APPLICATION_CREDENTIALS=$TRAVIS_BUILD_DIR/client_secrets.json #Other environment variables on same line + - PATH=$PATH:$HOME/gcloud/google-cloud-sdk/bin GOOGLE_APPLICATION_CREDENTIALS=$TRAVIS_BUILD_DIR/python-docs-samples.json PYTHONPATH=${HOME}/gcloud/google-cloud-sdk/platform/google_appengine #Other environment variables on same line before_install: #ENCRYPT YOUR PRIVATE KEY (If you need authentication) @@ -27,13 +28,14 @@ before_install: printf '\ny\n\ny\ny\n' | ./google-cloud-sdk/install.sh && cd $TRAVIS_BUILD_DIR; fi - - printf 'y\n' | gcloud components update - - if [ -a client_secrets.json ]; then - gcloud auth activate-service-account --key-file client_secrets.json; + - gcloud -q components update gae-python + - openssl aes-256-cbc -K $encrypted_4fda24e244ca_key -iv $encrypted_4fda24e244ca_iv -in python-docs-samples.json.enc -out python-docs-samples.json -d + - if [ -a python-docs-samples.json ]; then + gcloud auth activate-service-account --key-file python-docs-samples.json; fi install: - #Add app specific setup here + - pip install tox script: - #Test and/or deploy your app with gcloud commands here! + - tox diff --git a/README.md b/README.md index ce1ee24e787..8c9e56a6109 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,26 @@ For more detailed introduction to a product, check the README in the correspondi * See [CONTRIBUTING.md](CONTRIBUTING.md) +### How to run the test + +To run the tests, please install App Engine Python SDK and tox and run +tox with the environment variable PYTHONPATH to the App Engine Python +SDK. + +You can install App Engine Python SDK with +[Google Cloud SDK](https://cloud.google.com/sdk/) with the following +command: + + $ gcloud components update gae-python + +Here is instructions to run the tests with virtualenv, $GCLOUD is your +Google Cloud SDK installation path. + + $ virtualenv -p python2.7 --no-site-packages /some/where + $ source /some/where/bin/activate + $ pip install tox + $ env PYTHONPATH=${GCLOUD}/platform/google_appengine tox + ## Licensing * See [LICENSE](LICENSE) diff --git a/datastore/ndb/app.yaml b/app.yaml similarity index 82% rename from datastore/ndb/app.yaml rename to app.yaml index 031ed0fbca6..907c57fbff5 100644 --- a/datastore/ndb/app.yaml +++ b/app.yaml @@ -1,7 +1,5 @@ # dummy app.yaml for nosegae -application: ndb-snippets -version: 1 api_version: 1 runtime: python27 threadsafe: true diff --git a/bigquery/__init__.py b/bigquery/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/bigquery/samples/__init__.py b/bigquery/samples/__init__.py index b6953e496df..e69de29bb2d 100644 --- a/bigquery/samples/__init__.py +++ b/bigquery/samples/__init__.py @@ -1,13 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/bigquery/samples/async_query.py b/bigquery/samples/async_query.py index afd4b5d011e..50deb3a4731 100644 --- a/bigquery/samples/async_query.py +++ b/bigquery/samples/async_query.py @@ -1,20 +1,24 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and # limitations under the License. # from __future__ import print_function # For python 2/3 interoperability -from samples.utils import get_service, paging, poll_job -import uuid + import json +import uuid + +from bigquery.samples.utils import get_service +from bigquery.samples.utils import paging +from bigquery.samples.utils import poll_job # [START async_query] @@ -23,19 +27,19 @@ def async_query(service, project_id, query, batch=False, num_retries=5): # don't accidentally duplicate query job_data = { 'jobReference': { - 'projectId': project_id, - 'job_id': str(uuid.uuid4()) - }, + 'projectId': project_id, + 'job_id': str(uuid.uuid4()) + }, 'configuration': { - 'query': { - 'query': query, - 'priority': 'BATCH' if batch else 'INTERACTIVE', - }, - } + 'query': { + 'query': query, + 'priority': 'BATCH' if batch else 'INTERACTIVE' + } } + } return service.jobs().insert( - projectId=project_id, - body=job_data).execute(num_retries=num_retries) + projectId=project_id, + body=job_data).execute(num_retries=num_retries) # [END async_query] @@ -55,7 +59,6 @@ def run(project_id, query_string, batch, num_retries, interval): interval, num_retries) - for page in paging(service, service.jobs().getQueryResults, num_retries=num_retries, @@ -69,18 +72,13 @@ def run(project_id, query_string, batch, num_retries, interval): def main(): project_id = raw_input("Enter the project ID: ") query_string = raw_input("Enter the Bigquery SQL Query: ") - batch = raw_input("Run query as batch (y/n)?: ") in ('True', - 'true', - 'y', - 'Y', - 'yes', - 'Yes') - + batch = raw_input("Run query as batch (y/n)?: ") in ( + 'True', 'true', 'y', 'Y', 'yes', 'Yes') num_retries = raw_input( - "Enter number of times to retry in case of 500 error: ") + "Enter number of times to retry in case of 500 error: ") interval = raw_input( - "Enter how often to poll the query for completion (seconds): ") + "Enter how often to poll the query for completion (seconds): ") for result in run(project_id, query_string, batch, num_retries, interval): print(result) diff --git a/bigquery/samples/discovery_doc.py b/bigquery/samples/discovery_doc.py index 509d17a5c88..006ff46903f 100644 --- a/bigquery/samples/discovery_doc.py +++ b/bigquery/samples/discovery_doc.py @@ -1,35 +1,38 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and # limitations under the License. # -import os +""" +A module that takes care of caching and updating discovery docs for +google-api-python-clients (until such a feature is integrated). +""" + import json -import httplib2 +import os import time -# [START build_and_update] +import httplib2 -RESOURCE_PATH='..' #look for discovery docs in the parent folder -MAX_AGE = 86400 #update discovery docs older than a day +# [START build_and_update] -# A module that takes care of caching and updating discovery docs -# for google-api-python-clients (until such a feature is integrated) +RESOURCE_PATH = '..' # look for discovery docs in the parent folder +MAX_AGE = 86400 # update discovery docs older than a day +BIGQUERY_SCOPES = ['https://www.googleapis.com/auth/bigquery'] def build_and_update(api, version): from oauth2client.client import GoogleCredentials from googleapiclient.discovery import build_from_document - path = os.path.join(RESOURCE_PATH, '{}.{}'.format(api, version)) try: age = time.time() - os.path.getmtime(path) @@ -38,11 +41,14 @@ def build_and_update(api, version): except os.error: _update_discovery_doc(api, version, path) + credentials = GoogleCredentials.get_application_default() + if credentials.create_scoped_required(): + credentials = credentials.create_scoped(BIGQUERY_SCOPES) with open(path, 'r') as discovery_doc: return build_from_document(discovery_doc.read(), - http=httplib2.Http(), - credentials=GoogleCredentials - .get_application_default()) + http=httplib2.Http(), + credentials=credentials) + def _update_discovery_doc(api, version, path): from apiclient.discovery import DISCOVERY_URI @@ -61,5 +67,5 @@ def _update_discovery_doc(api, version, path): json.dump(discovery_json, discovery_doc) except ValueError: raise InvalidJsonError( - 'Bad JSON: %s from %s.' % (content, requested_url)) + 'Bad JSON: %s from %s.' % (content, requested_url)) # [END build_and_update] diff --git a/bigquery/samples/export_data_to_cloud_storage.py b/bigquery/samples/export_data_to_cloud_storage.py index d61e7b2f161..7b361e68d40 100644 --- a/bigquery/samples/export_data_to_cloud_storage.py +++ b/bigquery/samples/export_data_to_cloud_storage.py @@ -1,6 +1,21 @@ -from samples.utils import get_service, poll_job +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# import uuid +from bigquery.samples.utils import get_service +from bigquery.samples.utils import poll_job + # [START export_table] def export_table(service, cloud_storage_path, @@ -9,21 +24,21 @@ def export_table(service, cloud_storage_path, # Generate a unique job_id so retries # don't accidentally duplicate export job_data = { - 'jobReference': { + 'jobReference': { + 'projectId': projectId, + 'jobId': str(uuid.uuid4()) + }, + 'configuration': { + 'extract': { + 'sourceTable': { 'projectId': projectId, - 'jobId': str(uuid.uuid4()) - }, - 'configuration': { - 'extract': { - 'sourceTable': { - 'projectId': projectId, - 'datasetId': datasetId, - 'tableId': tableId, - }, - 'destinationUris': [cloud_storage_path], - } - } + 'datasetId': datasetId, + 'tableId': tableId, + }, + 'destinationUris': [cloud_storage_path], } + } + } return service.jobs().insert( projectId=projectId, body=job_data).execute(num_retries=num_retries) @@ -52,11 +67,11 @@ def main(): datasetId = raw_input("Enter a dataset ID: ") tableId = raw_input("Enter a table name to copy: ") cloud_storage_path = raw_input( - "Enter a Google Cloud Storage URI: ") + "Enter a Google Cloud Storage URI: ") interval = raw_input( - "Enter how often to poll the job (in seconds): ") + "Enter how often to poll the job (in seconds): ") num_retries = raw_input( - "Enter the number of retries in case of 500 error: ") + "Enter the number of retries in case of 500 error: ") run(cloud_storage_path, projectId, datasetId, tableId, diff --git a/bigquery/samples/load_data_by_post.py b/bigquery/samples/load_data_by_post.py index 8124e34dfbe..6c03885c4db 100644 --- a/bigquery/samples/load_data_by_post.py +++ b/bigquery/samples/load_data_by_post.py @@ -1,19 +1,22 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and # limitations under the License. # import json + +from bigquery.samples.utils import get_service, poll_job + import httplib2 -from samples.utils import get_service, poll_job + from oauth2client.client import GoogleCredentials @@ -64,7 +67,7 @@ def main(): datasetId = raw_input('Enter a dataset ID: ') tableId = raw_input('Enter a table name to load the data to: ') schema_path = raw_input( - 'Enter the path to the schema file for the table: ') + 'Enter the path to the schema file for the table: ') with open(schema_path, 'r') as schema_file: schema = schema_file.read() diff --git a/bigquery/samples/load_data_from_csv.py b/bigquery/samples/load_data_from_csv.py index a22eed7318d..73a108d3eee 100644 --- a/bigquery/samples/load_data_from_csv.py +++ b/bigquery/samples/load_data_from_csv.py @@ -1,20 +1,21 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and # limitations under the License. # -from samples.utils import get_service, poll_job import json import uuid +from bigquery.samples.utils import get_service, poll_job + # [START load_table] def load_table(service, source_schema, source_csv, @@ -22,24 +23,24 @@ def load_table(service, source_schema, source_csv, # Generate a unique job_id so retries # don't accidentally duplicate query job_data = { - 'jobReference': { + 'jobReference': { + 'projectId': projectId, + 'job_id': str(uuid.uuid4()) + }, + 'configuration': { + 'load': { + 'sourceUris': [source_csv], + 'schema': { + 'fields': source_schema + }, + 'destinationTable': { 'projectId': projectId, - 'job_id': str(uuid.uuid4()) - }, - 'configuration': { - 'load': { - 'sourceUris': [source_csv], - 'schema': { - 'fields': source_schema - }, - 'destinationTable': { - 'projectId': projectId, - 'datasetId': datasetId, - 'tableId': tableId - }, - } - } + 'datasetId': datasetId, + 'tableId': tableId + } } + } + } return service.jobs().insert( projectId=projectId, @@ -70,16 +71,16 @@ def main(): tableId = raw_input("Enter a destination table name: ") schema_file_path = raw_input( - "Enter the path to the table schema: ") + "Enter the path to the table schema: ") with open(schema_file_path, 'r') as schema_file: schema = json.load(schema_file) data_file_path = raw_input( - "Enter the Cloud Storage path for the CSV file: ") + "Enter the Cloud Storage path for the CSV file: ") num_retries = raw_input( - "Enter number of times to retry in case of 500 error: ") + "Enter number of times to retry in case of 500 error: ") interval = raw_input( - "Enter how often to poll the query for completion (seconds): ") + "Enter how often to poll the query for completion (seconds): ") run(schema, data_file_path, projectId, diff --git a/bigquery/samples/streaming.py b/bigquery/samples/streaming.py index 64b95ce28df..dd8de3783fe 100644 --- a/bigquery/samples/streaming.py +++ b/bigquery/samples/streaming.py @@ -1,22 +1,23 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and # limitations under the License. # from __future__ import print_function -from samples.utils import get_service import ast -import uuid import json +import uuid + +from bigquery.samples.utils import get_service # [START stream_row_to_bigquery] @@ -29,14 +30,14 @@ def stream_row_to_bigquery(service, # Generate a unique row id so retries # don't accidentally duplicate insert insert_all_data = { - 'insertId': str(uuid.uuid4()), - 'rows': [{'json': row}] - } + 'insertId': str(uuid.uuid4()), + 'rows': [{'json': row}] + } return service.tabledata().insertAll( - projectId=project_id, - datasetId=dataset_id, - tableId=table_id, - body=insert_all_data).execute(num_retries=num_retries) + projectId=project_id, + datasetId=dataset_id, + tableId=table_id, + body=insert_all_data).execute(num_retries=num_retries) # [END stream_row_to_bigquery] @@ -59,9 +60,8 @@ def get_rows(): line = raw_input("Enter a row (python dict) into the table: ") while line: yield ast.literal_eval(line) - line = raw_input( - "Enter another row into the table \n" + - "[hit enter to stop]: ") + line = raw_input("Enter another row into the table \n" + + "[hit enter to stop]: ") def main(): @@ -69,7 +69,7 @@ def main(): dataset_id = raw_input("Enter a dataset ID: ") table_id = raw_input("Enter a table ID : ") num_retries = int(raw_input( - "Enter number of times to retry in case of 500 error: ")) + "Enter number of times to retry in case of 500 error: ")) for result in run(project_id, dataset_id, table_id, get_rows(), num_retries): diff --git a/bigquery/samples/sync_query.py b/bigquery/samples/sync_query.py index 814f642628e..aab5a312365 100644 --- a/bigquery/samples/sync_query.py +++ b/bigquery/samples/sync_query.py @@ -1,30 +1,32 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and # limitations under the License. # from __future__ import print_function # For python 2/3 interoperability -from samples.utils import get_service, paging + import json +from bigquery.samples.utils import get_service, paging + # [START sync_query] def sync_query(service, project_id, query, timeout=10000, num_retries=5): query_data = { - 'query': query, - 'timeoutMs': timeout, - } + 'query': query, + 'timeoutMs': timeout, + } return service.jobs().query( - projectId=project_id, - body=query_data).execute(num_retries=num_retries) + projectId=project_id, + body=query_data).execute(num_retries=num_retries) # [END sync_query] @@ -50,13 +52,12 @@ def main(): project_id = raw_input("Enter the project ID: ") query_string = raw_input("Enter the Bigquery SQL Query: ") timeout = raw_input( - "Enter how long to wait for the query to complete in milliseconds" - "\n (if longer than 10 seconds, use an asynchronous query): ") + "Enter how long to wait for the query to complete in milliseconds" + "\n (if longer than 10 seconds, use an asynchronous query): ") num_retries = int(raw_input( - "Enter how many times to retry in case of server error")) + "Enter how many times to retry in case of server error")) for result in run(project_id, query_string, timeout, num_retries): print(result) - # [END main] diff --git a/bigquery/samples/utils.py b/bigquery/samples/utils.py index d328e20808d..d35d0c94713 100644 --- a/bigquery/samples/utils.py +++ b/bigquery/samples/utils.py @@ -1,30 +1,30 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and # limitations under the License. # + # [START get_service] def get_service(): from discovery_doc import build_and_update - return build_and_update('bigquery','v2') + return build_and_update('bigquery', 'v2') # [END get_service] + # [START poll_job] def poll_job(service, projectId, jobId, interval=5, num_retries=5): import time - job_get = service.jobs().get( - projectId=projectId, - jobId=jobId) + job_get = service.jobs().get(projectId=projectId, jobId=jobId) job_resource = job_get.execute(num_retries=num_retries) while not job_resource['status']['state'] == 'DONE': diff --git a/bigquery/test/__init__.py b/bigquery/test/__init__.py index e683428c124..6a70f3c2fc4 100644 --- a/bigquery/test/__init__.py +++ b/bigquery/test/__init__.py @@ -1,14 +1,14 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and # limitations under the License. # import os diff --git a/bigquery/test/base_test.py b/bigquery/test/base_test.py index e65a27a2192..9765b23a9c4 100644 --- a/bigquery/test/base_test.py +++ b/bigquery/test/base_test.py @@ -1,14 +1,35 @@ -import unittest -from test import RESOURCE_PATH +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# import json import os +import unittest + +from bigquery.test import RESOURCE_PATH class BaseBigqueryTest(unittest.TestCase): def setUp(self): + # A hack to prevent get_application_default to choose App Engine route. + self._server_software_org = os.environ.get('SERVER_SOFTWARE') + os.environ['SERVER_SOFTWARE'] = '' + with open( os.path.join(RESOURCE_PATH, 'constants.json'), 'r') as constants_file: self.constants = json.load(constants_file) + + def tearDown(self): + os.environ['SERVER_SOFTWARE'] = self._server_software_org diff --git a/bigquery/test/test_async_query.py b/bigquery/test/test_async_query.py index ab32d5aead4..3435542be54 100644 --- a/bigquery/test/test_async_query.py +++ b/bigquery/test/test_async_query.py @@ -1,21 +1,22 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and # limitations under the License. # -from test.base_test import BaseBigqueryTest -from samples.async_query import run import json import unittest +from bigquery.samples.async_query import run +from bigquery.test.base_test import BaseBigqueryTest + class TestAsyncQuery(BaseBigqueryTest): diff --git a/bigquery/test/test_export_data_to_cloud_storage.py b/bigquery/test/test_export_data_to_cloud_storage.py index 796c121f8a8..8b5c77f5a87 100644 --- a/bigquery/test/test_export_data_to_cloud_storage.py +++ b/bigquery/test/test_export_data_to_cloud_storage.py @@ -1,21 +1,24 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and # limitations under the License. # + """Tests for export_table_to_gcs.""" -from test.base_test import BaseBigqueryTest -from samples.export_data_to_cloud_storage import run + import unittest +from bigquery.samples.export_data_to_cloud_storage import run +from bigquery.test.base_test import BaseBigqueryTest + class TestExportTableToGCS(BaseBigqueryTest): @@ -27,5 +30,6 @@ def test_export_table(self): 5, 5) + if __name__ == '__main__': unittest.main() diff --git a/bigquery/test/test_load_data_from_csv.py b/bigquery/test/test_load_data_from_csv.py index 44ac69145cc..db9a3fc4dc9 100644 --- a/bigquery/test/test_load_data_from_csv.py +++ b/bigquery/test/test_load_data_from_csv.py @@ -1,25 +1,26 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and # limitations under the License. # """Tests for load_data_from_csv.""" -from test.base_test import BaseBigqueryTest -from test import RESOURCE_PATH -from samples.load_data_from_csv import run -import os import json +import os import unittest +from bigquery.samples.load_data_from_csv import run +from bigquery.test import RESOURCE_PATH +from bigquery.test.base_test import BaseBigqueryTest + class TestLoadDataFromCSV(BaseBigqueryTest): diff --git a/bigquery/test/test_streaming.py b/bigquery/test/test_streaming.py index 7bbcd057c7d..05dbe4a5865 100644 --- a/bigquery/test/test_streaming.py +++ b/bigquery/test/test_streaming.py @@ -1,24 +1,25 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and # limitations under the License. # """Tests for export_table_to_gcs.""" -from samples.streaming import run -from test.base_test import BaseBigqueryTest -from test import RESOURCE_PATH import json import os import unittest +from bigquery.samples.streaming import run +from bigquery.test import RESOURCE_PATH +from bigquery.test.base_test import BaseBigqueryTest + class TestStreaming(BaseBigqueryTest): diff --git a/bigquery/test/test_sync_query.py b/bigquery/test/test_sync_query.py index 68208848c00..74ab0cabd06 100644 --- a/bigquery/test/test_sync_query.py +++ b/bigquery/test/test_sync_query.py @@ -1,21 +1,21 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and # limitations under the License. # +import json import unittest -from samples.sync_query import run -from test.base_test import BaseBigqueryTest -import json +from bigquery.samples.sync_query import run +from bigquery.test.base_test import BaseBigqueryTest class TestSyncQuery(BaseBigqueryTest): diff --git a/datastore/__init__.py b/datastore/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/datastore/ndb/CONTRIBUTING.md b/datastore/ndb/CONTRIBUTING.md deleted file mode 100644 index 6736efd943c..00000000000 --- a/datastore/ndb/CONTRIBUTING.md +++ /dev/null @@ -1,35 +0,0 @@ -# How to become a contributor and submit your own code - -## Contributor License Agreements - -We'd love to accept your sample apps and patches! Before we can take them, we -have to jump a couple of legal hurdles. - -Please fill out either the individual or corporate Contributor License Agreement -(CLA). - - * If you are an individual writing original source code and you're sure you - own the intellectual property, then you'll need to sign an [individual CLA] - (https://developers.google.com/open-source/cla/individual). - * If you work for a company that wants to allow you to contribute your work, - then you'll need to sign a [corporate CLA] - (https://developers.google.com/open-source/cla/corporate). - -Follow either of the two links above to access the appropriate CLA and -instructions for how to sign and return it. Once we receive it, we'll be able to -accept your pull requests. - -## Contributing A Patch - -1. Submit an issue describing your proposed change to the repo in question. -1. The repo owner will respond to your issue promptly. -1. If your proposed change is accepted, and you haven't already done so, sign a - Contributor License Agreement (see details above). -1. Fork the desired repo, develop and test your code changes. -1. Ensure that your code adheres to the existing style in the sample to which - you are contributing. Refer to the - [Google Cloud Platform Samples Style Guide] - (https://github.com/GoogleCloudPlatform/Template/wiki/style.html) for the - recommended coding standards for this organization. -1. Ensure that your code has an appropriate set of unit tests which all pass. -1. Submit a pull request. diff --git a/datastore/ndb/LICENSE b/datastore/ndb/LICENSE deleted file mode 100644 index 4f73028109b..00000000000 --- a/datastore/ndb/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/datastore/ndb/README.md b/datastore/ndb/README.md index 25f67690ece..262b47ba03a 100644 --- a/datastore/ndb/README.md +++ b/datastore/ndb/README.md @@ -3,20 +3,3 @@ appengine-ndb-snippets Sample code snippets for NDB. -How to run the test -=================== - -To run the tests, please install App Engine Python SDK and tox and run -tox with the environment variable PYTHONPATH to the App Engine Python SDK. - -You can install App Engine Python SDK with [Google Cloud SDK](https://cloud.google.com/sdk/) with the following command: - - $ gcloud components update gae-python - -Here is instructions to run the tests with virtualenv, $GCLOUD is your -Google Cloud SDK installation path. - - $ virtualenv -p python2.7 --no-site-packages . - $ source bin/activate - $ pip install tox - $ env PYTHONPATH=${GCLOUD}/platform/google_appengine tox diff --git a/datastore/ndb/__init__.py b/datastore/ndb/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/datastore/ndb/modeling/__init__.py b/datastore/ndb/modeling/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/datastore/ndb/modeling/tests/__init__.py b/datastore/ndb/modeling/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/datastore/ndb/modeling/tests/test_base.py b/datastore/ndb/modeling/tests/test_base.py index 7b20270be41..267b2661f5a 100644 --- a/datastore/ndb/modeling/tests/test_base.py +++ b/datastore/ndb/modeling/tests/test_base.py @@ -17,8 +17,8 @@ import unittest -from google.appengine.ext import testbed from google.appengine.datastore import datastore_stub_util +from google.appengine.ext import testbed class TestCase(unittest.TestCase): diff --git a/datastore/ndb/modeling/tests/test_contact_with_group_models.py b/datastore/ndb/modeling/tests/test_contact_with_group_models.py index 2993d3228d7..68a4984ca92 100644 --- a/datastore/ndb/modeling/tests/test_contact_with_group_models.py +++ b/datastore/ndb/modeling/tests/test_contact_with_group_models.py @@ -17,9 +17,10 @@ import unittest +from datastore.ndb.modeling import contact_with_group_models as models + from google.appengine.ext import ndb -import contact_with_group_models as models import test_base diff --git a/datastore/ndb/modeling/tests/test_keyproperty_models.py b/datastore/ndb/modeling/tests/test_keyproperty_models.py index 07c9dabfd3e..78113189751 100644 --- a/datastore/ndb/modeling/tests/test_keyproperty_models.py +++ b/datastore/ndb/modeling/tests/test_keyproperty_models.py @@ -17,7 +17,8 @@ import unittest -import keyproperty_models as models +from datastore.ndb.modeling import keyproperty_models as models + import test_base diff --git a/datastore/ndb/modeling/tests/test_naive_models.py b/datastore/ndb/modeling/tests/test_naive_models.py index e2294f7648c..e6fb51aa739 100644 --- a/datastore/ndb/modeling/tests/test_naive_models.py +++ b/datastore/ndb/modeling/tests/test_naive_models.py @@ -17,7 +17,8 @@ import unittest -import naive_models as models +from datastore.ndb.modeling import naive_models as models + import test_base diff --git a/datastore/ndb/modeling/tests/test_parent_child_models.py b/datastore/ndb/modeling/tests/test_parent_child_models.py index 152776031ab..4f7b8864d1d 100644 --- a/datastore/ndb/modeling/tests/test_parent_child_models.py +++ b/datastore/ndb/modeling/tests/test_parent_child_models.py @@ -17,11 +17,12 @@ import unittest -import parent_child_models as models -import test_base +from datastore.ndb.modeling import parent_child_models as models from google.appengine.ext import ndb +import test_base + class ContactTestCase(test_base.TestCase): """A test case for the Contact model class with KeyProperty.""" diff --git a/datastore/ndb/modeling/tests/test_relation_model_models.py b/datastore/ndb/modeling/tests/test_relation_model_models.py index c0efc5b8ee3..8990dae6196 100644 --- a/datastore/ndb/modeling/tests/test_relation_model_models.py +++ b/datastore/ndb/modeling/tests/test_relation_model_models.py @@ -17,11 +17,12 @@ import unittest -import relation_model_models as models -import test_base +from datastore.ndb.modeling import relation_model_models as models from google.appengine.ext import ndb +import test_base + class ContactTestCase(test_base.TestCase): """A test case for the Contact model with relationship model.""" diff --git a/datastore/ndb/modeling/tests/test_structured_property_models.py b/datastore/ndb/modeling/tests/test_structured_property_models.py index 9bdac47ce2a..8af106894d0 100644 --- a/datastore/ndb/modeling/tests/test_structured_property_models.py +++ b/datastore/ndb/modeling/tests/test_structured_property_models.py @@ -17,7 +17,8 @@ import unittest -import structured_property_models as models +from datastore.ndb.modeling import structured_property_models as models + import test_base diff --git a/datastore/ndb/overview/__init__.py b/datastore/ndb/overview/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/datastore/ndb/overview/main.py b/datastore/ndb/overview/main.py index 91befce85f0..90024e7b6f4 100644 --- a/datastore/ndb/overview/main.py +++ b/datastore/ndb/overview/main.py @@ -16,10 +16,10 @@ import cgi import urllib -import webapp2 - from google.appengine.ext import ndb +import webapp2 + # [START greeting] class Greeting(ndb.Model): @@ -65,11 +65,13 @@ def post(self): # We set the parent key on each 'Greeting' to ensure each guestbook's # greetings are in the same entity group. guestbook_name = self.request.get('guestbook_name') - greeting = Greeting(parent=ndb.Key("Book", guestbook_name or "*notitle*"), + greeting = Greeting(parent=ndb.Key("Book", + guestbook_name or "*notitle*"), content=self.request.get('content')) greeting.put() # [END submit] - self.redirect('/?' + urllib.urlencode({'guestbook_name': guestbook_name})) + self.redirect('/?' + urllib.urlencode( + {'guestbook_name': guestbook_name})) app = webapp2.WSGIApplication([ diff --git a/datastore/ndb/tox.ini b/datastore/ndb/tox.ini deleted file mode 100644 index 9a17a256173..00000000000 --- a/datastore/ndb/tox.ini +++ /dev/null @@ -1,14 +0,0 @@ -[tox] -skipsdist = True -envlist = flake8-py2.7,py2.7 - -[testenv:flake8-py2.7] -deps = flake8 -commands = flake8 modeling - -[testenv:py2.7] -deps = - nose - nosegae -commands = - nosetests --with-gae modeling diff --git a/datastore/ndb/transactions/__init__.py b/datastore/ndb/transactions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/datastore/ndb/transactions/main.py b/datastore/ndb/transactions/main.py index 6f06bc067ff..4a077af565d 100644 --- a/datastore/ndb/transactions/main.py +++ b/datastore/ndb/transactions/main.py @@ -13,12 +13,14 @@ # limitations under the License. import cgi -import flask import random import urllib + +import flask + # [START taskq-imp] -from google.appengine.ext import ndb from google.appengine.api import taskqueue +from google.appengine.ext import ndb # [END taskq-imp] @@ -47,12 +49,14 @@ def main_page(): response += '

%s

' % cgi.escape(note.key.id()) response += '
%s
' % cgi.escape(note.content) - response += """ -
-
- Submit Note:
- -
""" % urllib.urlencode({'page_name': page_name}) + response += ( + """
+
+ Submit Note:
+ +
""" + % urllib.urlencode({'page_name': page_name})) response += """
Switch page: @@ -120,6 +124,7 @@ def insert_if_absent_indep(note_key, note): return True return False + # [START taskq] @ndb.transactional def insert_if_absent_taskq(note_key, note): @@ -137,6 +142,7 @@ def insert_if_absent_taskq(note_key, note): def taskq_worker(): pass + def pick_random_insert(note_key, note): choice = random.randint(0, 5) if choice == 0: @@ -149,7 +155,8 @@ def pick_random_insert(note_key, note): inserted = insert_if_absent_xg(note_key, note) elif choice == 3: # [START sometimes-call] - inserted = ndb.transaction(lambda: insert_if_absent_sometimes(note_key, note)) + inserted = ndb.transaction(lambda: + insert_if_absent_sometimes(note_key, note)) # [END sometimes-call] elif choice == 4: inserted = insert_if_absent_indep(note_key, note) @@ -174,13 +181,15 @@ def add_note(): note = Note(key=note_key, content=note_text) # [END calling] if pick_random_insert(note_key, note) is False: - return 'Already there
Return' % flask.url_for('main_page', page_name=page_name) + return ('Already there
Return' + % flask.url_for('main_page', page_name=page_name)) return flask.redirect(flask.url_for('main_page', page_name=page_name)) elif choice == 1: # Use get_or_insert, which is transactional note = Note.get_or_insert(note_title, parent=parent, content=note_text) if note.content != note_text: - return 'Already there
Return' % flask.url_for('main_page', page_name=page_name) + return ('Already there
Return' + % flask.url_for('main_page', page_name=page_name)) return flask.redirect(flask.url_for('main_page', page_name=page_name)) diff --git a/lib/README.md b/lib/README.md new file mode 100644 index 00000000000..9dbd703f9fb --- /dev/null +++ b/lib/README.md @@ -0,0 +1 @@ +This directory is just for preventing the vendor lib throwing an error. diff --git a/localtesting/__init__.py b/localtesting/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/localtesting/runner.py b/localtesting/runner.py index 6dcc2ff4f66..b9cb839e96c 100755 --- a/localtesting/runner.py +++ b/localtesting/runner.py @@ -15,9 +15,10 @@ # [START runner] import optparse +import os import sys import unittest -import os + USAGE = """%prog SDK_PATH TEST_PATH Run unit tests for App Engine apps. @@ -31,9 +32,9 @@ def main(sdk_path, test_path): # If the sdk path points to a google cloud sdk installation # then we should alter it to point to the GAE platform location. if os.path.exists(os.path.join(sdk_path, 'platform/google_appengine')): - sys.path.insert(0, os.path.join(sdk_path, 'platform/google_appengine')) + sys.path.insert(0, os.path.join(sdk_path, 'platform/google_appengine')) else: - sys.path.insert(0, sdk_path) + sys.path.insert(0, sdk_path) # Ensure that the google.appengine.* packages are available # in tests as well as all bundled third-party packages. @@ -44,10 +45,10 @@ def main(sdk_path, test_path): # changes to configuration there are available to all tests (e.g. # sys.path modifications, namespaces, etc.) try: - import appengine_config - (appengine_config) + import appengine_config + (appengine_config) except ImportError: - print "Note: unable to import appengine_config." + print "Note: unable to import appengine_config." # Discover and run tests. suite = unittest.loader.TestLoader().discover(test_path) diff --git a/localtesting/test_datastore.py b/localtesting/test_datastore.py index 6b5839001a4..512d32e6332 100644 --- a/localtesting/test_datastore.py +++ b/localtesting/test_datastore.py @@ -14,6 +14,7 @@ # [START imports] import unittest + from google.appengine.api import memcache from google.appengine.ext import ndb from google.appengine.ext import testbed @@ -22,139 +23,140 @@ # [START datastore_example_1] class TestModel(ndb.Model): - """A model class used for testing.""" - number = ndb.IntegerProperty(default=42) - text = ndb.StringProperty() + """A model class used for testing.""" + number = ndb.IntegerProperty(default=42) + text = ndb.StringProperty() class TestEntityGroupRoot(ndb.Model): - """Entity group root""" - pass + """Entity group root""" + pass def GetEntityViaMemcache(entity_key): - """Get entity from memcache if available, from datastore if not.""" - entity = memcache.get(entity_key) - if entity is not None: + """Get entity from memcache if available, from datastore if not.""" + entity = memcache.get(entity_key) + if entity is not None: + return entity + key = ndb.Key(urlsafe=entity_key) + entity = key.get() + if entity is not None: + memcache.set(entity_key, entity) return entity - key = ndb.Key(urlsafe=entity_key) - entity = key.get() - if entity is not None: - memcache.set(entity_key, entity) - return entity # [END datastore_example_1] # [START datastore_example_test] class DatstoreTestCase(unittest.TestCase): - def setUp(self): - # First, create an instance of the Testbed class. - self.testbed = testbed.Testbed() - # Then activate the testbed, which prepares the service stubs for use. - self.testbed.activate() - # Next, declare which service stubs you want to use. - self.testbed.init_datastore_v3_stub() - self.testbed.init_memcache_stub() - # Clear ndb's in-context cache between tests. - # This prevents data from leaking between tests. - # Alternatively, you could disable caching by - # using ndb.get_context().set_cache_policy(False) - ndb.get_context().clear_cache() + def setUp(self): + # First, create an instance of the Testbed class. + self.testbed = testbed.Testbed() + # Then activate the testbed, which prepares the service stubs for use. + self.testbed.activate() + # Next, declare which service stubs you want to use. + self.testbed.init_datastore_v3_stub() + self.testbed.init_memcache_stub() + # Clear ndb's in-context cache between tests. + # This prevents data from leaking between tests. + # Alternatively, you could disable caching by + # using ndb.get_context().set_cache_policy(False) + ndb.get_context().clear_cache() # [END datastore_example_test] - # [START datastore_example_teardown] - def tearDown(self): - self.testbed.deactivate() - # [END datastore_example_teardown] - - # [START datastore_example_insert] - def testInsertEntity(self): - TestModel().put() - self.assertEqual(1, len(TestModel.query().fetch(2))) - # [END datastore_example_insert] - - # [START datastore_example_filter] - def testFilterByNumber(self): - root = TestEntityGroupRoot(id="root") - TestModel(parent=root.key).put() - TestModel(number=17, parent=root.key).put() - query = TestModel.query(ancestor=root.key).filter(TestModel.number == 42) - results = query.fetch(2) - self.assertEqual(1, len(results)) - self.assertEqual(42, results[0].number) - # [END datastore_example_filter] - - # [START datastore_example_memcache] - def testGetEntityViaMemcache(self): - entity_key = TestModel(number=18).put().urlsafe() - retrieved_entity = GetEntityViaMemcache(entity_key) - self.assertNotEqual(None, retrieved_entity) - self.assertEqual(18, retrieved_entity.number) - # [END datastore_example_memcache] + # [START datastore_example_teardown] + def tearDown(self): + self.testbed.deactivate() + # [END datastore_example_teardown] + + # [START datastore_example_insert] + def testInsertEntity(self): + TestModel().put() + self.assertEqual(1, len(TestModel.query().fetch(2))) + # [END datastore_example_insert] + + # [START datastore_example_filter] + def testFilterByNumber(self): + root = TestEntityGroupRoot(id="root") + TestModel(parent=root.key).put() + TestModel(number=17, parent=root.key).put() + query = TestModel.query(ancestor=root.key).filter( + TestModel.number == 42) + results = query.fetch(2) + self.assertEqual(1, len(results)) + self.assertEqual(42, results[0].number) + # [END datastore_example_filter] + + # [START datastore_example_memcache] + def testGetEntityViaMemcache(self): + entity_key = TestModel(number=18).put().urlsafe() + retrieved_entity = GetEntityViaMemcache(entity_key) + self.assertNotEqual(None, retrieved_entity) + self.assertEqual(18, retrieved_entity.number) + # [END datastore_example_memcache] # [START HRD_example_1] -from google.appengine.datastore import datastore_stub_util +from google.appengine.datastore import datastore_stub_util # noqa class HighReplicationTestCaseOne(unittest.TestCase): - def setUp(self): - # First, create an instance of the Testbed class. - self.testbed = testbed.Testbed() - # Then activate the testbed, which prepares the service stubs for use. - self.testbed.activate() - # Create a consistency policy that will simulate the High Replication - # consistency model. - self.policy = datastore_stub_util.PseudoRandomHRConsistencyPolicy( - probability=0) - # Initialize the datastore stub with this policy. - self.testbed.init_datastore_v3_stub(consistency_policy=self.policy) - # Initialize memcache stub too, since ndb also uses memcache - self.testbed.init_memcache_stub() - # Clear in-context cache before each test. - ndb.get_context().clear_cache() - - def tearDown(self): - self.testbed.deactivate() - - def testEventuallyConsistentGlobalQueryResult(self): - class TestModel(ndb.Model): - pass - - user_key = ndb.Key('User', 'ryan') - - # Put two entities - ndb.put_multi([ - TestModel(parent=user_key), - TestModel(parent=user_key) - ]) - - # Global query doesn't see the data. - self.assertEqual(0, TestModel.query().count(3)) - # Ancestor query does see the data. - self.assertEqual(2, TestModel.query(ancestor=user_key).count(3)) + def setUp(self): + # First, create an instance of the Testbed class. + self.testbed = testbed.Testbed() + # Then activate the testbed, which prepares the service stubs for use. + self.testbed.activate() + # Create a consistency policy that will simulate the High Replication + # consistency model. + self.policy = datastore_stub_util.PseudoRandomHRConsistencyPolicy( + probability=0) + # Initialize the datastore stub with this policy. + self.testbed.init_datastore_v3_stub(consistency_policy=self.policy) + # Initialize memcache stub too, since ndb also uses memcache + self.testbed.init_memcache_stub() + # Clear in-context cache before each test. + ndb.get_context().clear_cache() + + def tearDown(self): + self.testbed.deactivate() + + def testEventuallyConsistentGlobalQueryResult(self): + class TestModel(ndb.Model): + pass + + user_key = ndb.Key('User', 'ryan') + + # Put two entities + ndb.put_multi([ + TestModel(parent=user_key), + TestModel(parent=user_key) + ]) + + # Global query doesn't see the data. + self.assertEqual(0, TestModel.query().count(3)) + # Ancestor query does see the data. + self.assertEqual(2, TestModel.query(ancestor=user_key).count(3)) # [END HRD_example_1] - # [START HRD_example_2] - def testDeterministicOutcome(self): - # 50% chance to apply. - self.policy.SetProbability(.5) - # Use the pseudo random sequence derived from seed=2. - self.policy.SetSeed(2) + # [START HRD_example_2] + def testDeterministicOutcome(self): + # 50% chance to apply. + self.policy.SetProbability(.5) + # Use the pseudo random sequence derived from seed=2. + self.policy.SetSeed(2) - class TestModel(ndb.Model): - pass + class TestModel(ndb.Model): + pass - TestModel().put() + TestModel().put() - self.assertEqual(0, TestModel.query().count(3)) - self.assertEqual(0, TestModel.query().count(3)) - # Will always be applied before the third query. - self.assertEqual(1, TestModel.query().count(3)) - # [END HRD_example_2] + self.assertEqual(0, TestModel.query().count(3)) + self.assertEqual(0, TestModel.query().count(3)) + # Will always be applied before the third query. + self.assertEqual(1, TestModel.query().count(3)) + # [END HRD_example_2] # [START main] diff --git a/localtesting/test_env_vars.py b/localtesting/test_env_vars.py index c68c3f162b6..d34ed283415 100644 --- a/localtesting/test_env_vars.py +++ b/localtesting/test_env_vars.py @@ -13,8 +13,9 @@ # limitations under the License. # [START env_example] -import unittest import os +import unittest + from google.appengine.ext import testbed diff --git a/localtesting/test_login.py b/localtesting/test_login.py index 5749e06090d..545ea2bd837 100644 --- a/localtesting/test_login.py +++ b/localtesting/test_login.py @@ -14,8 +14,9 @@ # [START login_example] import unittest -from google.appengine.ext import testbed + from google.appengine.api import users +from google.appengine.ext import testbed class LoginTestCase(unittest.TestCase): diff --git a/localtesting/test_mail.py b/localtesting/test_mail.py index 52ec939bb74..ead48353820 100644 --- a/localtesting/test_mail.py +++ b/localtesting/test_mail.py @@ -14,29 +14,30 @@ # [START mail_example] import unittest -from google.appengine.ext import testbed + from google.appengine.api import mail +from google.appengine.ext import testbed class MailTestCase(unittest.TestCase): - def setUp(self): - self.testbed = testbed.Testbed() - self.testbed.activate() - self.testbed.init_mail_stub() - self.mail_stub = self.testbed.get_stub(testbed.MAIL_SERVICE_NAME) - - def tearDown(self): - self.testbed.deactivate() - - def testMailSent(self): - mail.send_mail(to='alice@example.com', - subject='This is a test', - sender='bob@example.com', - body='This is a test e-mail') - messages = self.mail_stub.get_sent_messages(to='alice@example.com') - self.assertEqual(1, len(messages)) - self.assertEqual('alice@example.com', messages[0].to) + def setUp(self): + self.testbed = testbed.Testbed() + self.testbed.activate() + self.testbed.init_mail_stub() + self.mail_stub = self.testbed.get_stub(testbed.MAIL_SERVICE_NAME) + + def tearDown(self): + self.testbed.deactivate() + + def testMailSent(self): + mail.send_mail(to='alice@example.com', + subject='This is a test', + sender='bob@example.com', + body='This is a test e-mail') + messages = self.mail_stub.get_sent_messages(to='alice@example.com') + self.assertEqual(1, len(messages)) + self.assertEqual('alice@example.com', messages[0].to) # [END mail_example] if __name__ == '__main__': diff --git a/localtesting/test_task_queue.py b/localtesting/test_task_queue.py index 3dc5cac975b..ad1b40a9200 100644 --- a/localtesting/test_task_queue.py +++ b/localtesting/test_task_queue.py @@ -15,9 +15,10 @@ import operator # [START taskqueue] import unittest -from google.appengine.ext import testbed + from google.appengine.api import taskqueue from google.appengine.ext import deferred +from google.appengine.ext import testbed class TaskQueueTestCase(unittest.TestCase): diff --git a/python-docs-samples.json.enc b/python-docs-samples.json.enc new file mode 100644 index 00000000000..9c34df7c109 Binary files /dev/null and b/python-docs-samples.json.enc differ diff --git a/queue.yaml b/queue.yaml new file mode 100644 index 00000000000..fcdd5444dd9 --- /dev/null +++ b/queue.yaml @@ -0,0 +1,7 @@ +queue: +- name: default + rate: 5/s +- name: queue-1 + rate: 5/s +- name: queue-2 + rate: 5/s diff --git a/resources/constants.json b/resources/constants.json new file mode 100644 index 00000000000..66b2964baf1 --- /dev/null +++ b/resources/constants.json @@ -0,0 +1,9 @@ +{ + "projectId": "bigquery-devrel-samples", + "datasetId": "test_dataset", + "currentTableId": "test_table", + "newTableId": "test_table2", + "cloudStorageInputURI": "gs://bigquery-devrel-samples-bucket/data.csv", + "cloudStorageOutputURI": "gs://bigquery-devrel-samples-bucket/output.csv", + "query": "SELECT corpus FROM publicdata:samples.shakespeare GROUP BY corpus;" +} diff --git a/resources/data.csv b/resources/data.csv new file mode 100644 index 00000000000..230a96b559d --- /dev/null +++ b/resources/data.csv @@ -0,0 +1 @@ +Gandalf, 2000, 140.0, 1 diff --git a/resources/schema.json b/resources/schema.json new file mode 100644 index 00000000000..a48971ef857 --- /dev/null +++ b/resources/schema.json @@ -0,0 +1 @@ +[{"type": "STRING", "name": "Name"}, {"type": "INTEGER", "name": "Age"}, {"type": "FLOAT", "name": "Weight"}, {"type": "BOOLEAN", "name": "IsMagic"}] \ No newline at end of file diff --git a/resources/streamrows.json b/resources/streamrows.json new file mode 100644 index 00000000000..fbc7a392971 --- /dev/null +++ b/resources/streamrows.json @@ -0,0 +1,7 @@ +[ + {"Name": "test", "Age": 0, "Weight": 100.0, "IsMagic": false}, + {"Name": "test", "Age": 1, "Weight": 100.0, "IsMagic": false}, + {"Name": "test", "Age": 2, "Weight": 100.0, "IsMagic": false}, + {"Name": "test", "Age": 3, "Weight": 100.0, "IsMagic": false}, + {"Name": "test", "Age": 0, "Weight": 100.0, "IsMagic": false} +] diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000000..ad3c7765928 --- /dev/null +++ b/tox.ini @@ -0,0 +1,27 @@ +[tox] +skipsdist = True +envlist = {py27}-{nosetest,pep8}, cover + +[testenv] +basepython = + cover,py27: python2.7 +deps = + flask + google-api-python-client + pep8: flake8 + pep8: flake8-import-order + nosetest,cover: mock + nosetest,cover: nose + nosetest,cover: nosegae + cover: coverage + cover: coveralls + cover: nosexcover +setenv = + GOOGLE_APPLICATION_CREDENTIALS=python-docs-samples.json +commands = + cover: nosetests --with-gae --with-xunit --with-xcoverage \ + cover: --cover-tests --cover-branches --cover-min-percentage=70 \ + cover: --cover-inclusive --cover-erase + cover: coveralls + nosetest: nosetests --with-gae + pep8: flake8 --max-complexity=10 --import-order-style=google