diff --git a/.gitignore b/.gitignore index 850c3a9fab0..176a48ea6e7 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ secrets.tar .cache junit.xml credentials.dat +.nox diff --git a/.travis.yml b/.travis.yml index 4a147705cd5..8b43097f2c0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,15 +14,16 @@ env: - PATH=${PATH}:${HOME}/gcloud/google-cloud-sdk/bin - GOOGLE_APPLICATION_CREDENTIALS=${TRAVIS_BUILD_DIR}/testing/resources/service-account.json - GOOGLE_CLIENT_SECRETS=${TRAVIS_BUILD_DIR}/testing/resources/client-secrets.json - - GAE_PYTHONPATH=${HOME}/.cache/google_appengine - GAE_ROOT=${HOME}/.cache/ - secure: Orp9Et2TIwCG/Hf59aa0NUDF1pNcwcS4TFulXX175918cFREOzf/cNZNg+Ui585ZRFjbifZdc858tVuCVd8XlxQPXQgp7bwB7nXs3lby3LYg4+HD83Gaz7KOWxRLWVor6IVn8OxeCzwl6fJkdmffsTTO9csC4yZ7izHr+u7hiO4= before_install: +- pip install --upgrade pip wheel virtualenv - openssl aes-256-cbc -k "$secrets_password" -in secrets.tar.enc -out secrets.tar -d - tar xvf secrets.tar install: -- pip install tox coverage +- pip install nox-automation coverage script: - source ${TRAVIS_BUILD_DIR}/testing/resources/test-env.sh -- tox +- nox --stop-on-first-error -s lint travis +after_script: - coverage report diff --git a/conftest.py b/conftest.py index 92a808d07b0..8bf0e8d2dcf 100644 --- a/conftest.py +++ b/conftest.py @@ -1,3 +1,17 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import pytest diff --git a/container_engine/django_tutorial/requirements.txt b/container_engine/django_tutorial/requirements.txt index 377a14aab47..6d724096fce 100644 --- a/container_engine/django_tutorial/requirements.txt +++ b/container_engine/django_tutorial/requirements.txt @@ -1,4 +1,4 @@ -Django==1.9.2 +Django==1.9.3 mysqlclient==1.3.7 wheel==0.29.0 gunicorn==19.4.5 diff --git a/managed_vms/datastore/requirements.txt b/managed_vms/datastore/requirements.txt index 6a6bfb7c53f..ae27a8070a2 100644 --- a/managed_vms/datastore/requirements.txt +++ b/managed_vms/datastore/requirements.txt @@ -1,4 +1,4 @@ Flask==0.10.1 gcloud==0.10.1 gunicorn==19.4.5 -oauth2client==2.0.0.post1 +oauth2client==2.0.1 diff --git a/managed_vms/django_cloudsql/requirements.txt b/managed_vms/django_cloudsql/requirements.txt index 377a14aab47..6d724096fce 100644 --- a/managed_vms/django_cloudsql/requirements.txt +++ b/managed_vms/django_cloudsql/requirements.txt @@ -1,4 +1,4 @@ -Django==1.9.2 +Django==1.9.3 mysqlclient==1.3.7 wheel==0.29.0 gunicorn==19.4.5 diff --git a/managed_vms/hello_world_django/requirements.txt b/managed_vms/hello_world_django/requirements.txt index b68903a987a..55fd25b8803 100644 --- a/managed_vms/hello_world_django/requirements.txt +++ b/managed_vms/hello_world_django/requirements.txt @@ -1,2 +1,2 @@ -Django==1.9.2 +Django==1.9.3 gunicorn==19.4.5 diff --git a/managed_vms/kinto/requirements.txt b/managed_vms/kinto/requirements.txt index 3acab72551f..b4e154f0ff5 100644 --- a/managed_vms/kinto/requirements.txt +++ b/managed_vms/kinto/requirements.txt @@ -1,3 +1,3 @@ gunicorn==19.4.5 -oauth2client==2.0.0.post1 +oauth2client==2.0.1 kinto==1.11.2 diff --git a/managed_vms/pubsub/requirements.txt b/managed_vms/pubsub/requirements.txt index 6a6bfb7c53f..ae27a8070a2 100644 --- a/managed_vms/pubsub/requirements.txt +++ b/managed_vms/pubsub/requirements.txt @@ -1,4 +1,4 @@ Flask==0.10.1 gcloud==0.10.1 gunicorn==19.4.5 -oauth2client==2.0.0.post1 +oauth2client==2.0.1 diff --git a/managed_vms/sendgrid/requirements.txt b/managed_vms/sendgrid/requirements.txt index a2d9f40f85a..e8e67f01aa1 100644 --- a/managed_vms/sendgrid/requirements.txt +++ b/managed_vms/sendgrid/requirements.txt @@ -1,3 +1,3 @@ Flask==0.10.1 -sendgrid==1.6.22 +sendgrid==2.2.1 gunicorn==19.4.5 diff --git a/nox.py b/nox.py new file mode 100644 index 00000000000..a3d3f0a0e15 --- /dev/null +++ b/nox.py @@ -0,0 +1,145 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fnmatch +import os +import tempfile + +import nox + +REPO_TOOLS_REQ =\ + 'git+https://github.com/GoogleCloudPlatform/python-repo-tools.git' + +COMMON_PYTEST_ARGS = [ + '-x', '--no-success-flaky-report', '--cov', '--cov-config', + '.coveragerc', '--cov-append', '--cov-report='] + +SAMPLES = [ + 'bigquery/api', + 'blog/introduction_to_data_models_in_cloud_datastore', + 'cloud_logging/api', + 'compute/api', + 'compute/autoscaler/demo', + 'datastore/api', + 'managed_vms/cloudsql', + 'managed_vms/datastore', + 'managed_vms/disk', + 'managed_vms/extending_runtime', + 'managed_vms/hello_world', + 'managed_vms/hello_world_compat', + 'managed_vms/memcache', + 'managed_vms/pubsub', + 'managed_vms/static_files', + 'managed_vms/storage', + 'monitoring/api', + 'storage/api', +] + +GAE_SAMPLES = [ + 'appengine/app_identity/signing', + 'appengine/bigquery', + 'appengine/blobstore', + 'appengine/cloudsql', + 'appengine/images', + 'appengine/localtesting', + 'appengine/logging/reading_logs', + 'appengine/logging/writing_logs', + 'appengine/mailgun', + 'appengine/memcache/guestbook', + 'appengine/multitenancy', + 'appengine/ndb/modeling', + 'appengine/ndb/overview', + 'appengine/ndb/transactions', + 'appengine/storage', +] + + +def session_lint(session): + session.install('flake8', 'flake8-import-order') + session.run( + 'flake8', '--builtin=gettext', '--max-complexity=10', + '--import-order-style=google', + '--exclude', + 'container_engine/django_tutorial/polls/migrations/*,.nox,.cache,env', + *(session.posargs or ['.'])) + + +def list_files(folder, pattern): + for root, folders, files in os.walk(folder): + for filename in files: + if fnmatch.fnmatch(filename, pattern): + yield os.path.join(root, filename) + + +def session_reqcheck(session): + session.install(REPO_TOOLS_REQ) + + if 'update' in session.posargs: + command = 'update-requirements' + else: + command = 'check-requirements' + + for reqfile in list_files('.', 'requirements*.txt'): + session.run('gcprepotools', command, reqfile) + + +@nox.parametrize('interpreter', ['python2.7', 'python3.4']) +def session_tests(session, interpreter, extra_pytest_args=None): + session.interpreter = interpreter + session.install(REPO_TOOLS_REQ) + session.install('-r', 'requirements-{}-dev.txt'.format(interpreter)) + + # extra_pytest_args can be send by another session calling this session, + # see session_travis. + pytest_args = COMMON_PYTEST_ARGS + (extra_pytest_args or []) + + # session.posargs is any leftover arguments from the command line, which + # allows users to run a particular test instead of all of them. + for sample in (session.posargs or SAMPLES): + session.run( + 'py.test', sample, + *pytest_args, + success_codes=[0, 5]) # Treat no test collected as success. + + +def session_gae(session, extra_pytest_args=None): + session.interpreter = 'python2.7' + session.install(REPO_TOOLS_REQ) + session.install('-r', 'requirements-python2.7-dev.txt') + + # Install the app engine sdk and setup import paths. + gae_root = os.environ.get('GAE_ROOT', tempfile.gettempdir()) + session.env['PYTHONPATH'] = os.path.join(gae_root, 'google_appengine') + session.run('gcprepotools', 'download-appengine-sdk', gae_root) + + # Create a lib directory to prevent the GAE vendor library from + # complaining. + if not os.path.exists('lib'): + os.makedirs('lib') + + pytest_args = COMMON_PYTEST_ARGS + (extra_pytest_args or []) + + for sample in (session.posargs or GAE_SAMPLES): + session.run( + 'py.test', sample, + *pytest_args, + success_codes=[0, 5]) # Treat no test collected as success. + + +def session_travis(session): + """On travis, just run with python3.4 and don't run slow or flaky tests.""" + session_tests( + session, 'python3.4', extra_pytest_args=['-m not slow and not flaky']) + session_gae( + session, extra_pytest_args=['-m not slow and not flaky']) diff --git a/requirements-py27-dev.txt b/requirements-python2.7-dev.txt similarity index 86% rename from requirements-py27-dev.txt rename to requirements-python2.7-dev.txt index c8e31cca20b..f551457c1d9 100644 --- a/requirements-py27-dev.txt +++ b/requirements-python2.7-dev.txt @@ -1,6 +1,6 @@ gcloud==0.10.1 google-api-python-client==1.5.0 -oauth2client==2.0.0.post1 +oauth2client==2.0.1 requests[security]==2.9.1 beautifulsoup4==4.4.1 coverage==4.1b2 @@ -21,10 +21,10 @@ PyMySQL==0.7.2 pymemcache==1.3.5 PyCrypto==2.6.1 flaky==3.1.0 -Django==1.9.2 +Django==1.9.3 twilio==6.3.dev0 -sendgrid==1.6.22 +sendgrid==2.2.1 Flask-Sockets==0.2.0 mysql-python==1.2.5 -pytest==2.8.7 +pytest==2.9.0 pytest-cov==2.2.1 diff --git a/requirements-py34-dev.txt b/requirements-python3.4-dev.txt similarity index 85% rename from requirements-py34-dev.txt rename to requirements-python3.4-dev.txt index 968df9921a9..2c490ffe5ec 100644 --- a/requirements-py34-dev.txt +++ b/requirements-python3.4-dev.txt @@ -1,6 +1,6 @@ gcloud==0.10.1 google-api-python-client==1.5.0 -oauth2client==2.0.0.post1 +oauth2client==2.0.1 requests[security]==2.9.1 beautifulsoup4==4.4.1 coverage==4.1b2 @@ -21,8 +21,8 @@ PyMySQL==0.7.2 pymemcache==1.3.5 PyCrypto==2.6.1 flaky==3.1.0 -Django==1.9.2 +Django==1.9.3 twilio==6.3.dev0 -sendgrid==1.6.22 -pytest==2.8.7 +sendgrid==2.2.1 +pytest==2.9.0 pytest-cov==2.2.1 diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 5e8939b42b4..00000000000 --- a/tox.ini +++ /dev/null @@ -1,142 +0,0 @@ -[tox] -skipsdist = True -envlist = pep8, py27, py34, gae - -[testenv] -passenv = * -basepython = python2.7 -examples = - bigquery/api - blog/introduction_to_data_models_in_cloud_datastore - cloud_logging/api - compute/api - compute/autoscaler/demo - datastore/api - managed_vms/cloudsql - managed_vms/datastore - managed_vms/disk - managed_vms/extending_runtime - managed_vms/hello_world - managed_vms/hello_world_compat - managed_vms/memcache - managed_vms/pubsub - managed_vms/static_files - managed_vms/storage - monitoring/api - storage/api -setenv = - PYTHONPATH=. - -[testenv:pep8] -deps = - flake8 - flake8-import-order -commands = - flake8 --builtin=gettext --max-complexity=10 --import-order-style=google {posargs} - -[testenv:py27] -deps = - git+https://github.com/GoogleCloudPlatform/python-repo-tools - -rrequirements-py27-dev.txt -commands = - python scripts/run-tests.py {posargs:{[testenv]examples}} - -[testenv:py27-all] -deps = - {[testenv:py27]deps} -commands = - python scripts/run-tests.py \ - --junit \ - --run-slow \ - {posargs:{[testenv]examples}} - -[testenv:py34] -basepython = python3.4 -deps = - git+https://github.com/GoogleCloudPlatform/python-repo-tools - -rrequirements-py34-dev.txt -commands = - python scripts/run-tests.py {posargs:{[testenv]examples}} - -[testenv:py34-all] -basepython = python3.4 -deps = - {[testenv:py34]deps} -commands = - python scripts/run-tests.py \ - --junit \ - --run-slow \ - {posargs:{[testenv]examples}} - -[testenv:gae] -deps = - git+https://github.com/GoogleCloudPlatform/python-repo-tools - -rrequirements-py27-dev.txt -examples = - appengine/app_identity/signing - appengine/bigquery - appengine/blobstore - appengine/cloudsql - appengine/images - appengine/localtesting - appengine/logging/reading_logs - appengine/logging/writing_logs - appengine/mailgun - appengine/memcache/guestbook - appengine/multitenancy - appengine/ndb/modeling - appengine/ndb/overview - appengine/ndb/transactions - appengine/storage -commands = - gcprepotools download-appengine-sdk {env:GAE_ROOT} - # Create a lib directory, otherwise, the vendor library will explode. - mkdir -p lib - python scripts/run-tests.py {posargs:{[testenv:gae]examples}} -setenv = - PYTHONPATH=.:{env:GAE_ROOT:}/google_appengine -whitelist_externals = mkdir - -[testenv:gae-all] -deps = - {[testenv:gae]deps} -commands = - gcprepotools download-appengine-sdk {env:GAE_ROOT} - # Create a lib directory, otherwise, the vendor library will explode. - mkdir -p lib - python scripts/run-tests.py \ - --junit \ - --run-slow \ - {posargs:{[testenv:gae]examples}} -setenv = - PYTHONPATH=.:{env:GAE_ROOT:}/google_appengine -whitelist_externals = mkdir - -[testenv:reqcheck] -deps= - git+https://github.com/GoogleCloudPlatform/python-repo-tools -commands = - bash -c "find . -name requirements\*.txt |\ - xargs -P 4 -n 1 gcprepotools check-requirements" -whitelist_externals = - bash - find - xargs - -[testenv:requpdate] -deps= - git+https://github.com/GoogleCloudPlatform/python-repo-tools -commands = - bash -c "find . -name requirements\*.txt |\ - xargs -P 4 -n 1 gcprepotools update-requirements" -whitelist_externals = - bash - find - xargs - -[flake8] -exclude=container_engine/django_tutorial/polls/migrations/* - -[pytest] -addopts = -x --no-success-flaky-report --cov --cov-config .coveragerc --cov-append --cov-report= -