From d0cea7ded7c1792dd39d0e42cb3134c037274460 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Wed, 21 Jun 2023 14:56:47 +0000 Subject: [PATCH 01/25] saving state of noxfile prior to owlbot.py edits --- noxfile.py | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 94 insertions(+), 3 deletions(-) diff --git a/noxfile.py b/noxfile.py index e18072ca..4ee09f5b 100644 --- a/noxfile.py +++ b/noxfile.py @@ -31,6 +31,7 @@ DEFAULT_PYTHON_VERSION = "3.8" + UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] UNIT_TEST_STANDARD_DEPENDENCIES = [ "mock", @@ -51,6 +52,11 @@ "3.9": [], } +CONDA_TEST_PYTHON_VERSIONS = [ + UNIT_TEST_PYTHON_VERSIONS[0], + UNIT_TEST_PYTHON_VERSIONS[-1] +] + SYSTEM_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] SYSTEM_TEST_STANDARD_DEPENDENCIES = [ "mock", @@ -174,7 +180,8 @@ def default(session): # Run py.test against the unit tests. session.run( "py.test", - "--quiet", + # "--quiet", + "-vv", f"--junitxml=unit_{session.python}_sponge_log.xml", "--cov=pandas_gbq", "--cov=tests/unit", @@ -252,7 +259,8 @@ def system(session): if system_test_exists: session.run( "py.test", - "--quiet", + #"--quiet", + "-vv", f"--junitxml=system_{session.python}_sponge_log.xml", system_test_path, *session.posargs, @@ -260,7 +268,8 @@ def system(session): if system_test_folder_exists: session.run( "py.test", - "--quiet", + #"--quiet", + "-vv", f"--junitxml=system_{session.python}_sponge_log.xml", system_test_folder_path, *session.posargs, @@ -515,3 +524,85 @@ def prerelease_deps(session): system_test_folder_path, *session.posargs, ) + + +def install_conda_unittest_dependencies(session, standard_deps, conda_forge_packages): + """Installs packages from conda forge, pypi, and locally. + """ + + # Install from conda-forge and default conda package repos. + session.conda_install( + *conda_forge_packages, + channel=['defaults', 'conda-forge'] + ) + + # Install from pypi for packages not readily available on conda forge. + session.install( + *standard_deps, + ) + + # Install via pip from the local repo, avoid doing dependency resolution + # via pip, so that we don't override any conda resolved dependencies + session.install("-e", ".", "--no-deps") + + +@nox.session(python=CONDA_TEST_PYTHON_VERSIONS, venv_backend="mamba") +def conda_test(session): + """Run test suite in a conda virtual environment. + + Installs all test dependencies, then installs this package. + NOTE: Some of these libraries are not readily available on conda-forge + at this time and are thus installed using pip after the base install of + libraries from conda-forge. + + We decided that it was more important to prove a base ability to install + using conda than to complicate things with adding a whole nother + set of constraints just for a conda install, so this install does not + attempt to constrain packages (i.e. in a constraints-x.x.txt file) + manually. + """ + + standard_deps = ( + UNIT_TEST_STANDARD_DEPENDENCIES + + UNIT_TEST_DEPENDENCIES + + UNIT_TEST_EXTERNAL_DEPENDENCIES + ) + + conda_forge_packages = [ + 'db-dtypes', + 'google-api-core', + 'google-auth', + 'google-auth-oauthlib', + 'google-cloud-bigquery', + 'google-cloud-bigquery-storage', + 'numpy', + 'pandas', + 'pyarrow', + 'pydata-google-auth', + 'tqdm', + 'protobuf', + ] + + install_conda_unittest_dependencies(session, standard_deps, conda_forge_packages) + + # Provide a list of all installed packages (both from conda forge and pip) + # for troubleshooting purposes + session.run( + "mamba", + "list" + ) + + # Tests are limited to unit tests only, at this time. + session.run( + "py.test", + "--quiet", + f"--junitxml=unit_{session.python}_sponge_log.xml", + "--cov=pandas_gbq", + "--cov=tests/unit", + "--cov-append", + "--cov-config=.coveragerc", + "--cov-report=", + "--cov-fail-under=0", + os.path.join("tests", "unit"), + *session.posargs, + ) From 9dd6eb5a51c9db503cffd886be4c578b0c10b361 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Wed, 21 Jun 2023 16:52:11 +0000 Subject: [PATCH 02/25] minor clean up of leftover artifacts from various experiments --- noxfile.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/noxfile.py b/noxfile.py index 4ee09f5b..9cbb7891 100644 --- a/noxfile.py +++ b/noxfile.py @@ -180,8 +180,7 @@ def default(session): # Run py.test against the unit tests. session.run( "py.test", - # "--quiet", - "-vv", + "--quiet", f"--junitxml=unit_{session.python}_sponge_log.xml", "--cov=pandas_gbq", "--cov=tests/unit", @@ -259,8 +258,7 @@ def system(session): if system_test_exists: session.run( "py.test", - #"--quiet", - "-vv", + "--quiet", f"--junitxml=system_{session.python}_sponge_log.xml", system_test_path, *session.posargs, @@ -268,8 +266,7 @@ def system(session): if system_test_folder_exists: session.run( "py.test", - #"--quiet", - "-vv", + "--quiet", f"--junitxml=system_{session.python}_sponge_log.xml", system_test_folder_path, *session.posargs, From e8a55127c05f7ab279d8ee2e821658b76890f49f Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Wed, 21 Jun 2023 16:53:34 +0000 Subject: [PATCH 03/25] linting and black --- noxfile.py | 47 ++++++++++++++++++++--------------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/noxfile.py b/noxfile.py index 9cbb7891..bfaeefa1 100644 --- a/noxfile.py +++ b/noxfile.py @@ -54,7 +54,7 @@ CONDA_TEST_PYTHON_VERSIONS = [ UNIT_TEST_PYTHON_VERSIONS[0], - UNIT_TEST_PYTHON_VERSIONS[-1] + UNIT_TEST_PYTHON_VERSIONS[-1], ] SYSTEM_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] @@ -524,22 +524,18 @@ def prerelease_deps(session): def install_conda_unittest_dependencies(session, standard_deps, conda_forge_packages): - """Installs packages from conda forge, pypi, and locally. - """ + """Installs packages from conda forge, pypi, and locally.""" # Install from conda-forge and default conda package repos. - session.conda_install( - *conda_forge_packages, - channel=['defaults', 'conda-forge'] - ) + session.conda_install(*conda_forge_packages, channel=["defaults", "conda-forge"]) # Install from pypi for packages not readily available on conda forge. session.install( *standard_deps, ) - + # Install via pip from the local repo, avoid doing dependency resolution - # via pip, so that we don't override any conda resolved dependencies + # via pip, so that we don't override any conda resolved dependencies session.install("-e", ".", "--no-deps") @@ -549,8 +545,8 @@ def conda_test(session): Installs all test dependencies, then installs this package. NOTE: Some of these libraries are not readily available on conda-forge - at this time and are thus installed using pip after the base install of - libraries from conda-forge. + at this time and are thus installed using pip after the base install of + libraries from conda-forge. We decided that it was more important to prove a base ability to install using conda than to complicate things with adding a whole nother @@ -566,28 +562,25 @@ def conda_test(session): ) conda_forge_packages = [ - 'db-dtypes', - 'google-api-core', - 'google-auth', - 'google-auth-oauthlib', - 'google-cloud-bigquery', - 'google-cloud-bigquery-storage', - 'numpy', - 'pandas', - 'pyarrow', - 'pydata-google-auth', - 'tqdm', - 'protobuf', + "db-dtypes", + "google-api-core", + "google-auth", + "google-auth-oauthlib", + "google-cloud-bigquery", + "google-cloud-bigquery-storage", + "numpy", + "pandas", + "pyarrow", + "pydata-google-auth", + "tqdm", + "protobuf", ] install_conda_unittest_dependencies(session, standard_deps, conda_forge_packages) # Provide a list of all installed packages (both from conda forge and pip) # for troubleshooting purposes - session.run( - "mamba", - "list" - ) + session.run("mamba", "list") # Tests are limited to unit tests only, at this time. session.run( From e88a4f9e691d93a3d7f5021c3e5c0a56b26cc70f Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Fri, 23 Jun 2023 15:26:04 +0000 Subject: [PATCH 04/25] adds noxfile.py to excludes and removes noxfile edits in owlbot --- owlbot.py | 117 ++---------------------------------------------------- 1 file changed, 3 insertions(+), 114 deletions(-) diff --git a/owlbot.py b/owlbot.py index 9c9454f8..bbb078c7 100644 --- a/owlbot.py +++ b/owlbot.py @@ -52,10 +52,11 @@ excludes=[ # pandas-gbq was originally licensed BSD-3-Clause License "LICENSE", - # Mulit-processing note isn't relevant, as pandas_gbq is responsible for + # Multi-processing note isn't relevant, as pandas_gbq is responsible for # creating clients, not the end user. "docs/multiprocessing.rst", - "README.rst", + "noxfile.py", + "README.rst", ], ) @@ -63,118 +64,6 @@ # Fixup files # ---------------------------------------------------------------------------- -s.replace( - ["noxfile.py"], - r"import pathlib\s+import shutil", - "import pathlib\nimport re\nimport shutil", -) - -s.replace( - ["noxfile.py"], r"[\"']google[\"']", '"pandas_gbq"', -) - - -s.replace( - ["noxfile.py"], "--cov=google", "--cov=pandas_gbq", -) - -# Workaround for https://github.com/googleapis/synthtool/issues/1317 -s.replace( - ["noxfile.py"], r'extras = "\[\]"', 'extras = ""', -) - -s.replace( - ["noxfile.py"], - r"@nox.session\(python=DEFAULT_PYTHON_VERSION\)\s+def cover\(session\):", - r"""@nox.session(python=DEFAULT_PYTHON_VERSION) -def prerelease(session): - session.install( - "--extra-index-url", - "https://pypi.fury.io/arrow-nightlies/", - "--prefer-binary", - "--pre", - "--upgrade", - "pyarrow", - ) - session.install( - "--extra-index-url", - "https://pypi.anaconda.org/scipy-wheels-nightly/simple", - "--prefer-binary", - "--pre", - "--upgrade", - "pandas", - ) - session.install( - "--prefer-binary", - "--pre", - "--upgrade", - "google-api-core", - "google-cloud-bigquery", - "google-cloud-bigquery-storage", - "google-cloud-core", - "google-resumable-media", - # Exclude version 1.49.0rc1 which has a known issue. See https://github.com/grpc/grpc/pull/30642 - "grpcio!=1.49.0rc1", - ) - session.install( - "freezegun", - "google-cloud-datacatalog", - "google-cloud-storage", - "google-cloud-testutils", - "IPython", - "mock", - "psutil", - "pytest", - "pytest-cov", - ) - - # Because we test minimum dependency versions on the minimum Python - # version, the first version we test with in the unit tests sessions has a - # constraints file containing all dependencies and extras. - with open( - CURRENT_DIRECTORY - / "testing" - / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt", - encoding="utf-8", - ) as constraints_file: - constraints_text = constraints_file.read() - - # Ignore leading whitespace and comment lines. - deps = [ - match.group(1) - for match in re.finditer( - r"^\\s*(\\S+)(?===\\S+)", constraints_text, flags=re.MULTILINE - ) - ] - - # We use --no-deps to ensure that pre-release versions aren't overwritten - # by the version ranges in setup.py. - session.install(*deps) - session.install("--no-deps", "-e", ".[all]") - - # Print out prerelease package versions. - session.run("python", "-m", "pip", "freeze") - - # Run all tests, except a few samples tests which require extra dependencies. - session.run( - "py.test", - "--quiet", - f"--junitxml=prerelease_unit_{session.python}_sponge_log.xml", - os.path.join("tests", "unit"), - ) - session.run( - "py.test", - "--quiet", - f"--junitxml=prerelease_system_{session.python}_sponge_log.xml", - os.path.join("tests", "system"), - ) - - -@nox.session(python=DEFAULT_PYTHON_VERSION) -def cover(session):""", - re.MULTILINE, -) - s.replace( [".github/header-checker-lint.yml"], '"Google LLC"', '"pandas-gbq Authors"', ) From 53bd3d88bb170116dc9b21be40c68f16d71b776f Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Fri, 23 Jun 2023 16:11:43 +0000 Subject: [PATCH 05/25] Adds new kokoro file in presubmit --- .kokoro/presubmit/conda_test.cfg | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .kokoro/presubmit/conda_test.cfg diff --git a/.kokoro/presubmit/conda_test.cfg b/.kokoro/presubmit/conda_test.cfg new file mode 100644 index 00000000..6e3943f3 --- /dev/null +++ b/.kokoro/presubmit/conda_test.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "conda_test" +} From cb6ecfd0100d21a35731cb122d5ba6f3d2348145 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Tue, 27 Jun 2023 12:29:14 +0000 Subject: [PATCH 06/25] removes circle/ci config file --- .circleci/config.yml | 34 ---------------------------------- 1 file changed, 34 deletions(-) delete mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index e008054c..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -version: 2 -jobs: - # Conda - "conda-3.7": - docker: - - image: mambaorg/micromamba - environment: - PYTHON: "3.7" - PANDAS: "0.24.2" - steps: - - checkout - - run: ci/config_auth.sh - - run: ci/run_conda.sh - "conda-3.9": - docker: - - image: mambaorg/micromamba - environment: - PYTHON: "3.9" - PANDAS: "1.3.4" - steps: - - checkout - - run: ci/config_auth.sh - - run: ci/run_conda.sh - -workflows: - version: 2 - build: - jobs: - - "conda-3.7" - - "conda-3.9" From 9045c1cac5b4086ffc4fa7f6912f97df3eb31ec8 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Mon, 10 Jul 2023 08:24:10 -0400 Subject: [PATCH 07/25] Update noxfile.py --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 42bccef9..135ce432 100644 --- a/noxfile.py +++ b/noxfile.py @@ -578,7 +578,7 @@ def conda_test(session): install_conda_unittest_dependencies(session, standard_deps, conda_forge_packages) # Provide a list of all installed packages (both from conda forge and pip) - # for troubleshooting purposes + # for troubleshooting purposes. session.run("mamba", "list") # Tests are limited to unit tests only, at this time. From fcdd47712f22282eb2d65dffd18fa579f9fe1410 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Wed, 12 Jul 2023 09:12:30 -0400 Subject: [PATCH 08/25] Update noxfile.py --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 135ce432..683cbeb7 100644 --- a/noxfile.py +++ b/noxfile.py @@ -579,7 +579,7 @@ def conda_test(session): # Provide a list of all installed packages (both from conda forge and pip) # for troubleshooting purposes. - session.run("mamba", "list") + session.run("conda", "list") # Tests are limited to unit tests only, at this time. session.run( From 1a5e26a80e81f4aa1f1bee756a313d5b561c0f49 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Wed, 12 Jul 2023 09:31:51 -0400 Subject: [PATCH 09/25] Update noxfile.py --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 683cbeb7..135ce432 100644 --- a/noxfile.py +++ b/noxfile.py @@ -579,7 +579,7 @@ def conda_test(session): # Provide a list of all installed packages (both from conda forge and pip) # for troubleshooting purposes. - session.run("conda", "list") + session.run("mamba", "list") # Tests are limited to unit tests only, at this time. session.run( From b8d9b2a6e61a83a99493fc1623a428eb7c8b991f Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Wed, 12 Jul 2023 18:38:18 +0000 Subject: [PATCH 10/25] remove limit on printing only KOKORO env variables (for testing) --- .kokoro/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index d02a78db..956feb49 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -25,7 +25,7 @@ cd "${PROJECT_ROOT}" export PYTHONUNBUFFERED=1 # Debug: show build environment -env | grep KOKORO +env #| grep KOKORO # Setup service account credentials. export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json From 5c925c9397615cd9cbf9aea46e8dcf303ab6f6f7 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Thu, 13 Jul 2023 00:13:38 +0000 Subject: [PATCH 11/25] updates build.sh with conda environmental variables --- .kokoro/build.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 956feb49..ce3b5335 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -24,6 +24,16 @@ cd "${PROJECT_ROOT}" # Disable buffering, so that the logs stream through. export PYTHONUNBUFFERED=1 +# Set conda variables and update path +export CONDA_EXE=/root/conda/bin/conda +export CONDA_PREFIX=/root/conda +export CONDA_PROMPT_MODIFIER=(base) +export _CE_CONDA= +export CONDA_SHLVL=1 +export CONDA_PYTHON_EXE=/root/conda/bin/python +export CONDA_DEFAULT_ENV=base +export PATH=/root/conda/bin:/root/conda/condabin:$PATH + # Debug: show build environment env #| grep KOKORO From 90b72ff209890ca1b393f3eedee03547c3c924ac Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Thu, 13 Jul 2023 00:14:15 +0000 Subject: [PATCH 12/25] updates build.sh with conda environmental variables redux --- .kokoro/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index ce3b5335..d1fc058b 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -32,7 +32,7 @@ export _CE_CONDA= export CONDA_SHLVL=1 export CONDA_PYTHON_EXE=/root/conda/bin/python export CONDA_DEFAULT_ENV=base -export PATH=/root/conda/bin:/root/conda/condabin:$PATH +export PATH=/root/conda/bin:/root/conda/condabin:${PATH} # Debug: show build environment env #| grep KOKORO From 9d6bade960533ae294dea1b4afa73ac7a488eef4 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Thu, 13 Jul 2023 11:10:52 +0000 Subject: [PATCH 13/25] removes the ci folder and files related to circle ci --- ci/config_auth.sh | 13 ------------- ci/requirements-3.7-0.24.2.conda | 17 ----------------- ci/requirements-3.9-1.3.4.conda | 14 -------------- ci/run_conda.sh | 23 ----------------------- ci/run_tests.sh | 15 --------------- 5 files changed, 82 deletions(-) delete mode 100755 ci/config_auth.sh delete mode 100644 ci/requirements-3.7-0.24.2.conda delete mode 100644 ci/requirements-3.9-1.3.4.conda delete mode 100755 ci/run_conda.sh delete mode 100755 ci/run_tests.sh diff --git a/ci/config_auth.sh b/ci/config_auth.sh deleted file mode 100755 index cde115c7..00000000 --- a/ci/config_auth.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -set -e -# Don't set -x, because we don't want to leak keys. -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -# Write key to file if present. -if [ ! -z "$SERVICE_ACCOUNT_KEY" ] ; then - echo "$SERVICE_ACCOUNT_KEY" | base64 --decode > "$DIR"/service_account.json -fi diff --git a/ci/requirements-3.7-0.24.2.conda b/ci/requirements-3.7-0.24.2.conda deleted file mode 100644 index 2d61383e..00000000 --- a/ci/requirements-3.7-0.24.2.conda +++ /dev/null @@ -1,17 +0,0 @@ -codecov -coverage -db-dtypes -fastavro -flake8 -freezegun -numpy -google-api-core -google-auth -google-cloud-bigquery -google-cloud-bigquery-storage -pyarrow -pydata-google-auth -pytest -pytest-cov -requests-oauthlib -tqdm diff --git a/ci/requirements-3.9-1.3.4.conda b/ci/requirements-3.9-1.3.4.conda deleted file mode 100644 index 1411fe5b..00000000 --- a/ci/requirements-3.9-1.3.4.conda +++ /dev/null @@ -1,14 +0,0 @@ -codecov -coverage -db-dtypes -fastavro -flake8 -freezegun -google-cloud-bigquery -google-cloud-bigquery-storage -numpy -pyarrow -pydata-google-auth -pytest -pytest-cov -tqdm diff --git a/ci/run_conda.sh b/ci/run_conda.sh deleted file mode 100755 index 11b5b569..00000000 --- a/ci/run_conda.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -set -e -x -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -eval "$(micromamba shell hook --shell=bash)" -micromamba activate - -# Install dependencies using (micro)mamba -# https://github.com/mamba-org/micromamba-docker -REQ="ci/requirements-${PYTHON}-${PANDAS}" -micromamba install -q pandas=$PANDAS python=${PYTHON} -c conda-forge; -micromamba install -q --file "$REQ.conda" -c conda-forge; -micromamba list -micromamba info - -python setup.py develop --no-deps - -# Run the tests -$DIR/run_tests.sh diff --git a/ci/run_tests.sh b/ci/run_tests.sh deleted file mode 100755 index 8a1d7f91..00000000 --- a/ci/run_tests.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -set -e -x -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -if [ -f "$DIR/service_account.json" ]; then - export GOOGLE_APPLICATION_CREDENTIALS="$DIR/service_account.json" -fi - -# Install test requirements -pip install coverage pytest pytest-cov flake8 codecov google-cloud-testutils -pytest -v -m "not local_auth" --cov=pandas_gbq --cov-report xml:/tmp/pytest-cov.xml --cov-fail-under=0 tests From da3eaf2098cc71c76c255ee311416cbf321a2fb8 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Thu, 13 Jul 2023 11:21:10 +0000 Subject: [PATCH 14/25] updates env variable to display all vars not just KOKORO --- .kokoro/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index d1fc058b..94436ea7 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -35,7 +35,7 @@ export CONDA_DEFAULT_ENV=base export PATH=/root/conda/bin:/root/conda/condabin:${PATH} # Debug: show build environment -env #| grep KOKORO +env # Setup service account credentials. export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json From eeeff05eea46b61a2b151a5702499d3b028cb3a2 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Thu, 13 Jul 2023 13:24:47 +0000 Subject: [PATCH 15/25] removes additional content related to circle ci --- tests/system/conftest.py | 11 ----------- tests/system/test_auth.py | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/tests/system/conftest.py b/tests/system/conftest.py index 4ba8bf31..9690446d 100644 --- a/tests/system/conftest.py +++ b/tests/system/conftest.py @@ -16,17 +16,6 @@ REPO_DIR = pathlib.Path(__file__).parent.parent.parent -# TODO: remove when fully migrated off of Circle CI -@pytest.fixture(scope="session", autouse=True) -def default_credentials(): - """Setup application default credentials for use in code samples.""" - # Written by the 'ci/config_auth.sh' script. - path = REPO_DIR / "ci" / "service_account.json" - - if path.is_file() and "GOOGLE_APPLICATION_CREDENTIALS" not in os.environ: - os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = str(path) - - @pytest.fixture(scope="session", autouse=True) def cleanup_datasets(bigquery_client: bigquery.Client): for dataset in bigquery_client.list_datasets(): diff --git a/tests/system/test_auth.py b/tests/system/test_auth.py index d9f7d096..ecedd973 100644 --- a/tests/system/test_auth.py +++ b/tests/system/test_auth.py @@ -12,7 +12,7 @@ from pandas_gbq import auth -IS_RUNNING_ON_CI = "CIRCLE_BUILD_NUM" in os.environ or "KOKORO_BUILD_ID" in os.environ +IS_RUNNING_ON_CI = "KOKORO_BUILD_ID" in os.environ def mock_default_credentials(scopes=None, request=None): From 0db7bde66b3c3398ef312cfa19af1ff3921e573a Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Thu, 13 Jul 2023 17:18:29 +0000 Subject: [PATCH 16/25] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .kokoro/build.sh | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 94436ea7..d02a78db 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -24,18 +24,8 @@ cd "${PROJECT_ROOT}" # Disable buffering, so that the logs stream through. export PYTHONUNBUFFERED=1 -# Set conda variables and update path -export CONDA_EXE=/root/conda/bin/conda -export CONDA_PREFIX=/root/conda -export CONDA_PROMPT_MODIFIER=(base) -export _CE_CONDA= -export CONDA_SHLVL=1 -export CONDA_PYTHON_EXE=/root/conda/bin/python -export CONDA_DEFAULT_ENV=base -export PATH=/root/conda/bin:/root/conda/condabin:${PATH} - # Debug: show build environment -env +env | grep KOKORO # Setup service account credentials. export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json From cf2e40addac3d0a93602452c37770d8fb97c0a64 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Thu, 13 Jul 2023 18:03:18 +0000 Subject: [PATCH 17/25] updates owlbot with build.sh edits --- owlbot.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/owlbot.py b/owlbot.py index bbb078c7..6b86207d 100644 --- a/owlbot.py +++ b/owlbot.py @@ -71,6 +71,32 @@ # Work around bug in templates https://github.com/googleapis/synthtool/pull/1335 s.replace(".github/workflows/unittest.yml", "--fail-under=100", "--fail-under=96") +# Add environment variables to build.sh to support conda virtualenv +# installations +s.replace( + [".kokoro/build.sh"], + "export PYTHONUNBUFFERED=1", + r"""export PYTHONUNBUFFERED=1 +export CONDA_EXE=/root/conda/bin/conda +export CONDA_PREFIX=/root/conda +export CONDA_PROMPT_MODIFIER=(base) +export _CE_CONDA= +export CONDA_SHLVL=1 +export CONDA_PYTHON_EXE=/root/conda/bin/python +export CONDA_DEFAULT_ENV=base +export PATH=/root/conda/bin:/root/conda/condabin:${PATH} +""", +) + + +# Enable display of all environmental variables, not just KOKORO related vars +s.replace( + [".kokoro/build.sh"], + "env | grep KOKORO", + "env", +) + + # ---------------------------------------------------------------------------- # Samples templates # ---------------------------------------------------------------------------- From a07b46b98a537e871e879f948643088a1c8d05ff Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Thu, 13 Jul 2023 18:05:18 +0000 Subject: [PATCH 18/25] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .kokoro/build.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index d02a78db..f0b81e60 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -23,9 +23,18 @@ cd "${PROJECT_ROOT}" # Disable buffering, so that the logs stream through. export PYTHONUNBUFFERED=1 +export CONDA_EXE=/root/conda/bin/conda +export CONDA_PREFIX=/root/conda +export CONDA_PROMPT_MODIFIER=(base) +export _CE_CONDA= +export CONDA_SHLVL=1 +export CONDA_PYTHON_EXE=/root/conda/bin/python +export CONDA_DEFAULT_ENV=base +export PATH=/root/conda/bin:/root/conda/condabin:${PATH} + # Debug: show build environment -env | grep KOKORO +env|env # Setup service account credentials. export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json From 4bf7389509128595966545548a7cb24739fcda4c Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Thu, 13 Jul 2023 18:05:32 +0000 Subject: [PATCH 19/25] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .kokoro/build.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index d02a78db..f0b81e60 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -23,9 +23,18 @@ cd "${PROJECT_ROOT}" # Disable buffering, so that the logs stream through. export PYTHONUNBUFFERED=1 +export CONDA_EXE=/root/conda/bin/conda +export CONDA_PREFIX=/root/conda +export CONDA_PROMPT_MODIFIER=(base) +export _CE_CONDA= +export CONDA_SHLVL=1 +export CONDA_PYTHON_EXE=/root/conda/bin/python +export CONDA_DEFAULT_ENV=base +export PATH=/root/conda/bin:/root/conda/condabin:${PATH} + # Debug: show build environment -env | grep KOKORO +env|env # Setup service account credentials. export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json From 1882651adbe6fa46c48d49ce9ff3cc6da1db1f3a Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Thu, 13 Jul 2023 18:07:10 +0000 Subject: [PATCH 20/25] updates owlbot with build.sh minor tweak --- owlbot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/owlbot.py b/owlbot.py index 6b86207d..61e3f7b4 100644 --- a/owlbot.py +++ b/owlbot.py @@ -92,7 +92,7 @@ # Enable display of all environmental variables, not just KOKORO related vars s.replace( [".kokoro/build.sh"], - "env | grep KOKORO", + r"env | grep KOKORO", "env", ) From 4f375a23d0f9166529b84bee3190523a074bdc31 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Thu, 13 Jul 2023 18:10:17 +0000 Subject: [PATCH 21/25] revert build.sh --- .kokoro/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index f0b81e60..f33d45b1 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -34,7 +34,7 @@ export PATH=/root/conda/bin:/root/conda/condabin:${PATH} # Debug: show build environment -env|env +env | grep KOKORO # Setup service account credentials. export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json From 3f6d739025f6a6a6296665cac54a49795f79b71b Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Thu, 13 Jul 2023 18:12:35 +0000 Subject: [PATCH 22/25] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .kokoro/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index f33d45b1..f0b81e60 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -34,7 +34,7 @@ export PATH=/root/conda/bin:/root/conda/condabin:${PATH} # Debug: show build environment -env | grep KOKORO +env|env # Setup service account credentials. export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json From 1a7531a2bd746c90343d047d2d34be75b57eb7f9 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Thu, 13 Jul 2023 18:31:06 +0000 Subject: [PATCH 23/25] testing regex change --- owlbot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/owlbot.py b/owlbot.py index 61e3f7b4..e648334d 100644 --- a/owlbot.py +++ b/owlbot.py @@ -92,7 +92,7 @@ # Enable display of all environmental variables, not just KOKORO related vars s.replace( [".kokoro/build.sh"], - r"env | grep KOKORO", + r"env \| grep KOKORO", "env", ) From 5f6366e752292ed03a5c81cd1f7d69820e8d2612 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Thu, 13 Jul 2023 18:32:01 +0000 Subject: [PATCH 24/25] revert build.sh --- .kokoro/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index f0b81e60..f33d45b1 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -34,7 +34,7 @@ export PATH=/root/conda/bin:/root/conda/condabin:${PATH} # Debug: show build environment -env|env +env | grep KOKORO # Setup service account credentials. export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json From f0423fbded146e1ec73043ea677a96c4ec2e7d12 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Thu, 13 Jul 2023 18:33:36 +0000 Subject: [PATCH 25/25] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .kokoro/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index f33d45b1..204ea03c 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -34,7 +34,7 @@ export PATH=/root/conda/bin:/root/conda/condabin:${PATH} # Debug: show build environment -env | grep KOKORO +env # Setup service account credentials. export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json