diff --git a/bigquery/docs/snippets.py b/bigquery/docs/snippets.py index bb584fa0494a..44cbf0f40840 100644 --- a/bigquery/docs/snippets.py +++ b/bigquery/docs/snippets.py @@ -1255,110 +1255,6 @@ def test_extract_table_compressed(client, to_delete): to_delete.insert(0, blob) -def test_client_query_total_rows(client, capsys): - """Run a query and just check for how many rows.""" - # [START bigquery_query_total_rows] - # from google.cloud import bigquery - # client = bigquery.Client() - - query = ( - "SELECT name FROM `bigquery-public-data.usa_names.usa_1910_2013` " - 'WHERE state = "TX" ' - "LIMIT 100" - ) - query_job = client.query( - query, - # Location must match that of the dataset(s) referenced in the query. - location="US", - ) # API request - starts the query - - results = query_job.result() # Wait for query to complete. - print("Got {} rows.".format(results.total_rows)) - # [END bigquery_query_total_rows] - - out, _ = capsys.readouterr() - assert "Got 100 rows." in out - - -def test_manage_job(client): - sql = """ - SELECT corpus - FROM `bigquery-public-data.samples.shakespeare` - GROUP BY corpus; - """ - location = "us" - job = client.query(sql, location=location) - job_id = job.job_id - - # [START bigquery_cancel_job] - # TODO(developer): Uncomment the lines below and replace with your values. - # from google.cloud import bigquery - # client = bigquery.Client() - # job_id = 'bq-job-123x456-123y123z123c' # replace with your job ID - # location = 'us' # replace with your location - - job = client.cancel_job(job_id, location=location) - # [END bigquery_cancel_job] - - # [START bigquery_get_job] - # TODO(developer): Uncomment the lines below and replace with your values. - # from google.cloud import bigquery - # client = bigquery.Client() - # job_id = 'bq-job-123x456-123y123z123c' # replace with your job ID - # location = 'us' # replace with your location - - job = client.get_job(job_id, location=location) # API request - - # Print selected job properties - print("Details for job {} running in {}:".format(job_id, location)) - print( - "\tType: {}\n\tState: {}\n\tCreated: {}".format( - job.job_type, job.state, job.created - ) - ) - # [END bigquery_get_job] - - -def test_query_external_gcs_permanent_table(client, to_delete): - dataset_id = "query_external_gcs_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - client.create_dataset(dataset) - to_delete.append(dataset) - - # [START bigquery_query_external_gcs_perm] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_id = 'my_dataset' - - # Configure the external data source - dataset_ref = client.dataset(dataset_id) - table_id = "us_states" - schema = [ - bigquery.SchemaField("name", "STRING"), - bigquery.SchemaField("post_abbr", "STRING"), - ] - table = bigquery.Table(dataset_ref.table(table_id), schema=schema) - external_config = bigquery.ExternalConfig("CSV") - external_config.source_uris = [ - "gs://cloud-samples-data/bigquery/us-states/us-states.csv" - ] - external_config.options.skip_leading_rows = 1 # optionally skip header row - table.external_data_configuration = external_config - - # Create a permanent table linked to the GCS file - table = client.create_table(table) # API request - - # Example query to find states starting with 'W' - sql = 'SELECT * FROM `{}.{}` WHERE name LIKE "W%"'.format(dataset_id, table_id) - - query_job = client.query(sql) # API request - - w_states = list(query_job) # Waits for query to finish - print("There are {} states with names starting with W.".format(len(w_states))) - # [END bigquery_query_external_gcs_perm] - assert len(w_states) == 4 - - def test_ddl_create_view(client, to_delete, capsys): """Create a view via a DDL query.""" project = client.project diff --git a/bigquery/docs/usage/jobs.rst b/bigquery/docs/usage/jobs.rst index c3dd71031bfc..630bf298dc40 100644 --- a/bigquery/docs/usage/jobs.rst +++ b/bigquery/docs/usage/jobs.rst @@ -19,3 +19,27 @@ List jobs for a project with the :dedent: 4 :start-after: [START bigquery_list_jobs] :end-before: [END bigquery_list_jobs] + +Get a Job +^^^^^^^^^ + +Get a job resource with the +:func:`~google.cloud.bigquery.client.Client.get_job` method: + +.. literalinclude:: ../samples/get_job.py + :language: python + :dedent: 4 + :start-after: [START bigquery_get_job] + :end-before: [END bigquery_get_job] + +Cancel a Job +^^^^^^^^^^^^ + +Cancel a job with the +:func:`~google.cloud.bigquery.client.Client.cancel_job` method: + +.. literalinclude:: ../samples/cancel_job.py + :language: python + :dedent: 4 + :start-after: [START bigquery_cancel_job] + :end-before: [END bigquery_cancel_job] \ No newline at end of file diff --git a/bigquery/docs/usage/queries.rst b/bigquery/docs/usage/queries.rst index fc57e54de9df..9ea40ffa11ed 100644 --- a/bigquery/docs/usage/queries.rst +++ b/bigquery/docs/usage/queries.rst @@ -61,3 +61,12 @@ standard SQL :dedent: 4 :start-after: [START bigquery_query_script] :end-before: [END bigquery_query_script] + +Run a query to check number of rows +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. literalinclude:: ../samples/client_query_total_rows.py + :language: python + :dedent: 4 + :start-after: [START bigquery_query_total_rows] + :end-before: [END bigquery_query_total_rows] \ No newline at end of file diff --git a/bigquery/noxfile.py b/bigquery/noxfile.py index 17a2dee417c0..ce9971cf11d8 100644 --- a/bigquery/noxfile.py +++ b/bigquery/noxfile.py @@ -175,7 +175,7 @@ def lint_setup_py(session): session.run("python", "setup.py", "check", "--restructuredtext", "--strict") -@nox.session(python="3.6") +@nox.session(python="3.7") def blacken(session): """Run black. Format code to uniform standard. diff --git a/bigquery/samples/cancel_job.py b/bigquery/samples/cancel_job.py new file mode 100644 index 000000000000..555ec9b04760 --- /dev/null +++ b/bigquery/samples/cancel_job.py @@ -0,0 +1,36 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def cancel_job(): + + # [START bigquery_cancel_job] + from google.cloud import bigquery + + # Construct a BigQuery client object. + client = bigquery.Client() + + sql = """ + SELECT corpus + FROM `bigquery-public-data.samples.shakespeare` + GROUP BY corpus; + """ + location = "us" + job = client.query(sql, location=location) + job = client.cancel_job(job.job_id, location=location) + print("The job has been cancelled") + print( + "Type: {}, State: {}, Created: {}".format(job.job_type, job.state, job.created) + ) + # [END bigquery_cancel_job] diff --git a/bigquery/samples/client_query_total_rows.py b/bigquery/samples/client_query_total_rows.py new file mode 100644 index 000000000000..217970783766 --- /dev/null +++ b/bigquery/samples/client_query_total_rows.py @@ -0,0 +1,37 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def client_query_total_rows(): + + # [START bigquery_query_total_rows] + from google.cloud import bigquery + + # Construct a BigQuery client object. + client = bigquery.Client() + + query = ( + "SELECT name FROM `bigquery-public-data.usa_names.usa_1910_2013` " + 'WHERE state = "TX" ' + "LIMIT 100" + ) + query_job = client.query( + query, + # Must match the dataset(s) location referenced in the query. + location="US", + ) # Make an API request. - starts the query + + results = query_job.result() # Wait for the query to complete. + print("Got {} rows.".format(results.total_rows)) + # [END bigquery_query_total_rows] diff --git a/bigquery/samples/get_job.py b/bigquery/samples/get_job.py new file mode 100644 index 000000000000..dfa353782f1b --- /dev/null +++ b/bigquery/samples/get_job.py @@ -0,0 +1,36 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def get_job(): + + # [START bigquery_get_job] + from google.cloud import bigquery + + # Construct a BigQuery client object. + client = bigquery.Client() + + sql = """ + SELECT corpus + FROM `bigquery-public-data.samples.shakespeare` + GROUP BY corpus; + """ + location = "us" + job = client.query(sql, location=location) + job = client.get_job(job.job_id, location=location) # Make an API request. + print("Details for job {} running in {}:".format(job.job_id, location)) + print( + "Type: {}, State: {}, Created: {}".format(job.job_type, job.state, job.created) + ) + # [END bigquery_get_job] diff --git a/bigquery/samples/query_external_gcs_permanent_table.py b/bigquery/samples/query_external_gcs_permanent_table.py new file mode 100644 index 000000000000..0fd8eb3c627d --- /dev/null +++ b/bigquery/samples/query_external_gcs_permanent_table.py @@ -0,0 +1,57 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def query_external_gcs_permanent_table(table_id): + + # [START bigquery_query_external_gcs_perm] + from google.cloud import bigquery + + # Construct a BigQuery client object. + client = bigquery.Client() + + # TODO(developer): Set dataset_id to the ID of the dataset to create. + # dataset_id = "{}.your_dataset".format(client.project) + + # TODO(developer): Set table_id to the ID of the model to fetch. + # table_id = 'your-project.your_dataset.your_table' + + table = bigquery.Table( + table_id, + schema=[ + bigquery.SchemaField("name", "STRING"), + bigquery.SchemaField("post_abbr", "STRING"), + ], + ) + + external_config = bigquery.ExternalConfig("CSV") + external_config.source_uris = [ + "gs://cloud-samples-data/bigquery/us-states/us-states.csv" + ] + external_config.options.skip_leading_rows = 1 # optionally skip header row + table.external_data_configuration = external_config + + # Create a permanent table linked to the GCS file + table = client.create_table(table) # Make an API request. + # Example query to find states starting with 'W' + sql = 'SELECT * FROM `{}.{}` WHERE name LIKE "W%"'.format( + table.dataset_id, table.table_id + ) + + query_job = client.query(sql) # Make an API request. + + w_states = list(query_job) # Waits for query to finish + print("There are {} states with names starting with W.".format(len(w_states))) + + # [END bigquery_query_external_gcs_perm] diff --git a/bigquery/samples/tests/test_cancel_job.py b/bigquery/samples/tests/test_cancel_job.py new file mode 100644 index 000000000000..eebde4e50489 --- /dev/null +++ b/bigquery/samples/tests/test_cancel_job.py @@ -0,0 +1,23 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .. import cancel_job + + +def test_cancel_job(capsys): + + cancel_job.cancel_job() + out, _ = capsys.readouterr() + print("The job has been cancelled") + assert "Type: query, State: DONE," in out diff --git a/bigquery/samples/tests/test_client_query_total_rows.py b/bigquery/samples/tests/test_client_query_total_rows.py new file mode 100644 index 000000000000..11c8f194a12a --- /dev/null +++ b/bigquery/samples/tests/test_client_query_total_rows.py @@ -0,0 +1,22 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .. import client_query_total_rows + + +def test_client_query_total_rows(capsys): + + client_query_total_rows.client_query_total_rows() + out, _ = capsys.readouterr() + assert "Got 100 rows." in out diff --git a/bigquery/samples/tests/test_get_job.py b/bigquery/samples/tests/test_get_job.py new file mode 100644 index 000000000000..4b5e336c27a7 --- /dev/null +++ b/bigquery/samples/tests/test_get_job.py @@ -0,0 +1,23 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .. import get_job + + +def test_get_job(capsys): + + get_job.get_job() + out, _ = capsys.readouterr() + assert "Details for job" in out + assert "Type: query, State: DONE," in out diff --git a/bigquery/samples/tests/test_query_external_gcs_permanent_table.py b/bigquery/samples/tests/test_query_external_gcs_permanent_table.py new file mode 100644 index 000000000000..81b425e8f6db --- /dev/null +++ b/bigquery/samples/tests/test_query_external_gcs_permanent_table.py @@ -0,0 +1,23 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .. import query_external_gcs_permanent_table + + +def test_query_external_gcs_permanent_table(capsys, random_table_id): + query_external_gcs_permanent_table.query_external_gcs_permanent_table( + random_table_id + ) + out, _ = capsys.readouterr() + assert "There are 4 states with names starting with W." in out