diff --git a/bigquery/docs/snippets.py b/bigquery/docs/snippets.py index bb584fa0494a..e861d518cb14 100644 --- a/bigquery/docs/snippets.py +++ b/bigquery/docs/snippets.py @@ -122,45 +122,6 @@ def test_create_client_default_credentials(): assert client is not None -def test_create_table_nested_repeated_schema(client, to_delete): - dataset_id = "create_table_nested_repeated_{}".format(_millis()) - dataset_ref = client.dataset(dataset_id) - dataset = bigquery.Dataset(dataset_ref) - client.create_dataset(dataset) - to_delete.append(dataset) - - # [START bigquery_nested_repeated_schema] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_ref = client.dataset('my_dataset') - - schema = [ - bigquery.SchemaField("id", "STRING", mode="NULLABLE"), - bigquery.SchemaField("first_name", "STRING", mode="NULLABLE"), - bigquery.SchemaField("last_name", "STRING", mode="NULLABLE"), - bigquery.SchemaField("dob", "DATE", mode="NULLABLE"), - bigquery.SchemaField( - "addresses", - "RECORD", - mode="REPEATED", - fields=[ - bigquery.SchemaField("status", "STRING", mode="NULLABLE"), - bigquery.SchemaField("address", "STRING", mode="NULLABLE"), - bigquery.SchemaField("city", "STRING", mode="NULLABLE"), - bigquery.SchemaField("state", "STRING", mode="NULLABLE"), - bigquery.SchemaField("zip", "STRING", mode="NULLABLE"), - bigquery.SchemaField("numberOfYears", "STRING", mode="NULLABLE"), - ], - ), - ] - table_ref = dataset_ref.table("my_table") - table = bigquery.Table(table_ref, schema=schema) - table = client.create_table(table) # API request - - print("Created table {}".format(table.full_table_id)) - # [END bigquery_nested_repeated_schema] - - def test_create_table_cmek(client, to_delete): dataset_id = "create_table_cmek_{}".format(_millis()) dataset = bigquery.Dataset(client.dataset(dataset_id)) @@ -190,44 +151,6 @@ def test_create_table_cmek(client, to_delete): # [END bigquery_create_table_cmek] -def test_create_partitioned_table(client, to_delete): - dataset_id = "create_table_partitioned_{}".format(_millis()) - dataset_ref = bigquery.Dataset(client.dataset(dataset_id)) - dataset = client.create_dataset(dataset_ref) - to_delete.append(dataset) - - # [START bigquery_create_table_partitioned] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_ref = client.dataset('my_dataset') - - table_ref = dataset_ref.table("my_partitioned_table") - schema = [ - bigquery.SchemaField("name", "STRING"), - bigquery.SchemaField("post_abbr", "STRING"), - bigquery.SchemaField("date", "DATE"), - ] - table = bigquery.Table(table_ref, schema=schema) - table.time_partitioning = bigquery.TimePartitioning( - type_=bigquery.TimePartitioningType.DAY, - field="date", # name of column to use for partitioning - expiration_ms=7776000000, - ) # 90 days - - table = client.create_table(table) - - print( - "Created table {}, partitioned on column {}".format( - table.table_id, table.time_partitioning.field - ) - ) - # [END bigquery_create_table_partitioned] - - assert table.time_partitioning.type_ == "DAY" - assert table.time_partitioning.field == "date" - assert table.time_partitioning.expiration_ms == 7776000000 - - @pytest.mark.skip( reason=( "update_table() is flaky " @@ -369,48 +292,6 @@ def test_update_table_expiration(client, to_delete): # [END bigquery_update_table_expiration] -@pytest.mark.skip( - reason=( - "update_table() is flaky " - "https://github.com/GoogleCloudPlatform/google-cloud-python/issues/5589" - ) -) -def test_relax_column(client, to_delete): - """Updates a schema field from required to nullable.""" - dataset_id = "relax_column_dataset_{}".format(_millis()) - table_id = "relax_column_table_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - dataset = client.create_dataset(dataset) - to_delete.append(dataset) - - # [START bigquery_relax_column] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_id = 'my_dataset' - # table_id = 'my_table' - - original_schema = [ - bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), - bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), - ] - table_ref = client.dataset(dataset_id).table(table_id) - table = bigquery.Table(table_ref, schema=original_schema) - table = client.create_table(table) - assert all(field.mode == "REQUIRED" for field in table.schema) - - # SchemaField properties cannot be edited after initialization. - # To make changes, construct new SchemaField objects. - relaxed_schema = [ - bigquery.SchemaField("full_name", "STRING", mode="NULLABLE"), - bigquery.SchemaField("age", "INTEGER", mode="NULLABLE"), - ] - table.schema = relaxed_schema - table = client.update_table(table, ["schema"]) - - assert all(field.mode == "NULLABLE" for field in table.schema) - # [END bigquery_relax_column] - - @pytest.mark.skip( reason=( "update_table() is flaky " @@ -1007,151 +888,6 @@ def test_load_table_from_uri_truncate(client, to_delete, capsys): assert "Loaded 50 rows." in out -def test_load_table_add_column(client, to_delete): - dataset_id = "load_table_add_column_{}".format(_millis()) - dataset_ref = client.dataset(dataset_id) - dataset = bigquery.Dataset(dataset_ref) - dataset.location = "US" - dataset = client.create_dataset(dataset) - to_delete.append(dataset) - - snippets_dir = os.path.abspath(os.path.dirname(__file__)) - filepath = os.path.join( - snippets_dir, "..", "..", "bigquery", "tests", "data", "people.csv" - ) - table_ref = dataset_ref.table("my_table") - old_schema = [bigquery.SchemaField("full_name", "STRING", mode="REQUIRED")] - table = client.create_table(bigquery.Table(table_ref, schema=old_schema)) - - # [START bigquery_add_column_load_append] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_ref = client.dataset('my_dataset') - # filepath = 'path/to/your_file.csv' - - # Retrieves the destination table and checks the length of the schema - table_id = "my_table" - table_ref = dataset_ref.table(table_id) - table = client.get_table(table_ref) - print("Table {} contains {} columns.".format(table_id, len(table.schema))) - - # Configures the load job to append the data to the destination table, - # allowing field addition - job_config = bigquery.LoadJobConfig() - job_config.write_disposition = bigquery.WriteDisposition.WRITE_APPEND - job_config.schema_update_options = [ - bigquery.SchemaUpdateOption.ALLOW_FIELD_ADDITION - ] - # In this example, the existing table contains only the 'full_name' column. - # 'REQUIRED' fields cannot be added to an existing schema, so the - # additional column must be 'NULLABLE'. - job_config.schema = [ - bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), - bigquery.SchemaField("age", "INTEGER", mode="NULLABLE"), - ] - job_config.source_format = bigquery.SourceFormat.CSV - job_config.skip_leading_rows = 1 - - with open(filepath, "rb") as source_file: - job = client.load_table_from_file( - source_file, - table_ref, - location="US", # Must match the destination dataset location. - job_config=job_config, - ) # API request - - job.result() # Waits for table load to complete. - print( - "Loaded {} rows into {}:{}.".format( - job.output_rows, dataset_id, table_ref.table_id - ) - ) - - # Checks the updated length of the schema - table = client.get_table(table) - print("Table {} now contains {} columns.".format(table_id, len(table.schema))) - # [END bigquery_add_column_load_append] - assert len(table.schema) == 2 - assert table.num_rows > 0 - - -def test_load_table_relax_column(client, to_delete): - dataset_id = "load_table_relax_column_{}".format(_millis()) - dataset_ref = client.dataset(dataset_id) - dataset = bigquery.Dataset(dataset_ref) - dataset.location = "US" - dataset = client.create_dataset(dataset) - to_delete.append(dataset) - - snippets_dir = os.path.abspath(os.path.dirname(__file__)) - filepath = os.path.join( - snippets_dir, "..", "..", "bigquery", "tests", "data", "people.csv" - ) - table_ref = dataset_ref.table("my_table") - old_schema = [ - bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), - bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), - bigquery.SchemaField("favorite_color", "STRING", mode="REQUIRED"), - ] - table = client.create_table(bigquery.Table(table_ref, schema=old_schema)) - - # [START bigquery_relax_column_load_append] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_ref = client.dataset('my_dataset') - # filepath = 'path/to/your_file.csv' - - # Retrieves the destination table and checks the number of required fields - table_id = "my_table" - table_ref = dataset_ref.table(table_id) - table = client.get_table(table_ref) - original_required_fields = sum(field.mode == "REQUIRED" for field in table.schema) - # In this example, the existing table has 3 required fields. - print("{} fields in the schema are required.".format(original_required_fields)) - - # Configures the load job to append the data to a destination table, - # allowing field relaxation - job_config = bigquery.LoadJobConfig() - job_config.write_disposition = bigquery.WriteDisposition.WRITE_APPEND - job_config.schema_update_options = [ - bigquery.SchemaUpdateOption.ALLOW_FIELD_RELAXATION - ] - # In this example, the existing table contains three required fields - # ('full_name', 'age', and 'favorite_color'), while the data to load - # contains only the first two fields. - job_config.schema = [ - bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), - bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), - ] - job_config.source_format = bigquery.SourceFormat.CSV - job_config.skip_leading_rows = 1 - - with open(filepath, "rb") as source_file: - job = client.load_table_from_file( - source_file, - table_ref, - location="US", # Must match the destination dataset location. - job_config=job_config, - ) # API request - - job.result() # Waits for table load to complete. - print( - "Loaded {} rows into {}:{}.".format( - job.output_rows, dataset_id, table_ref.table_id - ) - ) - - # Checks the updated number of required fields - table = client.get_table(table) - current_required_fields = sum(field.mode == "REQUIRED" for field in table.schema) - print("{} fields in the schema are now required.".format(current_required_fields)) - # [END bigquery_relax_column_load_append] - assert original_required_fields - current_required_fields == 1 - assert len(table.schema) == 3 - assert table.schema[2].mode == "NULLABLE" - assert table.num_rows > 0 - - def test_extract_table(client, to_delete): bucket_name = "extract_shakespeare_{}".format(_millis()) storage_client = storage.Client() diff --git a/bigquery/docs/usage/tables.rst b/bigquery/docs/usage/tables.rst index b6f8dbdde646..451ac858b21e 100644 --- a/bigquery/docs/usage/tables.rst +++ b/bigquery/docs/usage/tables.rst @@ -67,6 +67,24 @@ Create an integer range partitioned table with the :start-after: [START bigquery_create_table_range_partitioned] :end-before: [END bigquery_create_table_range_partitioned] +Create a table partition with the +:func:`~google.cloud.bigquery.client.Client.create_table` method: + +.. literalinclude:: ../samples/create_partitioned_table.py + :language: python + :dedent: 4 + :start-after: [START bigquery_create_table_partitioned] + :end-before: [END bigquery_create_table_partitioned] + +Create a nested repeated schema table partition with the +:func:`~google.cloud.bigquery.client.Client.create_table` method: + +.. literalinclude:: ../samples/create_table_nested_repeated_schema.py + :language: python + :dedent: 4 + :start-after: [START bigquery_nested_repeated_schema] + :end-before: [END bigquery_nested_repeated_schema] + Load table data from a file with the :func:`~google.cloud.bigquery.client.Client.load_table_from_file` method: @@ -154,6 +172,38 @@ Add an empty column to the existing table with the :start-after: [START bigquery_add_empty_column] :end-before: [END bigquery_add_empty_column] +Update schema field from required to nullable: + +.. literalinclude:: ../samples/relax_column.py + :language: python + :dedent: 4 + :start-after: [START bigquery_relax_column] + :end-before: [END bigquery_relax_column] + +Add a column to the existing table: + +.. literalinclude:: ../samples/load_table_add_column.py + :language: python + :dedent: 4 + :start-after: [START bigquery_add_column_load_append] + :end-before: [END bigquery_add_column_load_append] + +Add a nullable column to the existing table: + +.. literalinclude:: ../samples/load_table_add_column.py + :language: python + :dedent: 4 + :start-after: [START bigquery_add_column_load_append] + :end-before: [END bigquery_add_column_load_append] + +Relaxing a required field in the original schema to nullable: + +.. literalinclude:: ../samples/load_table_relax_column.py + :language: python + :dedent: 4 + :start-after: [START bigquery_relax_column_load_append] + :end-before: [END bigquery_relax_column_load_append] + Copying a Table ^^^^^^^^^^^^^^^ diff --git a/bigquery/samples/create_partitioned_table.py b/bigquery/samples/create_partitioned_table.py new file mode 100644 index 000000000000..7fd8f6e17276 --- /dev/null +++ b/bigquery/samples/create_partitioned_table.py @@ -0,0 +1,48 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def create_partitioned_table(table_id): + + # [START bigquery_create_table_partitioned] + from google.cloud import bigquery + + # Construct a BigQuery client object. + client = bigquery.Client() + + # TODO(developer): Set table_id to the ID of the table to create. + # table_id = "your-project.your_dataset.your_table_name" + + table = bigquery.Table( + table_id, + schema=[ + bigquery.SchemaField("name", "STRING"), + bigquery.SchemaField("post_abbr", "STRING"), + bigquery.SchemaField("date", "DATE"), + ], + ) + table.time_partitioning = bigquery.TimePartitioning( + type_=bigquery.TimePartitioningType.DAY, + field="date", # name of column to use for partitioning + expiration_ms=7776000000, + ) # 90 days + + table = client.create_table(table) + print( + "Created table {}, partitioned on column {}".format( + table.table_id, table.time_partitioning.field + ) + ) + # [END bigquery_create_table_partitioned] + return table diff --git a/bigquery/samples/create_table_nested_repeated_schema.py b/bigquery/samples/create_table_nested_repeated_schema.py new file mode 100644 index 000000000000..ef4a2f1db60d --- /dev/null +++ b/bigquery/samples/create_table_nested_repeated_schema.py @@ -0,0 +1,53 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def create_table_nested_repeated_schema(table_id): + + # [START bigquery_nested_repeated_schema] + from google.cloud import bigquery + + # Construct a BigQuery client object. + client = bigquery.Client() + + # TODO(developer): Set table_id to the ID of the table to create. + # table_id = "your-project.your_dataset.your_table_name" + + schema = [ + bigquery.SchemaField("id", "STRING", mode="NULLABLE"), + bigquery.SchemaField("first_name", "STRING", mode="NULLABLE"), + bigquery.SchemaField("last_name", "STRING", mode="NULLABLE"), + bigquery.SchemaField("dob", "DATE", mode="NULLABLE"), + bigquery.SchemaField( + "addresses", + "RECORD", + mode="REPEATED", + fields=[ + bigquery.SchemaField("status", "STRING", mode="NULLABLE"), + bigquery.SchemaField("address", "STRING", mode="NULLABLE"), + bigquery.SchemaField("city", "STRING", mode="NULLABLE"), + bigquery.SchemaField("state", "STRING", mode="NULLABLE"), + bigquery.SchemaField("zip", "STRING", mode="NULLABLE"), + bigquery.SchemaField("numberOfYears", "STRING", mode="NULLABLE"), + ], + ), + ] + + table = bigquery.Table(table_id, schema=schema) + table = client.create_table(table) # Make an API request. + + print( + "Created table {}.{}.{}".format(table.project, table.dataset_id, table.table_id) + ) + # [END bigquery_nested_repeated_schema] diff --git a/bigquery/samples/load_table_add_column.py b/bigquery/samples/load_table_add_column.py new file mode 100644 index 000000000000..f9d800176ab3 --- /dev/null +++ b/bigquery/samples/load_table_add_column.py @@ -0,0 +1,70 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def load_table_add_column(file_path, table_id): + + # [START bigquery_add_column_load_append] + from google.cloud import bigquery + + # Construct a BigQuery client object. + client = bigquery.Client() + + # TODO(developer): Set table_id to the ID of the table to create. + # table_id = "your-project.your_dataset.your_table_name" + + old_schema = [bigquery.SchemaField("full_name", "STRING", mode="REQUIRED")] + table = client.create_table(bigquery.Table(table_id, schema=old_schema)) + + # Retrieves the destination table and checks the length of the schema + table = client.get_table(table_id) + print("Table {} contains {} columns.".format(table_id, len(table.schema))) + + # Configures the load job to append the data to the destination table, + # allowing field addition + job_config = bigquery.LoadJobConfig( + write_disposition=bigquery.WriteDisposition.WRITE_APPEND, + source_format=bigquery.SourceFormat.CSV, + skip_leading_rows=1, + # In this example, the existing table contains only the 'full_name' column. + # 'REQUIRED' fields cannot be added to an existing schema, so the + # additional column must be 'NULLABLE'. + schema=[ + bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), + bigquery.SchemaField("age", "INTEGER", mode="NULLABLE"), + ], + schema_update_options=[bigquery.SchemaUpdateOption.ALLOW_FIELD_ADDITION], + ) + + with open(file_path, "rb") as source_file: + job = client.load_table_from_file( + source_file, + table_id, + location="US", # Must match the destination dataset location. + job_config=job_config, + ) # Make an API request. + + job.result() # Waits for the job to complete. + print( + "Loaded {} rows into {}:{}.".format( + job.output_rows, table.dataset_id, table.table_id + ) + ) + + # Checks the updated length of the schema + table = client.get_table(table) + print("Table {} now contains {} columns.".format(table_id, len(table.schema))) + # [END bigquery_add_column_load_append] + + return table diff --git a/bigquery/samples/load_table_relax_column.py b/bigquery/samples/load_table_relax_column.py new file mode 100644 index 000000000000..1d4a9c8696b1 --- /dev/null +++ b/bigquery/samples/load_table_relax_column.py @@ -0,0 +1,76 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def load_table_relax_column(file_path, table_id): + + # [START bigquery_relax_column_load_append] + from google.cloud import bigquery + + # Construct a BigQuery client object. + client = bigquery.Client() + + # TODO(developer): Set table_id to the ID of the table to create. + # table_id = "your-project.your_dataset.your_table_name" + + old_schema = [ + bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), + bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), + bigquery.SchemaField("favorite_color", "STRING", mode="REQUIRED"), + ] + table = client.create_table(bigquery.Table(table_id, schema=old_schema)) + + # Checks the number of required fields + original_required_fields = sum(field.mode == "REQUIRED" for field in table.schema) + # In this example, the existing table has 3 required fields. + print("{} fields in the schema are required.".format(original_required_fields)) + + # Configures the load job to append the data to a destination table, + # allowing field relaxation + job_config = bigquery.LoadJobConfig( + write_disposition=bigquery.WriteDisposition.WRITE_APPEND, + # In this example, the existing table contains three required fields + # ('full_name', 'age', and 'favorite_color'), while the data to load + # contains only the first two fields. + schema=[ + bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), + bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), + ], + schema_update_options=[bigquery.SchemaUpdateOption.ALLOW_FIELD_RELAXATION], + source_format=bigquery.SourceFormat.CSV, + skip_leading_rows=1, + ) + + with open(file_path, "rb") as source_file: + job = client.load_table_from_file( + source_file, + table_id, + location="US", # Must match the destination dataset location. + job_config=job_config, + ) # Make an API request. + + job.result() # Waits for the job to complete. + print( + "Loaded {} rows into {}:{}.".format( + job.output_rows, table.dataset_id, table.table_id + ) + ) + + # Checks the updated number of required fields + table = client.get_table(table) + current_required_fields = sum(field.mode == "REQUIRED" for field in table.schema) + print("{} fields in the schema are now required.".format(current_required_fields)) + # [END bigquery_relax_column_load_append] + + return table diff --git a/bigquery/samples/relax_column.py b/bigquery/samples/relax_column.py new file mode 100644 index 000000000000..6a6d2b0985c2 --- /dev/null +++ b/bigquery/samples/relax_column.py @@ -0,0 +1,49 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def relax_column(table_id): + + # [START bigquery_relax_column] + from google.cloud import bigquery + + # Construct a BigQuery client object. + client = bigquery.Client() + + # TODO(developer): Set table_id to the ID of the table to create. + # table_id = "your-project.your_dataset.your_table_name" + + original_schema = [ + bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), + bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), + ] + table = bigquery.Table(table_id, schema=original_schema) + table = client.create_table(table) + # Checks the number of required fields + original_required_fields = sum(field.mode == "REQUIRED" for field in table.schema) + # In this example, the existing table has 2 required fields. + print("{} fields in the schema are required.".format(original_required_fields)) + + # SchemaField properties cannot be edited after initialization. + # To make changes, construct new SchemaField objects. + relaxed_schema = [ + bigquery.SchemaField("full_name", "STRING", mode="NULLABLE"), + bigquery.SchemaField("age", "INTEGER", mode="NULLABLE"), + ] + table.schema = relaxed_schema + table = client.update_table(table, ["schema"]) + relaxed_nullable_fields = sum(field.mode == "NULLABLE" for field in table.schema) + # In this example, the existing table has 2 nullable fields. + print("{} fields in the schema are now nullable.".format(relaxed_nullable_fields)) + # [END bigquery_relax_column] diff --git a/bigquery/samples/tests/test_create_partitioned_table.py b/bigquery/samples/tests/test_create_partitioned_table.py new file mode 100644 index 000000000000..807fc4b027ab --- /dev/null +++ b/bigquery/samples/tests/test_create_partitioned_table.py @@ -0,0 +1,24 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .. import create_partitioned_table + + +def test_create_partitioned_table(capsys, random_table_id): + table = create_partitioned_table.create_partitioned_table(random_table_id) + out, _ = capsys.readouterr() + assert "Created table {}, partitioned on column date".format(table.table_id) + assert table.time_partitioning.type_ == "DAY" + assert table.time_partitioning.field == "date" + assert table.time_partitioning.expiration_ms == 7776000000 diff --git a/bigquery/samples/tests/test_create_table_nested_repeated_schema.py b/bigquery/samples/tests/test_create_table_nested_repeated_schema.py new file mode 100644 index 000000000000..e710ecafcff0 --- /dev/null +++ b/bigquery/samples/tests/test_create_table_nested_repeated_schema.py @@ -0,0 +1,24 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .. import create_table_nested_repeated_schema + + +def test_create_table_nested_repeated_schema(capsys, random_table_id): + + create_table_nested_repeated_schema.create_table_nested_repeated_schema( + random_table_id + ) + out, err = capsys.readouterr() + assert "Created table {}".format(random_table_id) in out diff --git a/bigquery/samples/tests/test_load_table_add_column.py b/bigquery/samples/tests/test_load_table_add_column.py new file mode 100644 index 000000000000..100c83afd530 --- /dev/null +++ b/bigquery/samples/tests/test_load_table_add_column.py @@ -0,0 +1,33 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from .. import load_table_add_column + + +def test_load_table_add_column(capsys, random_table_id): + + samples_test_dir = os.path.abspath(os.path.dirname(__file__)) + file_path = os.path.join( + samples_test_dir, "..", "..", "tests", "data", "people.csv" + ) + table = load_table_add_column.load_table_add_column(file_path, random_table_id) + out, _ = capsys.readouterr() + assert "Table {} contains 1 columns.".format(random_table_id) in out + assert "Loaded 2 rows into {}:{}.".format(table.dataset_id, table.table_id) + assert "Table {} now contains 2 columns.".format(table.table_id) + + assert len(table.schema) == 2 + assert table.num_rows > 0 diff --git a/bigquery/samples/tests/test_load_table_relax_column.py b/bigquery/samples/tests/test_load_table_relax_column.py new file mode 100644 index 000000000000..387ec4fb19ab --- /dev/null +++ b/bigquery/samples/tests/test_load_table_relax_column.py @@ -0,0 +1,34 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from .. import load_table_relax_column + + +def test_load_table_relax_column(capsys, random_table_id): + + samples_test_dir = os.path.abspath(os.path.dirname(__file__)) + file_path = os.path.join( + samples_test_dir, "..", "..", "tests", "data", "people.csv" + ) + table = load_table_relax_column.load_table_relax_column(file_path, random_table_id) + out, _ = capsys.readouterr() + assert "3 fields in the schema are required." in out + assert "Loaded 2 rows into {}:{}.".format(table.dataset_id, table.table_id) in out + assert "2 fields in the schema are now required." in out + + assert len(table.schema) == 3 + assert table.schema[2].mode == "NULLABLE" + assert table.num_rows > 0 diff --git a/bigquery/samples/tests/test_relax_column.py b/bigquery/samples/tests/test_relax_column.py new file mode 100644 index 000000000000..56801278e74d --- /dev/null +++ b/bigquery/samples/tests/test_relax_column.py @@ -0,0 +1,23 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .. import relax_column + + +def test_relax_column(capsys, random_table_id): + + relax_column.relax_column(random_table_id) + out, _ = capsys.readouterr() + assert "2 fields in the schema are required." in out + assert "2 fields in the schema are now nullable." in out