From 57b1bd59789d36ec04e80d516f4eb20291dded77 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Fri, 7 Aug 2015 16:49:07 -0400 Subject: [PATCH 1/6] Initial system tests for BQ. --- scripts/run_system_tests.sh | 1 + system_tests/bigquery.py | 224 ++++++++++++++++++++++++++++++++ system_tests/run_system_test.py | 1 + 3 files changed, 226 insertions(+) create mode 100644 system_tests/bigquery.py diff --git a/scripts/run_system_tests.sh b/scripts/run_system_tests.sh index 3b625066208b..bedcbfd3dc6c 100755 --- a/scripts/run_system_tests.sh +++ b/scripts/run_system_tests.sh @@ -38,3 +38,4 @@ fi python system_tests/run_system_test.py --package datastore python system_tests/run_system_test.py --package storage python system_tests/run_system_test.py --package pubsub +python system_tests/run_system_test.py --package bigquery diff --git a/system_tests/bigquery.py b/system_tests/bigquery.py new file mode 100644 index 000000000000..ab445823879c --- /dev/null +++ b/system_tests/bigquery.py @@ -0,0 +1,224 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import operator +import time + +import unittest2 + +from gcloud import _helpers +from gcloud.environment_vars import TESTS_PROJECT +from gcloud import bigquery + + +_helpers.PROJECT = TESTS_PROJECT +CLIENT = bigquery.Client() + + +class TestPubsub(unittest2.TestCase): + + def setUp(self): + self.to_delete = [] + + def tearDown(self): + for doomed in self.to_delete: + doomed.delete() + + def test_create_dataset(self): + DATASET_NAME = 'system_tests' + dataset = CLIENT.dataset(DATASET_NAME) + self.assertFalse(dataset.exists()) + dataset.create() + self.to_delete.append(dataset) + self.assertTrue(dataset.exists()) + self.assertEqual(dataset.name, DATASET_NAME) + + def test_patch_dataset(self): + DATASET_NAME = 'system_tests' + dataset = CLIENT.dataset(DATASET_NAME) + self.assertFalse(dataset.exists()) + dataset.create() + self.to_delete.append(dataset) + self.assertTrue(dataset.exists()) + self.assertEqual(dataset.friendly_name, None) + self.assertEqual(dataset.description, None) + dataset.patch(friendly_name='Friendly', description='Description') + self.assertEqual(dataset.friendly_name, 'Friendly') + self.assertEqual(dataset.description, 'Description') + + def test_update_dataset(self): + DATASET_NAME = 'system_tests' + dataset = CLIENT.dataset(DATASET_NAME) + self.assertFalse(dataset.exists()) + dataset.create() + self.to_delete.append(dataset) + self.assertTrue(dataset.exists()) + after = [grant for grant in dataset.access_grants + if grant.entity_id != 'projectWriters'] + dataset.access_grants = after + dataset.update() + self.assertEqual(len(dataset.access_grants), len(after)) + for found, expected in zip(dataset.access_grants, after): + self.assertEqual(found.role, expected.role) + self.assertEqual(found.entity_type, expected.entity_type) + self.assertEqual(found.entity_id, expected.entity_id) + + def test_list_datasets(self): + datasets_to_create = [ + 'new%d' % (1000 * time.time(),), + 'newer%d' % (1000 * time.time(),), + 'newest%d' % (1000 * time.time(),), + ] + for dataset_name in datasets_to_create: + dataset = CLIENT.dataset(dataset_name) + dataset.create() + self.to_delete.append(dataset) + + # Retrieve the datasets. + all_datasets, _ = CLIENT.list_datasets() + created = [dataset for dataset in all_datasets + if dataset.name in datasets_to_create and + dataset.project == CLIENT.project] + self.assertEqual(len(created), len(datasets_to_create)) + + def test_create_table(self): + DATASET_NAME = 'system_tests' + dataset = CLIENT.dataset(DATASET_NAME) + self.assertFalse(dataset.exists()) + dataset.create() + self.to_delete.append(dataset) + TABLE_NAME = 'test_table' + full_name = bigquery.SchemaField('full_name', 'STRING', + mode='REQUIRED') + age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED') + table = dataset.table(TABLE_NAME, schema=[full_name, age]) + self.assertFalse(table.exists()) + table.create() + self.to_delete.insert(0, table) + self.assertTrue(table.exists()) + self.assertEqual(table.name, TABLE_NAME) + self.assertTrue(table._dataset is dataset) + + def test_list_tables(self): + DATASET_NAME = 'system_tests' + dataset = CLIENT.dataset(DATASET_NAME) + self.assertFalse(dataset.exists()) + dataset.create() + self.to_delete.append(dataset) + tables_to_create = [ + 'new%d' % (1000 * time.time(),), + 'newer%d' % (1000 * time.time(),), + 'newest%d' % (1000 * time.time(),), + ] + full_name = bigquery.SchemaField('full_name', 'STRING', + mode='REQUIRED') + age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED') + for table_name in tables_to_create: + table = dataset.table(table_name, schema=[full_name, age]) + table.create() + self.to_delete.insert(0, table) + + # Retrieve the tables. + all_tables, _ = dataset.list_tables() + created = [table for table in all_tables + if table.name in tables_to_create and + table._dataset.name == DATASET_NAME] + self.assertEqual(len(created), len(tables_to_create)) + + def test_patch_table(self): + DATASET_NAME = 'system_tests' + dataset = CLIENT.dataset(DATASET_NAME) + self.assertFalse(dataset.exists()) + dataset.create() + self.to_delete.append(dataset) + TABLE_NAME = 'test_table' + full_name = bigquery.SchemaField('full_name', 'STRING', + mode='REQUIRED') + age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED') + table = dataset.table(TABLE_NAME, schema=[full_name, age]) + self.assertFalse(table.exists()) + table.create() + self.to_delete.insert(0, table) + self.assertTrue(table.exists()) + self.assertEqual(table.friendly_name, None) + self.assertEqual(table.description, None) + table.patch(friendly_name='Friendly', description='Description') + self.assertEqual(table.friendly_name, 'Friendly') + self.assertEqual(table.description, 'Description') + + def test_update_table(self): + DATASET_NAME = 'system_tests' + dataset = CLIENT.dataset(DATASET_NAME) + self.assertFalse(dataset.exists()) + dataset.create() + self.to_delete.append(dataset) + TABLE_NAME = 'test_table' + full_name = bigquery.SchemaField('full_name', 'STRING', + mode='REQUIRED') + age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED') + table = dataset.table(TABLE_NAME, schema=[full_name, age]) + self.assertFalse(table.exists()) + table.create() + self.to_delete.insert(0, table) + self.assertTrue(table.exists()) + voter = bigquery.SchemaField('voter', 'BOOLEAN', mode='NULLABLE') + schema = table.schema + schema.append(voter) + table.schema = schema + table.update() + self.assertEqual(len(table.schema), len(schema)) + for found, expected in zip(table.schema, schema): + self.assertEqual(found.name, expected.name) + self.assertEqual(found.field_type, expected.field_type) + self.assertEqual(found.mode, expected.mode) + + def test_load_dump_table(self): + ROWS = [ + ('Phred Phlyntstone', 32), + ('Bharney Rhubble', 33), + ('Wylma Phlyntstone', 29), + ('Bhettye Rhubble', 27), + ] + ROW_IDS = range(len(ROWS)) + DATASET_NAME = 'system_tests' + dataset = CLIENT.dataset(DATASET_NAME) + self.assertFalse(dataset.exists()) + dataset.create() + self.to_delete.append(dataset) + TABLE_NAME = 'test_table' + full_name = bigquery.SchemaField('full_name', 'STRING', + mode='REQUIRED') + age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED') + table = dataset.table(TABLE_NAME, schema=[full_name, age]) + self.assertFalse(table.exists()) + table.create() + self.to_delete.insert(0, table) + self.assertTrue(table.exists()) + + errors = table.insert_data(ROWS, ROW_IDS) + self.assertEqual(len(errors), 0) + + rows = () + counter = 9 + # Allow for 90 seconds of "warm up" before rows visible. See: + # https://cloud.google.com/bigquery/streaming-data-into-bigquery#dataavailability + + while len(rows) == 0 and counter > 0: + rows, _, _ = table.fetch_data() + if len(rows) == 0: + time.sleep(10) + + by_age = operator.itemgetter(1) + self.assertEqual(sorted(rows, key=by_age), + sorted(ROWS, key=by_age)) diff --git a/system_tests/run_system_test.py b/system_tests/run_system_test.py index ab284ed5936c..448ab38d0870 100644 --- a/system_tests/run_system_test.py +++ b/system_tests/run_system_test.py @@ -24,6 +24,7 @@ 'datastore': ['dataset_id', 'credentials'], 'storage': ['project', 'credentials'], 'pubsub': ['project', 'credentials'], + 'bigquery': ['project', 'credentials'], } From 7102976467e050d802f8bb1361d458997c2204a7 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 12 Aug 2015 14:32:53 -0400 Subject: [PATCH 2/6] Fix testcase class name. --- system_tests/bigquery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system_tests/bigquery.py b/system_tests/bigquery.py index ab445823879c..1fd82e66e640 100644 --- a/system_tests/bigquery.py +++ b/system_tests/bigquery.py @@ -26,7 +26,7 @@ CLIENT = bigquery.Client() -class TestPubsub(unittest2.TestCase): +class TestBigQuery(unittest2.TestCase): def setUp(self): self.to_delete = [] From 75a5ef8e81787bdb22a21745b67696ad11b44d7f Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 12 Aug 2015 14:37:09 -0400 Subject: [PATCH 3/6] Exercise 'Datset.reload'. Addresses: https://github.com/GoogleCloudPlatform/gcloud-python/pull/1047#discussion_r36788878 --- system_tests/bigquery.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/system_tests/bigquery.py b/system_tests/bigquery.py index 1fd82e66e640..e42085607b75 100644 --- a/system_tests/bigquery.py +++ b/system_tests/bigquery.py @@ -44,6 +44,18 @@ def test_create_dataset(self): self.assertTrue(dataset.exists()) self.assertEqual(dataset.name, DATASET_NAME) + def test_reload_dataset(self): + DATASET_NAME = 'system_tests' + dataset = CLIENT.dataset(DATASET_NAME) + dataset.friendly_name = 'Friendly' + dataset.description = 'Description' + dataset.create() + self.to_delete.append(dataset) + other = CLIENT.dataset(DATASET_NAME) + other.reload() + self.assertEqual(other.friendly_name, 'Friendly') + self.assertEqual(other.description, 'Description') + def test_patch_dataset(self): DATASET_NAME = 'system_tests' dataset = CLIENT.dataset(DATASET_NAME) From ea4ebb7b87b9ff27ac9cae9cd7c0e5d3ad630dea Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 12 Aug 2015 15:01:02 -0400 Subject: [PATCH 4/6] Check that next page token is None for short list. Addresses: https://github.com/GoogleCloudPlatform/gcloud-python/pull/1047#discussion_r36789959 --- system_tests/bigquery.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/system_tests/bigquery.py b/system_tests/bigquery.py index e42085607b75..fb3a4aab3c17 100644 --- a/system_tests/bigquery.py +++ b/system_tests/bigquery.py @@ -98,7 +98,8 @@ def test_list_datasets(self): self.to_delete.append(dataset) # Retrieve the datasets. - all_datasets, _ = CLIENT.list_datasets() + all_datasets, token = CLIENT.list_datasets() + self.assertTrue(token is None) created = [dataset for dataset in all_datasets if dataset.name in datasets_to_create and dataset.project == CLIENT.project] @@ -142,7 +143,8 @@ def test_list_tables(self): self.to_delete.insert(0, table) # Retrieve the tables. - all_tables, _ = dataset.list_tables() + all_tables, token = dataset.list_tables() + self.assertTrue(token is None) created = [table for table in all_tables if table.name in tables_to_create and table._dataset.name == DATASET_NAME] From ecc47fb70aff1b0779fc3dd1e07add7bbe5a7c4c Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 12 Aug 2015 15:03:27 -0400 Subject: [PATCH 5/6] Rename testcase method for clarity. Addresses: https://github.com/GoogleCloudPlatform/gcloud-python/pull/1047#discussion_r36790292 --- system_tests/bigquery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system_tests/bigquery.py b/system_tests/bigquery.py index fb3a4aab3c17..a303cbee3b1e 100644 --- a/system_tests/bigquery.py +++ b/system_tests/bigquery.py @@ -197,7 +197,7 @@ def test_update_table(self): self.assertEqual(found.field_type, expected.field_type) self.assertEqual(found.mode, expected.mode) - def test_load_dump_table(self): + def test_load_table_then_dump_table(self): ROWS = [ ('Phred Phlyntstone', 32), ('Bharney Rhubble', 33), From 62509d2a9718a422ae1da642784bbd21f9b25346 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 12 Aug 2015 15:04:28 -0400 Subject: [PATCH 6/6] Decrement counter. Addresses: https://github.com/GoogleCloudPlatform/gcloud-python/pull/1047#discussion_r36790517 --- system_tests/bigquery.py | 1 + 1 file changed, 1 insertion(+) diff --git a/system_tests/bigquery.py b/system_tests/bigquery.py index a303cbee3b1e..31fc8b6f3071 100644 --- a/system_tests/bigquery.py +++ b/system_tests/bigquery.py @@ -229,6 +229,7 @@ def test_load_table_then_dump_table(self): # https://cloud.google.com/bigquery/streaming-data-into-bigquery#dataavailability while len(rows) == 0 and counter > 0: + counter -= 1 rows, _, _ = table.fetch_data() if len(rows) == 0: time.sleep(10)