diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index 202a39ac8447..8cebe9fbec01 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -15,9 +15,13 @@ """Shared helper functions for BigQuery API classes.""" from collections import OrderedDict +import datetime -from google.cloud._helpers import _datetime_from_microseconds from google.cloud._helpers import _date_from_iso8601_date +from google.cloud._helpers import _datetime_from_microseconds +from google.cloud._helpers import _datetime_to_rfc3339 +from google.cloud._helpers import _microseconds_from_datetime +from google.cloud._helpers import _RFC3339_NO_FRACTION def _not_null(value, field): @@ -43,13 +47,20 @@ def _bool_from_json(value, field): return value.lower() in ['t', 'true', '1'] -def _datetime_from_json(value, field): +def _timestamp_from_json(value, field): """Coerce 'value' to a datetime, if set or not nullable.""" if _not_null(value, field): # value will be a float in seconds, to microsecond precision, in UTC. return _datetime_from_microseconds(1e6 * float(value)) +def _datetime_from_json(value, field): + """Coerce 'value' to a datetime, if set or not nullable.""" + if _not_null(value, field): + # value will be a string, in YYYY-MM-DDTHH:MM:SS form. + return datetime.datetime.strptime(value, _RFC3339_NO_FRACTION) + + def _date_from_json(value, field): """Coerce 'value' to a datetime date, if set or not nullable""" if _not_null(value, field): @@ -83,13 +94,67 @@ def _string_from_json(value, _): 'FLOAT64': _float_from_json, 'BOOLEAN': _bool_from_json, 'BOOL': _bool_from_json, - 'TIMESTAMP': _datetime_from_json, + 'TIMESTAMP': _timestamp_from_json, + 'DATETIME': _datetime_from_json, 'DATE': _date_from_json, 'RECORD': _record_from_json, 'STRING': _string_from_json, } +def _int_to_json(value): + """Coerce 'value' to an JSON-compatible representation.""" + if isinstance(value, int): + value = str(value) + return value + + +def _float_to_json(value): + """Coerce 'value' to an JSON-compatible representation.""" + return value + + +def _bool_to_json(value): + """Coerce 'value' to an JSON-compatible representation.""" + if isinstance(value, bool): + value = 'true' if value else 'false' + return value + + +def _timestamp_to_json(value): + """Coerce 'value' to an JSON-compatible representation.""" + if isinstance(value, datetime.datetime): + value = _microseconds_from_datetime(value) / 1.0e6 + return value + + +def _datetime_to_json(value): + """Coerce 'value' to an JSON-compatible representation.""" + if isinstance(value, datetime.datetime): + value = _datetime_to_rfc3339(value) + return value + + +def _date_to_json(value): + """Coerce 'value' to an JSON-compatible representation.""" + if isinstance(value, datetime.date): + value = value.isoformat() + return value + + +_SCALAR_VALUE_TO_JSON = { + 'INTEGER': _int_to_json, + 'INT64': _int_to_json, + 'FLOAT': _float_to_json, + 'FLOAT64': _float_to_json, + 'BOOLEAN': _bool_to_json, + 'BOOL': _bool_to_json, + 'TIMESTAMP': _timestamp_to_json, + 'DATETIME': _datetime_to_json, + 'DATE': _date_to_json, +} + + def _row_from_json(row, schema): """Convert JSON row data to row with appropriate types. @@ -262,8 +327,8 @@ class ScalarQueryParameter(AbstractQueryParameter): paramter can only be addressed via position (``?``). :type type_: str - :param type_: name of parameter type. One of `'STRING'`, `'INT64'`, - `'FLOAT64'`, `'BOOL'`, `'TIMESTAMP'`, or `'DATE'`. + :param type_: name of parameter type. One of 'STRING', 'INT64', + 'FLOAT64', 'BOOL', 'TIMESTAMP', 'DATETIME', or 'DATE'. :type value: str, int, float, bool, :class:`datetime.datetime`, or :class:`datetime.date`. @@ -279,8 +344,9 @@ def positional(cls, type_, value): """Factory for positional paramters. :type type_: str - :param type_: name of paramter type. One of `'STRING'`, `'INT64'`, - `'FLOAT64'`, `'BOOL'`, `'TIMESTAMP'`, or `'DATE'`. + :param type_: + name of paramter type. One of 'STRING', 'INT64', + 'FLOAT64', 'BOOL', 'TIMESTAMP', 'DATETIME', or 'DATE'. :type value: str, int, float, bool, :class:`datetime.datetime`, or :class:`datetime.date`. @@ -313,12 +379,16 @@ def to_api_repr(self): :rtype: dict :returns: JSON mapping """ + value = self.value + converter = _SCALAR_VALUE_TO_JSON.get(self.type_) + if converter is not None: + value = converter(value) resource = { 'parameterType': { 'type': self.type_, }, 'parameterValue': { - 'value': self.value, + 'value': value, }, } if self.name is not None: @@ -386,12 +456,16 @@ def to_api_repr(self): :rtype: dict :returns: JSON mapping """ + values = self.values + converter = _SCALAR_VALUE_TO_JSON.get(self.array_type) + if converter is not None: + values = [converter(value) for value in values] resource = { 'parameterType': { 'arrayType': self.array_type, }, 'parameterValue': { - 'arrayValues': self.values, + 'arrayValues': values, }, } if self.name is not None: @@ -458,12 +532,19 @@ def to_api_repr(self): {'name': key, 'type': value} for key, value in self.struct_types.items() ] + values = {} + for name, value in self.struct_values.items(): + converter = _SCALAR_VALUE_TO_JSON.get(self.struct_types[name]) + if converter is not None: + value = converter(value) + values[name] = value + resource = { 'parameterType': { 'structTypes': types, }, 'parameterValue': { - 'structValues': self.struct_values, + 'structValues': values, }, } if self.name is not None: diff --git a/bigquery/unit_tests/test__helpers.py b/bigquery/unit_tests/test__helpers.py index b133e95d45a7..b3ccb1d715f5 100644 --- a/bigquery/unit_tests/test__helpers.py +++ b/bigquery/unit_tests/test__helpers.py @@ -105,11 +105,11 @@ def test_w_value_other(self): self.assertFalse(coerced) -class Test_datetime_from_json(unittest.TestCase): +class Test_timestamp_from_json(unittest.TestCase): def _call_fut(self, value, field): - from google.cloud.bigquery._helpers import _datetime_from_json - return _datetime_from_json(value, field) + from google.cloud.bigquery._helpers import _timestamp_from_json + return _timestamp_from_json(value, field) def test_w_none_nullable(self): self.assertIsNone(self._call_fut(None, _Field('NULLABLE'))) @@ -135,6 +135,27 @@ def test_w_float_value(self): _EPOCH + datetime.timedelta(seconds=1, microseconds=234567)) +class Test_datetime_from_json(unittest.TestCase): + + def _call_fut(self, value, field): + from google.cloud.bigquery._helpers import _datetime_from_json + return _datetime_from_json(value, field) + + def test_w_none_nullable(self): + self.assertIsNone(self._call_fut(None, _Field('NULLABLE'))) + + def test_w_none_required(self): + with self.assertRaises(TypeError): + self._call_fut(None, _Field('REQUIRED')) + + def test_w_string_value(self): + import datetime + coerced = self._call_fut('2016-12-02T18:51:33', object()) + self.assertEqual( + coerced, + datetime.datetime(2016, 12, 2, 18, 51, 33)) + + class Test_date_from_json(unittest.TestCase): def _call_fut(self, value, field): @@ -411,6 +432,96 @@ def test_w_int64_float64_bool(self): self.assertEqual(coerced, expected) +class Test_int_to_json(unittest.TestCase): + + def _call_fut(self, value): + from google.cloud.bigquery._helpers import _int_to_json + return _int_to_json(value) + + def test_w_int(self): + self.assertEqual(self._call_fut(123), '123') + + def test_w_string(self): + self.assertEqual(self._call_fut('123'), '123') + + +class Test_float_to_json(unittest.TestCase): + + def _call_fut(self, value): + from google.cloud.bigquery._helpers import _float_to_json + return _float_to_json(value) + + def test_w_float(self): + self.assertEqual(self._call_fut(1.23), 1.23) + + +class Test_bool_to_json(unittest.TestCase): + + def _call_fut(self, value): + from google.cloud.bigquery._helpers import _bool_to_json + return _bool_to_json(value) + + def test_w_true(self): + self.assertEqual(self._call_fut(True), 'true') + + def test_w_false(self): + self.assertEqual(self._call_fut(False), 'false') + + def test_w_string(self): + self.assertEqual(self._call_fut('false'), 'false') + + +class Test_timestamp_to_json(unittest.TestCase): + + def _call_fut(self, value): + from google.cloud.bigquery._helpers import _timestamp_to_json + return _timestamp_to_json(value) + + def test_w_float(self): + self.assertEqual(self._call_fut(1.234567), 1.234567) + + def test_w_datetime(self): + import datetime + from google.cloud._helpers import UTC + from google.cloud._helpers import _microseconds_from_datetime + when = datetime.datetime(2016, 12, 3, 14, 11, 27, tzinfo=UTC) + self.assertEqual(self._call_fut(when), + _microseconds_from_datetime(when) / 1e6) + + +class Test_datetime_to_json(unittest.TestCase): + + def _call_fut(self, value): + from google.cloud.bigquery._helpers import _datetime_to_json + return _datetime_to_json(value) + + def test_w_string(self): + RFC3339 = '2016-12-03T14:14:51Z' + self.assertEqual(self._call_fut(RFC3339), RFC3339) + + def test_w_datetime(self): + import datetime + from google.cloud._helpers import UTC + when = datetime.datetime(2016, 12, 3, 14, 11, 27, 123456, tzinfo=UTC) + self.assertEqual(self._call_fut(when), '2016-12-03T14:11:27.123456Z') + + +class Test_date_to_json(unittest.TestCase): + + def _call_fut(self, value): + from google.cloud.bigquery._helpers import _date_to_json + return _date_to_json(value) + + def test_w_string(self): + RFC3339 = '2016-12-03' + self.assertEqual(self._call_fut(RFC3339), RFC3339) + + def test_w_datetime(self): + import datetime + when = datetime.date(2016, 12, 3) + self.assertEqual(self._call_fut(when), '2016-12-03') + + class Test_ConfigurationProperty(unittest.TestCase): @staticmethod @@ -668,7 +779,7 @@ def test_to_api_repr_w_name(self): 'type': 'INT64', }, 'parameterValue': { - 'value': 123, + 'value': '123', }, } param = self._make_one(name='foo', type_='INT64', value=123) @@ -680,13 +791,150 @@ def test_to_api_repr_wo_name(self): 'type': 'INT64', }, 'parameterValue': { - 'value': 123, + 'value': '123', }, } klass = self._get_target_class() param = klass.positional(type_='INT64', value=123) self.assertEqual(param.to_api_repr(), EXPECTED) + def test_to_api_repr_w_float(self): + EXPECTED = { + 'parameterType': { + 'type': 'FLOAT64', + }, + 'parameterValue': { + 'value': 12.345, + }, + } + klass = self._get_target_class() + param = klass.positional(type_='FLOAT64', value=12.345) + self.assertEqual(param.to_api_repr(), EXPECTED) + + def test_to_api_repr_w_bool(self): + EXPECTED = { + 'parameterType': { + 'type': 'BOOL', + }, + 'parameterValue': { + 'value': 'false', + }, + } + klass = self._get_target_class() + param = klass.positional(type_='BOOL', value=False) + self.assertEqual(param.to_api_repr(), EXPECTED) + + def test_to_api_repr_w_timestamp_datetime(self): + import datetime + from google.cloud._helpers import _microseconds_from_datetime + now = datetime.datetime.utcnow() + seconds = _microseconds_from_datetime(now) / 1.0e6 + EXPECTED = { + 'parameterType': { + 'type': 'TIMESTAMP', + }, + 'parameterValue': { + 'value': seconds, + }, + } + klass = self._get_target_class() + param = klass.positional(type_='TIMESTAMP', value=now) + self.assertEqual(param.to_api_repr(), EXPECTED) + + def test_to_api_repr_w_timestamp_micros(self): + import datetime + from google.cloud._helpers import _microseconds_from_datetime + now = datetime.datetime.utcnow() + seconds = _microseconds_from_datetime(now) / 1.0e6 + EXPECTED = { + 'parameterType': { + 'type': 'TIMESTAMP', + }, + 'parameterValue': { + 'value': seconds, + }, + } + klass = self._get_target_class() + param = klass.positional(type_='TIMESTAMP', value=seconds) + self.assertEqual(param.to_api_repr(), EXPECTED) + + def test_to_api_repr_w_datetime_datetime(self): + import datetime + from google.cloud._helpers import _datetime_to_rfc3339 + now = datetime.datetime.utcnow() + EXPECTED = { + 'parameterType': { + 'type': 'DATETIME', + }, + 'parameterValue': { + 'value': _datetime_to_rfc3339(now), + }, + } + klass = self._get_target_class() + param = klass.positional(type_='DATETIME', value=now) + self.assertEqual(param.to_api_repr(), EXPECTED) + + def test_to_api_repr_w_datetime_string(self): + import datetime + from google.cloud._helpers import _datetime_to_rfc3339 + now = datetime.datetime.utcnow() + now_str = _datetime_to_rfc3339(now) + EXPECTED = { + 'parameterType': { + 'type': 'DATETIME', + }, + 'parameterValue': { + 'value': now_str, + }, + } + klass = self._get_target_class() + param = klass.positional(type_='DATETIME', value=now_str) + self.assertEqual(param.to_api_repr(), EXPECTED) + + def test_to_api_repr_w_date_date(self): + import datetime + today = datetime.date.today() + EXPECTED = { + 'parameterType': { + 'type': 'DATE', + }, + 'parameterValue': { + 'value': today.isoformat(), + }, + } + klass = self._get_target_class() + param = klass.positional(type_='DATE', value=today) + self.assertEqual(param.to_api_repr(), EXPECTED) + + def test_to_api_repr_w_date_string(self): + import datetime + today = datetime.date.today() + today_str = today.isoformat(), + EXPECTED = { + 'parameterType': { + 'type': 'DATE', + }, + 'parameterValue': { + 'value': today_str, + }, + } + klass = self._get_target_class() + param = klass.positional(type_='DATE', value=today_str) + self.assertEqual(param.to_api_repr(), EXPECTED) + + def test_to_api_repr_w_unknown_type(self): + EXPECTED = { + 'parameterType': { + 'type': 'UNKNOWN', + }, + 'parameterValue': { + 'value': 'unknown', + }, + } + klass = self._get_target_class() + param = klass.positional(type_='UNKNOWN', value='unknown') + self.assertEqual(param.to_api_repr(), EXPECTED) + class Test_ArrayQueryParameter(unittest.TestCase): @@ -749,7 +997,7 @@ def test_to_api_repr_w_name(self): 'arrayType': 'INT64', }, 'parameterValue': { - 'arrayValues': [1, 2], + 'arrayValues': ['1', '2'], }, } param = self._make_one(name='foo', array_type='INT64', values=[1, 2]) @@ -761,13 +1009,26 @@ def test_to_api_repr_wo_name(self): 'arrayType': 'INT64', }, 'parameterValue': { - 'arrayValues': [1, 2], + 'arrayValues': ['1', '2'], }, } klass = self._get_target_class() param = klass.positional(array_type='INT64', values=[1, 2]) self.assertEqual(param.to_api_repr(), EXPECTED) + def test_to_api_repr_w_unknown_type(self): + EXPECTED = { + 'parameterType': { + 'arrayType': 'UNKNOWN', + }, + 'parameterValue': { + 'arrayValues': ['unknown'], + }, + } + klass = self._get_target_class() + param = klass.positional(array_type='UNKNOWN', values=['unknown']) + self.assertEqual(param.to_api_repr(), EXPECTED) + class Test_StructQueryParameter(unittest.TestCase): @@ -848,7 +1109,7 @@ def test_to_api_repr_w_name(self): ], }, 'parameterValue': { - 'structValues': {'bar': 123, 'baz': 'abc'}, + 'structValues': {'bar': '123', 'baz': 'abc'}, }, } sub_1 = self._make_subparam('bar', 'INT64', 123) @@ -865,7 +1126,7 @@ def test_to_api_repr_wo_name(self): ], }, 'parameterValue': { - 'structValues': {'bar': 123, 'baz': 'abc'}, + 'structValues': {'bar': '123', 'baz': 'abc'}, }, } sub_1 = self._make_subparam('bar', 'INT64', 123) diff --git a/bigquery/unit_tests/test_job.py b/bigquery/unit_tests/test_job.py index 84ee25418491..a451e0040931 100644 --- a/bigquery/unit_tests/test_job.py +++ b/bigquery/unit_tests/test_job.py @@ -1624,7 +1624,7 @@ def test_begin_w_named_query_parameter(self): 'type': 'INT64', }, 'parameterValue': { - 'value': 123, + 'value': '123', }, }, ] @@ -1674,7 +1674,7 @@ def test_begin_w_positional_query_parameter(self): 'type': 'INT64', }, 'parameterValue': { - 'value': 123, + 'value': '123', }, }, ] diff --git a/bigquery/unit_tests/test_query.py b/bigquery/unit_tests/test_query.py index 3dfc795a6072..fa8dd401e435 100644 --- a/bigquery/unit_tests/test_query.py +++ b/bigquery/unit_tests/test_query.py @@ -419,7 +419,7 @@ def test_run_w_named_query_paramter(self): 'type': 'INT64', }, 'parameterValue': { - 'value': 123, + 'value': '123', }, }, ] @@ -453,7 +453,7 @@ def test_run_w_positional_query_paramter(self): 'type': 'INT64', }, 'parameterValue': { - 'value': 123, + 'value': '123', }, }, ]