From f6de120da1f3ec8ebfe0959fb6173ce75f01d4a7 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Mon, 10 Aug 2015 15:03:13 -0700 Subject: [PATCH 1/8] Moving _millis and _total_seconds methods from bigquery to core. --- gcloud/_helpers.py | 27 +++++++++++++++++++++++++++ gcloud/bigquery/_helpers.py | 25 +------------------------ gcloud/bigquery/test__helpers.py | 22 ++++------------------ gcloud/bigquery/test_table.py | 8 ++++---- gcloud/test__helpers.py | 14 ++++++++++++++ 5 files changed, 50 insertions(+), 46 deletions(-) diff --git a/gcloud/_helpers.py b/gcloud/_helpers.py index 7e0644ef7cb7..4860caac5774 100644 --- a/gcloud/_helpers.py +++ b/gcloud/_helpers.py @@ -19,6 +19,7 @@ import datetime import os import socket +import sys try: from threading import local as Local @@ -207,7 +208,33 @@ def _determine_default_project(project=None): return project +def _millis(when): + """Convert a zone-aware datetime to integer milliseconds. + + :type when: ``datetime.datetime`` + :param when: the datetime to convert + + :rtype: integer + :returns: milliseconds since epoch for ``when`` + """ + return int(_total_seconds(when - _EPOCH) * 1000) + + try: from pytz import UTC # pylint: disable=unused-import except ImportError: UTC = _UTC() # Singleton instance to be used throughout. + +# Need to define _EPOCH at the end of module since it relies on UTC. +_EPOCH = datetime.datetime.utcfromtimestamp(0).replace(tzinfo=UTC) + + +if sys.version_info[:2] < (2, 7): + def _total_seconds(offset): # pragma: NO COVER + """Backport of timedelta.total_seconds() from python 2.7+.""" + seconds = offset.days * 24 * 60 * 60 + offset.seconds + microseconds = seconds * 10**6 + offset.microseconds + return microseconds / (10**6 * 1.0) +else: + def _total_seconds(offset): + return offset.total_seconds() diff --git a/gcloud/bigquery/_helpers.py b/gcloud/bigquery/_helpers.py index df6215721c7d..659f108da610 100644 --- a/gcloud/bigquery/_helpers.py +++ b/gcloud/bigquery/_helpers.py @@ -16,26 +16,14 @@ import datetime -import sys from gcloud._helpers import UTC +from gcloud._helpers import _millis _EPOCH = datetime.datetime.utcfromtimestamp(0).replace(tzinfo=UTC) -def _millis(when): - """Convert a zone-aware datetime to integer milliseconds. - - :type when: ``datetime.datetime`` - :param when: the datetime to convert - - :rtype: integer - :returns: milliseconds since epoch for ``when`` - """ - return int(_total_seconds(when - _EPOCH) * 1000) - - def _datetime_from_prop(value): """Convert non-none timestamp to datetime, assuming UTC. @@ -66,14 +54,3 @@ def _prop_from_datetime(value): value = value.replace(tzinfo=UTC) # back-end wants timestamps as milliseconds since the epoch return _millis(value) - - -if sys.version_info[:2] < (2, 7): - def _total_seconds(offset): # pragma: NO COVER - """Backport of timedelta.total_seconds() from python 2.7+.""" - seconds = offset.days * 24 * 60 * 60 + offset.seconds - microseconds = seconds * 10**6 + offset.microseconds - return microseconds / (10**6 * 1.0) -else: - def _total_seconds(offset): - return offset.total_seconds() diff --git a/gcloud/bigquery/test__helpers.py b/gcloud/bigquery/test__helpers.py index ee97717ef9f6..3a6ac8f2183e 100644 --- a/gcloud/bigquery/test__helpers.py +++ b/gcloud/bigquery/test__helpers.py @@ -15,20 +15,6 @@ import unittest2 -class Test__millis(unittest2.TestCase): - - def _callFUT(self, value): - from gcloud.bigquery._helpers import _millis - return _millis(value) - - def test_one_second_from_epoch(self): - import datetime - from gcloud._helpers import UTC - - WHEN = datetime.datetime(1970, 1, 1, 0, 0, 1, tzinfo=UTC) - self.assertEqual(self._callFUT(WHEN), 1000) - - class Test__datetime_from_prop(unittest2.TestCase): def _callFUT(self, value): @@ -41,7 +27,7 @@ def test_w_none(self): def test_w_millis(self): import datetime from gcloud._helpers import UTC - from gcloud.bigquery._helpers import _total_seconds + from gcloud._helpers import _total_seconds NOW = datetime.datetime(2015, 7, 29, 17, 45, 21, 123456, tzinfo=UTC) @@ -62,7 +48,7 @@ def test_w_none(self): def test_w_utc_datetime(self): import datetime from gcloud._helpers import UTC - from gcloud.bigquery._helpers import _total_seconds + from gcloud._helpers import _total_seconds NOW = datetime.datetime.utcnow().replace(tzinfo=UTC) EPOCH = datetime.datetime(1970, 1, 1, tzinfo=UTC) @@ -75,7 +61,7 @@ def test_w_non_utc_datetime(self): import datetime from gcloud._helpers import UTC from gcloud._helpers import _UTC - from gcloud.bigquery._helpers import _total_seconds + from gcloud._helpers import _total_seconds class CET(_UTC): _tzname = 'CET' @@ -92,7 +78,7 @@ class CET(_UTC): def test_w_naive_datetime(self): import datetime from gcloud._helpers import UTC - from gcloud.bigquery._helpers import _total_seconds + from gcloud._helpers import _total_seconds NOW = datetime.datetime.utcnow() UTC_NOW = NOW.replace(tzinfo=UTC) diff --git a/gcloud/bigquery/test_table.py b/gcloud/bigquery/test_table.py index 3dd5394da5cd..8825d9b822e9 100644 --- a/gcloud/bigquery/test_table.py +++ b/gcloud/bigquery/test_table.py @@ -240,7 +240,7 @@ def test_schema_setter(self): def test_props_set_by_server(self): import datetime from gcloud._helpers import UTC - from gcloud.bigquery._helpers import _millis + from gcloud._helpers import _millis CREATED = datetime.datetime(2015, 7, 29, 12, 13, 22, tzinfo=UTC) MODIFIED = datetime.datetime(2015, 7, 29, 14, 47, 15, tzinfo=UTC) @@ -516,8 +516,8 @@ def test_create_w_bound_client(self): def test_create_w_alternate_client(self): import datetime from gcloud._helpers import UTC + from gcloud._helpers import _millis from gcloud.bigquery.table import SchemaField - from gcloud.bigquery._helpers import _millis PATH = 'projects/%s/datasets/%s/tables' % (self.PROJECT, self.DS_NAME) DESCRIPTION = 'DESCRIPTION' @@ -716,7 +716,7 @@ def test_patch_w_bound_client(self): def test_patch_w_alternate_client(self): import datetime from gcloud._helpers import UTC - from gcloud.bigquery._helpers import _millis + from gcloud._helpers import _millis from gcloud.bigquery.table import SchemaField PATH = 'projects/%s/datasets/%s/tables/%s' % ( @@ -825,7 +825,7 @@ def test_update_w_bound_client(self): def test_update_w_alternate_client(self): import datetime from gcloud._helpers import UTC - from gcloud.bigquery._helpers import _millis + from gcloud._helpers import _millis from gcloud.bigquery.table import SchemaField PATH = 'projects/%s/datasets/%s/tables/%s' % ( diff --git a/gcloud/test__helpers.py b/gcloud/test__helpers.py index 72715470f6f2..d89bd3d46f4f 100644 --- a/gcloud/test__helpers.py +++ b/gcloud/test__helpers.py @@ -251,6 +251,20 @@ def test_prod(self): self.assertEqual(callers, ['prod_mock']) +class Test__millis(unittest2.TestCase): + + def _callFUT(self, value): + from gcloud._helpers import _millis + return _millis(value) + + def test_one_second_from_epoch(self): + import datetime + from gcloud._helpers import UTC + + WHEN = datetime.datetime(1970, 1, 1, 0, 0, 1, tzinfo=UTC) + self.assertEqual(self._callFUT(WHEN), 1000) + + class _AppIdentity(object): def __init__(self, app_id): From 00583030be884e439e7434b3193b4020584ed329 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Mon, 10 Aug 2015 15:16:40 -0700 Subject: [PATCH 2/8] Giving proper definition for _total_seconds methods. This way we can test all code-paths, just not the branch where Python 2.6 is active. --- gcloud/_helpers.py | 39 ++++++++++++++++++++++++-------- gcloud/bigquery/test__helpers.py | 16 ++++++------- gcloud/test__helpers.py | 35 ++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 17 deletions(-) diff --git a/gcloud/_helpers.py b/gcloud/_helpers.py index 4860caac5774..5b4f2de1ae6a 100644 --- a/gcloud/_helpers.py +++ b/gcloud/_helpers.py @@ -208,16 +208,42 @@ def _determine_default_project(project=None): return project +def _manual_total_seconds(offset): + """Backport of timedelta.total_seconds() from python 2.7+. + + :type offset: :class:`datetime.timedelta` + :param offset: The value to convert into seconds. + + :rtype: float + :returns: The time offset, converted to total seconds. + """ + seconds = offset.days * 24 * 60 * 60 + offset.seconds + microseconds = seconds * 10**6 + offset.microseconds + return microseconds / (10**6 * 1.0) + + +def _total_seconds_from_type(offset): + """Basic wrapper around timedelta.total_seconds(). + + :type offset: :class:`datetime.timedelta` + :param offset: The value to convert into seconds. + + :rtype: float + :returns: The time offset, converted to total seconds. + """ + return offset.total_seconds() + + def _millis(when): """Convert a zone-aware datetime to integer milliseconds. - :type when: ``datetime.datetime`` + :type when: :class:`datetime.datetime` :param when: the datetime to convert :rtype: integer :returns: milliseconds since epoch for ``when`` """ - return int(_total_seconds(when - _EPOCH) * 1000) + return int(_TOTAL_SECONDS(when - _EPOCH) * 1000) try: @@ -230,11 +256,6 @@ def _millis(when): if sys.version_info[:2] < (2, 7): - def _total_seconds(offset): # pragma: NO COVER - """Backport of timedelta.total_seconds() from python 2.7+.""" - seconds = offset.days * 24 * 60 * 60 + offset.seconds - microseconds = seconds * 10**6 + offset.microseconds - return microseconds / (10**6 * 1.0) + _TOTAL_SECONDS = _manual_total_seconds # pragma: NO COVER else: - def _total_seconds(offset): - return offset.total_seconds() + _TOTAL_SECONDS = _total_seconds_from_type diff --git a/gcloud/bigquery/test__helpers.py b/gcloud/bigquery/test__helpers.py index 3a6ac8f2183e..ba632e3347f6 100644 --- a/gcloud/bigquery/test__helpers.py +++ b/gcloud/bigquery/test__helpers.py @@ -27,12 +27,12 @@ def test_w_none(self): def test_w_millis(self): import datetime from gcloud._helpers import UTC - from gcloud._helpers import _total_seconds + from gcloud._helpers import _TOTAL_SECONDS NOW = datetime.datetime(2015, 7, 29, 17, 45, 21, 123456, tzinfo=UTC) EPOCH = datetime.datetime(1970, 1, 1, tzinfo=UTC) - MILLIS = _total_seconds(NOW - EPOCH) * 1000 + MILLIS = _TOTAL_SECONDS(NOW - EPOCH) * 1000 self.assertEqual(self._callFUT(MILLIS), NOW) @@ -48,11 +48,11 @@ def test_w_none(self): def test_w_utc_datetime(self): import datetime from gcloud._helpers import UTC - from gcloud._helpers import _total_seconds + from gcloud._helpers import _TOTAL_SECONDS NOW = datetime.datetime.utcnow().replace(tzinfo=UTC) EPOCH = datetime.datetime(1970, 1, 1, tzinfo=UTC) - MILLIS = int(_total_seconds(NOW - EPOCH) * 1000) + MILLIS = int(_TOTAL_SECONDS(NOW - EPOCH) * 1000) result = self._callFUT(NOW) self.assertTrue(isinstance(result, int)) self.assertEqual(result, MILLIS) @@ -61,7 +61,7 @@ def test_w_non_utc_datetime(self): import datetime from gcloud._helpers import UTC from gcloud._helpers import _UTC - from gcloud._helpers import _total_seconds + from gcloud._helpers import _TOTAL_SECONDS class CET(_UTC): _tzname = 'CET' @@ -70,7 +70,7 @@ class CET(_UTC): zone = CET() NOW = datetime.datetime(2015, 7, 28, 16, 34, 47, tzinfo=zone) EPOCH = datetime.datetime(1970, 1, 1, tzinfo=UTC) - MILLIS = int(_total_seconds(NOW - EPOCH) * 1000) + MILLIS = int(_TOTAL_SECONDS(NOW - EPOCH) * 1000) result = self._callFUT(NOW) self.assertTrue(isinstance(result, int)) self.assertEqual(result, MILLIS) @@ -78,12 +78,12 @@ class CET(_UTC): def test_w_naive_datetime(self): import datetime from gcloud._helpers import UTC - from gcloud._helpers import _total_seconds + from gcloud._helpers import _TOTAL_SECONDS NOW = datetime.datetime.utcnow() UTC_NOW = NOW.replace(tzinfo=UTC) EPOCH = datetime.datetime(1970, 1, 1, tzinfo=UTC) - MILLIS = int(_total_seconds(UTC_NOW - EPOCH) * 1000) + MILLIS = int(_TOTAL_SECONDS(UTC_NOW - EPOCH) * 1000) result = self._callFUT(NOW) self.assertTrue(isinstance(result, int)) self.assertEqual(result, MILLIS) diff --git a/gcloud/test__helpers.py b/gcloud/test__helpers.py index d89bd3d46f4f..15cc42e1fc77 100644 --- a/gcloud/test__helpers.py +++ b/gcloud/test__helpers.py @@ -251,6 +251,41 @@ def test_prod(self): self.assertEqual(callers, ['prod_mock']) +class Test__manual_total_seconds(unittest2.TestCase): + + def _callFUT(self, value): + from gcloud._helpers import _manual_total_seconds + return _manual_total_seconds(value) + + def test_it(self): + import datetime + + delta = datetime.timedelta(minutes=1, seconds=1, + microseconds=10**(6 - 1)) + result = self._callFUT(delta) + self.assertEqual(result, 61.1) + + +class Test__total_seconds_from_type(unittest2.TestCase): + + def _callFUT(self, value): + from gcloud._helpers import _total_seconds_from_type + return _total_seconds_from_type(value) + + def test_it(self): + class FakeDelta(object): + + def __init__(self, total_seconds): + self._total_seconds = total_seconds + + def total_seconds(self): + return self._total_seconds + + value = object() + fake_delta = FakeDelta(value) + self.assertEqual(self._callFUT(fake_delta), value) + + class Test__millis(unittest2.TestCase): def _callFUT(self, value): From e0218c8ff6dcbb141bef8583381e501c18fdcac0 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Mon, 10 Aug 2015 15:25:16 -0700 Subject: [PATCH 3/8] Moving remaining datetime functions from bigquery helpers to core. --- gcloud/_helpers.py | 32 ++++++++++++ gcloud/bigquery/_helpers.py | 56 -------------------- gcloud/bigquery/dataset.py | 2 +- gcloud/bigquery/table.py | 4 +- gcloud/bigquery/test__helpers.py | 89 -------------------------------- gcloud/bigquery/test_table.py | 4 +- gcloud/test__helpers.py | 74 ++++++++++++++++++++++++++ 7 files changed, 111 insertions(+), 150 deletions(-) delete mode 100644 gcloud/bigquery/_helpers.py delete mode 100644 gcloud/bigquery/test__helpers.py diff --git a/gcloud/_helpers.py b/gcloud/_helpers.py index 5b4f2de1ae6a..b663397d6e3f 100644 --- a/gcloud/_helpers.py +++ b/gcloud/_helpers.py @@ -246,6 +246,38 @@ def _millis(when): return int(_TOTAL_SECONDS(when - _EPOCH) * 1000) +def _datetime_from_prop(value): + """Convert non-none timestamp to datetime, assuming UTC. + + :rtype: :class:`datetime.datetime`, or ``NoneType`` + """ + if value is not None: + # back-end returns timestamps as milliseconds since the epoch + seconds = int(value / 1000.0) + microseconds = 1000.0 * (value - 1000 * seconds) + return ( + _EPOCH + + datetime.timedelta(seconds=seconds, microseconds=microseconds) + ) + + +def _prop_from_datetime(value): + """Convert non-none datetime to timestamp, assuming UTC. + + :type value: :class:`datetime.datetime`, or None + :param value: the timestamp + + :rtype: integer, or ``NoneType`` + :returns: the timestamp, in milliseconds, or None + """ + if value is not None: + if value.tzinfo is None: + # Assume UTC + value = value.replace(tzinfo=UTC) + # back-end wants timestamps as milliseconds since the epoch + return _millis(value) + + try: from pytz import UTC # pylint: disable=unused-import except ImportError: diff --git a/gcloud/bigquery/_helpers.py b/gcloud/bigquery/_helpers.py deleted file mode 100644 index 659f108da610..000000000000 --- a/gcloud/bigquery/_helpers.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""BigQuery utility functions.""" - - -import datetime - -from gcloud._helpers import UTC -from gcloud._helpers import _millis - - -_EPOCH = datetime.datetime.utcfromtimestamp(0).replace(tzinfo=UTC) - - -def _datetime_from_prop(value): - """Convert non-none timestamp to datetime, assuming UTC. - - :rtype: ``datetime.datetime``, or ``NoneType`` - """ - if value is not None: - # back-end returns timestamps as milliseconds since the epoch - seconds = int(value / 1000.0) - microseconds = 1000.0 * (value - 1000 * seconds) - return ( - _EPOCH + - datetime.timedelta(seconds=seconds, microseconds=microseconds) - ) - - -def _prop_from_datetime(value): - """Convert non-none datetime to timestamp, assuming UTC. - - :type value: ``datetime.datetime``, or None - :param value: the timestamp - - :rtype: integer, or ``NoneType`` - :returns: the timestamp, in milliseconds, or None - """ - if value is not None: - if value.tzinfo is None: - # Assume UTC - value = value.replace(tzinfo=UTC) - # back-end wants timestamps as milliseconds since the epoch - return _millis(value) diff --git a/gcloud/bigquery/dataset.py b/gcloud/bigquery/dataset.py index 47d799e33186..fed19a17761b 100644 --- a/gcloud/bigquery/dataset.py +++ b/gcloud/bigquery/dataset.py @@ -15,8 +15,8 @@ """Define API Datasets.""" import six +from gcloud._helpers import _datetime_from_prop from gcloud.exceptions import NotFound -from gcloud.bigquery._helpers import _datetime_from_prop from gcloud.bigquery.table import Table diff --git a/gcloud/bigquery/table.py b/gcloud/bigquery/table.py index ee2d40cb8a2a..0294ceed3f2e 100644 --- a/gcloud/bigquery/table.py +++ b/gcloud/bigquery/table.py @@ -18,9 +18,9 @@ import six +from gcloud._helpers import _datetime_from_prop +from gcloud._helpers import _prop_from_datetime from gcloud.exceptions import NotFound -from gcloud.bigquery._helpers import _datetime_from_prop -from gcloud.bigquery._helpers import _prop_from_datetime _MARKER = object() diff --git a/gcloud/bigquery/test__helpers.py b/gcloud/bigquery/test__helpers.py deleted file mode 100644 index ba632e3347f6..000000000000 --- a/gcloud/bigquery/test__helpers.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest2 - - -class Test__datetime_from_prop(unittest2.TestCase): - - def _callFUT(self, value): - from gcloud.bigquery._helpers import _datetime_from_prop - return _datetime_from_prop(value) - - def test_w_none(self): - self.assertTrue(self._callFUT(None) is None) - - def test_w_millis(self): - import datetime - from gcloud._helpers import UTC - from gcloud._helpers import _TOTAL_SECONDS - - NOW = datetime.datetime(2015, 7, 29, 17, 45, 21, 123456, - tzinfo=UTC) - EPOCH = datetime.datetime(1970, 1, 1, tzinfo=UTC) - MILLIS = _TOTAL_SECONDS(NOW - EPOCH) * 1000 - self.assertEqual(self._callFUT(MILLIS), NOW) - - -class Test__prop_from_datetime(unittest2.TestCase): - - def _callFUT(self, value): - from gcloud.bigquery._helpers import _prop_from_datetime - return _prop_from_datetime(value) - - def test_w_none(self): - self.assertTrue(self._callFUT(None) is None) - - def test_w_utc_datetime(self): - import datetime - from gcloud._helpers import UTC - from gcloud._helpers import _TOTAL_SECONDS - - NOW = datetime.datetime.utcnow().replace(tzinfo=UTC) - EPOCH = datetime.datetime(1970, 1, 1, tzinfo=UTC) - MILLIS = int(_TOTAL_SECONDS(NOW - EPOCH) * 1000) - result = self._callFUT(NOW) - self.assertTrue(isinstance(result, int)) - self.assertEqual(result, MILLIS) - - def test_w_non_utc_datetime(self): - import datetime - from gcloud._helpers import UTC - from gcloud._helpers import _UTC - from gcloud._helpers import _TOTAL_SECONDS - - class CET(_UTC): - _tzname = 'CET' - _utcoffset = datetime.timedelta(hours=-1) - - zone = CET() - NOW = datetime.datetime(2015, 7, 28, 16, 34, 47, tzinfo=zone) - EPOCH = datetime.datetime(1970, 1, 1, tzinfo=UTC) - MILLIS = int(_TOTAL_SECONDS(NOW - EPOCH) * 1000) - result = self._callFUT(NOW) - self.assertTrue(isinstance(result, int)) - self.assertEqual(result, MILLIS) - - def test_w_naive_datetime(self): - import datetime - from gcloud._helpers import UTC - from gcloud._helpers import _TOTAL_SECONDS - - NOW = datetime.datetime.utcnow() - UTC_NOW = NOW.replace(tzinfo=UTC) - EPOCH = datetime.datetime(1970, 1, 1, tzinfo=UTC) - MILLIS = int(_TOTAL_SECONDS(UTC_NOW - EPOCH) * 1000) - result = self._callFUT(NOW) - self.assertTrue(isinstance(result, int)) - self.assertEqual(result, MILLIS) diff --git a/gcloud/bigquery/test_table.py b/gcloud/bigquery/test_table.py index 8825d9b822e9..1b4f92d528ed 100644 --- a/gcloud/bigquery/test_table.py +++ b/gcloud/bigquery/test_table.py @@ -914,7 +914,7 @@ def test_fetch_data_w_bound_client(self): import datetime from gcloud._helpers import UTC from gcloud.bigquery.table import SchemaField - from gcloud.bigquery._helpers import _prop_from_datetime + from gcloud._helpers import _prop_from_datetime PATH = 'projects/%s/datasets/%s/tables/%s/data' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME) @@ -1145,7 +1145,7 @@ def test_fetch_data_w_record_schema(self): def test_insert_data_w_bound_client(self): import datetime from gcloud._helpers import UTC - from gcloud.bigquery._helpers import _prop_from_datetime + from gcloud._helpers import _prop_from_datetime from gcloud.bigquery.table import SchemaField WHEN_TS = 1437767599.006 diff --git a/gcloud/test__helpers.py b/gcloud/test__helpers.py index 15cc42e1fc77..b3c11ba6c20e 100644 --- a/gcloud/test__helpers.py +++ b/gcloud/test__helpers.py @@ -300,6 +300,80 @@ def test_one_second_from_epoch(self): self.assertEqual(self._callFUT(WHEN), 1000) +class Test__datetime_from_prop(unittest2.TestCase): + + def _callFUT(self, value): + from gcloud._helpers import _datetime_from_prop + return _datetime_from_prop(value) + + def test_w_none(self): + self.assertTrue(self._callFUT(None) is None) + + def test_w_millis(self): + import datetime + from gcloud._helpers import UTC + from gcloud._helpers import _TOTAL_SECONDS + + NOW = datetime.datetime(2015, 7, 29, 17, 45, 21, 123456, + tzinfo=UTC) + EPOCH = datetime.datetime(1970, 1, 1, tzinfo=UTC) + MILLIS = _TOTAL_SECONDS(NOW - EPOCH) * 1000 + self.assertEqual(self._callFUT(MILLIS), NOW) + + +class Test__prop_from_datetime(unittest2.TestCase): + + def _callFUT(self, value): + from gcloud._helpers import _prop_from_datetime + return _prop_from_datetime(value) + + def test_w_none(self): + self.assertTrue(self._callFUT(None) is None) + + def test_w_utc_datetime(self): + import datetime + from gcloud._helpers import UTC + from gcloud._helpers import _TOTAL_SECONDS + + NOW = datetime.datetime.utcnow().replace(tzinfo=UTC) + EPOCH = datetime.datetime(1970, 1, 1, tzinfo=UTC) + MILLIS = int(_TOTAL_SECONDS(NOW - EPOCH) * 1000) + result = self._callFUT(NOW) + self.assertTrue(isinstance(result, int)) + self.assertEqual(result, MILLIS) + + def test_w_non_utc_datetime(self): + import datetime + from gcloud._helpers import UTC + from gcloud._helpers import _UTC + from gcloud._helpers import _TOTAL_SECONDS + + class CET(_UTC): + _tzname = 'CET' + _utcoffset = datetime.timedelta(hours=-1) + + zone = CET() + NOW = datetime.datetime(2015, 7, 28, 16, 34, 47, tzinfo=zone) + EPOCH = datetime.datetime(1970, 1, 1, tzinfo=UTC) + MILLIS = int(_TOTAL_SECONDS(NOW - EPOCH) * 1000) + result = self._callFUT(NOW) + self.assertTrue(isinstance(result, int)) + self.assertEqual(result, MILLIS) + + def test_w_naive_datetime(self): + import datetime + from gcloud._helpers import UTC + from gcloud._helpers import _TOTAL_SECONDS + + NOW = datetime.datetime.utcnow() + UTC_NOW = NOW.replace(tzinfo=UTC) + EPOCH = datetime.datetime(1970, 1, 1, tzinfo=UTC) + MILLIS = int(_TOTAL_SECONDS(UTC_NOW - EPOCH) * 1000) + result = self._callFUT(NOW) + self.assertTrue(isinstance(result, int)) + self.assertEqual(result, MILLIS) + + class _AppIdentity(object): def __init__(self, app_id): From 5cb3106dbecb090af5e849b80d902ac917da588a Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Mon, 10 Aug 2015 15:31:37 -0700 Subject: [PATCH 4/8] Renaming "_prop" datetime helpers to express use of millis. --- gcloud/_helpers.py | 4 ++-- gcloud/bigquery/dataset.py | 6 +++--- gcloud/bigquery/table.py | 20 ++++++++++---------- gcloud/bigquery/test_table.py | 12 ++++++------ gcloud/test__helpers.py | 12 ++++++------ 5 files changed, 27 insertions(+), 27 deletions(-) diff --git a/gcloud/_helpers.py b/gcloud/_helpers.py index b663397d6e3f..6255962c964d 100644 --- a/gcloud/_helpers.py +++ b/gcloud/_helpers.py @@ -246,7 +246,7 @@ def _millis(when): return int(_TOTAL_SECONDS(when - _EPOCH) * 1000) -def _datetime_from_prop(value): +def _datetime_from_millis(value): """Convert non-none timestamp to datetime, assuming UTC. :rtype: :class:`datetime.datetime`, or ``NoneType`` @@ -261,7 +261,7 @@ def _datetime_from_prop(value): ) -def _prop_from_datetime(value): +def _millis_from_datetime(value): """Convert non-none datetime to timestamp, assuming UTC. :type value: :class:`datetime.datetime`, or None diff --git a/gcloud/bigquery/dataset.py b/gcloud/bigquery/dataset.py index fed19a17761b..417eb17b1c60 100644 --- a/gcloud/bigquery/dataset.py +++ b/gcloud/bigquery/dataset.py @@ -15,7 +15,7 @@ """Define API Datasets.""" import six -from gcloud._helpers import _datetime_from_prop +from gcloud._helpers import _datetime_from_millis from gcloud.exceptions import NotFound from gcloud.bigquery.table import Table @@ -114,7 +114,7 @@ def created(self): :rtype: ``datetime.datetime``, or ``NoneType`` :returns: the creation time (None until set from the server). """ - return _datetime_from_prop(self._properties.get('creationTime')) + return _datetime_from_millis(self._properties.get('creationTime')) @property def dataset_id(self): @@ -141,7 +141,7 @@ def modified(self): :rtype: ``datetime.datetime``, or ``NoneType`` :returns: the modification time (None until set from the server). """ - return _datetime_from_prop(self._properties.get('lastModifiedTime')) + return _datetime_from_millis(self._properties.get('lastModifiedTime')) @property def self_link(self): diff --git a/gcloud/bigquery/table.py b/gcloud/bigquery/table.py index 0294ceed3f2e..d69e718cc3c6 100644 --- a/gcloud/bigquery/table.py +++ b/gcloud/bigquery/table.py @@ -18,8 +18,8 @@ import six -from gcloud._helpers import _datetime_from_prop -from gcloud._helpers import _prop_from_datetime +from gcloud._helpers import _datetime_from_millis +from gcloud._helpers import _millis_from_datetime from gcloud.exceptions import NotFound @@ -116,7 +116,7 @@ def created(self): :rtype: ``datetime.datetime``, or ``NoneType`` :returns: the creation time (None until set from the server). """ - return _datetime_from_prop(self._properties.get('creationTime')) + return _datetime_from_millis(self._properties.get('creationTime')) @property def etag(self): @@ -134,7 +134,7 @@ def modified(self): :rtype: ``datetime.datetime``, or ``NoneType`` :returns: the modification time (None until set from the server). """ - return _datetime_from_prop(self._properties.get('lastModifiedTime')) + return _datetime_from_millis(self._properties.get('lastModifiedTime')) @property def num_bytes(self): @@ -212,7 +212,7 @@ def expires(self): :rtype: ``datetime.datetime``, or ``NoneType`` :returns: the expiration time, or None """ - return _datetime_from_prop(self._properties.get('expirationTime')) + return _datetime_from_millis(self._properties.get('expirationTime')) @expires.setter def expires(self, value): @@ -223,7 +223,7 @@ def expires(self, value): """ if not isinstance(value, datetime.datetime) and value is not None: raise ValueError("Pass a datetime, or None") - self._properties['expirationTime'] = _prop_from_datetime(value) + self._properties['expirationTime'] = _millis_from_datetime(value) @property def friendly_name(self): @@ -405,7 +405,7 @@ def _build_resource(self): resource['description'] = self.description if self.expires is not None: - value = _prop_from_datetime(self.expires) + value = _millis_from_datetime(self.expires) resource['expirationTime'] = value if self.friendly_name is not None: @@ -518,7 +518,7 @@ def patch(self, if (not isinstance(expires, datetime.datetime) and expires is not None): raise ValueError("Pass a datetime, or None") - partial['expirationTime'] = _prop_from_datetime(expires) + partial['expirationTime'] = _millis_from_datetime(expires) if description is not _MARKER: partial['description'] = description @@ -678,7 +678,7 @@ def insert_data(self, for field, value in zip(self._schema, row): if field.field_type == 'TIMESTAMP': - value = _prop_from_datetime(value) + value = _millis_from_datetime(value) row_info[field.name] = value info = {'json': row_info} @@ -727,7 +727,7 @@ def _bool_from_json(value, field): def _datetime_from_json(value, field): if _not_null(value, field): - return _datetime_from_prop(float(value)) + return _datetime_from_millis(float(value)) def _record_from_json(value, field): diff --git a/gcloud/bigquery/test_table.py b/gcloud/bigquery/test_table.py index 1b4f92d528ed..530f55a9ead0 100644 --- a/gcloud/bigquery/test_table.py +++ b/gcloud/bigquery/test_table.py @@ -914,7 +914,7 @@ def test_fetch_data_w_bound_client(self): import datetime from gcloud._helpers import UTC from gcloud.bigquery.table import SchemaField - from gcloud._helpers import _prop_from_datetime + from gcloud._helpers import _millis_from_datetime PATH = 'projects/%s/datasets/%s/tables/%s/data' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME) @@ -932,17 +932,17 @@ def test_fetch_data_w_bound_client(self): {'f': [ {'v': 'Phred Phlyntstone'}, {'v': '32'}, - {'v': _prop_from_datetime(WHEN)}, + {'v': _millis_from_datetime(WHEN)}, ]}, {'f': [ {'v': 'Bharney Rhubble'}, {'v': '33'}, - {'v': _prop_from_datetime(WHEN_1)}, + {'v': _millis_from_datetime(WHEN_1)}, ]}, {'f': [ {'v': 'Wylma Phlyntstone'}, {'v': '29'}, - {'v': _prop_from_datetime(WHEN_2)}, + {'v': _millis_from_datetime(WHEN_2)}, ]}, {'f': [ {'v': 'Bhettye Rhubble'}, @@ -1145,7 +1145,7 @@ def test_fetch_data_w_record_schema(self): def test_insert_data_w_bound_client(self): import datetime from gcloud._helpers import UTC - from gcloud._helpers import _prop_from_datetime + from gcloud._helpers import _millis_from_datetime from gcloud.bigquery.table import SchemaField WHEN_TS = 1437767599.006 @@ -1171,7 +1171,7 @@ def test_insert_data_w_bound_client(self): def _row_data(row): return {'full_name': row[0], 'age': row[1], - 'joined': _prop_from_datetime(row[2])} + 'joined': _millis_from_datetime(row[2])} SENT = { 'rows': [{'json': _row_data(row)} for row in ROWS], diff --git a/gcloud/test__helpers.py b/gcloud/test__helpers.py index b3c11ba6c20e..6b46fe907068 100644 --- a/gcloud/test__helpers.py +++ b/gcloud/test__helpers.py @@ -300,11 +300,11 @@ def test_one_second_from_epoch(self): self.assertEqual(self._callFUT(WHEN), 1000) -class Test__datetime_from_prop(unittest2.TestCase): +class Test__datetime_from_millis(unittest2.TestCase): def _callFUT(self, value): - from gcloud._helpers import _datetime_from_prop - return _datetime_from_prop(value) + from gcloud._helpers import _datetime_from_millis + return _datetime_from_millis(value) def test_w_none(self): self.assertTrue(self._callFUT(None) is None) @@ -321,11 +321,11 @@ def test_w_millis(self): self.assertEqual(self._callFUT(MILLIS), NOW) -class Test__prop_from_datetime(unittest2.TestCase): +class Test__millis_from_datetime(unittest2.TestCase): def _callFUT(self, value): - from gcloud._helpers import _prop_from_datetime - return _prop_from_datetime(value) + from gcloud._helpers import _millis_from_datetime + return _millis_from_datetime(value) def test_w_none(self): self.assertTrue(self._callFUT(None) is None) From 41e8dc5c702cbf77e19ae744e20729088c85e89a Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 11 Aug 2015 13:10:26 -0700 Subject: [PATCH 5/8] Unifying datetime->timestamp conversions. Using a method to convert to microseconds (which handles both naive datetime objects and ones with timezones other than UTC) and then converts them to microseconds. All consumers then convert to the desired granularity. --- gcloud/_helpers.py | 59 ++++++++-------------- gcloud/credentials.py | 13 ++--- gcloud/datastore/helpers.py | 11 +---- gcloud/test__helpers.py | 99 +++++++++++++++---------------------- 4 files changed, 66 insertions(+), 116 deletions(-) diff --git a/gcloud/_helpers.py b/gcloud/_helpers.py index 6255962c964d..5b52f8e6668b 100644 --- a/gcloud/_helpers.py +++ b/gcloud/_helpers.py @@ -16,10 +16,10 @@ This module is not part of the public API surface of `gcloud`. """ +import calendar import datetime import os import socket -import sys try: from threading import local as Local @@ -208,32 +208,6 @@ def _determine_default_project(project=None): return project -def _manual_total_seconds(offset): - """Backport of timedelta.total_seconds() from python 2.7+. - - :type offset: :class:`datetime.timedelta` - :param offset: The value to convert into seconds. - - :rtype: float - :returns: The time offset, converted to total seconds. - """ - seconds = offset.days * 24 * 60 * 60 + offset.seconds - microseconds = seconds * 10**6 + offset.microseconds - return microseconds / (10**6 * 1.0) - - -def _total_seconds_from_type(offset): - """Basic wrapper around timedelta.total_seconds(). - - :type offset: :class:`datetime.timedelta` - :param offset: The value to convert into seconds. - - :rtype: float - :returns: The time offset, converted to total seconds. - """ - return offset.total_seconds() - - def _millis(when): """Convert a zone-aware datetime to integer milliseconds. @@ -243,7 +217,9 @@ def _millis(when): :rtype: integer :returns: milliseconds since epoch for ``when`` """ - return int(_TOTAL_SECONDS(when - _EPOCH) * 1000) + micros = _microseconds_from_datetime(when) + millis, _ = divmod(micros, 1000) + return millis def _datetime_from_millis(value): @@ -261,6 +237,23 @@ def _datetime_from_millis(value): ) +def _microseconds_from_datetime(value): + """Convert non-none datetime to microseconds. + + :type value: :class:`datetime.datetime` + :param value: The timestamp to convert. + + :rtype: integer + :returns: The timestamp, in microseconds. + """ + if not value.tzinfo: + value = value.replace(tzinfo=UTC) + # Regardless of what timezone is on the value, convert it to UTC. + value = value.astimezone(UTC) + # Convert the datetime to a microsecond timestamp. + return int(calendar.timegm(value.timetuple()) * 1e6) + value.microsecond + + def _millis_from_datetime(value): """Convert non-none datetime to timestamp, assuming UTC. @@ -271,10 +264,6 @@ def _millis_from_datetime(value): :returns: the timestamp, in milliseconds, or None """ if value is not None: - if value.tzinfo is None: - # Assume UTC - value = value.replace(tzinfo=UTC) - # back-end wants timestamps as milliseconds since the epoch return _millis(value) @@ -285,9 +274,3 @@ def _millis_from_datetime(value): # Need to define _EPOCH at the end of module since it relies on UTC. _EPOCH = datetime.datetime.utcfromtimestamp(0).replace(tzinfo=UTC) - - -if sys.version_info[:2] < (2, 7): - _TOTAL_SECONDS = _manual_total_seconds # pragma: NO COVER -else: - _TOTAL_SECONDS = _total_seconds_from_type diff --git a/gcloud/credentials.py b/gcloud/credentials.py index 843937021eef..eef7624f311b 100644 --- a/gcloud/credentials.py +++ b/gcloud/credentials.py @@ -15,7 +15,6 @@ """A simple wrapper around the OAuth2 credentials library.""" import base64 -import calendar import datetime import six from six.moves.urllib.parse import urlencode # pylint: disable=F0401 @@ -40,6 +39,7 @@ class _GAECreds(object): """Dummy class if not in App Engine environment.""" from gcloud._helpers import UTC +from gcloud._helpers import _microseconds_from_datetime def get_credentials(): @@ -280,15 +280,8 @@ def _get_expiration_seconds(expiration): # If it's a datetime, convert to a timestamp. if isinstance(expiration, datetime.datetime): - # Make sure the timezone on the value is UTC - # (either by converting or replacing the value). - if expiration.tzinfo: - expiration = expiration.astimezone(UTC) - else: - expiration = expiration.replace(tzinfo=UTC) - - # Turn the datetime into a timestamp (seconds, not microseconds). - expiration = int(calendar.timegm(expiration.timetuple())) + micros = _microseconds_from_datetime(expiration) + expiration, _ = divmod(micros, 10**6) if not isinstance(expiration, six.integer_types): raise TypeError('Expected an integer timestamp, datetime, or ' diff --git a/gcloud/datastore/helpers.py b/gcloud/datastore/helpers.py index ccf428c83a47..aec26e1a0acd 100644 --- a/gcloud/datastore/helpers.py +++ b/gcloud/datastore/helpers.py @@ -17,13 +17,13 @@ The non-private functions are part of the API. """ -import calendar import datetime from google.protobuf.internal.type_checkers import Int64ValueChecker import six from gcloud._helpers import UTC +from gcloud._helpers import _microseconds_from_datetime from gcloud.datastore import _datastore_v1_pb2 as datastore_pb from gcloud.datastore.entity import Entity from gcloud.datastore.key import Key @@ -182,14 +182,7 @@ def _pb_attr_value(val): if isinstance(val, datetime.datetime): name = 'timestamp_microseconds' - # If the datetime is naive (no timezone), consider that it was - # intended to be UTC and replace the tzinfo to that effect. - if not val.tzinfo: - val = val.replace(tzinfo=UTC) - # Regardless of what timezone is on the value, convert it to UTC. - val = val.astimezone(UTC) - # Convert the datetime to a microsecond timestamp. - value = int(calendar.timegm(val.timetuple()) * 1e6) + val.microsecond + value = _microseconds_from_datetime(val) elif isinstance(val, Key): name, value = 'key', val.to_protobuf() elif isinstance(val, bool): diff --git a/gcloud/test__helpers.py b/gcloud/test__helpers.py index 6b46fe907068..0b8dec4fcba4 100644 --- a/gcloud/test__helpers.py +++ b/gcloud/test__helpers.py @@ -251,41 +251,6 @@ def test_prod(self): self.assertEqual(callers, ['prod_mock']) -class Test__manual_total_seconds(unittest2.TestCase): - - def _callFUT(self, value): - from gcloud._helpers import _manual_total_seconds - return _manual_total_seconds(value) - - def test_it(self): - import datetime - - delta = datetime.timedelta(minutes=1, seconds=1, - microseconds=10**(6 - 1)) - result = self._callFUT(delta) - self.assertEqual(result, 61.1) - - -class Test__total_seconds_from_type(unittest2.TestCase): - - def _callFUT(self, value): - from gcloud._helpers import _total_seconds_from_type - return _total_seconds_from_type(value) - - def test_it(self): - class FakeDelta(object): - - def __init__(self, total_seconds): - self._total_seconds = total_seconds - - def total_seconds(self): - return self._total_seconds - - value = object() - fake_delta = FakeDelta(value) - self.assertEqual(self._callFUT(fake_delta), value) - - class Test__millis(unittest2.TestCase): def _callFUT(self, value): @@ -300,25 +265,21 @@ def test_one_second_from_epoch(self): self.assertEqual(self._callFUT(WHEN), 1000) -class Test__datetime_from_millis(unittest2.TestCase): +class Test__microseconds_from_datetime(unittest2.TestCase): def _callFUT(self, value): - from gcloud._helpers import _datetime_from_millis - return _datetime_from_millis(value) - - def test_w_none(self): - self.assertTrue(self._callFUT(None) is None) + from gcloud._helpers import _microseconds_from_datetime + return _microseconds_from_datetime(value) - def test_w_millis(self): + def test_it(self): import datetime - from gcloud._helpers import UTC - from gcloud._helpers import _TOTAL_SECONDS - NOW = datetime.datetime(2015, 7, 29, 17, 45, 21, 123456, - tzinfo=UTC) - EPOCH = datetime.datetime(1970, 1, 1, tzinfo=UTC) - MILLIS = _TOTAL_SECONDS(NOW - EPOCH) * 1000 - self.assertEqual(self._callFUT(MILLIS), NOW) + microseconds = 314159 + timestamp = datetime.datetime(1970, 1, 1, hour=0, + minute=0, second=0, + microsecond=microseconds) + result = self._callFUT(timestamp) + self.assertEqual(result, microseconds) class Test__millis_from_datetime(unittest2.TestCase): @@ -333,20 +294,19 @@ def test_w_none(self): def test_w_utc_datetime(self): import datetime from gcloud._helpers import UTC - from gcloud._helpers import _TOTAL_SECONDS + from gcloud._helpers import _microseconds_from_datetime NOW = datetime.datetime.utcnow().replace(tzinfo=UTC) - EPOCH = datetime.datetime(1970, 1, 1, tzinfo=UTC) - MILLIS = int(_TOTAL_SECONDS(NOW - EPOCH) * 1000) + NOW_MICROS = _microseconds_from_datetime(NOW) + MILLIS, _ = divmod(NOW_MICROS, 1000) result = self._callFUT(NOW) self.assertTrue(isinstance(result, int)) self.assertEqual(result, MILLIS) def test_w_non_utc_datetime(self): import datetime - from gcloud._helpers import UTC from gcloud._helpers import _UTC - from gcloud._helpers import _TOTAL_SECONDS + from gcloud._helpers import _microseconds_from_datetime class CET(_UTC): _tzname = 'CET' @@ -354,8 +314,8 @@ class CET(_UTC): zone = CET() NOW = datetime.datetime(2015, 7, 28, 16, 34, 47, tzinfo=zone) - EPOCH = datetime.datetime(1970, 1, 1, tzinfo=UTC) - MILLIS = int(_TOTAL_SECONDS(NOW - EPOCH) * 1000) + NOW_MICROS = _microseconds_from_datetime(NOW) + MILLIS, _ = divmod(NOW_MICROS, 1000) result = self._callFUT(NOW) self.assertTrue(isinstance(result, int)) self.assertEqual(result, MILLIS) @@ -363,17 +323,38 @@ class CET(_UTC): def test_w_naive_datetime(self): import datetime from gcloud._helpers import UTC - from gcloud._helpers import _TOTAL_SECONDS + from gcloud._helpers import _microseconds_from_datetime NOW = datetime.datetime.utcnow() UTC_NOW = NOW.replace(tzinfo=UTC) - EPOCH = datetime.datetime(1970, 1, 1, tzinfo=UTC) - MILLIS = int(_TOTAL_SECONDS(UTC_NOW - EPOCH) * 1000) + UTC_NOW_MICROS = _microseconds_from_datetime(UTC_NOW) + MILLIS, _ = divmod(UTC_NOW_MICROS, 1000) result = self._callFUT(NOW) self.assertTrue(isinstance(result, int)) self.assertEqual(result, MILLIS) +class Test__datetime_from_millis(unittest2.TestCase): + + def _callFUT(self, value): + from gcloud._helpers import _datetime_from_millis + return _datetime_from_millis(value) + + def test_w_none(self): + self.assertTrue(self._callFUT(None) is None) + + def test_w_millis(self): + import datetime + from gcloud._helpers import UTC + from gcloud._helpers import _microseconds_from_datetime + + NOW = datetime.datetime(2015, 7, 29, 17, 45, 21, 123456, + tzinfo=UTC) + NOW_MICROS = _microseconds_from_datetime(NOW) + MILLIS = NOW_MICROS / 1000.0 + self.assertEqual(self._callFUT(MILLIS), NOW) + + class _AppIdentity(object): def __init__(self, app_id): From 41e64395661a31af8352fb2358e5c95ff5437830 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 11 Aug 2015 13:33:55 -0700 Subject: [PATCH 6/8] Converting _datetime_from_millis to handle microseconds. Also making it require a float (or int) instead of bailing out for NoneType. Changing the use of the method in bigquery to not pass potentially null values and to multiply by 1000.0 (convert from millis to micros). Also updating micros -> datetime conversion in datastore to use the newly converted method. --- gcloud/_helpers.py | 19 ++++++++----------- gcloud/bigquery/dataset.py | 12 +++++++++--- gcloud/bigquery/table.py | 20 +++++++++++++++----- gcloud/datastore/helpers.py | 8 +++----- gcloud/test__helpers.py | 14 +++++--------- 5 files changed, 40 insertions(+), 33 deletions(-) diff --git a/gcloud/_helpers.py b/gcloud/_helpers.py index 5b52f8e6668b..83618911b3c9 100644 --- a/gcloud/_helpers.py +++ b/gcloud/_helpers.py @@ -222,19 +222,16 @@ def _millis(when): return millis -def _datetime_from_millis(value): - """Convert non-none timestamp to datetime, assuming UTC. +def _datetime_from_microseconds(value): + """Convert timestamp to datetime, assuming UTC. - :rtype: :class:`datetime.datetime`, or ``NoneType`` + :type value: float + :param value: The timestamp to convert + + :rtype: :class:`datetime.datetime` + :returns: The datetime object created from the value. """ - if value is not None: - # back-end returns timestamps as milliseconds since the epoch - seconds = int(value / 1000.0) - microseconds = 1000.0 * (value - 1000 * seconds) - return ( - _EPOCH + - datetime.timedelta(seconds=seconds, microseconds=microseconds) - ) + return _EPOCH + datetime.timedelta(microseconds=value) def _microseconds_from_datetime(value): diff --git a/gcloud/bigquery/dataset.py b/gcloud/bigquery/dataset.py index 417eb17b1c60..82d60c6dfec2 100644 --- a/gcloud/bigquery/dataset.py +++ b/gcloud/bigquery/dataset.py @@ -15,7 +15,7 @@ """Define API Datasets.""" import six -from gcloud._helpers import _datetime_from_millis +from gcloud._helpers import _datetime_from_microseconds from gcloud.exceptions import NotFound from gcloud.bigquery.table import Table @@ -114,7 +114,10 @@ def created(self): :rtype: ``datetime.datetime``, or ``NoneType`` :returns: the creation time (None until set from the server). """ - return _datetime_from_millis(self._properties.get('creationTime')) + creation_time = self._properties.get('creationTime') + if creation_time is not None: + # creation_time will be in milliseconds. + return _datetime_from_microseconds(1000.0 * creation_time) @property def dataset_id(self): @@ -141,7 +144,10 @@ def modified(self): :rtype: ``datetime.datetime``, or ``NoneType`` :returns: the modification time (None until set from the server). """ - return _datetime_from_millis(self._properties.get('lastModifiedTime')) + modified_time = self._properties.get('lastModifiedTime') + if modified_time is not None: + # modified_time will be in milliseconds. + return _datetime_from_microseconds(1000.0 * modified_time) @property def self_link(self): diff --git a/gcloud/bigquery/table.py b/gcloud/bigquery/table.py index d69e718cc3c6..6f1c09c2ad99 100644 --- a/gcloud/bigquery/table.py +++ b/gcloud/bigquery/table.py @@ -18,7 +18,7 @@ import six -from gcloud._helpers import _datetime_from_millis +from gcloud._helpers import _datetime_from_microseconds from gcloud._helpers import _millis_from_datetime from gcloud.exceptions import NotFound @@ -116,7 +116,10 @@ def created(self): :rtype: ``datetime.datetime``, or ``NoneType`` :returns: the creation time (None until set from the server). """ - return _datetime_from_millis(self._properties.get('creationTime')) + creation_time = self._properties.get('creationTime') + if creation_time is not None: + # creation_time will be in milliseconds. + return _datetime_from_microseconds(1000.0 * creation_time) @property def etag(self): @@ -134,7 +137,10 @@ def modified(self): :rtype: ``datetime.datetime``, or ``NoneType`` :returns: the modification time (None until set from the server). """ - return _datetime_from_millis(self._properties.get('lastModifiedTime')) + modified_time = self._properties.get('lastModifiedTime') + if modified_time is not None: + # modified_time will be in milliseconds. + return _datetime_from_microseconds(1000.0 * modified_time) @property def num_bytes(self): @@ -212,7 +218,10 @@ def expires(self): :rtype: ``datetime.datetime``, or ``NoneType`` :returns: the expiration time, or None """ - return _datetime_from_millis(self._properties.get('expirationTime')) + expiration_time = self._properties.get('expirationTime') + if expiration_time is not None: + # expiration_time will be in milliseconds. + return _datetime_from_microseconds(1000.0 * expiration_time) @expires.setter def expires(self, value): @@ -727,7 +736,8 @@ def _bool_from_json(value, field): def _datetime_from_json(value, field): if _not_null(value, field): - return _datetime_from_millis(float(value)) + # Field value will be in milliseconds. + return _datetime_from_microseconds(1000.0 * float(value)) def _record_from_json(value, field): diff --git a/gcloud/datastore/helpers.py b/gcloud/datastore/helpers.py index aec26e1a0acd..427df84e726f 100644 --- a/gcloud/datastore/helpers.py +++ b/gcloud/datastore/helpers.py @@ -22,7 +22,7 @@ from google.protobuf.internal.type_checkers import Int64ValueChecker import six -from gcloud._helpers import UTC +from gcloud._helpers import _datetime_from_microseconds from gcloud._helpers import _microseconds_from_datetime from gcloud.datastore import _datastore_v1_pb2 as datastore_pb from gcloud.datastore.entity import Entity @@ -224,9 +224,7 @@ def _get_value_from_value_pb(value_pb): result = None if value_pb.HasField('timestamp_microseconds_value'): microseconds = value_pb.timestamp_microseconds_value - naive = (datetime.datetime.utcfromtimestamp(0) + - datetime.timedelta(microseconds=microseconds)) - result = naive.replace(tzinfo=UTC) + result = _datetime_from_microseconds(microseconds) elif value_pb.HasField('key_value'): result = key_from_protobuf(value_pb.key_value) @@ -285,7 +283,7 @@ def _set_protobuf_value(value_pb, val): :type value_pb: :class:`gcloud.datastore._datastore_v1_pb2.Value` :param value_pb: The value protobuf to which the value is being assigned. - :type val: `datetime.datetime`, boolean, float, integer, string, + :type val: :class:`datetime.datetime`, boolean, float, integer, string, :class:`gcloud.datastore.key.Key`, :class:`gcloud.datastore.entity.Entity`, :param val: The value to be assigned. diff --git a/gcloud/test__helpers.py b/gcloud/test__helpers.py index 0b8dec4fcba4..6c4dca31a3a9 100644 --- a/gcloud/test__helpers.py +++ b/gcloud/test__helpers.py @@ -334,16 +334,13 @@ def test_w_naive_datetime(self): self.assertEqual(result, MILLIS) -class Test__datetime_from_millis(unittest2.TestCase): +class Test__datetime_from_microseconds(unittest2.TestCase): def _callFUT(self, value): - from gcloud._helpers import _datetime_from_millis - return _datetime_from_millis(value) + from gcloud._helpers import _datetime_from_microseconds + return _datetime_from_microseconds(value) - def test_w_none(self): - self.assertTrue(self._callFUT(None) is None) - - def test_w_millis(self): + def test_it(self): import datetime from gcloud._helpers import UTC from gcloud._helpers import _microseconds_from_datetime @@ -351,8 +348,7 @@ def test_w_millis(self): NOW = datetime.datetime(2015, 7, 29, 17, 45, 21, 123456, tzinfo=UTC) NOW_MICROS = _microseconds_from_datetime(NOW) - MILLIS = NOW_MICROS / 1000.0 - self.assertEqual(self._callFUT(MILLIS), NOW) + self.assertEqual(self._callFUT(NOW_MICROS), NOW) class _AppIdentity(object): From 4cb166edf01d6502522d91edfd05faef31cbda41 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 11 Aug 2015 13:45:31 -0700 Subject: [PATCH 7/8] Unifying use of and mocking out of utcnow. --- gcloud/_helpers.py | 1 + gcloud/credentials.py | 11 ++--------- gcloud/pubsub/topic.py | 4 +--- gcloud/test_credentials.py | 4 ++-- 4 files changed, 6 insertions(+), 14 deletions(-) diff --git a/gcloud/_helpers.py b/gcloud/_helpers.py index 83618911b3c9..2d7adc7494c8 100644 --- a/gcloud/_helpers.py +++ b/gcloud/_helpers.py @@ -37,6 +37,7 @@ class Local(object): from gcloud.environment_vars import PROJECT +_NOW = datetime.datetime.utcnow # To be replaced by tests. _RFC3339_MICROS = '%Y-%m-%dT%H:%M:%S.%fZ' diff --git a/gcloud/credentials.py b/gcloud/credentials.py index eef7624f311b..772b7b7b8c25 100644 --- a/gcloud/credentials.py +++ b/gcloud/credentials.py @@ -39,6 +39,7 @@ class _GAECreds(object): """Dummy class if not in App Engine environment.""" from gcloud._helpers import UTC +from gcloud._helpers import _NOW from gcloud._helpers import _microseconds_from_datetime @@ -256,14 +257,6 @@ def _get_signed_query_params(credentials, expiration, string_to_sign): } -def _utcnow(): # pragma: NO COVER testing replaces - """Returns current time as UTC datetime. - - NOTE: on the module namespace so tests can replace it. - """ - return datetime.datetime.utcnow() - - def _get_expiration_seconds(expiration): """Convert 'expiration' to a number of seconds in the future. @@ -275,7 +268,7 @@ def _get_expiration_seconds(expiration): """ # If it's a timedelta, add it to `now` in UTC. if isinstance(expiration, datetime.timedelta): - now = _utcnow().replace(tzinfo=UTC) + now = _NOW().replace(tzinfo=UTC) expiration = now + expiration # If it's a datetime, convert to a timestamp. diff --git a/gcloud/pubsub/topic.py b/gcloud/pubsub/topic.py index 78582c5a7aea..6ac05ee243f0 100644 --- a/gcloud/pubsub/topic.py +++ b/gcloud/pubsub/topic.py @@ -15,15 +15,13 @@ """Define API Topics.""" import base64 -import datetime +from gcloud._helpers import _NOW from gcloud._helpers import _RFC3339_MICROS from gcloud.exceptions import NotFound from gcloud.pubsub._helpers import topic_name_from_path from gcloud.pubsub.subscription import Subscription -_NOW = datetime.datetime.utcnow - class Topic(object): """Topics are targets to which messages can be published. diff --git a/gcloud/test_credentials.py b/gcloud/test_credentials.py index 33987f5db74c..06d4d50afb52 100644 --- a/gcloud/test_credentials.py +++ b/gcloud/test_credentials.py @@ -541,7 +541,7 @@ def test_w_timedelta_seconds(self): utc_seconds = self._utc_seconds(dummy_utcnow) expiration_as_delta = datetime.timedelta(seconds=10) - with _Monkey(MUT, _utcnow=lambda: dummy_utcnow): + with _Monkey(MUT, _NOW=lambda: dummy_utcnow): result = self._callFUT(expiration_as_delta) self.assertEqual(result, utc_seconds + 10) @@ -555,7 +555,7 @@ def test_w_timedelta_days(self): utc_seconds = self._utc_seconds(dummy_utcnow) expiration_as_delta = datetime.timedelta(days=1) - with _Monkey(MUT, _utcnow=lambda: dummy_utcnow): + with _Monkey(MUT, _NOW=lambda: dummy_utcnow): result = self._callFUT(expiration_as_delta) self.assertEqual(result, utc_seconds + 86400) From cb5e0d08b3d8a1d59f1d8c7d9b155253d1f42fba Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Wed, 12 Aug 2015 11:27:35 -0700 Subject: [PATCH 8/8] Replacing uses of divmod with // in datetime code. --- gcloud/_helpers.py | 3 +-- gcloud/credentials.py | 2 +- gcloud/test__helpers.py | 6 +++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/gcloud/_helpers.py b/gcloud/_helpers.py index 2d7adc7494c8..664c867b1c61 100644 --- a/gcloud/_helpers.py +++ b/gcloud/_helpers.py @@ -219,8 +219,7 @@ def _millis(when): :returns: milliseconds since epoch for ``when`` """ micros = _microseconds_from_datetime(when) - millis, _ = divmod(micros, 1000) - return millis + return micros // 1000 def _datetime_from_microseconds(value): diff --git a/gcloud/credentials.py b/gcloud/credentials.py index 772b7b7b8c25..56eca8fc6612 100644 --- a/gcloud/credentials.py +++ b/gcloud/credentials.py @@ -274,7 +274,7 @@ def _get_expiration_seconds(expiration): # If it's a datetime, convert to a timestamp. if isinstance(expiration, datetime.datetime): micros = _microseconds_from_datetime(expiration) - expiration, _ = divmod(micros, 10**6) + expiration = micros // 10**6 if not isinstance(expiration, six.integer_types): raise TypeError('Expected an integer timestamp, datetime, or ' diff --git a/gcloud/test__helpers.py b/gcloud/test__helpers.py index 6c4dca31a3a9..e248d687220d 100644 --- a/gcloud/test__helpers.py +++ b/gcloud/test__helpers.py @@ -298,7 +298,7 @@ def test_w_utc_datetime(self): NOW = datetime.datetime.utcnow().replace(tzinfo=UTC) NOW_MICROS = _microseconds_from_datetime(NOW) - MILLIS, _ = divmod(NOW_MICROS, 1000) + MILLIS = NOW_MICROS // 1000 result = self._callFUT(NOW) self.assertTrue(isinstance(result, int)) self.assertEqual(result, MILLIS) @@ -315,7 +315,7 @@ class CET(_UTC): zone = CET() NOW = datetime.datetime(2015, 7, 28, 16, 34, 47, tzinfo=zone) NOW_MICROS = _microseconds_from_datetime(NOW) - MILLIS, _ = divmod(NOW_MICROS, 1000) + MILLIS = NOW_MICROS // 1000 result = self._callFUT(NOW) self.assertTrue(isinstance(result, int)) self.assertEqual(result, MILLIS) @@ -328,7 +328,7 @@ def test_w_naive_datetime(self): NOW = datetime.datetime.utcnow() UTC_NOW = NOW.replace(tzinfo=UTC) UTC_NOW_MICROS = _microseconds_from_datetime(UTC_NOW) - MILLIS, _ = divmod(UTC_NOW_MICROS, 1000) + MILLIS = UTC_NOW_MICROS // 1000 result = self._callFUT(NOW) self.assertTrue(isinstance(result, int)) self.assertEqual(result, MILLIS)