Introduce Datetime type for ranges outside datetime.[MIN|MAX}YEAR

In Python `datetime.datetime` type year has to be in range
[MINYEAR, MAXYEAR]. This range is not the same as possible timestamps
in scylla. Previously if timestamp was outside this range it made
driver raise an Exception. It was not correct behavior. There was a
work around implemented in cqlsh.

This commit introduces a `Datetime` type to accommodate ranges outside
datetime.[MIN|MAX]YEAR. For Datetimes that cannot be represented as
a datetime.datetime (because datetime.MINYEAR, datetime.MAXYEAR),
this type falls back to printing milliseconds_from_epoch offset.

Fixes: #255

Author: sylwiaszunejko

cassandra/cqlengine/columns.py

@@ -22,7 +22,7 @@
 from cassandra.cqltypes import SimpleDateType, _cqltypes, UserType
 from cassandra.cqlengine import ValidationError
 from cassandra.cqlengine.functions import get_total_seconds
-from cassandra.util import Duration as _Duration
+from cassandra.util import Datetime, Duration as _Duration
 
 log = logging.getLogger(__name__)
 
@@ -542,17 +542,19 @@ class DateTime(Column):
     def to_python(self, value):
         if value is None:
             return
+        elif isinstance(value, Datetime):
+            return value
         if isinstance(value, datetime):
             if DateTime.truncate_microseconds:
                 us = value.microsecond
                 truncated_us = us // 1000 * 1000
-                return value - timedelta(microseconds=us - truncated_us)
+                return Datetime(value - timedelta(microseconds=us - truncated_us))
             else:
-                return value
+                return Datetime(value)
         elif isinstance(value, date):
-            return datetime(*(value.timetuple()[:6]))
+            return Datetime(datetime(*(value.timetuple()[:6])))
 
-        return datetime.utcfromtimestamp(value)
+        return Datetime(datetime.utcfromtimestamp(value))
 
     def to_database(self, value):
         value = super(DateTime, self).to_database(value)
@@ -561,6 +563,11 @@ def to_database(self, value):
         if not isinstance(value, datetime):
             if isinstance(value, date):
                 value = datetime(value.year, value.month, value.day)
+            elif isinstance(value, Datetime):
+                try:
+                    value = value.datetime()
+                except ValueError:
+                    return int(value.milliseconds_from_epoch)
             else:
                 raise ValidationError("{0} '{1}' is not a datetime object".format(self.column_name, value))
         epoch = datetime(1970, 1, 1, tzinfo=value.tzinfo)

cassandra/cqltypes.py

@@ -32,6 +32,7 @@
 from binascii import unhexlify
 import calendar
 from collections import namedtuple
+import datetime
 from decimal import Decimal
 import io
 from itertools import chain
@@ -642,17 +643,16 @@ def interpret_datestring(val):
     @staticmethod
     def deserialize(byts, protocol_version):
         timestamp = int64_unpack(byts) / 1000.0
-        return util.datetime_from_timestamp(timestamp)
+        return util.Datetime(util.Datetime(util.DATETIME_EPOC) + datetime.timedelta(seconds=timestamp))
 
     @staticmethod
     def serialize(v, protocol_version):
         try:
-            # v is datetime
-            timestamp_seconds = calendar.timegm(v.utctimetuple())
-            timestamp = timestamp_seconds * 1e3 + getattr(v, 'microsecond', 0) / 1e3
+            # v is Datetime
+            timestamp = v.milliseconds_from_epoch
         except AttributeError:
             try:
-                timestamp = calendar.timegm(v.timetuple()) * 1e3
+                timestamp = util.Datetime(v).milliseconds_from_epoch
             except AttributeError:
                 # Ints and floats are valid timestamps too
                 if type(v) not in _number_types:

cassandra/deserializers.pyx

@@ -17,7 +17,7 @@ from libc.stdint cimport int32_t, uint16_t
 
 include 'cython_marshal.pyx'
 from cassandra.buffer cimport Buffer, to_bytes, slice_buffer
-from cassandra.cython_utils cimport datetime_from_timestamp
+from cassandra.cython_utils cimport datetime_from_timestamp, DATETIME_EPOC
 
 from cython.view cimport array as cython_array
 from cassandra.tuple cimport tuple_new, tuple_set
@@ -140,7 +140,7 @@ cdef class DesCounterColumnType(DesLongType):
 cdef class DesDateType(Deserializer):
     cdef deserialize(self, Buffer *buf, int protocol_version):
         cdef double timestamp = unpack_num[int64_t](buf) / 1000.0
-        return datetime_from_timestamp(timestamp)
+        return util.Datetime(util.Datetime(DATETIME_EPOC) + datetime.timedelta(seconds=timestamp))
 
 
 cdef class TimestampType(DesDateType):

cassandra/encoder.py

@@ -29,7 +29,7 @@
 from uuid import UUID
 import six
 
-from cassandra.util import (OrderedDict, OrderedMap, OrderedMapSerializedKey,
+from cassandra.util import (Datetime, OrderedDict, OrderedMap, OrderedMapSerializedKey,
                             sortedset, Time, Date, Point, LineString, Polygon)
 
 if six.PY3:
@@ -80,6 +80,7 @@ def __init__(self):
             datetime.date: self.cql_encode_date,
             datetime.time: self.cql_encode_time,
             Date: self.cql_encode_date_ext,
+            Datetime: self.cql_encode_datetime_ext,
             Time: self.cql_encode_time,
             dict: self.cql_encode_map_collection,
             OrderedDict: self.cql_encode_map_collection,
@@ -171,6 +172,13 @@ def cql_encode_datetime(self, val):
         timestamp = calendar.timegm(val.utctimetuple())
         return str(long(timestamp * 1e3 + getattr(val, 'microsecond', 0) / 1e3))
 
+    def cql_encode_datetime_ext(self, val):
+        """
+        Encodes a :class:`cassandra.util.Datetime` object as an integer
+        """
+        # using the int form in case the Datetime exceeds datetime.[MIN|MAX]YEAR
+        return str(val.milliseconds_from_epoch)
+
     def cql_encode_date(self, val):
         """
         Converts a :class:`datetime.date` object to a string with format

cassandra/util.py

@@ -1150,6 +1150,131 @@ def __str__(self):
         except:
             # If we overflow datetime.[MIN|MAX]
             return str(self.days_from_epoch)
+        
+
+@total_ordering
+class Datetime(object):
+    '''
+    Idealized datetime: year, month, day, hour, minute, second, microsecond, and tzinfo
+
+    Offers wider year range than datetime.datetime. For Datetimess that cannot be represented
+    as a datetime.datetime (because datetime.MINYEAR, datetime.MAXYEAR), this type falls back
+    to printing milliseconds_from_epoch offset.
+    '''
+    
+    MICRO = 1000
+    MILLI = 1000 * MICRO
+    SECOND = 1000 * MILLI
+    MINUTE = 60
+    HOUR = 60 * MINUTE
+    DAY = 24 * HOUR
+
+    milliseconds_from_epoch = 0
+    tzinfo = None
+
+    def __init__(self, value):
+        """
+        Initializer value can be:
+
+        - number_type: milliseconds from epoch (1970, 1, 1). Can be negative.
+        - datetime.datetime: built-in datetime
+        """
+        if isinstance(value, (int, long, float)):
+            self.milliseconds_from_epoch = value
+        elif isinstance(value, datetime.datetime):
+            self._from_datetime(value)
+        elif isinstance(value, datetime.date):
+            self._from_timetuple(value.timetuple())
+        elif isinstance(value, Datetime):
+            self.milliseconds_from_epoch = value.milliseconds_from_epoch
+            self.tzinfo = value.tzinfo
+        else:
+            raise TypeError('Date arguments must be a whole number or datetime.datetime')
+
+    @property
+    def seconds(self):
+        """
+        Absolute seconds from epoch (can be negative)
+        """
+        return self.milliseconds_from_epoch // 1000
+     
+    @property
+    def days(self):
+        """
+        Absolute days from epoch (can be negative)
+        """
+        return self.seconds // Date.DAY
+
+    def datetime(self):
+        """
+        Return a built-in datetime.datetime for Dates falling in the years [datetime.MINYEAR, datetime.MAXYEAR]
+
+        ValueError is raised for Dates outside this range.
+        """
+        try:
+            dt = datetime.datetime(1970, 1, 1, tzinfo=self.tzinfo) + datetime.timedelta(milliseconds=self.milliseconds_from_epoch)
+            return dt
+        except Exception:
+            raise ValueError("%r exceeds ranges for built-in datetime.datetime" % self)
+        
+    def utctimetuple(self):
+        return self.datetime().utctimetuple()
+    
+    def timetuple(self):
+        return self.datetime().timetuple()
+
+    def isoformat(self, sep='T', timespec='auto'):
+        return self.datetime().isoformat(sep, timespec)
+
+    def _from_timetuple(self, t):
+        self.milliseconds_from_epoch = calendar.timegm(t) * 1000
+
+    def _from_datetime(self, v):
+        self.milliseconds_from_epoch = calendar.timegm(v.timetuple()) * 1000 + v.microsecond // 1000
+        self.tzinfo = v.tzinfo
+
+    def __hash__(self):
+        return self.milliseconds_from_epoch
+
+    def __eq__(self, other):
+        if isinstance(other, Datetime):
+            return self.milliseconds_from_epoch == other.milliseconds_from_epoch
+
+        if isinstance(other, (int, long, float)):
+            return self.milliseconds_from_epoch == other
+
+        try:
+            return self.datetime() == other
+        except Exception:
+            return False
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __lt__(self, other):
+        if not isinstance(other, Datetime):
+            return NotImplemented
+        return self.milliseconds_from_epoch < other.milliseconds_from_epoch
+
+    def __add__(self, other):
+        if isinstance(other, datetime.timedelta):
+            return Datetime(int(self.milliseconds_from_epoch + other.total_seconds() * 1000))
+        return self + other
+    
+    def __sub__(self, other):
+        if isinstance(other, Datetime):
+            return datetime.timedelta(milliseconds=self.milliseconds_from_epoch - other.milliseconds_from_epoch)
+        return self - other
+
+    def __repr__(self):
+        return "Datetime(%s)" % self.milliseconds_from_epoch
+
+    def __str__(self):
+        try:
+            dt = datetime.datetime(1970, 1, 1, tzinfo=self.tzinfo) + datetime.timedelta(milliseconds=self.milliseconds_from_epoch)
+            return "%04d-%02d-%02d %02d:%02d:%02d.%09d" % (dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, dt.microsecond)
+        except:
+            return str(self.milliseconds_from_epoch)
 
 import socket
 if hasattr(socket, 'inet_pton'):

tests/integration/cqlengine/columns/test_container_columns.py

@@ -26,6 +26,7 @@
 from cassandra.cqlengine.functions import get_total_seconds
 from cassandra.cqlengine.models import Model, ValidationError
 from cassandra.cqlengine.management import sync_table, drop_table
+from cassandra.util import Datetime
 
 from tests.integration import CASSANDRA_IP
 from tests.integration.cqlengine import is_prepend_reversed
@@ -390,7 +391,7 @@ def test_io_success(self):
         """ Tests that a basic usage works as expected """
         k1 = uuid4()
         k2 = uuid4()
-        now = datetime.now()
+        now = Datetime(datetime.now())
         then = now + timedelta(days=1)
         m1 = TestMapModel.create(int_map={1: k1, 2: k2},
                                  text_map={'now': now, 'then': then})

tests/integration/cqlengine/columns/test_validation.py

@@ -97,7 +97,7 @@ def test_datetime_timestamp(self):
         dt_value = 1454520554
         self.DatetimeTest.objects.create(test_id=5, created_at=dt_value)
         dt2 = self.DatetimeTest.objects(test_id=5).first()
-        self.assertEqual(dt2.created_at, datetime.utcfromtimestamp(dt_value))
+        self.assertEqual(dt2.created_at, util.Datetime(datetime.utcfromtimestamp(dt_value)))
 
     def test_datetime_large(self):
         dt_value = datetime(2038, 12, 31, 10, 10, 10, 123000)
@@ -318,12 +318,12 @@ class TestDateTime(DataType, BaseCassEngTestCase):
     def setUpClass(cls):
         cls.db_klass, cls.python_klass = (
             DateTime,
-            datetime
+            util.Datetime
         )
         cls.first_value, cls.second_value, cls.third_value = (
-            datetime(2017, 4, 13, 18, 34, 24, 317000),
-            datetime(1, 1, 1),
-            datetime(1, 1, 2)
+            util.Datetime(datetime(2017, 4, 13, 18, 34, 24, 317000)),
+            util.Datetime(datetime(1, 1, 1)),
+            util.Datetime(datetime(1, 1, 2))
         )
         super(TestDateTime, cls).setUpClass()
 

tests/integration/cqlengine/query/test_datetime_queries.py

@@ -15,6 +15,7 @@
 from datetime import datetime, timedelta
 from uuid import uuid4
 from cassandra.cqlengine.functions import get_total_seconds
+from cassandra.util import Datetime
 
 from tests.integration.cqlengine.base import BaseCassEngTestCase
 
@@ -65,7 +66,7 @@ def test_range_query(self):
     @execute_count(3)
     def test_datetime_precision(self):
         """ Tests that millisecond resolution is preserved when saving datetime objects """
-        now = datetime.now()
+        now = Datetime(datetime.now())
         pk = 1000
         obj = DateTimeQueryTestModel.create(user=pk, day=now, data='energy cheese')
         load = DateTimeQueryTestModel.get(user=pk)

tests/integration/datatype_utils.py

@@ -17,7 +17,7 @@
 from uuid import uuid1, uuid4
 import six
 
-from cassandra.util import OrderedMap, Date, Time, sortedset, Duration
+from cassandra.util import Datetime, OrderedMap, Date, Time, sortedset, Duration
 
 from tests.integration import get_server_versions
 
@@ -104,7 +104,7 @@ def get_sample_data():
             sample_data[datatype] = 'text'
 
         elif datatype == 'timestamp':
-            sample_data[datatype] = datetime(2013, 12, 31, 23, 59, 59, 999000)
+            sample_data[datatype] = Datetime(datetime(2013, 12, 31, 23, 59, 59, 999000))
 
         elif datatype == 'timeuuid':
             sample_data[datatype] = uuid1()

tests/integration/standard/test_query.py

@@ -87,21 +87,6 @@ def test_trace_prints_okay(self):
         for event in trace.events:
             str(event)
 
-    def test_row_error_message(self):
-        """
-        Test to validate, new column deserialization message
-        @since 3.7.0
-        @jira_ticket PYTHON-361
-        @expected_result Special failed decoding message should be present
-
-        @test_category tracing
-        """
-        self.session.execute("CREATE TABLE {0}.{1} (k int PRIMARY KEY, v timestamp)".format(self.keyspace_name,self.function_table_name))
-        ss = SimpleStatement("INSERT INTO {0}.{1} (k, v) VALUES (1, 1000000000000000)".format(self.keyspace_name, self.function_table_name))
-        self.session.execute(ss)
-        with self.assertRaises(DriverException) as context:
-            self.session.execute("SELECT * FROM {0}.{1}".format(self.keyspace_name, self.function_table_name))
-        self.assertIn("Failed decoding result column", str(context.exception))
 
     def test_trace_id_to_resultset(self):
 

tests/unit/cython/types_testhelper.pyx

@@ -23,6 +23,7 @@ import io
 from cassandra.cqltypes import DateType
 from cassandra.protocol import write_value
 from cassandra.deserializers import find_deserializer
+from cassandra.util import Datetime
 from cassandra.bytesio cimport BytesIOReader
 from cassandra.buffer cimport Buffer
 from cassandra.deserializers cimport from_binary, Deserializer
@@ -38,7 +39,7 @@ def test_datetype(assert_equal):
         cdef BytesIOReader reader
         cdef Buffer buf
 
-        dt = datetime.datetime.utcfromtimestamp(timestamp)
+        dt = Datetime(datetime.datetime.utcfromtimestamp(timestamp))
 
         bytes = io.BytesIO()
         write_value(bytes, DateType.serialize(dt, 0))
@@ -89,7 +90,7 @@ def test_date_side_by_side(assert_equal):
         bior = BytesIOReader(blob)
         buf.ptr = bior.read()
         buf.size = bior.size
-        cython_deserialized = from_binary(cython_deserializer, &buf, 0)
+        cython_deserialized = Datetime(from_binary(cython_deserializer, &buf, 0))
         python_deserialized = DateType.deserialize(blob, 0)
         assert_equal(cython_deserialized, python_deserialized)