Skip to content

BUG: TimeSeries compat from < 0.13 #4755

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 6, 2013
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 62 additions & 21 deletions pandas/compat/pickle_compat.py
Original file line number Diff line number Diff line change
@@ -1,62 +1,103 @@
""" support pre 0.12 series pickle compatibility """

import sys
import pickle
import numpy as np
import pandas
import pickle as pkl
from pandas import compat
from pandas.core.series import Series
from pandas.sparse.series import SparseSeries
from pandas.compat import u, string_types
from pandas.core.series import Series, TimeSeries
from pandas.sparse.series import SparseSeries, SparseTimeSeries

def load_reduce(self):
stack = self.stack
args = stack.pop()
func = stack[-1]
if type(args[0]) is type:
n = args[0].__name__
if n == 'DeprecatedSeries':
if n == u('DeprecatedSeries') or n == u('DeprecatedTimeSeries'):
stack[-1] = object.__new__(Series)
return
elif n == 'DeprecatedSparseSeries':
elif n == u('DeprecatedSparseSeries') or n == u('DeprecatedSparseTimeSeries'):
stack[-1] = object.__new__(SparseSeries)
return

try:
value = func(*args)
except:
print(sys.exc_info())
print(func, args)

# try to reencode the arguments
if self.encoding is not None:
args = tuple([ arg.encode(self.encoding) if isinstance(arg, string_types) else arg for arg in args ])
try:
stack[-1] = func(*args)
return
except:
pass

if self.is_verbose:
print(sys.exc_info())
print(func, args)
raise

stack[-1] = value

if compat.PY3:
class Unpickler(pickle._Unpickler):
class Unpickler(pkl._Unpickler):
pass
else:
class Unpickler(pickle.Unpickler):
class Unpickler(pkl.Unpickler):
pass

Unpickler.dispatch[pickle.REDUCE[0]] = load_reduce
Unpickler.dispatch[pkl.REDUCE[0]] = load_reduce

def load(fh, encoding=None, compat=False, is_verbose=False):
"""
load a pickle, with a provided encoding

def load(file):
# try to load a compatibility pickle
# fake the old class hierarchy
# if it works, then return the new type objects
if compat is True:
fake the old class hierarchy
if it works, then return the new type objects

Parameters
----------
fh: a filelike object
encoding: an optional encoding
compat: provide Series compatibility mode, boolean, default False
is_verbose: show exception output
"""

try:
pandas.core.series.Series = DeprecatedSeries
pandas.sparse.series.SparseSeries = DeprecatedSparseSeries
with open(file,'rb') as fh:
return Unpickler(fh).load()
if compat:
pandas.core.series.Series = DeprecatedSeries
pandas.core.series.TimeSeries = DeprecatedTimeSeries
pandas.sparse.series.SparseSeries = DeprecatedSparseSeries
pandas.sparse.series.SparseTimeSeries = DeprecatedSparseTimeSeries
fh.seek(0)
if encoding is not None:
up = Unpickler(fh, encoding=encoding)
else:
up = Unpickler(fh)
up.is_verbose = is_verbose

return up.load()
except:
raise
finally:
pandas.core.series.Series = Series
pandas.sparse.series.SparseSeries = SparseSeries
if compat:
pandas.core.series.Series = Series
pandas.core.series.Series = TimeSeries
pandas.sparse.series.SparseSeries = SparseSeries
pandas.sparse.series.SparseTimeSeries = SparseTimeSeries

class DeprecatedSeries(Series, np.ndarray):
class DeprecatedSeries(np.ndarray, Series):
pass

class DeprecatedTimeSeries(DeprecatedSeries):
pass

class DeprecatedSparseSeries(DeprecatedSeries):
pass

class DeprecatedSparseTimeSeries(DeprecatedSparseSeries):
pass
5 changes: 5 additions & 0 deletions pandas/core/internals.py
Original file line number Diff line number Diff line change
Expand Up @@ -1845,6 +1845,11 @@ def __setstate__(self, state):

blocks = []
for values, items in zip(bvalues, bitems):

# numpy < 1.7 pickle compat
if values.dtype == 'M8[us]':
values = values.astype('M8[ns]')

blk = make_block(values, items, self.axes[0])
blocks.append(blk)
self.blocks = blocks
Expand Down
22 changes: 17 additions & 5 deletions pandas/io/pickle.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from pandas.compat import cPickle as pkl, PY3
from pandas.compat import cPickle as pkl, pickle_compat as pc, PY3

def to_pickle(obj, path):
"""
Expand Down Expand Up @@ -31,11 +31,23 @@ def read_pickle(path):
-------
unpickled : type of object stored in file
"""

def try_read(path, encoding=None):
# try with current pickle, if we have a Type Error then
# try with the compat pickle to handle subclass changes
# pass encoding only if its not None as py2 doesn't handle
# the param
try:
with open(path,'rb') as fh:
with open(path,'rb') as fh:
return pc.load(fh, encoding=encoding, compat=False)
except:
with open(path,'rb') as fh:
return pc.load(fh, encoding=encoding, compat=True)

try:
with open(path, 'rb') as fh:
return pkl.load(fh)
return try_read(path)
except:
if PY3:
with open(path, 'rb') as fh:
return pkl.load(fh, encoding='latin1')
return try_read(path, encoding='latin1')
raise
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
66 changes: 38 additions & 28 deletions pandas/io/tests/generate_legacy_pickles.py
Original file line number Diff line number Diff line change
@@ -1,26 +1,47 @@
""" self-contained to write legacy pickle files """
from __future__ import print_function

from pandas.compat import zip, cPickle as pickle
# make sure we are < 0.13 compat (in py3)
try:
from pandas.compat import zip, cPickle as pickle
except:
import pickle

def _create_sp_series():

import numpy as np
from pandas import bdate_range, SparseSeries
from pandas import SparseSeries

nan = np.nan

# nan-based
arr = np.arange(15, dtype=float)
arr = np.arange(15, dtype=np.float64)
index = np.arange(15)
arr[7:12] = nan
arr[-1:] = nan

date_index = bdate_range('1/1/2011', periods=len(index))
bseries = SparseSeries(arr, index=index, kind='block')
bseries = SparseSeries(arr, kind='block')
bseries.name = 'bseries'
return bseries

def _create_sp_tsseries():

import numpy as np
from pandas import bdate_range, SparseTimeSeries

nan = np.nan

# nan-based
arr = np.arange(15, dtype=np.float64)
index = np.arange(15)
arr[7:12] = nan
arr[-1:] = nan

date_index = bdate_range('1/1/2011', periods=len(index))
bseries = SparseTimeSeries(arr, index=date_index, kind='block')
bseries.name = 'btsseries'
return bseries

def _create_sp_frame():
import numpy as np
from pandas import bdate_range, SparseDataFrame
Expand All @@ -29,7 +50,7 @@ def _create_sp_frame():

data = {'A': [nan, nan, nan, 0, 1, 2, 3, 4, 5, 6],
'B': [0, 1, 2, nan, nan, nan, 3, 4, 5, 6],
'C': np.arange(10),
'C': np.arange(10).astype(np.int64),
'D': [0, 1, 2, 3, 4, 5, nan, nan, nan, nan]}

dates = bdate_range('1/1/2011', periods=10)
Expand All @@ -40,8 +61,8 @@ def create_data():

import numpy as np
import pandas
from pandas import (Series,DataFrame,Panel,
SparseSeries,SparseDataFrame,SparsePanel,
from pandas import (Series,TimeSeries,DataFrame,Panel,
SparseSeries,SparseTimeSeries,SparseDataFrame,SparsePanel,
Index,MultiIndex,PeriodIndex,
date_range,bdate_range,Timestamp)
nan = np.nan
Expand All @@ -61,10 +82,11 @@ def create_data():
names=['first', 'second']))
series = dict(float = Series(data['A']),
int = Series(data['B']),
mixed = Series(data['E']))
mixed = Series(data['E']),
ts = TimeSeries(np.arange(10).astype(np.int64),index=date_range('20130101',periods=10)))
frame = dict(float = DataFrame(dict(A = series['float'], B = series['float'] + 1)),
int = DataFrame(dict(A = series['int'] , B = series['int'] + 1)),
mixed = DataFrame(dict([ (k,data[k]) for k in ['A','B','C','D']])))
int = DataFrame(dict(A = series['int'] , B = series['int'] + 1)),
mixed = DataFrame(dict([ (k,data[k]) for k in ['A','B','C','D']])))
panel = dict(float = Panel(dict(ItemA = frame['float'], ItemB = frame['float']+1)))


Expand All @@ -74,7 +96,8 @@ def create_data():
panel = panel,
index = index,
mi = mi,
sp_series = dict(float = _create_sp_series()),
sp_series = dict(float = _create_sp_series(),
ts = _create_sp_tsseries()),
sp_frame = dict(float = _create_sp_frame())
)

Expand All @@ -92,24 +115,11 @@ def write_legacy_pickles():

print("This script generates a pickle file for the current arch, system, and python version")

base_dir, _ = os.path.split(os.path.abspath(__file__))
base_dir = os.path.join(base_dir,'data/legacy_pickle')

# could make this a parameter?
version = None


if version is None:
version = pandas.__version__
pth = os.path.join(base_dir, str(version))
try:
os.mkdir(pth)
except:
pass
version = pandas.__version__

# construct a reasonable platform name
f = '_'.join([ str(pl.machine()), str(pl.system().lower()), str(pl.python_version()) ])
pth = os.path.abspath(os.path.join(pth,'%s.pickle' % f))
f = '_'.join([ str(version), str(pl.machine()), str(pl.system().lower()), str(pl.python_version()) ])
pth = '{0}.pickle'.format(f)

fh = open(pth,'wb')
pickle.dump(create_data(),fh,pickle.HIGHEST_PROTOCOL)
Expand Down
43 changes: 16 additions & 27 deletions pandas/io/tests/test_pickle.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

from datetime import datetime, timedelta
import operator
import pickle
import pickle as pkl
import unittest
import nose
import os
Expand All @@ -29,25 +29,11 @@ def compare(self, vf):

# py3 compat when reading py2 pickle
try:
with open(vf,'rb') as fh:
data = pickle.load(fh)
except ValueError as detail:

# we are trying to read a py3 pickle in py2.....
data = pandas.read_pickle(vf)
except (ValueError) as detail:
# trying to read a py3 pickle in py2
return

# we have a deprecated klass
except TypeError as detail:

from pandas.compat.pickle_compat import load
data = load(vf)

except:
if not compat.PY3:
raise
with open(vf,'rb') as fh:
data = pickle.load(fh, encoding='latin1')

for typ, dv in data.items():
for dt, result in dv.items():

Expand All @@ -64,23 +50,26 @@ def compare(self, vf):
comparator = getattr(tm,"assert_%s_equal" % typ)
comparator(result,expected)

def test_read_pickles_0_10_1(self):
def read_pickles(self, version):
if not is_little_endian():
raise nose.SkipTest("known failure of test_read_pickles_0_10_1 on non-little endian")
raise nose.SkipTest("known failure on non-little endian")

pth = tm.get_data_path('legacy_pickle/0.10.1')
pth = tm.get_data_path('legacy_pickle/{0}'.format(str(version)))
for f in os.listdir(pth):
vf = os.path.join(pth,f)
self.compare(vf)

def test_read_pickles_0_10_1(self):
self.read_pickles('0.10.1')

def test_read_pickles_0_11_0(self):
if not is_little_endian():
raise nose.SkipTest("known failure of test_read_pickles_0_11_0 on non-little endian")
self.read_pickles('0.11.0')

pth = tm.get_data_path('legacy_pickle/0.11.0')
for f in os.listdir(pth):
vf = os.path.join(pth,f)
self.compare(vf)
def test_read_pickles_0_12_0(self):
self.read_pickles('0.12.0')

def test_read_pickles_0_13_0(self):
self.read_pickles('0.13.0')

if __name__ == '__main__':
import nose
Expand Down
6 changes: 6 additions & 0 deletions pandas/tseries/index.py
Original file line number Diff line number Diff line change
Expand Up @@ -533,6 +533,12 @@ def __setstate__(self, state):
self.offset = own_state[1]
self.tz = own_state[2]
np.ndarray.__setstate__(self, nd_state)

# provide numpy < 1.7 compat
if nd_state[2] == 'M8[us]':
new_state = np.ndarray.__reduce__(self.values.astype('M8[ns]'))
np.ndarray.__setstate__(self, new_state[2])

else: # pragma: no cover
np.ndarray.__setstate__(self, state)

Expand Down
2 changes: 2 additions & 0 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -526,6 +526,8 @@ def pxd(name):
package_data={'pandas.io': ['tests/data/legacy_hdf/*.h5',
'tests/data/legacy_pickle/0.10.1/*.pickle',
'tests/data/legacy_pickle/0.11.0/*.pickle',
'tests/data/legacy_pickle/0.12.0/*.pickle',
'tests/data/legacy_pickle/0.13.0/*.pickle',
'tests/data/*.csv',
'tests/data/*.dta',
'tests/data/*.txt',
Expand Down