Skip to content

Read very old stata DTA files #12233

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Read very old stata DTA files #12233

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion doc/source/whatsnew/v0.18.0.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -694,7 +694,7 @@ Bug Fixes
- Bug in getitem when the values of a ``Series`` were tz-aware (:issue:`12089`)
- Bug in ``Series.str.get_dummies`` when one of the variables was 'name' (:issue:`12180`)
- Bug in ``pd.concat`` while concatenating tz-aware NaT series. (:issue:`11693`, :issue:`11755`)

- Bug in ``pd.read_stata`` with version <= 108 files (:issue:`12232`)


- Bug in ``Timedelta.round`` with negative values (:issue:`11690`)
Expand Down
73 changes: 35 additions & 38 deletions pandas/io/stata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -851,23 +851,24 @@ def __init__(self, encoding):
float32_max = b'\xff\xff\xff\x7e'
float64_min = b'\xff\xff\xff\xff\xff\xff\xef\xff'
float64_max = b'\xff\xff\xff\xff\xff\xff\xdf\x7f'
self.VALID_RANGE = \
{
'b': (-127, 100),
'h': (-32767, 32740),
'l': (-2147483647, 2147483620),
'f': (np.float32(struct.unpack('<f', float32_min)[0]),
np.float32(struct.unpack('<f', float32_max)[0])),
'd': (np.float64(struct.unpack('<d', float64_min)[0]),
np.float64(struct.unpack('<d', float64_max)[0]))
}

self.OLD_TYPE_MAPPING = \
{
'i': 252,
'f': 254,
'b': 251
}
self.VALID_RANGE = {
'b': (-127, 100),
'h': (-32767, 32740),
'l': (-2147483647, 2147483620),
'f': (np.float32(struct.unpack('<f', float32_min)[0]),
np.float32(struct.unpack('<f', float32_max)[0])),
'd': (np.float64(struct.unpack('<d', float64_min)[0]),
np.float64(struct.unpack('<d', float64_max)[0]))
}

self.OLD_TYPE_MAPPING = {
98: 251, # byte
105: 252, # int
108: 253, # long
102: 254 # float
# don't know old code for double
}

# These missing values are the generic '.' in Stata, and are used
# to replace nans
self.MISSING_VALUES = {
Expand All @@ -878,15 +879,14 @@ def __init__(self, encoding):
'd': np.float64(
struct.unpack('<d', b'\x00\x00\x00\x00\x00\x00\xe0\x7f')[0])
}
self.NUMPY_TYPE_MAP = \
{
'b': 'i1',
'h': 'i2',
'l': 'i4',
'f': 'f4',
'd': 'f8',
'Q': 'u8'
}
self.NUMPY_TYPE_MAP = {
'b': 'i1',
'h': 'i2',
'l': 'i4',
'f': 'f4',
'd': 'f8',
'Q': 'u8'
}

# Reserved words cannot be used as variable names
self.RESERVED_WORDS = ('aggregate', 'array', 'boolean', 'break',
Expand All @@ -900,12 +900,6 @@ def __init__(self, encoding):
'protected', 'quad', 'rowvector', 'short',
'typedef', 'typename', 'virtual')

def _decode_bytes(self, str, errors=None):
if compat.PY3 or self._encoding is not None:
return str.decode(self._encoding, errors)
else:
return str


class StataReader(StataParser):
__doc__ = _stata_reader_doc
Expand Down Expand Up @@ -1201,11 +1195,14 @@ def _read_old_header(self, first_char):
typlist = [ord(self.path_or_buf.read(1))
for i in range(self.nvar)]
else:
typlist = [
self.OLD_TYPE_MAPPING[
self._decode_bytes(self.path_or_buf.read(1))
] for i in range(self.nvar)
]
buf = self.path_or_buf.read(self.nvar)
typlistb = np.frombuffer(buf, dtype=np.uint8)
typlist = []
for tp in typlistb:
if tp in self.OLD_TYPE_MAPPING:
typlist.append(self.OLD_TYPE_MAPPING[tp])
else:
typlist.append(tp - 127) # string

try:
self.typlist = [self.TYPE_MAP[typ] for typ in typlist]
Expand Down Expand Up @@ -1533,7 +1530,7 @@ def read(self, nrows=None, convert_dates=None,
data[col],
self.fmtlist[i])

if convert_categoricals and self.value_label_dict:
if convert_categoricals and self.format_version > 108:
data = self._do_convert_categoricals(data,
self.value_label_dict,
self.lbllist,
Expand Down
Binary file added pandas/io/tests/data/S4_EDUC1.DTA
View file
Open in desktop
Binary file not shown.
20 changes: 17 additions & 3 deletions pandas/io/tests/test_stata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -409,9 +409,9 @@ def test_read_write_dta12(self):
written_and_read_again.set_index('index'), formatted)

def test_read_write_dta13(self):
s1 = Series(2**9, dtype=np.int16)
s2 = Series(2**17, dtype=np.int32)
s3 = Series(2**33, dtype=np.int64)
s1 = Series(2 ** 9, dtype=np.int16)
s2 = Series(2 ** 17, dtype=np.int32)
s3 = Series(2 ** 33, dtype=np.int64)
original = DataFrame({'int16': s1, 'int32': s2, 'int64': s3})
original.index.name = 'index'

Expand Down Expand Up @@ -568,6 +568,20 @@ def test_dates_invalid_column(self):
tm.assert_frame_equal(written_and_read_again.set_index('index'),
modified)

def test_105(self):
# Data obtained from:
# http://go.worldbank.org/ZXY29PVJ21
dpath = os.path.join(self.dirpath, 'S4_EDUC1.DTA')
df = pd.read_stata(dpath)
df0 = [[1, 1, 3, -2], [2, 1, 2, -2], [4, 1, 1, -2]]
df0 = pd.DataFrame(df0)
df0.columns = ["clustnum", "pri_schl", "psch_num", "psch_dis"]
df0['clustnum'] = df0["clustnum"].astype(np.int16)
df0['pri_schl'] = df0["pri_schl"].astype(np.int8)
df0['psch_num'] = df0["psch_num"].astype(np.int8)
df0['psch_dis'] = df0["psch_dis"].astype(np.float32)
tm.assert_frame_equal(df.head(3), df0)

def test_date_export_formats(self):
columns = ['tc', 'td', 'tw', 'tm', 'tq', 'th', 'ty']
conversions = dict(((c, c) for c in columns))
Expand Down