Skip to content

Column aliases #1523

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 23 additions & 10 deletions pandas/core/frame.py
Original file line number Diff line number Diff line change
Expand Up @@ -1021,15 +1021,17 @@ def to_panel(self):

to_wide = deprecate('to_wide', to_panel)

def _helper_csvexcel(self, writer, na_rep=None, cols=None, header=True,
index=True, index_label=None):
def _helper_csvexcel(self, writer, na_rep=None, cols=None,
header=True, index=True, index_label=None):
if cols is None:
cols = self.columns

series = {}
for k, v in self._series.iteritems():
series[k] = v.values
if header:

has_aliases = isinstance(header, (tuple, list, np.ndarray))
if has_aliases or header:
if index:
# should write something for index label
if index_label is None:
Expand All @@ -1050,7 +1052,15 @@ def _helper_csvexcel(self, writer, na_rep=None, cols=None, header=True,
index_label = [index_label]

encoded_labels = list(index_label)
encoded_cols = list(cols)
if has_aliases:
if len(header) != len(cols):
raise ValueError(('Writing %d cols but got %d aliases'
% (len(cols), len(header))))
else:
write_cols = header
else:
write_cols = cols
encoded_cols = list(write_cols)

writer.writerow(encoded_labels + encoded_cols)
else:
Expand All @@ -1075,8 +1085,8 @@ def _helper_csvexcel(self, writer, na_rep=None, cols=None, header=True,
writer.writerow(row_fields)

def to_csv(self, path_or_buf, sep=",", na_rep='', cols=None,
header=True, index=True, index_label=None, mode='w',
nanRep=None, encoding=None):
header=True, index=True, index_label=None,
mode='w', nanRep=None, encoding=None):
"""
Write DataFrame to a comma-separated values (csv) file

Expand All @@ -1088,8 +1098,9 @@ def to_csv(self, path_or_buf, sep=",", na_rep='', cols=None,
Missing data representation
cols : sequence, optional
Columns to write
header : boolean, default True
Write out column names
header : boolean or list of string, default True
Write out column names. If a list of string is given it is
assumed to be aliases for the column names
index : boolean, default True
Write row names (index)
index_label : string or sequence, default None
Expand Down Expand Up @@ -1125,6 +1136,7 @@ def to_csv(self, path_or_buf, sep=",", na_rep='', cols=None,
self._helper_csvexcel(csvout, na_rep=na_rep, cols=cols,
header=header, index=index,
index_label=index_label)

finally:
if close:
f.close()
Expand All @@ -1144,8 +1156,9 @@ def to_excel(self, excel_writer, sheet_name='sheet1', na_rep='',
Missing data rep'n
cols : sequence, optional
Columns to write
header : boolean, default True
Write out column names
header : boolean or list of string, default True
Write out column names. If a list of string is given it is
assumed to be aliases for the column names
index : boolean, default True
Write row names (index)
index_label : string or sequence, default None
Expand Down
4 changes: 2 additions & 2 deletions pandas/io/parsers.py
Original file line number Diff line number Diff line change
Expand Up @@ -1029,9 +1029,9 @@ def _convert_types(values, na_values):
return values, na_count

try:
result = lib.maybe_convert_numeric(values, na_values)
result = lib.maybe_convert_numeric(values, na_values, False)
except Exception:
na_count = lib.sanitize_objects(values, na_values)
na_count = lib.sanitize_objects(values, na_values, False)
result = values

if result.dtype == np.object_:
Expand Down
26 changes: 26 additions & 0 deletions pandas/io/tests/test_parsers.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,32 @@ def setUp(self):
self.csv2 = os.path.join(self.dirpath, 'test2.csv')
self.xls1 = os.path.join(self.dirpath, 'test.xls')

def test_empty_string(self):
data = """\
One,Two,Three
a,1,one
b,2,two
,3,three
d,4,nan
e,5,five
nan,6,
g,7,seven
"""
df = read_csv(StringIO(data))
xp = DataFrame({'One' : ['a', 'b', np.nan, 'd', 'e', np.nan, 'g'],
'Two' : [1,2,3,4,5,6,7],
'Three' : ['one', 'two', 'three', np.nan, 'five',
np.nan, 'seven']})
assert_frame_equal(xp.reindex(columns=df.columns), df)

df = read_csv(StringIO(data), na_values={'One': [], 'Three': []})
xp = DataFrame({'One' : ['a', 'b', '', 'd', 'e', 'nan', 'g'],
'Two' : [1,2,3,4,5,6,7],
'Three' : ['one', 'two', 'three', 'nan', 'five',
'', 'seven']})
assert_frame_equal(xp.reindex(columns=df.columns), df)


def test_read_csv(self):
pass

Expand Down
15 changes: 10 additions & 5 deletions pandas/src/inference.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -241,7 +241,8 @@ def is_date_array(ndarray[object] values):
return True


def maybe_convert_numeric(ndarray[object] values, set na_values):
def maybe_convert_numeric(ndarray[object] values, set na_values,
convert_empty=True):
'''
Type inference function-- convert strings to numeric (potentially) and
convert to proper dtype array
Expand Down Expand Up @@ -275,8 +276,11 @@ def maybe_convert_numeric(ndarray[object] values, set na_values):
floats[i] = complexes[i] = nan
seen_float = 1
elif len(val) == 0:
floats[i] = complexes[i] = nan
seen_float = 1
if convert_empty:
floats[i] = complexes[i] = nan
seen_float = 1
else:
raise ValueError('Empty string encountered')
elif util.is_complex_object(val):
complexes[i] = val
seen_complex = 1
Expand Down Expand Up @@ -573,7 +577,8 @@ def try_parse_datetime_components(ndarray[object] years, ndarray[object] months,

return result

def sanitize_objects(ndarray[object] values, set na_values):
def sanitize_objects(ndarray[object] values, set na_values,
convert_empty=True):
cdef:
Py_ssize_t i, n
object val, onan
Expand All @@ -585,7 +590,7 @@ def sanitize_objects(ndarray[object] values, set na_values):

for i from 0 <= i < n:
val = values[i]
if val == '' or val in na_values:
if (convert_empty and val == '') or (val in na_values):
values[i] = onan
na_count += 1
elif val in memo:
Expand Down
21 changes: 21 additions & 0 deletions pandas/tests/test_frame.py
Original file line number Diff line number Diff line change
Expand Up @@ -3078,6 +3078,19 @@ def test_to_csv_from_csv(self):
result = DataFrame.from_csv(path, index_col=[0, 1, 2],
parse_dates=False)
assert_frame_equal(result, df)

# column aliases
col_aliases = Index(['AA', 'X', 'Y', 'Z'])
self.frame2.to_csv(path, header=col_aliases)
rs = DataFrame.from_csv(path)
xp = self.frame2.copy()
xp.columns = col_aliases

assert_frame_equal(xp, rs)

self.assertRaises(ValueError, self.frame2.to_csv, path,
header=['AA', 'X'])

os.remove(path)

def test_to_csv_multiindex(self):
Expand Down Expand Up @@ -3262,6 +3275,14 @@ def test_to_excel_from_excel(self):
np.testing.assert_equal('test1', reader.sheet_names[0])
np.testing.assert_equal('test2', reader.sheet_names[1])

# column aliases
col_aliases = Index(['AA', 'X', 'Y', 'Z'])
self.frame2.to_excel(path, 'test1', header=col_aliases)
reader = ExcelFile(path)
rs = reader.parse('test1', index_col=0)
xp = self.frame2.copy()
xp.columns = col_aliases
assert_frame_equal(xp, rs)

os.remove(path)

Expand Down