From 3cf0099d4846b29d0fdca75174be4e501cd91a32 Mon Sep 17 00:00:00 2001 From: Robert de Vries Date: Wed, 25 Mar 2020 21:38:56 +0100 Subject: [PATCH 1/7] WIP: use exec to generate functions --- pandas/io/parsers.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/pandas/io/parsers.py b/pandas/io/parsers.py index 2df81ba0aa51a..f5b85ba2dbd32 100755 --- a/pandas/io/parsers.py +++ b/pandas/io/parsers.py @@ -528,9 +528,13 @@ def _read(filepath_or_buffer: FilePathOrBuffer, kwds): def _make_parser_function(name, default_sep=","): - def parser_f( - filepath_or_buffer: FilePathOrBuffer, - sep=default_sep, + exec(f""" +from pandas._typing import FilePathOrBuffer +import csv + +def {name}( + filepath_or_buffer : FilePathOrBuffer, + sep="{default_sep}", delimiter=None, # Column and Index Locations and Names header="infer", @@ -583,7 +587,7 @@ def parser_f( warn_bad_lines=True, # Internal delim_whitespace=False, - low_memory=_c_parser_defaults["low_memory"], + low_memory={_c_parser_defaults["low_memory"]}, memory_map=False, float_precision=None, ): @@ -674,13 +678,10 @@ def parser_f( ) return _read(filepath_or_buffer, kwds) - - parser_f.__name__ = name - - return parser_f + """, globals()) -read_csv = _make_parser_function("read_csv", default_sep=",") +_make_parser_function("read_csv", default_sep=",") read_csv = Appender( _doc_read_csv_and_table.format( func_name="read_csv", @@ -689,7 +690,7 @@ def parser_f( ) )(read_csv) -read_table = _make_parser_function("read_table", default_sep="\t") +_make_parser_function("read_table", default_sep="\t") read_table = Appender( _doc_read_csv_and_table.format( func_name="read_table", From 228bc6f84b7db0f7eb9335519b6d459a762510f0 Mon Sep 17 00:00:00 2001 From: Robert de Vries Date: Wed, 25 Mar 2020 22:39:49 +0100 Subject: [PATCH 2/7] Add missing default_sep replacements --- pandas/io/parsers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/io/parsers.py b/pandas/io/parsers.py index f5b85ba2dbd32..a674b845df515 100755 --- a/pandas/io/parsers.py +++ b/pandas/io/parsers.py @@ -605,7 +605,7 @@ def {name}( # the comparison to dialect values by checking if default values # for BOTH "delimiter" and "sep" were provided. if dialect is not None: - sep_override = delimiter is None and sep == default_sep + sep_override = delimiter is None and sep == "{default_sep}" kwds = dict(sep_override=sep_override) else: kwds = dict() @@ -614,7 +614,7 @@ def {name}( if delimiter is None: delimiter = sep - if delim_whitespace and delimiter != default_sep: + if delim_whitespace and delimiter != "{default_sep}": raise ValueError( "Specified a delimiter with both sep and " "delim_whitespace=True; you can only specify one." From 857685b72883ee4e82f579c797e4e721c0d53b7e Mon Sep 17 00:00:00 2001 From: Robert de Vries Date: Wed, 25 Mar 2020 22:48:40 +0100 Subject: [PATCH 3/7] Add regression test --- pandas/tests/io/parser/test_common.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pandas/tests/io/parser/test_common.py b/pandas/tests/io/parser/test_common.py index 5bf9587a6ca22..93b0696772f0f 100644 --- a/pandas/tests/io/parser/test_common.py +++ b/pandas/tests/io/parser/test_common.py @@ -2126,3 +2126,10 @@ def test_no_header_two_extra_columns(all_parsers): parser = all_parsers df = parser.read_csv(stream, header=None, names=column_names, index_col=False) tm.assert_frame_equal(df, ref) + +def test_unexpected_keyword_parameter_exception(all_parsers): + # GH 25648 + parser = all_parsers + msg = "read_csv\\(\\) got an unexpected keyword argument 'foo'" + with pytest.raises(TypeError, match=msg): + parser.read_csv("foo.csv", foo=1) From 19c503ca14a0a74d50446268cb3f89c2641c38aa Mon Sep 17 00:00:00 2001 From: Robert de Vries Date: Wed, 25 Mar 2020 22:52:31 +0100 Subject: [PATCH 4/7] Add whatsnew entry --- doc/source/whatsnew/v1.1.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v1.1.0.rst b/doc/source/whatsnew/v1.1.0.rst index dcbfe6aeb9a12..59c14c53b4c3e 100644 --- a/doc/source/whatsnew/v1.1.0.rst +++ b/doc/source/whatsnew/v1.1.0.rst @@ -355,6 +355,7 @@ I/O - Bug in :meth:`read_csv` was causing a segfault when there were blank lines between the header and data rows (:issue:`28071`) - Bug in :meth:`read_csv` was raising a misleading exception on a permissions issue (:issue:`23784`) - Bug in :meth:`read_csv` was raising an ``IndexError`` when header=None and 2 extra data columns +- `TypeError` exceptions raised by :meth:`read_csv` and :meth:`read_table` where showing as ``parser_f`` when an unexpected keyword argument was passed (:issue:`25648`) Plotting From 89f3fd8f2068d69a1d32ad9f60ec44f69e0ca195 Mon Sep 17 00:00:00 2001 From: Robert de Vries Date: Wed, 25 Mar 2020 22:53:18 +0100 Subject: [PATCH 5/7] black reformatter fixes --- pandas/io/parsers.py | 7 +++++-- pandas/tests/io/parser/test_common.py | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pandas/io/parsers.py b/pandas/io/parsers.py index a674b845df515..28b4683294bdd 100755 --- a/pandas/io/parsers.py +++ b/pandas/io/parsers.py @@ -528,7 +528,8 @@ def _read(filepath_or_buffer: FilePathOrBuffer, kwds): def _make_parser_function(name, default_sep=","): - exec(f""" + exec( + f""" from pandas._typing import FilePathOrBuffer import csv @@ -678,7 +679,9 @@ def {name}( ) return _read(filepath_or_buffer, kwds) - """, globals()) + """, + globals(), + ) _make_parser_function("read_csv", default_sep=",") diff --git a/pandas/tests/io/parser/test_common.py b/pandas/tests/io/parser/test_common.py index 93b0696772f0f..0dd8c6111b025 100644 --- a/pandas/tests/io/parser/test_common.py +++ b/pandas/tests/io/parser/test_common.py @@ -2127,6 +2127,7 @@ def test_no_header_two_extra_columns(all_parsers): df = parser.read_csv(stream, header=None, names=column_names, index_col=False) tm.assert_frame_equal(df, ref) + def test_unexpected_keyword_parameter_exception(all_parsers): # GH 25648 parser = all_parsers From 8b15883bf40a7407805eda5bbd19cab9ec4c532a Mon Sep 17 00:00:00 2001 From: Robert de Vries Date: Wed, 25 Mar 2020 23:02:07 +0100 Subject: [PATCH 6/7] Fix flake8 false positives --- pandas/io/parsers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pandas/io/parsers.py b/pandas/io/parsers.py index 28b4683294bdd..37876c249ac42 100755 --- a/pandas/io/parsers.py +++ b/pandas/io/parsers.py @@ -691,7 +691,8 @@ def {name}( summary="Read a comma-separated values (csv) file into DataFrame.", _default_sep="','", ) -)(read_csv) +)(read_csv) # noqa F821 + _make_parser_function("read_table", default_sep="\t") read_table = Appender( @@ -700,7 +701,7 @@ def {name}( summary="Read general delimited file into DataFrame.", _default_sep=r"'\\t' (tab-stop)", ) -)(read_table) +)(read_table) # noqa F821 def read_fwf( From 4240785c57fab899bcce8945ba7ee6d57fbf686d Mon Sep 17 00:00:00 2001 From: Robert de Vries Date: Sat, 28 Mar 2020 12:43:29 +0100 Subject: [PATCH 7/7] Fix spelling error in whatsnew Co-Authored-By: gfyoung --- doc/source/whatsnew/v1.1.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.1.0.rst b/doc/source/whatsnew/v1.1.0.rst index 59c14c53b4c3e..0f5372330970a 100644 --- a/doc/source/whatsnew/v1.1.0.rst +++ b/doc/source/whatsnew/v1.1.0.rst @@ -355,7 +355,7 @@ I/O - Bug in :meth:`read_csv` was causing a segfault when there were blank lines between the header and data rows (:issue:`28071`) - Bug in :meth:`read_csv` was raising a misleading exception on a permissions issue (:issue:`23784`) - Bug in :meth:`read_csv` was raising an ``IndexError`` when header=None and 2 extra data columns -- `TypeError` exceptions raised by :meth:`read_csv` and :meth:`read_table` where showing as ``parser_f`` when an unexpected keyword argument was passed (:issue:`25648`) +- `TypeError` exceptions raised by :meth:`read_csv` and :meth:`read_table` were showing as ``parser_f`` when an unexpected keyword argument was passed (:issue:`25648`) Plotting