ENH: Improve to_csv and from_{fwf,csv,table} (#248)

* ENH: Improve read_{csv/fwf/table}

* MAINT: Move parsers->readers

* ENH: Move complex type to _typing

* MAINT: Correct typing issues

* TYP: Remove None from quoting

* CLN: Fix merge issues

Co-authored-by: Kevin Sheppard <[email protected]>

Author: bashtage

pandas-stubs/_typing.pyi

@@ -14,6 +14,7 @@ from typing import (
     AnyStr,
     Callable,
     Hashable,
+    Iterator,
     Literal,
     Mapping,
     Optional,
@@ -66,6 +67,12 @@ class BaseBuffer(Protocol): ...
 class ReadBuffer(BaseBuffer, Protocol[AnyStr_cov]): ...
 class WriteBuffer(BaseBuffer, Protocol[AnyStr_cov]): ...
 
+class ReadCsvBuffer(ReadBuffer[AnyStr_cov], Protocol[AnyStr_cov]):
+    def __iter__(self) -> Iterator[AnyStr_cov]: ...
+    def readline(self) -> AnyStr_cov: ...
+    @property
+    def closed(self) -> bool: ...
+
 class WriteExcelBuffer(WriteBuffer[bytes], Protocol):
     def truncate(self, size: Union[int, None] = ...) -> int: ...
 
@@ -242,6 +249,7 @@ JsonSeriesOrient = Literal["split", "records", "index"]
 TimestampConvention = Literal["start", "end", "s", "e"]
 
 CSVEngine = Literal["c", "python", "pyarrow", "python-fwf"]
+CSVQuoting = Literal[0, 1, 2, 3]
 
 HDFCompLib = Literal["zlib", "lzo", "bzip2", "blosc"]
 ParquetEngine = Literal["auto", "pyarrow", "fastparquet"]

pandas-stubs/core/frame.pyi

@@ -1917,55 +1917,6 @@ class DataFrame(NDFrame, OpsMixin):
         self, excel: _bool = ..., sep: _str | None = ..., **kwargs
     ) -> None: ...
     @overload
-    def to_csv(
-        self,
-        path_or_buf: FilePathOrBuffer | None,
-        sep: _str = ...,
-        na_rep: _str = ...,
-        float_format: _str | None = ...,
-        columns: Sequence[Hashable] | None = ...,
-        header: _bool | list[_str] = ...,
-        index: _bool = ...,
-        index_label: _bool | _str | Sequence[Hashable] | None = ...,
-        mode: _str = ...,
-        encoding: _str | None = ...,
-        compression: _str | Mapping[_str, _str] = ...,
-        quoting: Literal[0, 1, 2, 3] | None = ...,
-        quotechar: _str = ...,
-        line_terminator: _str | None = ...,
-        chunksize: int | None = ...,
-        date_format: _str | None = ...,
-        doublequote: _bool = ...,
-        escapechar: _str | None = ...,
-        decimal: _str = ...,
-        errors: _str = ...,
-        storage_options: dict[_str, Any] | None = ...,
-    ) -> None: ...
-    @overload
-    def to_csv(
-        self,
-        sep: _str = ...,
-        na_rep: _str = ...,
-        float_format: _str | None = ...,
-        columns: Sequence[Hashable] | None = ...,
-        header: _bool | list[_str] = ...,
-        index: _bool = ...,
-        index_label: _bool | _str | Sequence[Hashable] | None = ...,
-        mode: _str = ...,
-        encoding: _str | None = ...,
-        compression: _str | Mapping[_str, _str] = ...,
-        quoting: Literal[0, 1, 2, 3] | None = ...,
-        quotechar: _str = ...,
-        line_terminator: _str | None = ...,
-        chunksize: int | None = ...,
-        date_format: _str | None = ...,
-        doublequote: _bool = ...,
-        escapechar: _str | None = ...,
-        decimal: _str = ...,
-        errors: _str = ...,
-        storage_options: dict[_str, Any] | None = ...,
-    ) -> _str: ...
-    @overload
     def to_json(
         self,
         path_or_buf: FilePathOrBuffer | None,

pandas-stubs/core/generic.pyi

@@ -22,6 +22,7 @@ from pandas._typing import (
     ArrayLike,
     Axis,
     CompressionOptions,
+    CSVQuoting,
     Dtype,
     DtypeArg,
     FilePath,
@@ -231,18 +232,18 @@ class NDFrame(PandasObject, indexing.IndexingMixin):
     @overload
     def to_csv(
         self,
-        path_or_buf: FilePathOrBuffer | None,
+        path_or_buf: FilePathOrBuffer,
         sep: _str = ...,
         na_rep: _str = ...,
-        float_format: _str | None = ...,
-        columns: Sequence[Hashable] | None = ...,
+        float_format: _str | Callable[[object], _str] | None = ...,
+        columns: list[HashableT] | None = ...,
         header: _bool | list[_str] = ...,
         index: _bool = ...,
-        index_label: _bool | _str | Sequence[Hashable] | None = ...,
-        mode: _str = ...,
+        index_label: Literal[False] | _str | list[HashableT] | None = ...,
+        mode: FileWriteMode = ...,
         encoding: _str | None = ...,
-        compression: _str | Mapping[_str, _str] = ...,
-        quoting: Literal[0, 1, 2, 3] | None = ...,
+        compression: CompressionOptions = ...,
+        quoting: CSVQuoting = ...,
         quotechar: _str = ...,
         line_terminator: _str | None = ...,
         chunksize: int | None = ...,
@@ -251,22 +252,23 @@ class NDFrame(PandasObject, indexing.IndexingMixin):
         escapechar: _str | None = ...,
         decimal: _str = ...,
         errors: _str = ...,
-        storage_options: dict[_str, Any] | None = ...,
+        storage_options: StorageOptions = ...,
     ) -> None: ...
     @overload
     def to_csv(
         self,
+        path_or_buf: None = ...,
         sep: _str = ...,
         na_rep: _str = ...,
-        float_format: _str | None = ...,
-        columns: Sequence[Hashable] | None = ...,
+        float_format: _str | Callable[[object], _str] | None = ...,
+        columns: list[HashableT] | None = ...,
         header: _bool | list[_str] = ...,
         index: _bool = ...,
-        index_label: _bool | _str | Sequence[Hashable] | None = ...,
-        mode: _str = ...,
+        index_label: Literal[False] | _str | list[HashableT] | None = ...,
+        mode: FileWriteMode = ...,
         encoding: _str | None = ...,
-        compression: _str | Mapping[_str, _str] = ...,
-        quoting: Literal[0, 1, 2, 3] | None = ...,
+        compression: CompressionOptions = ...,
+        quoting: CSVQuoting = ...,
         quotechar: _str = ...,
         line_terminator: _str | None = ...,
         chunksize: int | None = ...,
@@ -275,7 +277,7 @@ class NDFrame(PandasObject, indexing.IndexingMixin):
         escapechar: _str | None = ...,
         decimal: _str = ...,
         errors: _str = ...,
-        storage_options: dict[_str, Any] | None = ...,
+        storage_options: StorageOptions = ...,
     ) -> _str: ...
     def take(
         self, indices, axis=..., is_copy: _bool | None = ..., **kwargs

pandas-stubs/io/clipboards.pyi

@@ -15,6 +15,7 @@ from pandas.core.series import Series
 from pandas._typing import (
     CompressionOptions,
     CSVEngine,
+    CSVQuoting,
     DtypeArg,
     StorageOptions,
     npt,
@@ -71,7 +72,7 @@ def read_clipboard(
     decimal: str = ...,
     lineterminator: str | None = ...,
     quotechar: str = ...,
-    quoting: Literal[0, 1, 2, 3] | None = ...,
+    quoting: CSVQuoting = ...,
     doublequote: bool = ...,
     escapechar: str | None = ...,
     comment: str | None = ...,
@@ -137,7 +138,7 @@ def read_clipboard(
     decimal: str = ...,
     lineterminator: str | None = ...,
     quotechar: str = ...,
-    quoting: Literal[0, 1, 2, 3] | None = ...,
+    quoting: CSVQuoting = ...,
     doublequote: bool = ...,
     escapechar: str | None = ...,
     comment: str | None = ...,
@@ -203,7 +204,7 @@ def read_clipboard(
     decimal: str = ...,
     lineterminator: str | None = ...,
     quotechar: str = ...,
-    quoting: Literal[0, 1, 2, 3] | None = ...,
+    quoting: CSVQuoting = ...,
     doublequote: bool = ...,
     escapechar: str | None = ...,
     comment: str | None = ...,

pandas-stubs/io/common.pyi

@@ -1,17 +1,17 @@
-from typing import (
-    IO,
-    AnyStr,
-    Generic,
-)
-
-from pandas._typing import CompressionDict
-
-class IOHandles(Generic[AnyStr]):
-    handle: IO[AnyStr]
-    compression: CompressionDict
-    created_handles: list[IO[AnyStr]]
-    is_wrapped: bool
-    def close(self) -> None: ...
-    def __enter__(self) -> IOHandles[AnyStr]: ...
-    def __exit__(self, *args: object) -> None: ...
-    def __init__(self, handle, compression, created_handles, is_wrapped) -> None: ...
+from typing import (
+    IO,
+    AnyStr,
+    Generic,
+)
+
+from pandas._typing import CompressionDict
+
+class IOHandles(Generic[AnyStr]):
+    handle: IO[AnyStr]
+    compression: CompressionDict
+    created_handles: list[IO[AnyStr]]
+    is_wrapped: bool
+    def close(self) -> None: ...
+    def __enter__(self) -> IOHandles[AnyStr]: ...
+    def __exit__(self, *args: object) -> None: ...
+    def __init__(self, handle, compression, created_handles, is_wrapped) -> None: ...