ENH: add calamine excel reader (close #50395)

dimastbk · dimastbk · commit b6701f022592 · 2023-09-05T23:06:09.000+06:00
Co-author: Kostya Farber (#50581)
diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml
@@ -58,4 +58,5 @@ dependencies:
 
   - pip:
     - pyqt5>=5.15.6
+    - python-calamine>=0.1.6
     - tzdata>=2022.1
diff --git a/ci/deps/actions-311-downstream_compat.yaml b/ci/deps/actions-311-downstream_compat.yaml
@@ -73,4 +73,5 @@ dependencies:
   - pip:
     - dataframe-api-compat>=0.1.7
     - pyqt5>=5.15.6
+    - python-calamine>=0.1.6
     - tzdata>=2022.1
diff --git a/ci/deps/actions-311.yaml b/ci/deps/actions-311.yaml
@@ -58,4 +58,5 @@ dependencies:
 
   - pip:
     - pyqt5>=5.15.6
+    - python-calamine>=0.1.6
     - tzdata>=2022.1
diff --git a/ci/deps/actions-39-minimum_versions.yaml b/ci/deps/actions-39-minimum_versions.yaml
@@ -61,4 +61,5 @@ dependencies:
   - pip:
     - dataframe-api-compat==0.1.7
     - pyqt5==5.15.6
+    - python-calamine==0.1.6
     - tzdata==2022.1
diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml
@@ -58,4 +58,5 @@ dependencies:
 
   - pip:
     - pyqt5>=5.15.6
+    - python-calamine>=0.1.6
     - tzdata>=2022.1
diff --git a/ci/deps/circle-310-arm64.yaml b/ci/deps/circle-310-arm64.yaml
@@ -56,3 +56,6 @@ dependencies:
   - xlrd>=2.0.1
   - xlsxwriter>=3.0.3
   - zstandard>=0.17.0
+
+  - pip:
+    - python-calamine>=0.1.6
diff --git a/doc/source/getting_started/install.rst b/doc/source/getting_started/install.rst
@@ -281,6 +281,7 @@ xlrd                      2.0.1              excel           Reading Excel
 xlsxwriter                3.0.3              excel           Writing Excel
 openpyxl                  3.0.10             excel           Reading / writing for xlsx files
 pyxlsb                    1.0.9              excel           Reading for xlsb files
+python-calamine           0.1.6              excel           Reading for xls/xlsx/xlsb/ods files
 ========================= ================== =============== =============================================================
 
 HTML
diff --git a/doc/source/user_guide/io.rst b/doc/source/user_guide/io.rst
@@ -3453,7 +3453,8 @@ Excel files
 The :func:`~pandas.read_excel` method can read Excel 2007+ (``.xlsx``) files
 using the ``openpyxl`` Python module. Excel 2003 (``.xls``) files
 can be read using ``xlrd``. Binary Excel (``.xlsb``)
-files can be read using ``pyxlsb``.
+files can be read using ``pyxlsb``. Also, all this formats can be read using ``python-calamine``,
+but this library has some different behavior from other libraries (mostly for ``.ods``).
 The :meth:`~DataFrame.to_excel` instance method is used for
 saving a ``DataFrame`` to Excel.  Generally the semantics are
 similar to working with :ref:`csv<io.read_csv_table>` data.
diff --git a/doc/source/whatsnew/v2.2.0.rst b/doc/source/whatsnew/v2.2.0.rst
@@ -28,7 +28,7 @@ enhancement2
 
 Other enhancements
 ^^^^^^^^^^^^^^^^^^
--
+- Added ``calamine`` as an engine to ``read_excel`` (:issue:`50395`)
 -
 
 .. ---------------------------------------------------------------------------
diff --git a/environment.yml b/environment.yml
@@ -115,5 +115,6 @@ dependencies:
   - pip:
       - dataframe-api-compat>=0.1.7
       - sphinx-toggleprompt  # conda-forge version has stricter pins on jinja2
+      - python-calamine>=0.1.6
       - typing_extensions; python_version<"3.11"
       - tzdata>=2022.1
diff --git a/pandas/compat/_optional.py b/pandas/compat/_optional.py
@@ -37,6 +37,7 @@
     "pyarrow": "7.0.0",
     "pyreadstat": "1.1.5",
     "pytest": "7.3.2",
+    "python-calamine": "0.1.6",
     "pyxlsb": "1.0.9",
     "s3fs": "2022.05.0",
     "scipy": "1.8.1",
@@ -62,6 +63,7 @@
     "lxml.etree": "lxml",
     "odf": "odfpy",
     "pandas_gbq": "pandas-gbq",
+    "python_calamine": "python-calamine",
     "sqlalchemy": "SQLAlchemy",
     "tables": "pytables",
 }
diff --git a/pandas/core/config_init.py b/pandas/core/config_init.py
@@ -513,11 +513,11 @@ def use_inf_as_na_cb(key) -> None:
     auto, {others}.
 """
 
-_xls_options = ["xlrd"]
-_xlsm_options = ["xlrd", "openpyxl"]
-_xlsx_options = ["xlrd", "openpyxl"]
-_ods_options = ["odf"]
-_xlsb_options = ["pyxlsb"]
+_xls_options = ["xlrd", "calamine"]
+_xlsm_options = ["xlrd", "openpyxl", "calamine"]
+_xlsx_options = ["xlrd", "openpyxl", "calamine"]
+_ods_options = ["odf", "calamine"]
+_xlsb_options = ["pyxlsb", "calamine"]
 
 
 with cf.config_prefix("io.excel.xls"):
diff --git a/pandas/io/excel/_base.py b/pandas/io/excel/_base.py
@@ -160,13 +160,15 @@
     of dtype conversion.
 engine : str, default None
     If io is not a buffer or path, this must be set to identify io.
-    Supported engines: "xlrd", "openpyxl", "odf", "pyxlsb".
+    Supported engines: "xlrd", "openpyxl", "odf", "pyxlsb", "calamine".
     Engine compatibility :
 
     - "xlrd" supports old-style Excel files (.xls).
     - "openpyxl" supports newer Excel file formats.
     - "odf" supports OpenDocument file formats (.odf, .ods, .odt).
     - "pyxlsb" supports Binary Excel files.
+    - "calamine" supports Excel (.xls, .xlsx, .xlsm, .xlsb)
+      and OpenDocument (.ods) file formats.
 
     .. versionchanged:: 1.2.0
         The engine `xlrd <https://xlrd.readthedocs.io/en/latest/>`_
@@ -395,7 +397,7 @@ def read_excel(
     | Callable[[str], bool]
     | None = ...,
     dtype: DtypeArg | None = ...,
-    engine: Literal["xlrd", "openpyxl", "odf", "pyxlsb"] | None = ...,
+    engine: Literal["xlrd", "openpyxl", "odf", "pyxlsb", "calamine"] | None = ...,
     converters: dict[str, Callable] | dict[int, Callable] | None = ...,
     true_values: Iterable[Hashable] | None = ...,
     false_values: Iterable[Hashable] | None = ...,
@@ -434,7 +436,7 @@ def read_excel(
     | Callable[[str], bool]
     | None = ...,
     dtype: DtypeArg | None = ...,
-    engine: Literal["xlrd", "openpyxl", "odf", "pyxlsb"] | None = ...,
+    engine: Literal["xlrd", "openpyxl", "odf", "pyxlsb", "calamine"] | None = ...,
     converters: dict[str, Callable] | dict[int, Callable] | None = ...,
     true_values: Iterable[Hashable] | None = ...,
     false_values: Iterable[Hashable] | None = ...,
@@ -473,7 +475,7 @@ def read_excel(
     | Callable[[str], bool]
     | None = None,
     dtype: DtypeArg | None = None,
-    engine: Literal["xlrd", "openpyxl", "odf", "pyxlsb"] | None = None,
+    engine: Literal["xlrd", "openpyxl", "odf", "pyxlsb", "calamine"] | None = None,
     converters: dict[str, Callable] | dict[int, Callable] | None = None,
     true_values: Iterable[Hashable] | None = None,
     false_values: Iterable[Hashable] | None = None,
@@ -1463,13 +1465,15 @@ class ExcelFile:
         .xls, .xlsx, .xlsb, .xlsm, .odf, .ods, or .odt file.
     engine : str, default None
         If io is not a buffer or path, this must be set to identify io.
-        Supported engines: ``xlrd``, ``openpyxl``, ``odf``, ``pyxlsb``
+        Supported engines: ``xlrd``, ``openpyxl``, ``odf``, ``pyxlsb``, ``calamine``
         Engine compatibility :
 
         - ``xlrd`` supports old-style Excel files (.xls).
         - ``openpyxl`` supports newer Excel file formats.
         - ``odf`` supports OpenDocument file formats (.odf, .ods, .odt).
         - ``pyxlsb`` supports Binary Excel files.
+        - ``calamine`` supports Excel (.xls, .xlsx, .xlsm, .xlsb)
+          and OpenDocument (.ods) file formats.
 
         .. versionchanged:: 1.2.0
 
@@ -1505,6 +1509,7 @@ class ExcelFile:
     ...     df1 = pd.read_excel(xls, "Sheet1")  # doctest: +SKIP
     """
 
+    from pandas.io.excel._calamine import CalamineReader
     from pandas.io.excel._odfreader import ODFReader
     from pandas.io.excel._openpyxl import OpenpyxlReader
     from pandas.io.excel._pyxlsb import PyxlsbReader
@@ -1515,6 +1520,7 @@ class ExcelFile:
         "openpyxl": OpenpyxlReader,
         "odf": ODFReader,
         "pyxlsb": PyxlsbReader,
+        "calamine": CalamineReader,
     }
 
     def __init__(
diff --git a/pandas/io/excel/_calamine.py b/pandas/io/excel/_calamine.py
@@ -0,0 +1,127 @@
+from __future__ import annotations
+
+from datetime import (
+    date,
+    datetime,
+    time,
+    timedelta,
+)
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Union,
+    cast,
+)
+
+from pandas._typing import Scalar
+from pandas.compat._optional import import_optional_dependency
+from pandas.util._decorators import doc
+
+import pandas as pd
+from pandas.core.shared_docs import _shared_docs
+
+from pandas.io.excel._base import BaseExcelReader
+
+if TYPE_CHECKING:
+    from python_calamine import (
+        CalamineSheet,
+        CalamineWorkbook,
+    )
+
+    from pandas._typing import (
+        FilePath,
+        ReadBuffer,
+        StorageOptions,
+    )
+
+_CellValueT = Union[int, float, str, bool, time, date, datetime, timedelta]
+
+
+class CalamineReader(BaseExcelReader["CalamineWorkbook"]):
+    @doc(storage_options=_shared_docs["storage_options"])
+    def __init__(
+        self,
+        filepath_or_buffer: FilePath | ReadBuffer[bytes],
+        storage_options: StorageOptions | None = None,
+        engine_kwargs: dict | None = None,
+    ) -> None:
+        """
+        Reader using calamine engine (xlsx/xls/xlsb/ods).
+
+        Parameters
+        ----------
+        filepath_or_buffer : str, path to be parsed or
+            an open readable stream.
+        {storage_options}
+        engine_kwargs : dict, optional
+            Arbitrary keyword arguments passed to excel engine.
+        """
+        import_optional_dependency("python_calamine")
+        super().__init__(
+            filepath_or_buffer,
+            storage_options=storage_options,
+            engine_kwargs=engine_kwargs,
+        )
+
+    @property
+    def _workbook_class(self) -> type[CalamineWorkbook]:
+        from python_calamine import CalamineWorkbook
+
+        return CalamineWorkbook
+
+    def load_workbook(
+        self, filepath_or_buffer: FilePath | ReadBuffer[bytes], engine_kwargs: Any
+    ) -> CalamineWorkbook:
+        from python_calamine import load_workbook
+
+        return load_workbook(
+            filepath_or_buffer, **engine_kwargs  # type: ignore[arg-type]
+        )
+
+    @property
+    def sheet_names(self) -> list[str]:
+        from python_calamine import SheetTypeEnum
+
+        return [
+            sheet.name
+            for sheet in self.book.sheets_metadata
+            if sheet.typ == SheetTypeEnum.WorkSheet
+        ]
+
+    def get_sheet_by_name(self, name: str) -> CalamineSheet:
+        self.raise_if_bad_sheet_by_name(name)
+        return self.book.get_sheet_by_name(name)
+
+    def get_sheet_by_index(self, index: int) -> CalamineSheet:
+        self.raise_if_bad_sheet_by_index(index)
+        return self.book.get_sheet_by_index(index)
+
+    def get_sheet_data(
+        self, sheet: CalamineSheet, file_rows_needed: int | None = None
+    ) -> list[list[Scalar]]:
+        def _convert_cell(value: _CellValueT) -> Scalar:
+            if isinstance(value, float):
+                val = int(value)
+                if val == value:
+                    return val
+                else:
+                    return value
+            elif isinstance(value, date):
+                return pd.Timestamp(value)
+            elif isinstance(value, timedelta):
+                return pd.Timedelta(value)
+            elif isinstance(value, time):
+                # cast needed here because Scalar doesn't include datetime.time
+                return cast(Scalar, value)
+
+            return value
+
+        rows: list[list[_CellValueT]] = sheet.to_python(skip_empty_area=False)
+        data: list[list[Scalar]] = []
+
+        for row in rows:
+            data.append([_convert_cell(cell) for cell in row])
+            if file_rows_needed is not None and len(data) >= file_rows_needed:
+                break
+
+        return data
diff --git a/pandas/tests/io/excel/test_readers.py b/pandas/tests/io/excel/test_readers.py
diff --git a/pyproject.toml b/pyproject.toml
diff --git a/requirements-dev.txt b/requirements-dev.txt
diff --git a/scripts/tests/data/deps_expected_random.yaml b/scripts/tests/data/deps_expected_random.yaml
diff --git a/scripts/tests/data/deps_minimum.toml b/scripts/tests/data/deps_minimum.toml
diff --git a/scripts/tests/data/deps_unmodified_random.yaml b/scripts/tests/data/deps_unmodified_random.yaml

Original file line number	Diff line number	Diff line change
`@@ -28,7 +28,7 @@ enhancement2`
`28`	`28`
`29`	`29`	`Other enhancements`
`30`	`30`	`^^^^^^^^^^^^^^^^^^`
`31`		`--`
	`31`	+- Added ``calamine`` as an engine to ``read_excel`` (:issue:`50395`)
`32`	`32`	`-`
`33`	`33`
`34`	`34`	`.. ---------------------------------------------------------------------------`