diff --git a/pandas/core/frame.py b/pandas/core/frame.py index 6c36c7e71759c..dc0e68f36c877 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -129,6 +129,7 @@ ) from pandas.core.ops.missing import dispatch_fill_zeros from pandas.core.series import Series +from pandas.protocol.wrapper import DataFrame as DataFrameWrapper from pandas.io.common import get_filepath_or_buffer from pandas.io.formats import console, format as fmt @@ -138,6 +139,7 @@ if TYPE_CHECKING: from pandas.core.groupby.generic import DataFrameGroupBy from pandas.io.formats.style import Styler + from pandas.wesm import dataframe as dataframe_protocol # noqa: F401 # --------------------------------------------------------------------- # Docstring templates @@ -435,6 +437,32 @@ def __init__( if isinstance(data, DataFrame): data = data._data + elif hasattr(data, "__dataframe__"): + # construct using dict of numpy arrays + # TODO(simonjayhawkins) index, columns, dtype and copy arguments + obj = cast("dataframe_protocol.DataFrame", data.__dataframe__) + + def _get_column(col): + try: + return col.to_numpy() + except NotImplementedError: + return col.to_arrow() + + data = { + column_name: _get_column(obj[column_name]) + for column_name in obj.column_names + } + + if not index: + try: + index = MultiIndex.from_tuples(obj.row_names) + except TypeError: + index = obj.row_names + except NotImplementedError: + # It is not necessary to implement row_names in the + # dataframe interchange protocol + pass + if isinstance(data, BlockManager): mgr = self._init_mgr( data, axes=dict(index=index, columns=columns), dtype=dtype, copy=copy @@ -520,6 +548,13 @@ def __init__( NDFrame.__init__(self, mgr) + @property + def __dataframe__(self) -> DataFrameWrapper: + """ + DataFrame interchange protocol + """ + return DataFrameWrapper(self) + # ---------------------------------------------------------------------- @property diff --git a/pandas/protocol/__init__.py b/pandas/protocol/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/pandas/protocol/wrapper.py b/pandas/protocol/wrapper.py new file mode 100644 index 0000000000000..e4652800421e4 --- /dev/null +++ b/pandas/protocol/wrapper.py @@ -0,0 +1,91 @@ +from typing import TYPE_CHECKING, Any, Hashable, Iterable, Sequence + +from pandas.wesm import dataframe as dataframe_protocol +from pandas.wesm.example_dict_of_ndarray import NumPyColumn + +if TYPE_CHECKING: + import pandas as pd + + +class Column(NumPyColumn): + """ + Construct generic column from pandas Series + + Parameters + ---------- + ser : pd.Series + """ + + _ser: "pd.Series" + + def __init__(self, ser: "pd.Series"): + self._ser = ser + super().__init__(ser.name, ser.to_numpy()) + + +class DataFrame(dataframe_protocol.DataFrame): + """ + Construct generic data frame from pandas DataFrame + + Parameters + ---------- + df : pd.DataFrame + """ + + _df: "pd.DataFrame" + + def __init__(self, df: "pd.DataFrame"): + self._df = df + + def __str__(self) -> str: + return str(self._df) + + def __repr__(self) -> str: + return repr(self._df) + + def column_by_index(self, i: int) -> dataframe_protocol.Column: + """ + Return the column at the indicated position. + """ + return Column(self._df.iloc[:, i]) + + def column_by_name(self, key: Hashable) -> dataframe_protocol.Column: + """ + Return the column whose name is the indicated key. + """ + return Column(self._df[key]) + + @property + def column_names(self) -> Sequence[Any]: + """ + Return the column names as a materialized sequence. + """ + return self._df.columns.to_list() + + @property + def row_names(self) -> Sequence[Any]: + """ + Return the row names (if any) as a materialized sequence. It is not + necessary to implement this method + """ + return self._df.index.to_list() + + def iter_column_names(self) -> Iterable[Any]: + """ + Return the column names as an iterable. + """ + return self.column_names + + @property + def num_columns(self) -> int: + """ + Return the number of columns in the DataFrame. + """ + return self._df.shape[1] + + @property + def num_rows(self) -> int: + """ + Return the number of rows in the DataFrame. + """ + return len(self._df) diff --git a/pandas/tests/api/test_api.py b/pandas/tests/api/test_api.py index 5aab5b814bae7..97209b64afb8f 100644 --- a/pandas/tests/api/test_api.py +++ b/pandas/tests/api/test_api.py @@ -203,6 +203,8 @@ class TestPDApi(Base): "_tslib", "_typing", "_version", + "protocol", + "wesm", ] def test_api(self): diff --git a/pandas/tests/test_downstream.py b/pandas/tests/test_downstream.py index 122ef1f47968e..66893819eac0f 100644 --- a/pandas/tests/test_downstream.py +++ b/pandas/tests/test_downstream.py @@ -10,6 +10,8 @@ from pandas import DataFrame import pandas._testing as tm +from pandas.protocol.wrapper import DataFrame as DataFrameWrapper +from pandas.wesm import dataframe as dataframe_protocol, example_dict_of_ndarray def import_module(name): @@ -147,3 +149,100 @@ def test_missing_required_dependency(): output = exc.value.stdout.decode() for name in ["numpy", "pytz", "dateutil"]: assert name in output + + +# ----------------------------------------------------------------------------- +# DataFrame interchange protocol +# ----------------------------------------------------------------------------- + + +class TestDataFrameProtocol: + def test_interface_smoketest(self): + df = DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) + + result = df.__dataframe__ + assert isinstance(result, dataframe_protocol.DataFrame) + assert isinstance(result["a"], dataframe_protocol.Column) + assert isinstance(result.column_by_index(0), dataframe_protocol.Column) + assert isinstance(result["a"].type, dataframe_protocol.DataType) + + assert result.num_rows == 3 + assert result.num_columns == 2 + assert result.column_names == ["a", "b"] + assert list(result.iter_column_names()) == ["a", "b"] + assert result.row_names == [0, 1, 2] + + expected = np.array([1, 2, 3], dtype=np.int64) + res = result["a"].to_numpy() + tm.assert_numpy_array_equal(res, expected) + res = result.column_by_index(0).to_numpy() + tm.assert_numpy_array_equal(res, expected) + + assert result["a"].name == "a" + assert result.column_by_index(0).name == "a" + + expected_type = dataframe_protocol.Int64() + assert result["a"].type == expected_type + assert result.column_by_index(0).type == expected_type + + def test_pandas_dataframe_constructor(self): + # TODO(simonjayhawkins): move to test_constructors.py + df = DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) + + result = DataFrame(df) + tm.assert_frame_equal(result, df) + assert result is not df + + result = DataFrame(df.__dataframe__) + tm.assert_frame_equal(result, df) + assert result is not df + + # It is not necessary to implement row_names in the + # dataframe interchange protocol + + # TODO(simonjayhawkins) how to monkeypatch property with pytest + # raises AttributeError: can't set attribute + + class _DataFrameWrapper(DataFrameWrapper): + @property + def row_names(self): + raise NotImplementedError("row_names") + + result = _DataFrameWrapper(df) + with pytest.raises(NotImplementedError, match="row_names"): + result.row_names + + result = DataFrame(result) + tm.assert_frame_equal(result, df) + + def test_multiindex(self): + df = ( + DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) + .reset_index() + .set_index(["index", "a"]) + ) + result = df.__dataframe__ + + assert result.row_names == [(0, 1), (1, 2), (2, 3)] + + # TODO(simonjayhawkins) split this test and move to test_constructors.py + result = DataFrame(result) + # index and column names are not available from the protocol api + tm.assert_frame_equal(result, df, check_names=False) + + df = df.unstack() + result = df.__dataframe__ + + assert result.column_names == [("b", 1), ("b", 2), ("b", 3)] + + # TODO(simonjayhawkins) split this test and move to test_constructors.py + result = DataFrame(result) + # index and column names are not available from the protocol api + tm.assert_frame_equal(result, df, check_names=False) + + def test_example_dict_of_ndarray(self): + data, names, df = example_dict_of_ndarray.get_example() + df = DataFrame(df) + expected = DataFrame(data) + tm.assert_frame_equal(df, expected) + assert df.columns.to_list() == names diff --git a/pandas/wesm/__init__.py b/pandas/wesm/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/pandas/wesm/dataframe.py b/pandas/wesm/dataframe.py new file mode 100644 index 0000000000000..d8e1b5b63dc3d --- /dev/null +++ b/pandas/wesm/dataframe.py @@ -0,0 +1,272 @@ +# MIT License +# +# Copyright (c) 2020 Wes McKinney + +from abc import ABC, abstractmethod +from collections import abc +from typing import Any, Hashable, Iterable, Optional, Sequence + +# ---------------------------------------------------------------------- +# A simple data type class hierarchy for illustration + + +class DataType(ABC): + """ + A metadata object representing the logical value type of a cell in a data + frame column. This metadata does not guarantee an specific underlying data + representation + """ + + def __eq__(self, other: "DataType"): # type: ignore + return self.equals(other) + + def __str__(self): + return self.to_string() + + def __repr__(self): + return str(self) + + @abstractmethod + def to_string(self) -> str: + """ + Return human-readable representation of the data type + """ + + @abstractmethod + def equals(self, other: "DataType") -> bool: + """ + Return true if other DataType contains the same metadata as this + DataType + """ + pass + + +class PrimitiveType(DataType): + def equals(self, other: DataType) -> bool: + return type(self) == type(other) + + +class NullType(PrimitiveType): + """ + A data type whose values are always null + """ + + def to_string(self): + return "null" + + +class Boolean(PrimitiveType): + def to_string(self): + return "bool" + + +class NumberType(PrimitiveType): + pass + + +class IntegerType(NumberType): + pass + + +class SignedIntegerType(IntegerType): + pass + + +class Int8(SignedIntegerType): + def to_string(self): + return "int8" + + +class Int16(SignedIntegerType): + def to_string(self): + return "int16" + + +class Int32(SignedIntegerType): + def to_string(self): + return "int32" + + +class Int64(SignedIntegerType): + def to_string(self): + return "int64" + + +class Binary(PrimitiveType): + """ + A variable-size binary (bytes) value + """ + + def to_string(self): + return "binary" + + +class String(PrimitiveType): + """ + A UTF8-encoded string value + """ + + def to_string(self): + return "string" + + +class Object(PrimitiveType): + """ + Any PyObject value + """ + + def to_string(self): + return "object" + + +class Categorical(DataType): + """ + A categorical value is an ordinal (integer) value that references a + sequence of category values of an arbitrary data type + """ + + def __init__( + self, index_type: IntegerType, category_type: DataType, ordered: bool = False + ): + self.index_type = index_type + self.category_type = category_type + self.ordered = ordered + + def equals(self, other: DataType) -> bool: + return ( + isinstance(other, Categorical) + and self.index_type == other.index_type + and self.category_type == other.category_type + and self.ordered == other.ordered + ) + + def to_string(self): + return "categorical(indices={}, categories={}, ordered={})".format( + str(self.index_type), str(self.category_type), self.ordered + ) + + +# ---------------------------------------------------------------------- +# Classes representing a column in a DataFrame + + +class Column(ABC): + @property + @abstractmethod + def name(self) -> Optional[Hashable]: + pass + + @property + @abstractmethod + def type(self) -> DataType: + """ + Return the logical type of each column cell value + """ + pass + + def to_numpy(self): + """ + Access column's data as a NumPy array. Recommended to return a view if + able but not required + """ + raise NotImplementedError("Conversion to NumPy not available") + + def to_arrow(self, **kwargs): + """ + Access column's data in the Apache Arrow format as pyarrow.Array or + ChunkedArray. Recommended to return a view if able but not required + """ + raise NotImplementedError("Conversion to Arrow not available") + + +# ---------------------------------------------------------------------- +# DataFrame: the main public API + + +class DataFrame(ABC, abc.Mapping): + """ + An abstract data frame base class. + + A "data frame" represents an ordered collection of named columns. A + column's "name" is permitted to be any hashable Python value, but strings + are common. Names are not required to be unique. Columns may be accessed by + name (when the name is unique) or by position. + """ + + @property + def __dataframe__(self): + """ + Idempotence of data frame protocol + """ + return self + + def __iter__(self): + # TBD: Decide what iterating should return + return iter(self.column_names) + + def __len__(self): + return self.num_rows + + @property + @abstractmethod + def num_columns(self) -> int: + """ + Return the number of columns in the DataFrame + """ + pass + + @property + @abstractmethod + def num_rows(self) -> Optional[int]: + """ + Return the number of rows in the DataFrame (if known) + """ + pass + + @abstractmethod + def iter_column_names(self) -> Iterable[Any]: + """ + Return the column names as an iterable + """ + pass + + # TODO: Should this be a method or property? + @property + @abstractmethod + def column_names(self) -> Sequence[Any]: + """ + Return the column names as a materialized sequence + """ + pass + + # TODO: Should this be a method or property? + @property + def row_names(self) -> Sequence[Any]: + """ + Return the row names (if any) as a materialized sequence. It is not + necessary to implement this method + """ + raise NotImplementedError("row_names") + + def __getitem__(self, key: Hashable) -> Column: + return self.column_by_name(key) + + @abstractmethod + def column_by_name(self, key: Hashable) -> Column: + """ + Return the column whose name is the indicated key + """ + pass + + @abstractmethod + def column_by_index(self, i: int) -> Column: + """ + Return the column at the indicated position + """ + pass + + +class MutableDataFrame(DataFrame, abc.MutableMapping): + # TODO: Mutable data frames are fraught at this interface level and + # need more discussion + pass diff --git a/pandas/wesm/example_dict_of_ndarray.py b/pandas/wesm/example_dict_of_ndarray.py new file mode 100644 index 0000000000000..ba22ed5ea9a1a --- /dev/null +++ b/pandas/wesm/example_dict_of_ndarray.py @@ -0,0 +1,154 @@ +# MIT License +# +# Copyright (c) 2020 Wes McKinney + +from typing import Any, Dict, Hashable, Optional, Sequence + +import numpy as np + +import pandas.wesm.dataframe as dataframe + +_numeric_types = { + "int8": dataframe.Int8(), + "int16": dataframe.Int16(), + "int32": dataframe.Int32(), + "int64": dataframe.Int64(), +} + + +def _integer_factory(dtype): + return _numeric_types[dtype.name] + + +def _constant_factory(type_instance): + def factory(*unused): + return type_instance + + return factory + + +_type_factories = { + "b": _constant_factory(dataframe.Boolean()), + "i": _integer_factory, + "O": _constant_factory(dataframe.Object()), + "S": _constant_factory(dataframe.Binary()), + "U": _constant_factory(dataframe.String()), +} + + +class NumPyColumn(dataframe.Column): + def __init__(self, name, data): + self._name = name + self._data = data + + @property + def name(self) -> Hashable: + return self._name + + @property + def type(self) -> dataframe.DataType: + factory = _type_factories.get(self._data.dtype.kind) + if factory is None: + raise NotImplementedError( + "Data frame type for NumPy Type {} " + "not known".format(str(self._data.dtype)) + ) + return factory(self._data.dtype) + + def to_numpy(self): + return self._data + + +class DictDataFrame(dataframe.DataFrame): + """ + Construct data frame from dict of NumPy arrays + + Parameters + ---------- + data : dict + names : sequence, default None + If not passed, the names will be determined by the data's keys + num_rows : int, default None + If not passed, determined from the data + """ + + _num_rows: Optional[int] + + def __init__( + self, + columns: Dict[Hashable, np.ndarray], + names: Optional[Sequence[Hashable]] = None, + num_rows: Optional[int] = None, + ): + if names is None: + names = list(columns.keys()) + + assert len(columns) == len(names) + + self._columns = columns.copy() + self._names = list(names) + # self._name_to_index = {i: k for i, k in enumerate(self._names)} + + if len(columns) > 0: + assert num_rows is None + self._num_rows = len(next(iter(columns.values()))) + else: + self._num_rows = num_rows + + @property + def num_columns(self): + return len(self._columns) + + @property + def num_rows(self): + return self._num_rows + + def iter_column_names(self): + return iter(self._names) + + @property + def column_names(self): + return self._names + + def column_by_name(self, key: Hashable) -> NumPyColumn: + return NumPyColumn(key, self._columns[key]) + + def column_by_index(self, i: int) -> NumPyColumn: + return NumPyColumn(self._names[i], self._columns[self._names[i]]) + + +def get_example(): + data: Dict[Hashable, Any] = { + "a": np.array([1, 2, 3, 4, 5], dtype="int64"), + "b": np.array(["a", "b", "c", "d", "e"]), + "c": np.array([True, False, True, False, True]), + } + names = ["a", "b", "c"] + return data, names, DictDataFrame(data, names=names) + + +def test_basic_behavior(): + raw_data, names, df = get_example() + + assert len(df) == 5 + assert df.num_columns == 3 + assert df.num_rows == 5 + + for i, name in enumerate(df.column_names): + assert name == names[i] + + for i, name in enumerate(df.iter_column_names()): + assert name == names[i] + + expected_types = { + "a": dataframe.Int64(), + "b": dataframe.String(), + "c": dataframe.Boolean(), + } + + for i, name in enumerate(names): + col = df[name] + assert col.name == name + assert col.type == expected_types[name] + assert col.to_numpy() is raw_data[name] + assert df.column_by_index(i).name == col.name