Pull request #14: More work on np.array

Merge in ~STEPAN.SINDELAR_ORACLE.COM/numpy-hpy from ss/array_array2 to labs-hpy-port

* commit 'bebf094010c275b747e1115881d67fa85a9f6751':
  Remove dtypemeta_dealloc, visit the new HPyFields in DTypeMeta_traverse_impl
  Fix more C API usages in HPyArray_AssignArray
  Last C API usage removed from HPyArray_CheckCastSafety
  NPY_DType_Slots.within_dtype_castingimpl is HPyField
  NPY_DType_Slots.castingimpls is HPyField
  HPyArray_CheckCastSafety (partial port)

Author: steve-s

numpy/core/src/common/array_assign.c

@@ -64,6 +64,7 @@ broadcast_strides(int ndim, npy_intp const *shape,
     return 0;
 
 broadcast_error: {
+        CAPI_WARN("broadcast_error");
         PyObject *shape1 = convert_shape_to_string(strides_ndim,
                                                    strides_shape, "");
         if (shape1 == NULL) {
@@ -178,3 +179,18 @@ arrays_overlap(PyArrayObject *arr1, PyArrayObject *arr2)
         return 1;
     }
 }
+
+/* Returns 1 if the arrays have overlapping data, 0 otherwise */
+NPY_NO_EXPORT int
+hpy_arrays_overlap(HPyContext *ctx, HPy arr1, HPy arr2)
+{
+    mem_overlap_t result;
+
+    result = hpy_solve_may_share_memory(ctx, arr1, arr2, NPY_MAY_SHARE_BOUNDS);
+    if (result == MEM_OVERLAP_NO) {
+        return 0;
+    }
+    else {
+        return 1;
+    }
+}

numpy/core/src/common/array_assign.h

@@ -123,5 +123,8 @@ HIsUintAligned(HPyContext *ctx, HPy arr, PyArrayObject *arr_data);
 NPY_NO_EXPORT int
 arrays_overlap(PyArrayObject *arr1, PyArrayObject *arr2);
 
+NPY_NO_EXPORT int
+hpy_arrays_overlap(HPyContext *ctx, HPy arr1, HPy arr2);
+
 
 #endif  /* NUMPY_CORE_SRC_COMMON_ARRAY_ASSIGN_H_ */

numpy/core/src/multiarray/_multiarray_tests.c.src

@@ -1057,7 +1057,7 @@ get_all_cast_information(PyObject *NPY_UNUSED(mod), PyObject *NPY_UNUSED(args))
         PyObject *to_dtype, *cast_obj;
         Py_ssize_t pos = 0;
 
-        while (PyDict_Next(NPY_DT_SLOTS(from_dtype)->castingimpls,
+        while (PyDict_Next(DTYPE_SLOTS_CASTINGIMPL(from_dtype),
                            &pos, &to_dtype, &cast_obj)) {
             if (cast_obj == Py_None) {
                 continue;

numpy/core/src/multiarray/array_assign_array.c

@@ -511,12 +511,12 @@ HPyArray_AssignArray(HPyContext *ctx, HPy h_dst, HPy h_src,
      * of overlapping data. For bigger ndim and opposite-strided 1D
      * data, we make a temporary copy of 'src' if 'src' and 'dst' overlap.'
      */
-    capi_warn("HPyArray_AssignArray: arrays_overlap and reminder of this function...");
     if (((PyArray_NDIM(dst) == 1 && PyArray_NDIM(src) >= 1 &&
                     PyArray_STRIDES(dst)[0] *
                             PyArray_STRIDES(src)[PyArray_NDIM(src) - 1] < 0) ||
                     PyArray_NDIM(dst) > 1 || PyArray_HASFIELDS(dst)) &&
-                    arrays_overlap(src, dst)) {
+                    hpy_arrays_overlap(ctx, h_src, h_dst)) {
+        CAPI_WARN("HPyArray_AssignArray: overlapping arrays");
         PyArrayObject *tmp;
 
         /*
@@ -570,21 +570,21 @@ HPyArray_AssignArray(HPyContext *ctx, HPy h_dst, HPy h_src,
 
     PyArrayObject *wheremask = PyArrayObject_AsStruct(ctx, h_wheremask);
     /* optimization: scalar boolean mask */
-    if (wheremask != NULL &&
+    if (!HPy_IsNull(h_wheremask) &&
             PyArray_NDIM(wheremask) == 0 &&
-            PyArray_DESCR(wheremask)->type_num == NPY_BOOL) {
+            PyArray_Descr_AsStruct(ctx, HPyArray_DESCR(ctx, h_wheremask, wheremask))->type_num == NPY_BOOL) {
         npy_bool value = *(npy_bool *)PyArray_DATA(wheremask);
         if (value) {
             /* where=True is the same as no where at all */
-            wheremask = NULL;
+            h_wheremask = HPy_NULL;
         }
         else {
             /* where=False copies nothing */
             return 0;
         }
     }
 
-    if (wheremask == NULL) {
+    if (HPy_IsNull(h_wheremask)) {
         /* A straightforward value assignment */
         /* Do the assignment with raw array iteration */
         if (raw_array_assign_array(PyArray_NDIM(dst), PyArray_DIMS(dst),
@@ -606,6 +606,7 @@ HPyArray_AssignArray(HPyContext *ctx, HPy h_dst, HPy h_src,
 
         /* A straightforward where-masked assignment */
          /* Do the masked assignment with raw array iteration */
+        CAPI_WARN("non-straightforward value assignment that needs raw_array_wheremasked_assign_array");
         if (raw_array_wheremasked_assign_array(
                 PyArray_NDIM(dst), PyArray_DIMS(dst),
                 PyArray_DESCR(dst), PyArray_DATA(dst), PyArray_STRIDES(dst),
@@ -617,12 +618,16 @@ HPyArray_AssignArray(HPyContext *ctx, HPy h_dst, HPy h_src,
     }
 
     if (copied_src) {
+        CAPI_WARN("HPyArray_AssignArray: overlapping arrays");
+        // See the branch for overlapping arrays above that sets copied_src = true
         Py_DECREF(src);
     }
     return 0;
 
 fail:
     if (copied_src) {
+        CAPI_WARN("HPyArray_AssignArray: overlapping arrays");
+        // See the branch for overlapping arrays above that sets copied_src = true
         Py_DECREF(src);
     }
     return -1;

numpy/core/src/multiarray/convert_datatype.c

@@ -71,10 +71,10 @@ PyArray_GetCastingImpl(PyArray_DTypeMeta *from, PyArray_DTypeMeta *to)
 {
     PyObject *res;
     if (from == to) {
-        res = NPY_DT_SLOTS(from)->within_dtype_castingimpl;
+        res = DTYPE_SLOTS_WITHIN_DTYPE_CASTINGIMPL(from);
     }
     else {
-        res = PyDict_GetItemWithError(NPY_DT_SLOTS(from)->castingimpls, (PyObject *)to);
+        res = PyDict_GetItemWithError(DTYPE_SLOTS_CASTINGIMPL(from), (PyObject *)to);
     }
     if (res != NULL || PyErr_Occurred()) {
         Py_XINCREF(res);
@@ -120,7 +120,7 @@ PyArray_GetCastingImpl(PyArray_DTypeMeta *from, PyArray_DTypeMeta *to)
             if (castfunc == NULL) {
                 PyErr_Clear();
                 /* Remember that this cast is not possible */
-                if (PyDict_SetItem(NPY_DT_SLOTS(from)->castingimpls,
+                if (PyDict_SetItem(DTYPE_SLOTS_CASTINGIMPL(from),
                             (PyObject *) to, Py_None) < 0) {
                     return NULL;
                 }
@@ -149,7 +149,7 @@ PyArray_GetCastingImpl(PyArray_DTypeMeta *from, PyArray_DTypeMeta *to)
         Py_DECREF(res);
         return NULL;
     }
-    if (PyDict_SetItem(NPY_DT_SLOTS(from)->castingimpls,
+    if (PyDict_SetItem(DTYPE_SLOTS_CASTINGIMPL(from),
                 (PyObject *)to, res) < 0) {
         Py_DECREF(res);
         return NULL;
@@ -161,15 +161,12 @@ NPY_NO_EXPORT HPy
 HPyArray_GetCastingImpl(HPyContext *ctx, HPy from, HPy to)
 {
     PyArray_DTypeMeta *from_data = PyArray_DTypeMeta_AsStruct(ctx, from);
-    PyArray_DTypeMeta *to_data = PyArray_DTypeMeta_AsStruct(ctx, to);
     HPy res;
     if (HPy_Is(ctx, from, to)) {
-        CAPI_WARN("HPyArray_GetCastingImpl: using PyArray_DTypeMeta slots");
-        res = HPy_FromPyObject(ctx, NPY_DT_SLOTS(from_data)->within_dtype_castingimpl);
+        res = HPY_DTYPE_SLOTS_WITHIN_DTYPE_CASTINGIMPL(ctx, from, from_data);
     }
     else {
-        CAPI_WARN("HPyArray_GetCastingImpl: using PyArray_DTypeMeta slots");
-        HPy tmp = HPy_FromPyObject(ctx, NPY_DT_SLOTS(from_data)->castingimpls);
+        HPy tmp = HPY_DTYPE_SLOTS_CASTINGIMPL(ctx, from, from_data);
         res = HPy_GetItem(ctx, tmp, to);
         HPy_Close(ctx, tmp);
     }
@@ -728,6 +725,48 @@ PyArray_CheckCastSafety(NPY_CASTING casting,
 }
 
 
+NPY_NO_EXPORT int
+HPyArray_CheckCastSafety(HPyContext *ctx, NPY_CASTING casting,
+        HPy h_from, HPy h_to, HPy h_to_dtype_in)
+{
+    HPy h_to_dtype;
+    if (HPy_IsNull(h_to)) {
+        h_to_dtype = HPy_Type(ctx, h_to);
+    } else {
+        h_to_dtype = HPy_Dup(ctx, h_to_dtype_in);
+    }
+    HPy h_from_dtype = HPy_Type(ctx, h_from);
+    HPy meth = HPyArray_GetCastingImpl(ctx, h_from_dtype, h_to_dtype);
+    HPy_Close(ctx, h_from_dtype);
+    if (HPy_IsNull(meth)) {
+        return -1;
+    }
+    if (HPy_Is(ctx, meth, ctx->h_None)) {
+        HPy_Close(ctx, meth);
+        return -1;
+    }
+    
+    PyArrayMethodObject *castingimpl = PyArrayMethodObject_AsStruct(ctx, meth);
+    if (PyArray_MinCastSafety(castingimpl->casting, casting) == casting) {
+        /* No need to check using `castingimpl.resolve_descriptors()` */
+        HPy_Close(ctx, meth);
+        return 1;
+    }
+
+    hpy_abort_not_implemented("HPyArray_CheckCastSafety: generic code path");
+    // PyArray_DTypeMeta *dtypes[2] = {PyArray_DTypeMeta_AsStruct(ctx, h_from_dtype), PyArray_DTypeMeta_AsStruct(ctx, h_to_dtype)};
+    // npy_intp view_offset;
+    // NPY_CASTING safety = _get_cast_safety_from_castingimpl(castingimpl,
+    //         dtypes, PyArray_Descr_AsStruct(ctx, h_from), PyArray_Descr_AsStruct(ctx, h_to), &view_offset);
+    // Py_DECREF(meth);
+    // /* If casting is the smaller (or equal) safety we match */
+    // if (safety < 0) {
+    //     return -1;
+    // }
+    // return PyArray_MinCastSafety(safety, casting) == casting;
+}
+
+
 /*NUMPY_API
  *Check the type coercion rules.
  */
@@ -921,12 +960,8 @@ HPyArray_CanCastTypeTo(HPyContext *ctx, HPy h_from, HPy h_to,
         to = NULL;  /* consider mainly S0 and U0 as S and U */
     }
 
-    capi_warn("HPyArray_CanCastTypeTo -> PyArray_CheckCastSafety");
     HPy to_meta = HPy_Type(ctx, h_to);
-    int is_valid = PyArray_CheckCastSafety(casting,
-            PyArray_Descr_AsStruct(ctx, h_from), 
-            PyArray_Descr_AsStruct(ctx, h_to),
-            PyArray_DTypeMeta_AsStruct(ctx, to_meta));
+    int is_valid = HPyArray_CheckCastSafety(ctx, casting, h_from, h_to, to_meta);
     HPy_Close(ctx, to_meta);
     /* Clear any errors and consider this unsafe (should likely be changed) */
     if (is_valid < 0) {
@@ -2513,26 +2548,31 @@ PyArray_AddCastingImplementation(PyBoundArrayMethodObject *meth)
                     meth->method->name);
             return -1;
         }
-        if (NPY_DT_SLOTS(meth->dtypes[0])->within_dtype_castingimpl != NULL) {
+        if (DTYPE_SLOTS_WITHIN_DTYPE_CASTINGIMPL(meth->dtypes[0]) != NULL) {
             PyErr_Format(PyExc_RuntimeError,
                     "A cast was already added for %S -> %S. (method: %s)",
                     meth->dtypes[0], meth->dtypes[1], meth->method->name);
             return -1;
         }
         Py_INCREF(meth->method);
-        NPY_DT_SLOTS(meth->dtypes[0])->within_dtype_castingimpl = (
-                (PyObject *)meth->method);
+
+        HPyContext *ctx = npy_get_context();
+        HPy owner = HPy_FromPyObject(ctx, meth->dtypes[0]);
+        HPy value = HPy_FromPyObject(ctx, meth->method);
+        HPyField_Store(ctx, owner, &NPY_DT_SLOTS(meth->dtypes[0])->within_dtype_castingimpl, value);
+        HPy_Close(ctx, owner);
+        HPy_Close(ctx, value);
 
         return 0;
     }
-    if (PyDict_Contains(NPY_DT_SLOTS(meth->dtypes[0])->castingimpls,
+    if (PyDict_Contains(DTYPE_SLOTS_CASTINGIMPL(meth->dtypes[0]),
             (PyObject *)meth->dtypes[1])) {
         PyErr_Format(PyExc_RuntimeError,
                 "A cast was already added for %S -> %S. (method: %s)",
                 meth->dtypes[0], meth->dtypes[1], meth->method->name);
         return -1;
     }
-    if (PyDict_SetItem(NPY_DT_SLOTS(meth->dtypes[0])->castingimpls,
+    if (PyDict_SetItem(DTYPE_SLOTS_CASTINGIMPL(meth->dtypes[0]),
             (PyObject *)meth->dtypes[1], (PyObject *)meth->method) < 0) {
         return -1;
     }

numpy/core/src/multiarray/dtypemeta.c

@@ -20,19 +20,6 @@
 
 #include <assert.h>
 
-static void
-dtypemeta_dealloc(PyArray_DTypeMeta *self) {
-    /* Do not accidentally delete a statically defined DType: */
-    assert(((PyTypeObject *)self)->tp_flags & Py_TPFLAGS_HEAPTYPE);
-
-    Py_XDECREF(self->scalar_type);
-    Py_XDECREF(self->singleton);
-    Py_XDECREF(NPY_DT_SLOTS(self)->castingimpls);
-    NPY_DT_SLOTS(self)->castingimpls = NULL;
-    PyMem_Free(self->dt_slots);
-    PyType_Type.tp_dealloc((PyObject *) self);
-}
-
 static PyObject *
 dtypemeta_new(PyTypeObject *NPY_UNUSED(type),
         PyObject *NPY_UNUSED(args), PyObject *NPY_UNUSED(kwds))
@@ -72,7 +59,7 @@ dtypemeta_is_gc(PyObject *dtype_class)
 }
 
 HPyDef_SLOT(DTypeMeta_traverse, DTypeMeta_traverse_impl, HPy_tp_traverse)
-static int DTypeMeta_traverse_impl(void *self, HPyFunc_visitproc visit, void *arg) {
+static int DTypeMeta_traverse_impl(void *self_p, HPyFunc_visitproc visit, void *arg) {
     // HPY TODO: implement
     /*
      * We have to traverse the base class (if it is a HeapType).
@@ -87,6 +74,11 @@ static int DTypeMeta_traverse_impl(void *self, HPyFunc_visitproc visit, void *ar
     // Py_VISIT(type->singleton);
     // Py_VISIT(type->scalar_type);
     // return PyType_Type.tp_traverse((PyObject *)type, visit, arg);
+    PyArray_DTypeMeta *self = (PyArray_DTypeMeta*) self_p;
+    if (NPY_DT_SLOTS(self)) {
+        HPy_VISIT(&NPY_DT_SLOTS(self)->castingimpls);
+        HPy_VISIT(&NPY_DT_SLOTS(self)->within_dtype_castingimpl);
+    }
     return 0;
 }
 
@@ -589,11 +581,14 @@ dtypemeta_wrap_legacy_descriptor(HPyContext *ctx, PyArray_Descr *descr)
     }
 
     PyArray_DTypeMeta *new_dtype_data = PyArray_DTypeMeta_AsStruct(ctx, h_new_dtype_type);
+    new_dtype_data->dt_slots = dt_slots;
 
-    dt_slots->castingimpls = PyDict_New();
-    if (dt_slots->castingimpls == NULL) {
+    HPy h_castingimpls = HPyDict_New(ctx);
+    if (HPy_IsNull(h_castingimpls)) {
         goto cleanup;
     }
+    HPyField_Store(ctx, h_new_dtype_type, &dt_slots->castingimpls, h_castingimpls);
+    HPy_Close(ctx, h_castingimpls);
 
     /*
      * Fill DTypeMeta information that varies between DTypes, any variable
@@ -603,7 +598,6 @@ dtypemeta_wrap_legacy_descriptor(HPyContext *ctx, PyArray_Descr *descr)
     Py_INCREF(descr->typeobj);
     new_dtype_data->scalar_type = descr->typeobj;
     new_dtype_data->type_num = descr->type_num;
-    new_dtype_data->dt_slots = dt_slots;
     new_dtype_data->flags = NPY_DT_LEGACY;
     dt_slots->f = *(descr->f);
 

numpy/core/src/multiarray/dtypemeta.h

@@ -54,13 +54,13 @@ typedef struct {
      * The casting implementation (ArrayMethod) to convert between two
      * instances of this DType, stored explicitly for fast access:
      */
-    PyObject *within_dtype_castingimpl;
+    HPyField within_dtype_castingimpl;
     /*
      * Dictionary of ArrayMethods representing most possible casts
      * (structured and object are exceptions).
      * This should potentially become a weak mapping in the future.
      */
-    PyObject *castingimpls;
+    HPyField castingimpls;
 
     /*
      * Storage for `descr->f`, since we may need to allow some customizatoin
@@ -70,7 +70,6 @@ typedef struct {
     PyArray_ArrFuncs f;
 } NPY_DType_Slots;
 
-
 #define NPY_DTYPE(descr) ((PyArray_DTypeMeta *)Py_TYPE(descr))
 #define HNPY_DTYPE(ctx, descr) (HPy_Type(ctx, descr))
 #define NPY_DT_SLOTS(dtype) ((NPY_DType_Slots *)(dtype)->dt_slots)
@@ -79,6 +78,48 @@ typedef struct {
 #define NPY_DT_is_abstract(dtype) (((dtype)->flags & NPY_DT_ABSTRACT) != 0)
 #define NPY_DT_is_parametric(dtype) (((dtype)->flags & NPY_DT_PARAMETRIC) != 0)
 
+static inline PyObject* DTYPE_SLOTS_CASTINGIMPL(PyArray_DTypeMeta *meta) {
+    if (HPyField_IsNull(NPY_DT_SLOTS(meta)->castingimpls)) {
+        return NULL;
+    }
+    HPyContext *ctx = npy_get_context();
+    HPy owner = HPy_FromPyObject(ctx, (PyObject*) meta);
+    HPy res = HPyField_Load(ctx, owner, NPY_DT_SLOTS(meta)->castingimpls);
+    PyObject *py_res = HPy_AsPyObject(ctx, res);
+    HPy_Close(ctx, owner);
+    HPy_Close(ctx, res);
+    Py_DECREF(py_res);  // to simulate the borrowed reference...
+    return py_res;
+}
+
+static inline PyObject* DTYPE_SLOTS_WITHIN_DTYPE_CASTINGIMPL(PyArray_DTypeMeta *meta) {
+    if (HPyField_IsNull(NPY_DT_SLOTS(meta)->within_dtype_castingimpl)) {
+        return NULL;
+    }
+    HPyContext *ctx = npy_get_context();
+    HPy owner = HPy_FromPyObject(ctx, (PyObject*) meta);
+    HPy res = HPyField_Load(ctx, owner, NPY_DT_SLOTS(meta)->within_dtype_castingimpl);
+    PyObject *py_res = HPy_AsPyObject(ctx, res);
+    HPy_Close(ctx, owner);
+    HPy_Close(ctx, res);
+    Py_DECREF(py_res);  // to simulate the borrowed reference...
+    return py_res;
+}
+
+static inline HPy HPY_DTYPE_SLOTS_CASTINGIMPL(HPyContext *ctx, HPy h_meta, PyArray_DTypeMeta *meta) {
+    if (HPyField_IsNull(NPY_DT_SLOTS(meta)->castingimpls)) {
+        return HPy_NULL;
+    }
+    return HPyField_Load(ctx, h_meta, NPY_DT_SLOTS(meta)->castingimpls);
+}
+
+static inline HPy HPY_DTYPE_SLOTS_WITHIN_DTYPE_CASTINGIMPL(HPyContext *ctx, HPy h_meta, PyArray_DTypeMeta *meta) {
+    if (HPyField_IsNull(NPY_DT_SLOTS(meta)->within_dtype_castingimpl)) {
+        return HPy_NULL;
+    }
+    return HPyField_Load(ctx, h_meta, NPY_DT_SLOTS(meta)->within_dtype_castingimpl);
+}
+
 /*
  * Macros for convenient classmethod calls, since these require
  * the DType both for the slot lookup and as first arguments.