Pull request #10: Port part of the np.array function

Merge in ~STEPAN.SINDELAR_ORACLE.COM/numpy-hpy from ss/array_array to labs-hpy-port

* commit '5222b83f0faa5dd99b1a3c6a659940831c1873a1':
  HPyArray_AssignArray
  Add forgotten capi_warn
  Make PyArray_Descr memcopy work for both legacy/pure layout
  Add HPyType helpers generation for scalars
  Replace h_array_type_global with HPyArray_Type
  HPyArray_SetBaseObject
  HPyDataMem_UserNEW, HPyDataMem_UserNEW_ZEROED
  HPyArray_NewFromDescr_int: check PyCapsule using HPy
  Fully ported HPyArray_NewFromDescr_int (missing: PyArray_SetBaseObject and PyCapsule)
  HPyArray_NewLikeArrayWithShape
  Use HPy Type helpers instead of HPy_AsStruct everywhere
  Fix: use HPyArray_GetNDim
  Call new HPyArray_NewLikeArray
  HPyArray_NewCopy
  Checkpoint: np.array works
  Checkpoint in debugging
  Port np.array to HPy: args parsing
  Set legacy slots of PyCLongDoubleArrType, PyCDoubleArrType, PyCFloatArrType

Author: fangerer

numpy/core/include/numpy/ndarrayobject.h

@@ -26,6 +26,7 @@ extern "C" {
 #define PyArray_DescrCheck(op) PyObject_TypeCheck(op, &PyArrayDescr_Type)
 #define HPyArray_DescrCheck(ctx, op) HPy_TypeCheck(ctx, op, HPyArrayDescr_Type)
 
+
 #define PyArray_Check(op) PyObject_TypeCheck(op, &PyArray_Type)
 #define PyArray_CheckExact(op) (((PyObject*)(op))->ob_type == &PyArray_Type)
 #define HPyArray_Check(ctx, op) HPy_TypeCheck(ctx, op, HPyArray_Type)
@@ -184,6 +185,13 @@ PyArray_DiscardWritebackIfCopy(PyArrayObject *arr)
                 descr = _new_; \
         } while(0)
 
+#define HPyArray_DESCR_REPLACE(ctx, descr) do { \
+                HPy _new_; \
+                _new_ = HPyArray_DescrNew(ctx, descr); \
+                HPy_Close(ctx, descr); \
+                descr = _new_; \
+        } while(0)
+
 /* Copy should always return contiguous array */
 #define PyArray_Copy(obj) PyArray_NewCopy(obj, NPY_CORDER)
 
@@ -252,6 +260,7 @@ NPY_TITLE_KEY_check(PyObject *key, PyObject *value)
 #define NPY_TITLE_KEY(key, value) (NPY_TITLE_KEY_check((key), (value)))
 
 #define DEPRECATE(msg) PyErr_WarnEx(PyExc_DeprecationWarning,msg,1)
+#define HPY_DEPRECATE(ctx, msg) HPyErr_WarnEx(ctx,ctx->h_DeprecationWarning,msg,1)
 #define DEPRECATE_FUTUREWARNING(msg) PyErr_WarnEx(PyExc_FutureWarning,msg,1)
 
 #ifdef __cplusplus

numpy/core/include/numpy/ndarraytypes.h

@@ -613,9 +613,15 @@ typedef struct {
 #define PyDataType_FLAGCHK(dtype, flag) \
         (((dtype)->flags & (flag)) == (flag))
 
+#define HPyDataType_FLAGCHK(ctx, dtype, flag) \
+        ((PyArray_Descr_AsStruct(ctx, dtype)->flags & (flag)) == (flag))
+
 #define PyDataType_REFCHK(dtype) \
         PyDataType_FLAGCHK(dtype, NPY_ITEM_REFCOUNT)
 
+#define HPyDataType_REFCHK(ctx, dtype) \
+        HPyDataType_FLAGCHK(ctx, dtype, NPY_ITEM_REFCOUNT)
+
 typedef struct _PyArray_Descr {
         PyObject_HEAD
         /*
@@ -1584,13 +1590,19 @@ PyArray_DESCR(const PyArrayObject *arr)
 }
 
 static NPY_INLINE HPy
-HPyArray_GetBase(HPyContext *ctx, HPy arr)
+HPyArray_BASE(HPyContext *ctx, HPy h_arr, PyArrayObject *arr_obj)
 {
-    HPyField f_base = HPyArray_AsFields(ctx, arr)->f_base;
-    if (f_base._i == 0) {
+    PyArrayObject_fields *arr = (PyArrayObject_fields*) arr_obj;
+    if (HPyField_IsNull(arr->f_base)) {
         return HPy_NULL;
     }
-    return HPyField_Load(ctx, arr, HPyArray_AsFields(ctx, arr)->f_base);
+    return HPyField_Load(ctx, h_arr, arr->f_base);
+}
+
+static NPY_INLINE HPy
+HPyArray_GetBase(HPyContext *ctx, HPy arr)
+{
+    return HPyArray_BASE(ctx, arr, (PyArrayObject*) HPyArray_AsFields(ctx, arr));
 }
 
 static NPY_INLINE void
@@ -1653,7 +1665,7 @@ PyArray_NDIM(const PyArrayObject *arr)
 static NPY_INLINE int
 HPyArray_GetNDim(HPyContext *ctx, HPy arr)
 {
-    return ((PyArrayObject_fields *) HPy_AsStructLegacy(ctx, arr))->nd;
+    return ((PyArrayObject_fields *) PyArrayObject_AsStruct(ctx, arr))->nd;
 }
 
 static NPY_INLINE void *
@@ -2070,6 +2082,8 @@ typedef void (PyDataMem_EventHookFunc)(void *inp, void *outp, size_t size,
         void *reserved[3];
     } PyArray_DTypeMeta;
 
+    HPyType_LEGACY_HELPERS(PyArray_DTypeMeta);
+
 #endif  /* NPY_INTERNAL_BUILD */
 
 

numpy/core/src/common/array_assign.h

@@ -30,6 +30,12 @@ PyArray_AssignArray(PyArrayObject *dst, PyArrayObject *src,
                     PyArrayObject *wheremask,
                     NPY_CASTING casting);
 
+
+NPY_NO_EXPORT int
+HPyArray_AssignArray(HPyContext *ctx, HPy h_dst, HPy h_src,
+                    HPy h_wheremask,
+                    NPY_CASTING casting);
+
 NPY_NO_EXPORT int
 PyArray_AssignRawScalar(PyArrayObject *dst,
                         PyArray_Descr *src_dtype, char *src_data,

numpy/core/src/common/npy_import.h

@@ -29,4 +29,18 @@ npy_cache_import(const char *module, const char *attr, PyObject **cache)
     }
 }
 
+// HPY TODO: global cache...
+NPY_INLINE static void
+npy_hpy_cache_import(HPyContext *ctx, const char *module, const char *attr, HPy *cache)
+{
+    if (NPY_UNLIKELY(HPy_IsNull(*cache))) {
+        HPy mod = HPyImport_ImportModule(ctx, module);
+
+        if (HPy_IsNull(mod)) {
+            *cache = HPy_GetAttr_s(ctx, mod, attr);
+            HPy_Close(ctx, mod);
+        }
+    }
+}
+
 #endif  /* NUMPY_CORE_SRC_COMMON_NPY_IMPORT_H_ */

numpy/core/src/multiarray/abstractdtypes.c

@@ -125,7 +125,7 @@ initialize_and_map_pytypes_to_dtypes(HPyContext *ctx)
         HPy_Close(ctx, h_PyArray_PyIntAbstractDType);
         goto cleanup;
     }
-    PyArray_DTypeMeta *int_abstract_dtype_data = (PyArray_DTypeMeta *) HPy_AsStructLegacy(ctx, h_PyArray_PyIntAbstractDType);
+    PyArray_DTypeMeta *int_abstract_dtype_data = PyArray_DTypeMeta_AsStruct(ctx, h_PyArray_PyIntAbstractDType);
     int_abstract_dtype_data->dt_slots = &pyintabstractdtype_slots;
     int_abstract_dtype_data->flags = NPY_DT_ABSTRACT;
     int_abstract_dtype_data->scalar_type = &PyLong_Type;
@@ -137,7 +137,7 @@ initialize_and_map_pytypes_to_dtypes(HPyContext *ctx)
         HPy_Close(ctx, h_PyArray_PyFloatAbstractDType);
         goto cleanup;
     }
-    PyArray_DTypeMeta *float_abstract_dtype_data = (PyArray_DTypeMeta *) HPy_AsStructLegacy(ctx, h_PyArray_PyFloatAbstractDType);
+    PyArray_DTypeMeta *float_abstract_dtype_data = PyArray_DTypeMeta_AsStruct(ctx, h_PyArray_PyFloatAbstractDType);
     float_abstract_dtype_data->dt_slots = &pyfloatabstractdtype_slots;
     float_abstract_dtype_data->flags = NPY_DT_ABSTRACT;
     float_abstract_dtype_data->scalar_type = &PyFloat_Type;
@@ -149,7 +149,7 @@ initialize_and_map_pytypes_to_dtypes(HPyContext *ctx)
         HPy_Close(ctx, h_PyArray_PyComplexAbstractDType);
         goto cleanup;
     }
-    PyArray_DTypeMeta *complex_abstract_dtype_data = (PyArray_DTypeMeta *) HPy_AsStructLegacy(ctx, h_PyArray_PyComplexAbstractDType);
+    PyArray_DTypeMeta *complex_abstract_dtype_data = PyArray_DTypeMeta_AsStruct(ctx, h_PyArray_PyComplexAbstractDType);
     complex_abstract_dtype_data->dt_slots = &pycomplexabstractdtype_slots;
     complex_abstract_dtype_data->flags = NPY_DT_ABSTRACT;
     complex_abstract_dtype_data->scalar_type = &PyComplex_Type;

numpy/core/src/multiarray/alloc.c

@@ -407,7 +407,7 @@ PyDataMem_Handler default_handler = {
 PyObject *PyDataMem_DefaultHandler;
 
 #if (!defined(PYPY_VERSION_NUM) || PYPY_VERSION_NUM >= 0x07030600)
-PyObject *current_handler;
+HPy current_handler;
 #endif
 
 int uo_index=0;   /* user_override index */
@@ -416,15 +416,26 @@ int uo_index=0;   /* user_override index */
 
 NPY_NO_EXPORT void *
 PyDataMem_UserNEW(size_t size, PyObject *mem_handler)
+{
+    HPyContext *ctx = npy_get_context();
+    HPy h = HPy_FromPyObject(ctx, mem_handler);
+    void *res = HPyDataMem_UserNEW(ctx, size, h);
+    HPy_Close(ctx, h);
+    return res;
+}
+
+NPY_NO_EXPORT void *
+HPyDataMem_UserNEW(HPyContext *ctx, size_t size, HPy mem_handler)
 {
     void *result;
-    PyDataMem_Handler *handler = (PyDataMem_Handler *) PyCapsule_GetPointer(mem_handler, "mem_handler");
+    PyDataMem_Handler *handler = (PyDataMem_Handler *) HPyCapsule_GetPointer(ctx, mem_handler, "mem_handler");
     if (handler == NULL) {
         return NULL;
     }
     assert(size != 0);
     result = handler->allocator.malloc(handler->allocator.ctx, size);
     if (_PyDataMem_eventhook != NULL) {
+        capi_warn("HPyDataMem_UserNEW_ZEROED: GIL usage if _PyDataMem_eventhook is set");
         NPY_ALLOW_C_API_DEF
         NPY_ALLOW_C_API
         if (_PyDataMem_eventhook != NULL) {
@@ -439,14 +450,25 @@ PyDataMem_UserNEW(size_t size, PyObject *mem_handler)
 
 NPY_NO_EXPORT void *
 PyDataMem_UserNEW_ZEROED(size_t nmemb, size_t size, PyObject *mem_handler)
+{
+    HPyContext *ctx = npy_get_context();
+    HPy h = HPy_FromPyObject(ctx, mem_handler);
+    void *res = HPyDataMem_UserNEW_ZEROED(ctx, nmemb, size, h);
+    HPy_Close(ctx, h);
+    return res;
+}
+
+NPY_NO_EXPORT void *
+HPyDataMem_UserNEW_ZEROED(HPyContext *ctx, size_t nmemb, size_t size, HPy mem_handler)
 {
     void *result;
-    PyDataMem_Handler *handler = (PyDataMem_Handler *) PyCapsule_GetPointer(mem_handler, "mem_handler");
+    PyDataMem_Handler *handler = (PyDataMem_Handler *) HPyCapsule_GetPointer(ctx, mem_handler, "mem_handler");
     if (handler == NULL) {
         return NULL;
     }
     result = handler->allocator.calloc(handler->allocator.ctx, nmemb, size);
     if (_PyDataMem_eventhook != NULL) {
+        capi_warn("HPyDataMem_UserNEW_ZEROED: GIL usage if _PyDataMem_eventhook is set");
         NPY_ALLOW_C_API_DEF
         NPY_ALLOW_C_API
         if (_PyDataMem_eventhook != NULL) {
@@ -519,77 +541,76 @@ PyDataMem_UserRENEW(void *ptr, size_t size, PyObject *mem_handler)
 NPY_NO_EXPORT PyObject *
 PyDataMem_SetHandler(PyObject *handler)
 {
-    PyObject *old_handler;
-#if (!defined(PYPY_VERSION_NUM) || PYPY_VERSION_NUM >= 0x07030600)
-    PyObject *token;
-    if (PyContextVar_Get(current_handler, NULL, &old_handler)) {
-        return NULL;
-    }
-    if (handler == NULL) {
-        handler = PyDataMem_DefaultHandler;
-    }
-    token = PyContextVar_Set(current_handler, handler);
-    if (token == NULL) {
-        Py_DECREF(old_handler);
-        return NULL;
-    }
-    Py_DECREF(token);
-    return old_handler;
-#else
-    PyObject *p;
-    p = PyThreadState_GetDict();
-    if (p == NULL) {
-        return NULL;
-    }
-    old_handler = PyDict_GetItemString(p, "current_allocator");
-    if (old_handler == NULL) {
-        old_handler = PyDataMem_DefaultHandler
-    }
-    Py_INCREF(old_handler);
-    if (handler == NULL) {
-        handler = PyDataMem_DefaultHandler;
-    }
-    const int error = PyDict_SetItemString(p, "current_allocator", handler);
-    if (error) {
-        Py_DECREF(old_handler);
-        return NULL;
-    }
-    return old_handler;
-#endif
+    hpy_abort_not_implemented("PyDataMem_SetHandler");
+//     PyObject *old_handler;
+// #if (!defined(PYPY_VERSION_NUM) || PYPY_VERSION_NUM >= 0x07030600)
+//     PyObject *token;
+//     if (PyContextVar_Get(current_handler, NULL, &old_handler)) {
+//         return NULL;
+//     }
+//     if (handler == NULL) {
+//         handler = PyDataMem_DefaultHandler;
+//     }
+//     token = PyContextVar_Set(current_handler, handler);
+//     if (token == NULL) {
+//         Py_DECREF(old_handler);
+//         return NULL;
+//     }
+//     Py_DECREF(token);
+//     return old_handler;
+// #else
+//     PyObject *p;
+//     p = PyThreadState_GetDict();
+//     if (p == NULL) {
+//         return NULL;
+//     }
+//     old_handler = PyDict_GetItemString(p, "current_allocator");
+//     if (old_handler == NULL) {
+//         old_handler = PyDataMem_DefaultHandler
+//     }
+//     Py_INCREF(old_handler);
+//     if (handler == NULL) {
+//         handler = PyDataMem_DefaultHandler;
+//     }
+//     const int error = PyDict_SetItemString(p, "current_allocator", handler);
+//     if (error) {
+//         Py_DECREF(old_handler);
+//         return NULL;
+//     }
+//     return old_handler;
+// #endif
 }
 
-/*NUMPY_API
+/*
  * Return the policy that will be used to allocate data
  * for the next PyArrayObject. On failure, return NULL.
  */
-NPY_NO_EXPORT PyObject *
-PyDataMem_GetHandler()
+NPY_NO_EXPORT HPy
+HPyDataMem_GetHandler(HPyContext *ctx)
 {
-    PyObject *handler;
+    HPy handler;
 #if (!defined(PYPY_VERSION_NUM) || PYPY_VERSION_NUM >= 0x07030600)
-    if (PyContextVar_Get(current_handler, NULL, &handler)) {
-        return NULL;
+    if (HPyContextVar_Get(ctx, current_handler, HPy_NULL, &handler)) {
+        return HPy_NULL;
     }
     return handler;
 #else
-    PyObject *p = PyThreadState_GetDict();
-    if (p == NULL) {
-        return NULL;
-    }
-    handler = PyDict_GetItemString(p, "current_allocator");
-    if (handler == NULL) {
-        handler = PyCapsule_New(&default_handler, "mem_handler", NULL);
-        if (handler == NULL) {
-            return NULL;
-        }
-    }
-    else {
-        Py_INCREF(handler);
-    }
-    return handler;
+    hpy_abort_not_implemented("HPyDataMem_GetHandler on older Pypy versions");
 #endif
 }
 
+/*NUMPY_API
+ */
+NPY_NO_EXPORT PyObject *
+PyDataMem_GetHandler()
+{
+    HPyContext *ctx = npy_get_context();
+    HPy h = HPyDataMem_GetHandler(ctx);
+    PyObject *res = HPy_AsPyObject(ctx, h);
+    HPy_Close(ctx, h);
+    return res;
+}
+
 NPY_NO_EXPORT PyObject *
 get_handler_name(PyObject *NPY_UNUSED(self), PyObject *args)
 {

numpy/core/src/multiarray/alloc.h

@@ -16,6 +16,12 @@ PyDataMem_UserNEW(npy_uintp sz, PyObject *mem_handler);
 NPY_NO_EXPORT void *
 PyDataMem_UserNEW_ZEROED(size_t nmemb, size_t size, PyObject *mem_handler);
 
+NPY_NO_EXPORT void *
+HPyDataMem_UserNEW(HPyContext *ctx, size_t size, HPy mem_handler);
+
+NPY_NO_EXPORT void *
+HPyDataMem_UserNEW_ZEROED(HPyContext *ctx, size_t nmemb, size_t size, HPy mem_handler);
+
 NPY_NO_EXPORT void
 PyDataMem_UserFREE(void * p, npy_uintp sd, PyObject *mem_handler);
 
@@ -42,12 +48,15 @@ npy_free_cache_dim_array(PyArrayObject * arr)
 
 extern PyDataMem_Handler default_handler;
 #if (!defined(PYPY_VERSION_NUM) || PYPY_VERSION_NUM >= 0x07030600)
-extern PyObject *current_handler; /* PyContextVar/PyCapsule */
+extern HPy current_handler; /* PyContextVar/PyCapsule */
 #endif
 
 NPY_NO_EXPORT PyObject *
 get_handler_name(PyObject *NPY_UNUSED(self), PyObject *obj);
 NPY_NO_EXPORT PyObject *
 get_handler_version(PyObject *NPY_UNUSED(self), PyObject *obj);
 
+NPY_NO_EXPORT HPy
+HPyDataMem_GetHandler(HPyContext *ctx);
+
 #endif  /* NUMPY_CORE_SRC_MULTIARRAY_ALLOC_H_ */