Pull request #13: Migrate NpyIter implementation

Merge in ~STEPAN.SINDELAR_ORACLE.COM/numpy-hpy from fa/nditer2 to labs-hpy-port

* commit 'b06d48d5f27887fe147801235142468758ab648b': (46 commits)
  Revert "Temporarily add missing scalarapi.h"
  Do appropriate include for alloca
  Introduce HNPY_cast_info, use HPyField in PyArrayMethodObject, migrate function signature 'resolve_descriptors_function' to HPy.
  Convert PyArrayMethod_Type to legacy HPy type
  Migrate define_cast_for_descrs, init_cast_info, PyArray_GetCastingImpl
  Use HPyArray_AdaptDescriptorToArray in hnpyiter_prepare_one_operand
  Do minimal migration of find_descriptor_from_array
  Migrate HPyArray_ExtractDTypeAndDescriptor to use HPyGlobal
  Migrate nditer_constr to use HPyGlobal
  Include hpy_utils.h in common.h
  Add HPyGlobal_Is to hpy_utils.c
  Migrate PyArray_ExtractDTypeAndDescriptor
  Introduce HPyGlobal HPyArrayDTypeMeta_Type
  Migrate PyArray_FailUnlessWriteable
  Introduce legacy accessors NIT_PY_OPERANDS and NIT_PY_DTYPES
  Finish minimal migration of HNpyIter_AdvancedNew
  Migrate PyArray_GetMaskedDTypeTransferFunction and PyArray_GetDTypeTransferFunction
  Temporarily add missing scalarapi.h
  Use HPyArray_NewFromDescr
  Add HPyArray_NewFromDescr to ctors.h
  ...

Author: fangerer

numpy/core/code_generators/numpy_api.py

@@ -74,7 +74,7 @@
     'PyDatetimeArrType_Type':           (215, None, '&PyDatetimeArrType_Type'),
     'PyTimedeltaArrType_Type':          (216, None, '&PyTimedeltaArrType_Type'),
     'PyHalfArrType_Type':               (217, None, '&PyHalfArrType_Type'),
-    'NpyIter_Type':                     (218,),
+    'NpyIter_Type':                     (218, None, "_NpyIter_Type_p"),
     # End 1.6 API
 }
 

numpy/core/include/numpy/experimental_dtype_api.h

@@ -168,6 +168,8 @@ typedef struct {
     void *reserved[3];
 } PyArray_DTypeMeta;
 
+HPyType_LEGACY_HELPERS(PyArray_DTypeMeta)
+
 
 /*
  * ******************************************************

numpy/core/include/numpy/ndarrayobject.h

@@ -203,6 +203,26 @@ PyArray_DiscardWritebackIfCopy(PyArrayObject *arr)
     }
 }
 
+static NPY_INLINE void
+HPyArray_DiscardWritebackIfCopy(HPyContext *ctx, HPy h_arr)
+{
+    if (!HPy_IsNull(h_arr)) {
+        HPy h_base = HPyArray_GetBase(ctx, h_arr);
+        if (HPy_IsNull(h_base)) {
+            HPy_Close(ctx, h_arr);
+            return;
+        }
+        int flags = HPyArray_FLAGS(ctx, h_arr);
+        if (flags & NPY_ARRAY_WRITEBACKIFCOPY) {
+            HPyArray_ENABLEFLAGS(ctx, h_base, NPY_ARRAY_WRITEABLE);
+            HPyArray_SetBase(ctx, h_arr, HPy_NULL);
+            HPyArray_CLEARFLAGS(ctx, h_arr, NPY_ARRAY_WRITEBACKIFCOPY);
+        }
+        HPy_Close(ctx, h_base);
+        HPy_Close(ctx, h_arr);
+    }
+}
+
 #define PyArray_DESCR_REPLACE(descr) do { \
                 PyArray_Descr *_new_; \
                 _new_ = PyArray_DescrNew(descr); \
@@ -251,6 +271,10 @@ PyArray_DiscardWritebackIfCopy(PyArrayObject *arr)
         PyArray_FromDimsAndDataAndDescr(nd, d, PyArray_DescrFromType(type),   \
                                         data)
 
+/* Numpy HPy API */
+
+NPY_NO_EXPORT int
+HPyArray_ResolveWritebackIfCopy(HPyContext *ctx, HPy self);
 
 /*
    Check to see if this key in the dictionary is the "title"

numpy/core/include/numpy/ndarraytypes.h

@@ -24,6 +24,13 @@ static NPY_INLINE
 void capi_warn(const char *where) {
         printf("DEBUG WARNING: leaving to CPython API in %s\n", where);
 }
+
+static NPY_INLINE
+void capi_warn0(const char *caller_name, const char *file, int lineno) {
+        printf("DEBUG WARNING: (%s:%d) leaving to CPython API in %s\n", file, lineno, caller_name);
+}
+
+#define CAPI_WARN(_x) capi_warn0(_x, __FILE__, __LINE__)
 static __attribute__ ((noreturn)) NPY_INLINE
 void hpy_abort_not_implemented(const char *where) {
         printf("FATAL ERROR: not implemented in HPY: %s\n", where);

numpy/core/setup.py

@@ -927,6 +927,7 @@ def gl_if_msvc(build_cmd):
             join('src', 'multiarray', 'flagsobject.c'),
             join('src', 'multiarray', 'getset.c'),
             join('src', 'multiarray', 'hashdescr.c'),
+            join('src', 'multiarray', 'hpy_utils.c'),
             join('src', 'multiarray', 'item_selection.c'),
             join('src', 'multiarray', 'iterators.c'),
             join('src', 'multiarray', 'legacy_dtype_implementation.c'),

numpy/core/src/common/array_assign.c

@@ -152,6 +152,16 @@ IsUintAligned(PyArrayObject *ap)
                                 npy_uint_alignment(PyArray_DESCR(ap)->elsize));
 }
 
+NPY_NO_EXPORT int
+HIsUintAligned(HPyContext *ctx, HPy arr, PyArrayObject *arr_data)
+{
+    HPy descr = HPyArray_DESCR(ctx, arr, arr_data);
+    int res = raw_array_is_aligned(PyArray_NDIM(arr_data), PyArray_DIMS(arr_data),
+                                PyArray_DATA(arr_data), PyArray_STRIDES(arr_data),
+                                npy_uint_alignment(PyArray_Descr_AsStruct(ctx, descr)->elsize));
+    HPy_Close(ctx, descr);
+    return res;
+}
 
 
 /* Returns 1 if the arrays have overlapping data, 0 otherwise */

numpy/core/src/common/array_assign.h

@@ -116,6 +116,9 @@ IsAligned(PyArrayObject *ap);
 NPY_NO_EXPORT int
 IsUintAligned(PyArrayObject *ap);
 
+NPY_NO_EXPORT int
+HIsUintAligned(HPyContext *ctx, HPy arr, PyArrayObject *arr_data);
+
 /* Returns 1 if the arrays have overlapping data, 0 otherwise */
 NPY_NO_EXPORT int
 arrays_overlap(PyArrayObject *arr1, PyArrayObject *arr2);

numpy/core/src/common/lowlevel_strided_loops.h

@@ -199,6 +199,13 @@ PyArray_GetDTypeTransferFunction(int aligned,
                             int *out_needs_api);
 
 NPY_NO_EXPORT int
+HPyArray_GetDTypeTransferFunction(HPyContext *ctx, int aligned,
+                            npy_intp src_stride, npy_intp dst_stride,
+                            HPy src_dtype, HPy dst_dtype,
+                            int move_references,
+                            HNPY_cast_info *cast_info,
+                            int *out_needs_api);
+NPY_NO_EXPORT int
 get_fields_transfer_function(int aligned,
         npy_intp src_stride, npy_intp dst_stride,
         PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype,
@@ -243,6 +250,18 @@ PyArray_GetMaskedDTypeTransferFunction(int aligned,
                             NPY_cast_info *cast_info,
                             int *out_needs_api);
 
+NPY_NO_EXPORT int
+HPyArray_GetMaskedDTypeTransferFunction(HPyContext *ctx, int aligned,
+                            npy_intp src_stride,
+                            npy_intp dst_stride,
+                            npy_intp mask_stride,
+                            HPy src_dtype,
+                            HPy dst_dtype,
+                            HPy mask_dtype,
+                            int move_references,
+                            NPY_cast_info *cast_info,
+                            int *out_needs_api);
+
 /*
  * Casts the specified number of elements from 'src' with data type
  * 'src_dtype' to 'dst' with 'dst_dtype'. See

numpy/core/src/common/mem_overlap.c

@@ -709,6 +709,27 @@ get_array_memory_extents(PyArrayObject *arr,
     }
 }
 
+static void
+hpy_get_array_memory_extents(HPyContext *ctx, HPy arr,
+                         npy_uintp *out_start, npy_uintp *out_end,
+                         npy_uintp *num_bytes)
+{
+    npy_intp low, upper;
+    int j;
+    PyArrayObject *arr_data = PyArrayObject_AsStruct(ctx, arr);
+    npy_intp itemsize = HPyArray_ITEMSIZE(ctx, arr, arr_data);
+    offset_bounds_from_strides(itemsize, PyArray_NDIM(arr_data),
+                               PyArray_DIMS(arr_data), PyArray_STRIDES(arr_data),
+                               &low, &upper);
+    *out_start = (npy_uintp)PyArray_DATA(arr_data) + (npy_uintp)low;
+    *out_end = (npy_uintp)PyArray_DATA(arr_data) + (npy_uintp)upper;
+
+    *num_bytes = itemsize;
+    for (j = 0; j < PyArray_NDIM(arr_data); ++j) {
+        *num_bytes *= PyArray_DIM(arr_data, j);
+    }
+}
+
 
 static int
 strides_to_terms(PyArrayObject *arr, diophantine_term_t *terms,
@@ -837,6 +858,94 @@ solve_may_share_memory(PyArrayObject *a, PyArrayObject *b,
     return solve_diophantine(nterms, terms, rhs, max_work, 0, x);
 }
 
+/* PyArrayObject *a, PyArrayObject *b */
+NPY_VISIBILITY_HIDDEN mem_overlap_t
+hpy_solve_may_share_memory(HPyContext *ctx, HPy a, HPy b,
+                       HPy_ssize_t max_work)
+{
+    npy_int64 rhs;
+    diophantine_term_t terms[2*NPY_MAXDIMS + 2];
+    npy_uintp start1 = 0, end1 = 0, size1 = 0;
+    npy_uintp start2 = 0, end2 = 0, size2 = 0;
+    npy_uintp uintp_rhs;
+    npy_int64 x[2*NPY_MAXDIMS + 2];
+    unsigned int nterms;
+
+    hpy_get_array_memory_extents(ctx, a, &start1, &end1, &size1);
+    hpy_get_array_memory_extents(ctx, b, &start2, &end2, &size2);
+
+    if (!(start1 < end2 && start2 < end1 && start1 < end1 && start2 < end2)) {
+        /* Memory extents don't overlap */
+        return MEM_OVERLAP_NO;
+    }
+
+    if (max_work == 0) {
+        /* Too much work required, give up */
+        return MEM_OVERLAP_TOO_HARD;
+    }
+
+    /* Convert problem to Diophantine equation form with positive coefficients.
+       The bounds computed by offset_bounds_from_strides correspond to
+       all-positive strides.
+
+       start1 + sum(abs(stride1)*x1)
+       == start2 + sum(abs(stride2)*x2)
+       == end1 - 1 - sum(abs(stride1)*x1')
+       == end2 - 1 - sum(abs(stride2)*x2')
+
+       <=>
+
+       sum(abs(stride1)*x1) + sum(abs(stride2)*x2')
+       == end2 - 1 - start1
+
+       OR
+
+       sum(abs(stride1)*x1') + sum(abs(stride2)*x2)
+       == end1 - 1 - start2
+
+       We pick the problem with the smaller RHS (they are non-negative due to
+       the extent check above.)
+    */
+
+    uintp_rhs = MIN(end2 - 1 - start1, end1 - 1 - start2);
+    if (uintp_rhs > NPY_MAX_INT64) {
+        /* Integer overflow */
+        return MEM_OVERLAP_OVERFLOW;
+    }
+    rhs = (npy_int64)uintp_rhs;
+
+    nterms = 0;
+    PyArrayObject *a_data = PyArrayObject_AsStruct(ctx, a);
+    if (strides_to_terms(a_data, terms, &nterms, 1)) {
+        return MEM_OVERLAP_OVERFLOW;
+    }
+    PyArrayObject *b_data = PyArrayObject_AsStruct(ctx, b);
+    if (strides_to_terms(b_data, terms, &nterms, 1)) {
+        return MEM_OVERLAP_OVERFLOW;
+    }
+    npy_intp a_itemsize = HPyArray_ITEMSIZE(ctx, a, a_data);
+    if (a_itemsize > 1) {
+        terms[nterms].a = 1;
+        terms[nterms].ub = a_itemsize - 1;
+        ++nterms;
+    }
+    npy_intp b_itemsize = HPyArray_ITEMSIZE(ctx, b, b_data);
+    if (b_itemsize > 1) {
+        terms[nterms].a = 1;
+        terms[nterms].ub = b_itemsize - 1;
+        ++nterms;
+    }
+
+    /* Simplify, if possible */
+    if (diophantine_simplify(&nterms, terms, rhs)) {
+        /* Integer overflow */
+        return MEM_OVERLAP_OVERFLOW;
+    }
+
+    /* Solve */
+    return solve_diophantine(nterms, terms, rhs, (Py_ssize_t)max_work, 0, x);
+}
+
 
 /**
  * Determine whether an array has internal overlap.

numpy/core/src/multiarray/_datetime.h

@@ -38,6 +38,9 @@ create_datetime_dtype_with_unit(int type_num, NPY_DATETIMEUNIT unit);
 NPY_NO_EXPORT PyArray_DatetimeMetaData *
 get_datetime_metadata_from_dtype(PyArray_Descr *dtype);
 
+NPY_NO_EXPORT PyArray_DatetimeMetaData *
+h_get_datetime_metadata_from_dtype(HPyContext *ctx, PyArray_Descr *dtype_data);
+
 NPY_NO_EXPORT int
 find_string_array_datetime64_type(PyArrayObject *arr,
         PyArray_DatetimeMetaData *meta);