From 68cfbd8d2f41a793ec226369834e714b35a667d8 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Sun, 30 Aug 2020 12:05:16 +0100 Subject: [PATCH 001/409] MAINT: Use utf8 strings in more of datetime MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fixes an omission in a previous patch that did not allow μs in datetime_as_string. This also will very slightly speed up most uses, since all but very unusual code will be passing in `str` not `bytes`. --- numpy/core/src/multiarray/datetime_busday.c | 20 ++++----- .../core/src/multiarray/datetime_busdaycal.c | 25 ++++++----- numpy/core/src/multiarray/datetime_strings.c | 43 +++++++++++-------- numpy/core/tests/test_datetime.py | 5 ++- 4 files changed, 49 insertions(+), 44 deletions(-) diff --git a/numpy/core/src/multiarray/datetime_busday.c b/numpy/core/src/multiarray/datetime_busday.c index d3cce8a370da..5acba600afa6 100644 --- a/numpy/core/src/multiarray/datetime_busday.c +++ b/numpy/core/src/multiarray/datetime_busday.c @@ -834,24 +834,24 @@ static int PyArray_BusDayRollConverter(PyObject *roll_in, NPY_BUSDAY_ROLL *roll) { PyObject *obj = roll_in; - char *str; - Py_ssize_t len; - /* Make obj into an ASCII string */ - Py_INCREF(obj); - if (PyUnicode_Check(obj)) { - /* accept unicode input */ - PyObject *obj_str; - obj_str = PyUnicode_AsASCIIString(obj); + /* Make obj into an UTF8 string */ + if (PyBytes_Check(obj)) { + /* accept bytes input */ + PyObject *obj_str = PyUnicode_FromEncodedObject(obj, NULL, NULL); if (obj_str == NULL) { Py_DECREF(obj); return 0; } - Py_DECREF(obj); obj = obj_str; } + else { + Py_INCREF(obj); + } - if (PyBytes_AsStringAndSize(obj, &str, &len) < 0) { + Py_ssize_t len; + char const *str = PyUnicode_AsUTF8AndSize(obj, &len); + if (str == NULL) { Py_DECREF(obj); return 0; } diff --git a/numpy/core/src/multiarray/datetime_busdaycal.c b/numpy/core/src/multiarray/datetime_busdaycal.c index 2374eaa63453..906e3b539d4a 100644 --- a/numpy/core/src/multiarray/datetime_busdaycal.c +++ b/numpy/core/src/multiarray/datetime_busdaycal.c @@ -30,33 +30,32 @@ PyArray_WeekMaskConverter(PyObject *weekmask_in, npy_bool *weekmask) { PyObject *obj = weekmask_in; - /* Make obj into an ASCII string if it is UNICODE */ - Py_INCREF(obj); - if (PyUnicode_Check(obj)) { - /* accept unicode input */ - PyObject *obj_str; - obj_str = PyUnicode_AsASCIIString(obj); + /* Make obj into an UTF8 string */ + if (PyBytes_Check(obj)) { + /* accept bytes input */ + PyObject *obj_str = PyUnicode_FromEncodedObject(obj, NULL, NULL); if (obj_str == NULL) { Py_DECREF(obj); return 0; } - Py_DECREF(obj); obj = obj_str; } + else { + Py_INCREF(obj); + } + if (PyBytes_Check(obj)) { - char *str; Py_ssize_t len; - int i; - - if (PyBytes_AsStringAndSize(obj, &str, &len) < 0) { + char const *str = PyUnicode_AsUTF8AndSize(obj, &len); + if (str == NULL) { Py_DECREF(obj); return 0; } /* Length 7 is a string like "1111100" */ if (len == 7) { - for (i = 0; i < 7; ++i) { + for (int i = 0; i < 7; ++i) { switch(str[i]) { case '0': weekmask[i] = 0; @@ -75,7 +74,7 @@ PyArray_WeekMaskConverter(PyObject *weekmask_in, npy_bool *weekmask) general_weekmask_string: /* a string like "SatSun" or "Mon Tue Wed" */ memset(weekmask, 0, 7); - for (i = 0; i < len; i += 3) { + for (int i = 0; i < len; i += 3) { while (isspace(str[i])) ++i; diff --git a/numpy/core/src/multiarray/datetime_strings.c b/numpy/core/src/multiarray/datetime_strings.c index f847c7ea8e9c..49c7e81eb377 100644 --- a/numpy/core/src/multiarray/datetime_strings.c +++ b/numpy/core/src/multiarray/datetime_strings.c @@ -1385,21 +1385,23 @@ array_datetime_as_string(PyObject *NPY_UNUSED(self), PyObject *args, /* Parse the input unit if provided */ if (unit_in != NULL && unit_in != Py_None) { PyObject *strobj; - char *str = NULL; - Py_ssize_t len = 0; - if (PyUnicode_Check(unit_in)) { - strobj = PyUnicode_AsASCIIString(unit_in); - if (strobj == NULL) { - goto fail; + if (PyBytes_Check(unit_in)) { + /* accept bytes input */ + PyObject *obj_str = PyUnicode_FromEncodedObject(unit_in, NULL, NULL); + if (obj_str == NULL) { + return 0; } + strobj = obj_str; } else { + Py_INCREF(unit_in); strobj = unit_in; - Py_INCREF(strobj); } - if (PyBytes_AsStringAndSize(strobj, &str, &len) < 0) { + Py_ssize_t len = 0; + char *str = PyUnicode_AsUTF8AndSize(strobj, &len); + if (str == NULL) { Py_DECREF(strobj); goto fail; } @@ -1434,24 +1436,27 @@ array_datetime_as_string(PyObject *NPY_UNUSED(self), PyObject *args, /* Get the input time zone */ if (timezone_obj != NULL) { - /* Convert to ASCII if it's unicode */ - if (PyUnicode_Check(timezone_obj)) { - /* accept unicode input */ - PyObject *obj_str; - obj_str = PyUnicode_AsASCIIString(timezone_obj); + PyObject *strobj; + if (PyBytes_Check(timezone_obj)) { + /* accept bytes input */ + PyObject *obj_str = PyUnicode_FromEncodedObject(timezone_obj, NULL, NULL); if (obj_str == NULL) { goto fail; } - Py_DECREF(timezone_obj); - timezone_obj = obj_str; + strobj = obj_str; } + else { + Py_INCREF(timezone_obj); + strobj = unit_in; + } + + Py_SETREF(timezone_obj, strobj); /* Check for the supported string inputs */ - if (PyBytes_Check(timezone_obj)) { - char *str; + if (PyUnicode_Check(timezone_obj)) { Py_ssize_t len; - - if (PyBytes_AsStringAndSize(timezone_obj, &str, &len) < 0) { + char const *str = PyUnicode_AsUTF8AndSize(timezone_obj, &len); + if (str == NULL) { goto fail; } diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index f725091c5067..f58cf307a0de 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -1654,8 +1654,9 @@ def test_datetime_as_string(self): '1959-10-13T12:34:56') assert_equal(np.datetime_as_string(np.datetime64(datetime, 'ms')), '1959-10-13T12:34:56.789') - assert_equal(np.datetime_as_string(np.datetime64(datetime, 'us')), - '1959-10-13T12:34:56.789012') + for us in ['us', 'μs', b'us']: # check non-ascii and bytes too + assert_equal(np.datetime_as_string(np.datetime64(datetime, us)), + '1959-10-13T12:34:56.789012') datetime = '1969-12-31T23:34:56.789012345678901234' From a7e1d210cdf5cb9b24d3f0fed218e8afab914282 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Sun, 30 Aug 2020 13:31:36 +0100 Subject: [PATCH 002/409] Update numpy/core/src/multiarray/datetime_busday.c --- numpy/core/src/multiarray/datetime_busday.c | 1 - 1 file changed, 1 deletion(-) diff --git a/numpy/core/src/multiarray/datetime_busday.c b/numpy/core/src/multiarray/datetime_busday.c index 5acba600afa6..2cf1575515bc 100644 --- a/numpy/core/src/multiarray/datetime_busday.c +++ b/numpy/core/src/multiarray/datetime_busday.c @@ -840,7 +840,6 @@ PyArray_BusDayRollConverter(PyObject *roll_in, NPY_BUSDAY_ROLL *roll) /* accept bytes input */ PyObject *obj_str = PyUnicode_FromEncodedObject(obj, NULL, NULL); if (obj_str == NULL) { - Py_DECREF(obj); return 0; } obj = obj_str; From cd7acfd8d8a6d43dc4f64f4552a10a589cb30e97 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Sun, 30 Aug 2020 18:59:26 +0100 Subject: [PATCH 003/409] Update numpy/core/src/multiarray/datetime_busdaycal.c --- numpy/core/src/multiarray/datetime_busdaycal.c | 1 - 1 file changed, 1 deletion(-) diff --git a/numpy/core/src/multiarray/datetime_busdaycal.c b/numpy/core/src/multiarray/datetime_busdaycal.c index 906e3b539d4a..341b80a86f3e 100644 --- a/numpy/core/src/multiarray/datetime_busdaycal.c +++ b/numpy/core/src/multiarray/datetime_busdaycal.c @@ -35,7 +35,6 @@ PyArray_WeekMaskConverter(PyObject *weekmask_in, npy_bool *weekmask) /* accept bytes input */ PyObject *obj_str = PyUnicode_FromEncodedObject(obj, NULL, NULL); if (obj_str == NULL) { - Py_DECREF(obj); return 0; } obj = obj_str; From 911fa12539ca438142409eaabcf3404557aca650 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Mon, 31 Aug 2020 09:52:01 +0100 Subject: [PATCH 004/409] Apply suggestions from code review --- numpy/core/src/multiarray/datetime_strings.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/core/src/multiarray/datetime_strings.c b/numpy/core/src/multiarray/datetime_strings.c index 49c7e81eb377..8665d329579f 100644 --- a/numpy/core/src/multiarray/datetime_strings.c +++ b/numpy/core/src/multiarray/datetime_strings.c @@ -1400,7 +1400,7 @@ array_datetime_as_string(PyObject *NPY_UNUSED(self), PyObject *args, } Py_ssize_t len = 0; - char *str = PyUnicode_AsUTF8AndSize(strobj, &len); + char const *str = PyUnicode_AsUTF8AndSize(strobj, &len); if (str == NULL) { Py_DECREF(strobj); goto fail; @@ -1447,7 +1447,7 @@ array_datetime_as_string(PyObject *NPY_UNUSED(self), PyObject *args, } else { Py_INCREF(timezone_obj); - strobj = unit_in; + strobj = timezone_obj; } Py_SETREF(timezone_obj, strobj); From cdd5becba9a39d054f0ae34a493dd16d613f8cb9 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Mon, 31 Aug 2020 10:57:06 +0100 Subject: [PATCH 005/409] Update numpy/core/src/multiarray/datetime_busdaycal.c --- numpy/core/src/multiarray/datetime_busdaycal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/src/multiarray/datetime_busdaycal.c b/numpy/core/src/multiarray/datetime_busdaycal.c index 341b80a86f3e..cec7135ffea3 100644 --- a/numpy/core/src/multiarray/datetime_busdaycal.c +++ b/numpy/core/src/multiarray/datetime_busdaycal.c @@ -44,7 +44,7 @@ PyArray_WeekMaskConverter(PyObject *weekmask_in, npy_bool *weekmask) } - if (PyBytes_Check(obj)) { + if (PyUnicode_Check(obj)) { Py_ssize_t len; char const *str = PyUnicode_AsUTF8AndSize(obj, &len); if (str == NULL) { From b0dd380451085c250f6239dabb30d4e7235ac41b Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 21 Aug 2020 16:40:22 -0500 Subject: [PATCH 006/409] MAINT: Move dtype instance to DType class cast This function is better housed in convert_datatype.c and was only in array_coercion, because we did not use it anywhere else before. This also somewhat modifies the logic and cleans up use-cases of it in array_coercion.c --- numpy/core/src/multiarray/array_coercion.c | 61 ++++---------------- numpy/core/src/multiarray/convert_datatype.c | 49 ++++++++++++++++ numpy/core/src/multiarray/convert_datatype.h | 3 + 3 files changed, 62 insertions(+), 51 deletions(-) diff --git a/numpy/core/src/multiarray/array_coercion.c b/numpy/core/src/multiarray/array_coercion.c index ffb5bd632111..4e2354991034 100644 --- a/numpy/core/src/multiarray/array_coercion.c +++ b/numpy/core/src/multiarray/array_coercion.c @@ -306,51 +306,6 @@ discover_dtype_from_pyobject( } -/* - * This function should probably become public API eventually. At this - * time it is implemented by falling back to `PyArray_AdaptFlexibleDType`. - * We will use `CastingImpl[from, to].adjust_descriptors(...)` to implement - * this logic. - */ -static NPY_INLINE PyArray_Descr * -cast_descriptor_to_fixed_dtype( - PyArray_Descr *descr, PyArray_DTypeMeta *fixed_DType) -{ - if (fixed_DType == NULL) { - /* Nothing to do, we only need to promote the new dtype */ - Py_INCREF(descr); - return descr; - } - - if (!fixed_DType->parametric) { - /* - * Don't actually do anything, the default is always the result - * of any cast. - */ - return fixed_DType->default_descr(fixed_DType); - } - if (PyObject_TypeCheck((PyObject *)descr, (PyTypeObject *)fixed_DType)) { - Py_INCREF(descr); - return descr; - } - /* - * TODO: When this is implemented for all dtypes, the special cases - * can be removed... - */ - if (fixed_DType->legacy && fixed_DType->parametric && - NPY_DTYPE(descr)->legacy) { - PyArray_Descr *flex_dtype = PyArray_DescrFromType(fixed_DType->type_num); - return PyArray_AdaptFlexibleDType(descr, flex_dtype); - } - - PyErr_SetString(PyExc_NotImplementedError, - "Must use casting to find the correct dtype, this is " - "not yet implemented! " - "(It should not be possible to hit this code currently!)"); - return NULL; -} - - /** * Discover the correct descriptor from a known DType class and scalar. * If the fixed DType can discover a dtype instance/descr all is fine, @@ -392,7 +347,7 @@ find_scalar_descriptor( return descr; } - Py_SETREF(descr, cast_descriptor_to_fixed_dtype(descr, fixed_DType)); + Py_SETREF(descr, PyArray_CastDescrToDType(descr, fixed_DType)); return descr; } @@ -727,8 +682,13 @@ find_descriptor_from_array( enum _dtype_discovery_flags flags = 0; *out_descr = NULL; - if (NPY_UNLIKELY(DType != NULL && DType->parametric && - PyArray_ISOBJECT(arr))) { + if (DType == NULL) { + *out_descr = PyArray_DESCR(arr); + Py_INCREF(*out_descr); + return 0; + } + + if (NPY_UNLIKELY(DType->parametric && PyArray_ISOBJECT(arr))) { /* * We have one special case, if (and only if) the input array is of * object DType and the dtype is not fixed already but parametric. @@ -777,7 +737,7 @@ find_descriptor_from_array( } Py_DECREF(iter); } - else if (DType != NULL && NPY_UNLIKELY(DType->type_num == NPY_DATETIME) && + else if (NPY_UNLIKELY(DType->type_num == NPY_DATETIME) && PyArray_ISSTRING(arr)) { /* * TODO: This branch should be deprecated IMO, the workaround is @@ -806,8 +766,7 @@ find_descriptor_from_array( * If this is not an object array figure out the dtype cast, * or simply use the returned DType. */ - *out_descr = cast_descriptor_to_fixed_dtype( - PyArray_DESCR(arr), DType); + *out_descr = PyArray_CastDescrToDType(PyArray_DESCR(arr), DType); if (*out_descr == NULL) { return -1; } diff --git a/numpy/core/src/multiarray/convert_datatype.c b/numpy/core/src/multiarray/convert_datatype.c index d9121707b21a..0ccc0b538e8f 100644 --- a/numpy/core/src/multiarray/convert_datatype.c +++ b/numpy/core/src/multiarray/convert_datatype.c @@ -1031,6 +1031,55 @@ ensure_dtype_nbo(PyArray_Descr *type) } } + +/** + * This function should possibly become public API eventually. At this + * time it is implemented by falling back to `PyArray_AdaptFlexibleDType`. + * We will use `CastingImpl[from, to].adjust_descriptors(...)` to implement + * this logic. + * Before that, the API needs to be reviewed though. + * + * WARNING: This function currently does not guarantee that `descr` can + * actually be cast to the given DType. + * + * @param descr The dtype instance to adapt "cast" + * @param given_DType The DType class for which we wish to find an instance able + * to represent `descr`. + * @returns Instance of `given_DType`. If `given_DType` is parametric the + * descr may be adapted to hold it. + */ +NPY_NO_EXPORT PyArray_Descr * +PyArray_CastDescrToDType(PyArray_Descr *descr, PyArray_DTypeMeta *given_DType) +{ + if (NPY_DTYPE(descr) == given_DType) { + Py_INCREF(descr); + return descr; + } + if (!given_DType->parametric) { + /* + * Don't actually do anything, the default is always the result + * of any cast. + */ + return given_DType->default_descr(given_DType); + } + if (PyObject_TypeCheck((PyObject *)descr, (PyTypeObject *)given_DType)) { + Py_INCREF(descr); + return descr; + } + + if (!given_DType->legacy) { + PyErr_SetString(PyExc_NotImplementedError, + "Must use casting to find the correct DType for a parametric " + "user DType. This is not yet implemented (this error should be " + "unreachable)."); + return NULL; + } + + PyArray_Descr *flex_dtype = PyArray_DescrNew(given_DType->singleton); + return PyArray_AdaptFlexibleDType(descr, flex_dtype); +} + + /*NUMPY_API * Produces the smallest size and lowest kind type to which both * input types can be cast. diff --git a/numpy/core/src/multiarray/convert_datatype.h b/numpy/core/src/multiarray/convert_datatype.h index 9b7f39db2e33..eef2f7313390 100644 --- a/numpy/core/src/multiarray/convert_datatype.h +++ b/numpy/core/src/multiarray/convert_datatype.h @@ -49,4 +49,7 @@ npy_set_invalid_cast_error( NPY_NO_EXPORT PyArray_Descr * PyArray_AdaptFlexibleDType(PyArray_Descr *data_dtype, PyArray_Descr *flex_dtype); +NPY_NO_EXPORT PyArray_Descr * +PyArray_CastDescrToDType(PyArray_Descr *descr, PyArray_DTypeMeta *given_DType); + #endif From 6b1c643d82e5cb39498f791c0dc5020890229f6e Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 21 Aug 2020 16:51:42 -0500 Subject: [PATCH 007/409] MAINT: Use existing `ensure_dtype_nbo` in ufunc resolution There were two versions of this, since the merger of umath and multiarraymodule, this is unnecessary. --- numpy/core/src/multiarray/convert_datatype.c | 2 +- numpy/core/src/multiarray/convert_datatype.h | 3 +++ numpy/core/src/umath/ufunc_type_resolution.c | 15 --------------- 3 files changed, 4 insertions(+), 16 deletions(-) diff --git a/numpy/core/src/multiarray/convert_datatype.c b/numpy/core/src/multiarray/convert_datatype.c index 0ccc0b538e8f..3d81edc172ce 100644 --- a/numpy/core/src/multiarray/convert_datatype.c +++ b/numpy/core/src/multiarray/convert_datatype.c @@ -1019,7 +1019,7 @@ promote_types(PyArray_Descr *type1, PyArray_Descr *type2, * Returns a new reference to type if it is already NBO, otherwise * returns a copy converted to NBO. */ -static PyArray_Descr * +NPY_NO_EXPORT PyArray_Descr * ensure_dtype_nbo(PyArray_Descr *type) { if (PyArray_ISNBO(type->byteorder)) { diff --git a/numpy/core/src/multiarray/convert_datatype.h b/numpy/core/src/multiarray/convert_datatype.h index eef2f7313390..507a72266977 100644 --- a/numpy/core/src/multiarray/convert_datatype.h +++ b/numpy/core/src/multiarray/convert_datatype.h @@ -18,6 +18,9 @@ NPY_NO_EXPORT npy_bool can_cast_scalar_to(PyArray_Descr *scal_type, char *scal_data, PyArray_Descr *to, NPY_CASTING casting); +NPY_NO_EXPORT PyArray_Descr * +ensure_dtype_nbo(PyArray_Descr *type); + NPY_NO_EXPORT int should_use_min_scalar(npy_intp narrs, PyArrayObject **arr, npy_intp ndtypes, PyArray_Descr **dtypes); diff --git a/numpy/core/src/umath/ufunc_type_resolution.c b/numpy/core/src/umath/ufunc_type_resolution.c index aa6f34d59119..3abeb2c5a202 100644 --- a/numpy/core/src/umath/ufunc_type_resolution.c +++ b/numpy/core/src/umath/ufunc_type_resolution.c @@ -236,21 +236,6 @@ PyUFunc_ValidateCasting(PyUFuncObject *ufunc, return 0; } -/* - * Returns a new reference to type if it is already NBO, otherwise - * returns a copy converted to NBO. - */ -static PyArray_Descr * -ensure_dtype_nbo(PyArray_Descr *type) -{ - if (PyArray_ISNBO(type->byteorder)) { - Py_INCREF(type); - return type; - } - else { - return PyArray_DescrNewByteorder(type, NPY_NATIVE); - } -} /*UFUNC_API * From d9075b77586e0c7b536d5ec684bfd93c5bcd9439 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 21 Aug 2020 17:03:59 -0500 Subject: [PATCH 008/409] MAINT: Always define `default_descr()` and simplify code --- numpy/core/src/multiarray/array_coercion.c | 12 +++--------- numpy/core/src/multiarray/dtypemeta.c | 10 ++++++++++ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/numpy/core/src/multiarray/array_coercion.c b/numpy/core/src/multiarray/array_coercion.c index 4e2354991034..4f8456fb9499 100644 --- a/numpy/core/src/multiarray/array_coercion.c +++ b/numpy/core/src/multiarray/array_coercion.c @@ -1284,15 +1284,9 @@ PyArray_DiscoverDTypeAndShape( * the correct default. */ if (fixed_DType != NULL) { - if (fixed_DType->default_descr == NULL) { - Py_INCREF(fixed_DType->singleton); - *out_descr = fixed_DType->singleton; - } - else { - *out_descr = fixed_DType->default_descr(fixed_DType); - if (*out_descr == NULL) { - goto fail; - } + *out_descr = fixed_DType->default_descr(fixed_DType); + if (*out_descr == NULL) { + goto fail; } } } diff --git a/numpy/core/src/multiarray/dtypemeta.c b/numpy/core/src/multiarray/dtypemeta.c index 3026e68e9af0..6e5bf840e018 100644 --- a/numpy/core/src/multiarray/dtypemeta.c +++ b/numpy/core/src/multiarray/dtypemeta.c @@ -193,6 +193,14 @@ discover_datetime_and_timedelta_from_pyobject( } +static PyArray_Descr * +nonparametric_default_descr(PyArray_DTypeMeta *cls) +{ + Py_INCREF(cls->singleton); + return cls->singleton; +} + + static PyArray_Descr * flexible_default_descr(PyArray_DTypeMeta *cls) { @@ -399,6 +407,8 @@ dtypemeta_wrap_legacy_descriptor(PyArray_Descr *descr) dtype_class->kind = descr->kind; /* Strings and voids have (strange) logic around scalars. */ + dtype_class->default_descr = nonparametric_default_descr; + dtype_class->is_known_scalar_type = python_builtins_are_known_scalar_types; if (PyTypeNum_ISDATETIME(descr->type_num)) { From 76253895fbfbd2b6f5a5d4a5d2c6b96ff2dc5a0c Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 21 Aug 2020 17:04:41 -0500 Subject: [PATCH 009/409] API,MAINT: Rewrite promotion using common DType and common instance This defines `common_dtype` and `common_instance` (only for parametric DTypes), and uses them to implement the `PyArray_CommonDType` operation. `PyArray_CommonDType()` together with the `common_instance` method then define the existing PromoteTypes. This does not (yet) affect "value based promotion" as defined by `PyArray_ResultType()`. We also require the step of casting to the common DType to define this type of example: ``` np.promote_types("S1", "i8") == np.dtype('S21') ``` This steps requires finding the string length corresponding to the integer (21 characters). This is here handled by the `PyArray_CastDescrToDType` function. However, that function still relies on `PyArray_AdaptFlexibleDType` and thus does not generalize to arbitrary DTypes. See NEP 42 (currently "Common DType Operations" section): https://numpy.org/neps/nep-0042-new-dtypes.html#common-dtype-operations --- numpy/core/include/numpy/ndarraytypes.h | 6 + numpy/core/src/multiarray/convert_datatype.c | 370 ++++--------------- numpy/core/src/multiarray/convert_datatype.h | 3 + numpy/core/src/multiarray/dtypemeta.c | 137 ++++++- numpy/core/src/multiarray/dtypemeta.h | 16 + numpy/core/src/multiarray/usertypes.c | 121 ++++++ numpy/core/src/multiarray/usertypes.h | 4 + 7 files changed, 350 insertions(+), 307 deletions(-) diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index bbcf468c1f35..df480f96dae5 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -1839,6 +1839,10 @@ typedef void (PyDataMem_EventHookFunc)(void *inp, void *outp, size_t size, PyArray_DTypeMeta *cls, PyTypeObject *obj); typedef PyArray_Descr *(default_descr_function)(PyArray_DTypeMeta *cls); + typedef PyArray_DTypeMeta *(common_dtype_function)( + PyArray_DTypeMeta *dtype1, PyArray_DTypeMeta *dtyep2); + typedef PyArray_Descr *(common_instance_function)( + PyArray_Descr *dtype1, PyArray_Descr *dtyep2); /* * While NumPy DTypes would not need to be heap types the plan is to @@ -1894,6 +1898,8 @@ typedef void (PyDataMem_EventHookFunc)(void *inp, void *outp, size_t size, discover_descr_from_pyobject_function *discover_descr_from_pyobject; is_known_scalar_type_function *is_known_scalar_type; default_descr_function *default_descr; + common_dtype_function *common_dtype; + common_instance_function *common_instance; }; #endif /* NPY_INTERNAL_BUILD */ diff --git a/numpy/core/src/multiarray/convert_datatype.c b/numpy/core/src/multiarray/convert_datatype.c index 3d81edc172ce..f700bdc990df 100644 --- a/numpy/core/src/multiarray/convert_datatype.c +++ b/numpy/core/src/multiarray/convert_datatype.c @@ -1080,6 +1080,50 @@ PyArray_CastDescrToDType(PyArray_Descr *descr, PyArray_DTypeMeta *given_DType) } +/** + * This function defines the common DType operator. + * + * Note that the common DType will not be "object" (unless one of the dtypes + * is object), even though object can technically represent all values + * correctly. + * + * TODO: Before exposure, we should review the return value (e.g. no error + * when no common DType is found). + * + * @param dtype1 DType class to find the common type for. + * @param dtype2 Second DType class. + * @return The common DType or NULL with an error set + */ +NPY_NO_EXPORT PyArray_DTypeMeta * +PyArray_CommonDType(PyArray_DTypeMeta *dtype1, PyArray_DTypeMeta *dtype2) +{ + if (dtype1 == dtype2) { + Py_INCREF(dtype1); + return dtype1; + } + + PyArray_DTypeMeta *common_dtype; + + common_dtype = dtype1->common_dtype(dtype1, dtype2); + if (common_dtype == (PyArray_DTypeMeta *)Py_NotImplemented) { + Py_DECREF(common_dtype); + common_dtype = dtype2->common_dtype(dtype2, dtype1); + } + if (common_dtype == NULL) { + return NULL; + } + if (common_dtype == (PyArray_DTypeMeta *)Py_NotImplemented) { + Py_DECREF(Py_NotImplemented); + PyErr_Format(PyExc_TypeError, + "The DTypes %S and %S do not have a common DType. " + "For example they cannot be stored in a single array unless " + "the dtype is `object`.", dtype1, dtype2); + return NULL; + } + return common_dtype; +} + + /*NUMPY_API * Produces the smallest size and lowest kind type to which both * input types can be cast. @@ -1087,320 +1131,48 @@ PyArray_CastDescrToDType(PyArray_Descr *descr, PyArray_DTypeMeta *given_DType) NPY_NO_EXPORT PyArray_Descr * PyArray_PromoteTypes(PyArray_Descr *type1, PyArray_Descr *type2) { - int type_num1, type_num2, ret_type_num; + PyArray_DTypeMeta *common_dtype; + PyArray_Descr *res; - /* - * Fast path for identical dtypes. - * - * Non-native-byte-order types are converted to native ones below, so we - * can't quit early. - */ + /* Fast path for identical inputs (NOTE: This path preserves metadata!) */ if (type1 == type2 && PyArray_ISNBO(type1->byteorder)) { Py_INCREF(type1); return type1; } - type_num1 = type1->type_num; - type_num2 = type2->type_num; - - /* If they're built-in types, use the promotion table */ - if (type_num1 < NPY_NTYPES && type_num2 < NPY_NTYPES) { - ret_type_num = _npy_type_promotion_table[type_num1][type_num2]; - /* - * The table doesn't handle string/unicode/void/datetime/timedelta, - * so check the result - */ - if (ret_type_num >= 0) { - return PyArray_DescrFromType(ret_type_num); - } - } - /* If one or both are user defined, calculate it */ - else { - int skind1 = NPY_NOSCALAR, skind2 = NPY_NOSCALAR, skind; - - if (PyArray_CanCastTo(type2, type1)) { - /* Promoted types are always native byte order */ - return ensure_dtype_nbo(type1); - } - else if (PyArray_CanCastTo(type1, type2)) { - /* Promoted types are always native byte order */ - return ensure_dtype_nbo(type2); - } - - /* Convert the 'kind' char into a scalar kind */ - switch (type1->kind) { - case 'b': - skind1 = NPY_BOOL_SCALAR; - break; - case 'u': - skind1 = NPY_INTPOS_SCALAR; - break; - case 'i': - skind1 = NPY_INTNEG_SCALAR; - break; - case 'f': - skind1 = NPY_FLOAT_SCALAR; - break; - case 'c': - skind1 = NPY_COMPLEX_SCALAR; - break; - } - switch (type2->kind) { - case 'b': - skind2 = NPY_BOOL_SCALAR; - break; - case 'u': - skind2 = NPY_INTPOS_SCALAR; - break; - case 'i': - skind2 = NPY_INTNEG_SCALAR; - break; - case 'f': - skind2 = NPY_FLOAT_SCALAR; - break; - case 'c': - skind2 = NPY_COMPLEX_SCALAR; - break; - } - - /* If both are scalars, there may be a promotion possible */ - if (skind1 != NPY_NOSCALAR && skind2 != NPY_NOSCALAR) { - - /* Start with the larger scalar kind */ - skind = (skind1 > skind2) ? skind1 : skind2; - ret_type_num = _npy_smallest_type_of_kind_table[skind]; - - for (;;) { - - /* If there is no larger type of this kind, try a larger kind */ - if (ret_type_num < 0) { - ++skind; - /* Use -1 to signal no promoted type found */ - if (skind < NPY_NSCALARKINDS) { - ret_type_num = _npy_smallest_type_of_kind_table[skind]; - } - else { - break; - } - } - - /* If we found a type to which we can promote both, done! */ - if (PyArray_CanCastSafely(type_num1, ret_type_num) && - PyArray_CanCastSafely(type_num2, ret_type_num)) { - return PyArray_DescrFromType(ret_type_num); - } - - /* Try the next larger type of this kind */ - ret_type_num = _npy_next_larger_type_table[ret_type_num]; - } - - } - - PyErr_SetString(PyExc_TypeError, - "invalid type promotion with custom data type"); + common_dtype = PyArray_CommonDType(NPY_DTYPE(type1), NPY_DTYPE(type2)); + if (common_dtype == NULL) { return NULL; } - switch (type_num1) { - /* BOOL can convert to anything except datetime/void */ - case NPY_BOOL: - if (type_num2 == NPY_STRING || type_num2 == NPY_UNICODE) { - int char_size = 1; - if (type_num2 == NPY_UNICODE) { - char_size = 4; - } - if (type2->elsize < 5 * char_size) { - PyArray_Descr *ret = NULL; - PyArray_Descr *temp = PyArray_DescrNew(type2); - ret = ensure_dtype_nbo(temp); - ret->elsize = 5 * char_size; - Py_DECREF(temp); - return ret; - } - return ensure_dtype_nbo(type2); - } - else if (type_num2 != NPY_DATETIME && type_num2 != NPY_VOID) { - return ensure_dtype_nbo(type2); - } - break; - /* For strings and unicodes, take the larger size */ - case NPY_STRING: - if (type_num2 == NPY_STRING) { - if (type1->elsize > type2->elsize) { - return ensure_dtype_nbo(type1); - } - else { - return ensure_dtype_nbo(type2); - } - } - else if (type_num2 == NPY_UNICODE) { - if (type2->elsize >= type1->elsize * 4) { - return ensure_dtype_nbo(type2); - } - else { - PyArray_Descr *d = PyArray_DescrNewFromType(NPY_UNICODE); - if (d == NULL) { - return NULL; - } - d->elsize = type1->elsize * 4; - return d; - } - } - /* Allow NUMBER -> STRING */ - else if (PyTypeNum_ISNUMBER(type_num2)) { - PyArray_Descr *ret = NULL; - PyArray_Descr *temp = PyArray_DescrNew(type1); - PyDataType_MAKEUNSIZED(temp); - - temp = PyArray_AdaptFlexibleDType(type2, temp); - if (temp == NULL) { - return NULL; - } - if (temp->elsize > type1->elsize) { - ret = ensure_dtype_nbo(temp); - } - else { - ret = ensure_dtype_nbo(type1); - } - Py_DECREF(temp); - return ret; - } - break; - case NPY_UNICODE: - if (type_num2 == NPY_UNICODE) { - if (type1->elsize > type2->elsize) { - return ensure_dtype_nbo(type1); - } - else { - return ensure_dtype_nbo(type2); - } - } - else if (type_num2 == NPY_STRING) { - if (type1->elsize >= type2->elsize * 4) { - return ensure_dtype_nbo(type1); - } - else { - PyArray_Descr *d = PyArray_DescrNewFromType(NPY_UNICODE); - if (d == NULL) { - return NULL; - } - d->elsize = type2->elsize * 4; - return d; - } - } - /* Allow NUMBER -> UNICODE */ - else if (PyTypeNum_ISNUMBER(type_num2)) { - PyArray_Descr *ret = NULL; - PyArray_Descr *temp = PyArray_DescrNew(type1); - PyDataType_MAKEUNSIZED(temp); - temp = PyArray_AdaptFlexibleDType(type2, temp); - if (temp == NULL) { - return NULL; - } - if (temp->elsize > type1->elsize) { - ret = ensure_dtype_nbo(temp); - } - else { - ret = ensure_dtype_nbo(type1); - } - Py_DECREF(temp); - return ret; - } - break; - case NPY_DATETIME: - case NPY_TIMEDELTA: - if (type_num2 == NPY_DATETIME || type_num2 == NPY_TIMEDELTA) { - return datetime_type_promotion(type1, type2); - } - break; + if (!common_dtype->parametric) { + res = common_dtype->default_descr(common_dtype); + Py_DECREF(common_dtype); + return res; } - switch (type_num2) { - /* BOOL can convert to almost anything */ - case NPY_BOOL: - if (type_num2 == NPY_STRING || type_num2 == NPY_UNICODE) { - int char_size = 1; - if (type_num2 == NPY_UNICODE) { - char_size = 4; - } - if (type2->elsize < 5 * char_size) { - PyArray_Descr *ret = NULL; - PyArray_Descr *temp = PyArray_DescrNew(type2); - ret = ensure_dtype_nbo(temp); - ret->elsize = 5 * char_size; - Py_DECREF(temp); - return ret; - } - return ensure_dtype_nbo(type2); - } - else if (type_num1 != NPY_DATETIME && type_num1 != NPY_TIMEDELTA && - type_num1 != NPY_VOID) { - return ensure_dtype_nbo(type1); - } - break; - case NPY_STRING: - /* Allow NUMBER -> STRING */ - if (PyTypeNum_ISNUMBER(type_num1)) { - PyArray_Descr *ret = NULL; - PyArray_Descr *temp = PyArray_DescrNew(type2); - PyDataType_MAKEUNSIZED(temp); - temp = PyArray_AdaptFlexibleDType(type1, temp); - if (temp == NULL) { - return NULL; - } - if (temp->elsize > type2->elsize) { - ret = ensure_dtype_nbo(temp); - } - else { - ret = ensure_dtype_nbo(type2); - } - Py_DECREF(temp); - return ret; - } - break; - case NPY_UNICODE: - /* Allow NUMBER -> UNICODE */ - if (PyTypeNum_ISNUMBER(type_num1)) { - PyArray_Descr *ret = NULL; - PyArray_Descr *temp = PyArray_DescrNew(type2); - PyDataType_MAKEUNSIZED(temp); - temp = PyArray_AdaptFlexibleDType(type1, temp); - if (temp == NULL) { - return NULL; - } - if (temp->elsize > type2->elsize) { - ret = ensure_dtype_nbo(temp); - } - else { - ret = ensure_dtype_nbo(type2); - } - Py_DECREF(temp); - return ret; - } - break; - case NPY_TIMEDELTA: - if (PyTypeNum_ISSIGNED(type_num1)) { - return ensure_dtype_nbo(type2); - } - break; + /* Cast the input types to the common DType if necessary */ + type1 = PyArray_CastDescrToDType(type1, common_dtype); + if (type1 == NULL) { + Py_DECREF(common_dtype); + return NULL; } - - /* For types equivalent up to endianness, can return either */ - if (PyArray_CanCastTypeTo(type1, type2, NPY_EQUIV_CASTING)) { - return ensure_dtype_nbo(type1); + type2 = PyArray_CastDescrToDType(type2, common_dtype); + if (type2 == NULL) { + Py_DECREF(type1); + Py_DECREF(common_dtype); + return NULL; } - /* TODO: Also combine fields, subarrays, strings, etc */ - /* - printf("invalid type promotion: "); - PyObject_Print(type1, stdout, 0); - printf(" "); - PyObject_Print(type2, stdout, 0); - printf("\n"); - */ - PyErr_SetString(PyExc_TypeError, "invalid type promotion"); - return NULL; + * And find the common instance of the two inputs + * NOTE: Common instance preserves metadata (normally and of one input) + */ + res = common_dtype->common_instance(type1, type2); + Py_DECREF(type1); + Py_DECREF(type2); + Py_DECREF(common_dtype); + return res; } /* diff --git a/numpy/core/src/multiarray/convert_datatype.h b/numpy/core/src/multiarray/convert_datatype.h index 507a72266977..a2b36b497fbe 100644 --- a/numpy/core/src/multiarray/convert_datatype.h +++ b/numpy/core/src/multiarray/convert_datatype.h @@ -10,6 +10,9 @@ PyArray_ObjectType(PyObject *op, int minimum_type); NPY_NO_EXPORT PyArrayObject ** PyArray_ConvertToCommonType(PyObject *op, int *retn); +NPY_NO_EXPORT PyArray_DTypeMeta * +PyArray_CommonDType(PyArray_DTypeMeta *dtype1, PyArray_DTypeMeta *dtype2); + NPY_NO_EXPORT int PyArray_ValidType(int type); diff --git a/numpy/core/src/multiarray/dtypemeta.c b/numpy/core/src/multiarray/dtypemeta.c index 6e5bf840e018..92f50247e28a 100644 --- a/numpy/core/src/multiarray/dtypemeta.c +++ b/numpy/core/src/multiarray/dtypemeta.c @@ -15,6 +15,9 @@ #include "dtypemeta.h" #include "_datetime.h" #include "array_coercion.h" +#include "scalartypes.h" +#include "convert_datatype.h" +#include "usertypes.h" static void @@ -216,6 +219,34 @@ flexible_default_descr(PyArray_DTypeMeta *cls) } +static PyArray_Descr * +string_unicode_common_instance(PyArray_Descr *descr1, PyArray_Descr *descr2) +{ + if (descr1->elsize >= descr2->elsize) { + return ensure_dtype_nbo(descr1); + } + else { + return ensure_dtype_nbo(descr2); + } +} + + +static PyArray_Descr * +void_common_instance(PyArray_Descr *descr1, PyArray_Descr *descr2) +{ + /* + * We currently do not support promotion of void types unless they + * are equivalent. + */ + if (!PyArray_CanCastTypeTo(descr1, descr2, NPY_EQUIV_CASTING)) { + PyErr_SetString(PyExc_TypeError, + "invalid type promotion with structured or void datatype(s)."); + return NULL; + } + Py_INCREF(descr1); + return descr1; +} + static int python_builtins_are_known_scalar_types( PyArray_DTypeMeta *NPY_UNUSED(cls), PyTypeObject *pytype) @@ -289,6 +320,86 @@ string_known_scalar_types( } +/* + * The following set of functions define the common dtype operator for + * the builtin types. + */ +static PyArray_DTypeMeta * +default_builtin_common_dtype(PyArray_DTypeMeta *cls, PyArray_DTypeMeta *other) +{ + assert(cls->type_num < NPY_NTYPES); + if (!other->legacy || other->type_num > cls->type_num) { + /* Let the more generic (larger type number) DType handle this */ + Py_INCREF(Py_NotImplemented); + return (PyArray_DTypeMeta *)Py_NotImplemented; + } + + /* + * Note: The use of the promotion table should probably be revised at + * some point. It may be most useful to remove it entirely and then + * consider adding a fast path/cache `PyArray_CommonDType()` itself. + */ + int common_num = _npy_type_promotion_table[cls->type_num][other->type_num]; + if (common_num < 0) { + Py_INCREF(Py_NotImplemented); + return (PyArray_DTypeMeta *)Py_NotImplemented; + } + return PyArray_DTypeFromTypeNum(common_num); +} + + +static PyArray_DTypeMeta * +string_unicode_common_dtype(PyArray_DTypeMeta *cls, PyArray_DTypeMeta *other) +{ + assert(cls->type_num < NPY_NTYPES); + if (!other->legacy || other->type_num > cls->type_num || + other->type_num == NPY_OBJECT) { + /* Let the more generic (larger type number) DType handle this */ + Py_INCREF(Py_NotImplemented); + return (PyArray_DTypeMeta *)Py_NotImplemented; + } + /* + * The builtin types are ordered by complexity (aside from object) here. + * Arguably, we should not consider numbers and strings "common", but + * we currently do. + */ + Py_INCREF(cls); + return cls; +} + +static PyArray_DTypeMeta * +datetime_common_dtype(PyArray_DTypeMeta *cls, PyArray_DTypeMeta *other) +{ + if (cls->type_num == NPY_DATETIME && other->type_num == NPY_TIMEDELTA) { + /* + * TODO: We actually currently do allow promotion here. This is + * currently relied on within `np.add(datetime, timedelta)`, + * while for concatenation the cast step will fail. + */ + Py_INCREF(cls); + return cls; + } + return default_builtin_common_dtype(cls, other); +} + + + +static PyArray_DTypeMeta * +object_common_dtype( + PyArray_DTypeMeta *cls, PyArray_DTypeMeta *NPY_UNUSED(other)) +{ + /* + * The object DType is special in that it can represent everything, + * including all potential user DTypes. + * One reason to defer (or error) here might be if the other DType + * does not support scalars so that e.g. `arr1d[0]` returns a 0-D array + * and `arr.astype(object)` would fail. But object casts are special. + */ + Py_INCREF(cls); + return cls; +} + + /** * This function takes a PyArray_Descr and replaces its base class with * a newly created dtype subclass (DTypeMeta instances). @@ -406,16 +517,28 @@ dtypemeta_wrap_legacy_descriptor(PyArray_Descr *descr) dtype_class->f = descr->f; dtype_class->kind = descr->kind; - /* Strings and voids have (strange) logic around scalars. */ + /* Set default functions (correct for most dtypes, override below) */ dtype_class->default_descr = nonparametric_default_descr; - + dtype_class->discover_descr_from_pyobject = ( + nonparametric_discover_descr_from_pyobject); dtype_class->is_known_scalar_type = python_builtins_are_known_scalar_types; + dtype_class->common_dtype = default_builtin_common_dtype; + dtype_class->common_instance = NULL; - if (PyTypeNum_ISDATETIME(descr->type_num)) { + if (PyTypeNum_ISUSERDEF(descr->type_num)) { + dtype_class->common_dtype = legacy_userdtype_common_dtype_function; + } + else if (descr->type_num == NPY_OBJECT) { + dtype_class->common_dtype = object_common_dtype; + } + else if (PyTypeNum_ISDATETIME(descr->type_num)) { /* Datetimes are flexible, but were not considered previously */ dtype_class->parametric = NPY_TRUE; + dtype_class->default_descr = flexible_default_descr; dtype_class->discover_descr_from_pyobject = ( discover_datetime_and_timedelta_from_pyobject); + dtype_class->common_dtype = datetime_common_dtype; + dtype_class->common_instance = datetime_type_promotion; if (descr->type_num == NPY_DATETIME) { dtype_class->is_known_scalar_type = datetime_known_scalar_types; } @@ -426,18 +549,16 @@ dtypemeta_wrap_legacy_descriptor(PyArray_Descr *descr) if (descr->type_num == NPY_VOID) { dtype_class->discover_descr_from_pyobject = ( void_discover_descr_from_pyobject); + dtype_class->common_instance = void_common_instance; } else { dtype_class->is_known_scalar_type = string_known_scalar_types; dtype_class->discover_descr_from_pyobject = ( string_discover_descr_from_pyobject); + dtype_class->common_dtype = string_unicode_common_dtype; + dtype_class->common_instance = string_unicode_common_instance; } } - else { - /* nonparametric case */ - dtype_class->discover_descr_from_pyobject = ( - nonparametric_discover_descr_from_pyobject); - } if (_PyArray_MapPyTypeToDType(dtype_class, descr->typeobj, PyTypeNum_ISUSERDEF(dtype_class->type_num)) < 0) { diff --git a/numpy/core/src/multiarray/dtypemeta.h b/numpy/core/src/multiarray/dtypemeta.h index e0909a7eb4b2..83cf7c07e944 100644 --- a/numpy/core/src/multiarray/dtypemeta.h +++ b/numpy/core/src/multiarray/dtypemeta.h @@ -2,6 +2,22 @@ #define _NPY_DTYPEMETA_H #define NPY_DTYPE(descr) ((PyArray_DTypeMeta *)Py_TYPE(descr)) +/* + * This function will hopefully be phased out or replaced, but was convenient + * for incremental implementation of new DTypes based on DTypeMeta. + * (Error checking is not required for DescrFromType, assuming that the + * type is valid.) + */ +static NPY_INLINE PyArray_DTypeMeta * +PyArray_DTypeFromTypeNum(int typenum) +{ + PyArray_Descr *descr = PyArray_DescrFromType(typenum); + PyArray_DTypeMeta *dtype = NPY_DTYPE(descr); + Py_INCREF(dtype); + Py_DECREF(descr); + return dtype; +} + NPY_NO_EXPORT int dtypemeta_wrap_legacy_descriptor(PyArray_Descr *dtypem); diff --git a/numpy/core/src/multiarray/usertypes.c b/numpy/core/src/multiarray/usertypes.c index 6b6c6bd9d057..265ec4be42e3 100644 --- a/numpy/core/src/multiarray/usertypes.c +++ b/numpy/core/src/multiarray/usertypes.c @@ -38,6 +38,7 @@ maintainer email: oliphant.travis@ieee.org #include "usertypes.h" #include "dtypemeta.h" +#include "scalartypes.h" NPY_NO_EXPORT PyArray_Descr **userdescrs=NULL; @@ -347,3 +348,123 @@ PyArray_RegisterCanCast(PyArray_Descr *descr, int totype, return _append_new(&descr->f->cancastscalarkindto[scalar], totype); } } + + +/* + * Legacy user DTypes implemented the common DType operation + * (as used in type promotion/result_type, and e.g. the type for + * concatenation), by using "safe cast" logic. + * + * New DTypes do have this behaviour generally, but we use can-cast + * when legacy user dtypes are involved. + */ +NPY_NO_EXPORT PyArray_DTypeMeta * +legacy_userdtype_common_dtype_function( + PyArray_DTypeMeta *cls, PyArray_DTypeMeta *other) +{ + int skind1 = NPY_NOSCALAR, skind2 = NPY_NOSCALAR, skind; + + if (!other->legacy) { + /* legacy DTypes can always defer to new style ones */ + Py_INCREF(Py_NotImplemented); + return (PyArray_DTypeMeta *)Py_NotImplemented; + } + /* Defer so that only one of the types handles the cast */ + if (cls->type_num < other->type_num) { + Py_INCREF(Py_NotImplemented); + return (PyArray_DTypeMeta *)Py_NotImplemented; + } + + /* Check whether casting is possible from one type to the other */ + if (PyArray_CanCastSafely(cls->type_num, other->type_num)) { + Py_INCREF(other); + return other; + } + if (PyArray_CanCastSafely(other->type_num, cls->type_num)) { + Py_INCREF(cls); + return cls; + } + + /* + * The following code used to be part of PyArray_PromoteTypes(). + * We can expect that this code is never used. + * In principle, it allows for promotion of two different user dtypes + * to a single NumPy dtype of the same "kind". In practice + * using the same `kind` as NumPy was never possible due to an + * simplification where `PyArray_EquivTypes(descr1, descr2)` will + * return True if both kind and element size match (e.g. bfloat16 and + * float16 would be equivalent). + * The option is also very obscure and not used in the examples. + */ + + /* Convert the 'kind' char into a scalar kind */ + switch (cls->kind) { + case 'b': + skind1 = NPY_BOOL_SCALAR; + break; + case 'u': + skind1 = NPY_INTPOS_SCALAR; + break; + case 'i': + skind1 = NPY_INTNEG_SCALAR; + break; + case 'f': + skind1 = NPY_FLOAT_SCALAR; + break; + case 'c': + skind1 = NPY_COMPLEX_SCALAR; + break; + } + switch (other->kind) { + case 'b': + skind2 = NPY_BOOL_SCALAR; + break; + case 'u': + skind2 = NPY_INTPOS_SCALAR; + break; + case 'i': + skind2 = NPY_INTNEG_SCALAR; + break; + case 'f': + skind2 = NPY_FLOAT_SCALAR; + break; + case 'c': + skind2 = NPY_COMPLEX_SCALAR; + break; + } + + /* If both are scalars, there may be a promotion possible */ + if (skind1 != NPY_NOSCALAR && skind2 != NPY_NOSCALAR) { + + /* Start with the larger scalar kind */ + skind = (skind1 > skind2) ? skind1 : skind2; + int ret_type_num = _npy_smallest_type_of_kind_table[skind]; + + for (;;) { + + /* If there is no larger type of this kind, try a larger kind */ + if (ret_type_num < 0) { + ++skind; + /* Use -1 to signal no promoted type found */ + if (skind < NPY_NSCALARKINDS) { + ret_type_num = _npy_smallest_type_of_kind_table[skind]; + } + else { + break; + } + } + + /* If we found a type to which we can promote both, done! */ + if (PyArray_CanCastSafely(cls->type_num, ret_type_num) && + PyArray_CanCastSafely(other->type_num, ret_type_num)) { + return PyArray_DTypeFromTypeNum(ret_type_num); + } + + /* Try the next larger type of this kind */ + ret_type_num = _npy_next_larger_type_table[ret_type_num]; + } + } + + Py_INCREF(Py_NotImplemented); + return (PyArray_DTypeMeta *)Py_NotImplemented; +} diff --git a/numpy/core/src/multiarray/usertypes.h b/numpy/core/src/multiarray/usertypes.h index b3e386c5c671..1b323d458e88 100644 --- a/numpy/core/src/multiarray/usertypes.h +++ b/numpy/core/src/multiarray/usertypes.h @@ -17,4 +17,8 @@ NPY_NO_EXPORT int PyArray_RegisterCastFunc(PyArray_Descr *descr, int totype, PyArray_VectorUnaryFunc *castfunc); +NPY_NO_EXPORT PyArray_DTypeMeta * +legacy_userdtype_common_dtype_function( + PyArray_DTypeMeta *cls, PyArray_DTypeMeta *other); + #endif From 07c2e66efbf8d98af0da573f0e7a90765b6b979c Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 25 Aug 2020 19:14:51 -0500 Subject: [PATCH 010/409] TST: Test void promotion uses equivalent casting --- numpy/core/tests/test_numeric.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/numpy/core/tests/test_numeric.py b/numpy/core/tests/test_numeric.py index ae5ee4c88340..89a82d3b6fcf 100644 --- a/numpy/core/tests/test_numeric.py +++ b/numpy/core/tests/test_numeric.py @@ -1025,6 +1025,24 @@ def test_promote_types_metadata(self, dtype1, dtype2): assert res_bs == res assert res_bs.metadata == res.metadata + @pytest.mark.parametrize(["dtype1", "dtype2"], + [[np.dtype("V6"), np.dtype("V10")], + [np.dtype([("name1", "i8")]), np.dtype([("name2", "i8")])], + [np.dtype("i8,i8"), np.dtype("i4,i4")], + ]) + def test_invalid_void_promotion(self, dtype1, dtype2): + # Mainly test structured void promotion, which currently allows + # byte-swapping, but nothing else: + with pytest.raises(TypeError): + np.promote_types(dtype1, dtype2) + + @pytest.mark.parametrize(["dtype1", "dtype2"], + [[np.dtype("V10"), np.dtype("V10")], + [np.dtype([("name1", "i8")])], + [np.dtype("i8,i8"), np.dtype("i8,>i8")], + ]) + def test_valid_void_promotion(self, dtype1, dtype2): + assert np.promote_types(dtype1, dtype2) is dtype1 def test_can_cast(self): assert_(np.can_cast(np.int32, np.int64)) From b40f6bb22d7e71533e0b450493530e8fdd08afa5 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 2 Sep 2020 13:42:35 -0500 Subject: [PATCH 011/409] TST: Adapt metadata-promotion tests to new implementation --- numpy/core/tests/test_numeric.py | 39 ++++++++------------------------ 1 file changed, 9 insertions(+), 30 deletions(-) diff --git a/numpy/core/tests/test_numeric.py b/numpy/core/tests/test_numeric.py index 89a82d3b6fcf..f5428f98cb54 100644 --- a/numpy/core/tests/test_numeric.py +++ b/numpy/core/tests/test_numeric.py @@ -941,7 +941,7 @@ def test_promote_identical_types_metadata(self, dtype): return res = np.promote_types(dtype, dtype) - if res.char in "?bhilqpBHILQPefdgFDGOmM": + if res.char in "?bhilqpBHILQPefdgFDGOmM" or dtype.type is rational: # Metadata is lost for simple promotions (they create a new dtype) assert res.metadata is None else: @@ -976,41 +976,20 @@ def test_promote_types_metadata(self, dtype1, dtype2): # Promotion failed, this test only checks metadata return - # The rules for when metadata is preserved and which dtypes metadta - # will be used are very confusing and depend on multiple paths. - # This long if statement attempts to reproduce this: - if dtype1.type is rational or dtype2.type is rational: - # User dtype promotion preserves byte-order here: - if np.can_cast(res, dtype1): - assert res.metadata == dtype1.metadata - else: - assert res.metadata == dtype2.metadata - - elif res.char in "?bhilqpBHILQPefdgFDGOmM": + if res.char in "?bhilqpBHILQPefdgFDGOmM" or res.type is rational: # All simple types lose metadata (due to using promotion table): assert res.metadata is None - elif res.kind in "SU" and dtype1 == dtype2: - # Strings give precedence to the second dtype: - assert res is dtype2 elif res == dtype1: # If one result is the result, it is usually returned unchanged: assert res is dtype1 elif res == dtype2: - # If one result is the result, it is usually returned unchanged: - assert res is dtype2 - elif dtype1.kind == "S" and dtype2.kind == "U": - # Promotion creates a new unicode dtype from scratch - assert res.metadata is None - elif dtype1.kind == "U" and dtype2.kind == "S": - # Promotion creates a new unicode dtype from scratch - assert res.metadata is None - elif res.kind in "SU" and dtype2.kind != res.kind: - # We build on top of dtype1: - assert res.metadata == dtype1.metadata - elif res.kind in "SU" and res.kind == dtype1.kind: - assert res.metadata == dtype1.metadata - elif res.kind in "SU" and res.kind == dtype2.kind: - assert res.metadata == dtype2.metadata + # dtype1 may have been cast to the same type/kind as dtype2. + # If the resulting dtype is identical we currently pick the cast + # version of dtype1, which lost the metadata: + if np.promote_types(dtype1, dtype2.kind) == dtype2: + res.metadata is None + else: + res.metadata == metadata2 else: assert res.metadata is None From 0a921a8d282e41127c85d504e2569bfea05a7ec0 Mon Sep 17 00:00:00 2001 From: MelissaWM Date: Wed, 2 Sep 2020 16:12:04 -0300 Subject: [PATCH 012/409] DOC: Added templates for different types of issues. --- .github/ISSUE_TEMPLATE/bug-report.md | 30 +++++++++++++++++++ .github/ISSUE_TEMPLATE/documentation.md | 21 +++++++++++++ .github/ISSUE_TEMPLATE/feature-request.md | 15 ++++++++++ .../ISSUE_TEMPLATE/questions-help-support.md | 16 ++++++++++ .github/PULL_REQUEST_TEMPLATE.md | 4 +++ 5 files changed, 86 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug-report.md create mode 100644 .github/ISSUE_TEMPLATE/documentation.md create mode 100644 .github/ISSUE_TEMPLATE/feature-request.md create mode 100644 .github/ISSUE_TEMPLATE/questions-help-support.md diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md new file mode 100644 index 000000000000..123498224627 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -0,0 +1,30 @@ +--- +name: "Bug Report" +about: Submit a bug report to help us improve NumPy + +--- + + + +### Reproducing code example: + + + +```python +import numpy as np +<< your code here >> +``` + +### Error message: + + + + + +### Numpy/Python version information: + + + diff --git a/.github/ISSUE_TEMPLATE/documentation.md b/.github/ISSUE_TEMPLATE/documentation.md new file mode 100644 index 000000000000..cc249e33fab2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation.md @@ -0,0 +1,21 @@ +--- +name: "Documentation" +about: Report an issue related to the NumPy documentation + +--- + +## Documentation + + + + + + + diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md new file mode 100644 index 000000000000..96faf6aab148 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -0,0 +1,15 @@ +--- +name: "Feature Request" +about: Check instructions for submitting your idea on the mailing list first. + +--- + +## Feature + +If you're looking to request a new feature or change in functionality, please +post your idea on the [numpy-discussion mailing list] +(https://mail.python.org/mailman/listinfo/numpy-discussion) to explain your +reasoning before you open an issue or pull request. You can also check out our +[Contributor Guide] +(https://github.com/numpy/numpy/blob/master/doc/source/dev/index.rst) if you +need more information. diff --git a/.github/ISSUE_TEMPLATE/questions-help-support.md b/.github/ISSUE_TEMPLATE/questions-help-support.md new file mode 100644 index 000000000000..c170ed3740e3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/questions-help-support.md @@ -0,0 +1,16 @@ +--- +name: "Questions/Help/Support" +about: Resources to get help and support about NumPy. + +--- + +## Questions and Help + +### Please note that this issue tracker is not a help form and this issue will +be closed. + +If you need help, we have a set of [listed resources available on the website] +(https://numpy.org/gethelp/). + +For NumPy development-related matters, please see [Community] +(https://numpy.org/community/). diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index e12eea7bd763..1f313fd75027 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -5,3 +5,7 @@ http://www.numpy.org/devdocs/dev/development_workflow.html + + From 199448294c937e82127cb648fd4dd380a46d8d58 Mon Sep 17 00:00:00 2001 From: MelissaWM Date: Wed, 2 Sep 2020 19:22:02 -0300 Subject: [PATCH 013/409] Responding to PR suggestions. --- .github/ISSUE_TEMPLATE/config.yml | 8 ++++++++ .github/ISSUE_TEMPLATE/documentation.md | 7 +++---- .github/ISSUE_TEMPLATE/feature-request.md | 3 ++- .github/ISSUE_TEMPLATE/questions-help-support.md | 16 ---------------- 4 files changed, 13 insertions(+), 21 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/config.yml delete mode 100644 .github/ISSUE_TEMPLATE/questions-help-support.md diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000000..3027d66506a7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: Question/Help/Support + url: https://numpy.org/gethelp/ + about: "If you have a question, please look at the listed resources available on the website." + - name: Development-related matters + url: https://numpy.org/community/ + about: "If you would like to discuss development-related matters or need help from the NumPy team, see our community's communication channels." diff --git a/.github/ISSUE_TEMPLATE/documentation.md b/.github/ISSUE_TEMPLATE/documentation.md index cc249e33fab2..b0435a5654d5 100644 --- a/.github/ISSUE_TEMPLATE/documentation.md +++ b/.github/ISSUE_TEMPLATE/documentation.md @@ -9,13 +9,12 @@ about: Report an issue related to the NumPy documentation +referring to. You can also check the development version of the documentation +and see if this issue has already been addressed: https://numpy.org/devdocs/ +--> - - diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md index 96faf6aab148..38d64710bbb2 100644 --- a/.github/ISSUE_TEMPLATE/feature-request.md +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -6,7 +6,8 @@ about: Check instructions for submitting your idea on the mailing list first. ## Feature -If you're looking to request a new feature or change in functionality, please +If you're looking to request a new feature or change in functionality, including +adding or changing the meaning of arguments to an existing function, please post your idea on the [numpy-discussion mailing list] (https://mail.python.org/mailman/listinfo/numpy-discussion) to explain your reasoning before you open an issue or pull request. You can also check out our diff --git a/.github/ISSUE_TEMPLATE/questions-help-support.md b/.github/ISSUE_TEMPLATE/questions-help-support.md deleted file mode 100644 index c170ed3740e3..000000000000 --- a/.github/ISSUE_TEMPLATE/questions-help-support.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -name: "Questions/Help/Support" -about: Resources to get help and support about NumPy. - ---- - -## Questions and Help - -### Please note that this issue tracker is not a help form and this issue will -be closed. - -If you need help, we have a set of [listed resources available on the website] -(https://numpy.org/gethelp/). - -For NumPy development-related matters, please see [Community] -(https://numpy.org/community/). From e9e89e073e46f8af801a0940a276d3ed55c818c2 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Mon, 3 Aug 2020 13:20:33 -0400 Subject: [PATCH 014/409] DOC: Revise glossary page Entries added, explanations revised, Python-only entries now point to Python glossary. --- doc/source/glossary.rst | 780 ++++++++++++++++++++++------------------ 1 file changed, 429 insertions(+), 351 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index fb4e0137add2..8a7ebde827d1 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -2,474 +2,552 @@ Glossary ******** -.. toctree:: +.. glossary:: -.. glossary:: + (`n`,) - along an axis - Axes are defined for arrays with more than one dimension. A - 2-dimensional array has two corresponding axes: the first running - vertically downwards across rows (axis 0), and the second running - horizontally across columns (axis 1). + A tuple with one element.The trailing comma distinguishes a one-element + tuple from a parenthesized ``n``. - Many operations can take place along one of these axes. For example, - we can sum each row of an array, in which case we operate along - columns, or axis 1:: - >>> x = np.arange(12).reshape((3,4)) + -1 - >>> x - array([[ 0, 1, 2, 3], - [ 4, 5, 6, 7], - [ 8, 9, 10, 11]]) + Used as a dimension entry, ``-1`` instructs NumPy to choose the length + that will keep the total number of elements the same. - >>> x.sum(axis=1) - array([ 6, 22, 38]) - array - A homogeneous container of numerical elements. Each element in the - array occupies a fixed amount of memory (hence homogeneous), and - can be a numerical element of a single type (such as float, int - or complex) or a combination (such as ``(float, int, float)``). Each - array has an associated data-type (or ``dtype``), which describes - the numerical type of its elements:: + ``...`` - >>> x = np.array([1, 2, 3], float) + **When indexing an array**, shorthand that the missing axes, if they + exist, are full slices. - >>> x - array([ 1., 2., 3.]) + .. code:: python - >>> x.dtype # floating point number, 64 bits of memory per element - dtype('float64') + >>> a.shape + (2, 3, 4) + >>> a[...].shape + (2, 3, 4) - # More complicated data type: each array element is a combination of - # and integer and a floating point number - >>> np.array([(1, 2.0), (3, 4.0)], dtype=[('x', np.int64), ('y', float)]) - array([(1, 2.), (3, 4.)], dtype=[('x', '>> a[...,0].shape + (2, 3) - Fast element-wise operations, called a :term:`ufunc`, operate on arrays. + >>> a[0,...].shape + (3, 4) - array_like - Any sequence that can be interpreted as an ndarray. This includes - nested lists, tuples, scalars and existing arrays. + >>> a[0,...,0].shape + (3,) - attribute - A property of an object that can be accessed using ``obj.attribute``, - e.g., ``shape`` is an attribute of an array:: + It can be used at most once: :: - >>> x = np.array([1, 2, 3]) - >>> x.shape - (3,) + >>> a[0,...,0,...].shape + --------------------------------------------------------------------------- + IndexError Traceback (most recent call last) + in + ----> 1 a[0,...,0,...].shape - big-endian - When storing a multi-byte value in memory as a sequence of bytes, the - sequence addresses/sends/stores the most significant byte first (lowest - address) and the least significant byte last (highest address). Common in - micro-processors and used for transmission of data over network protocols. + IndexError: an index can only have a single ellipsis ('...') - BLAS - `Basic Linear Algebra Subprograms `_ + For more, see the article on + :doc:`Indexing. ` - broadcast - NumPy can do operations on arrays whose shapes are mismatched:: + **In printouts**, NumPy substitutes ``...`` for the middle elements of + large arrays. To see the entire array, use + :doc:`numpy.printoptions. ` - >>> x = np.array([1, 2]) - >>> y = np.array([[3], [4]]) - >>> x - array([1, 2]) + ``:`` - >>> y - array([[3], - [4]]) + The Python + `slice `_ + operator. In ndarrays, slicing can be applied to every + axis: - >>> x + y - array([[4, 5], - [5, 6]]) + .. code:: python - See `basics.broadcasting` for more information. + >>> a = np.arange(24).reshape(2,3,4) + a + array([[[ 0, 1, 2, 3], + [ 4, 5, 6, 7], + [ 8, 9, 10, 11]], - C order - See `row-major` + [[12, 13, 14, 15], + [16, 17, 18, 19], + [20, 21, 22, 23]]]) - column-major - A way to represent items in a N-dimensional array in the 1-dimensional - computer memory. In column-major order, the leftmost index "varies the - fastest": for example the array:: + >>> a[1:,-2:,:-1] + array([[[16, 17, 18], + [20, 21, 22]]]) - [[1, 2, 3], - [4, 5, 6]] + Trailing slices can be omitted: - is represented in the column-major order as:: + .. code:: python - [1, 4, 2, 5, 3, 6] + >>> a[1] == a[1,:,:] + array([[ True, True, True, True], + [ True, True, True, True], + [ True, True, True, True]]) - Column-major order is also known as the Fortran order, as the Fortran - programming language uses it. + In contrast to Python, where slicing creates a copy, in NumPy slicing + creates a :term:`view`. - decorator - An operator that transforms a function. For example, a ``log`` - decorator may be defined to print debugging information upon - function execution:: + For more, see + `Indexing. `_ - >>> def log(f): - ... def new_logging_func(*args, **kwargs): - ... print("Logging call with parameters:", args, kwargs) - ... return f(*args, **kwargs) - ... - ... return new_logging_func + ``<`` - Now, when we define a function, we can "decorate" it using ``log``:: + In a dtype declaration, indicates that the data is + `little-endian `_ + (the bracket is big on the right). :: - >>> @log - ... def add(a, b): - ... return a + b + >>> dt = np.dtype('>> add(1, 2) - Logging call with parameters: (1, 2) {} - 3 + ``>`` - dictionary - Resembling a language dictionary, which provides a mapping between - words and descriptions thereof, a Python dictionary is a mapping - between two objects:: + In a dtype declaration, indicates that the data is + `big-endian `_ + (the bracket is big on the left). :: - >>> x = {1: 'one', 'two': [1, 2]} + >>> dt = np.dtype('>H') # big-endian unsigned short - Here, `x` is a dictionary mapping keys to values, in this case - the integer 1 to the string "one", and the string "two" to - the list ``[1, 2]``. The values may be accessed using their - corresponding keys:: - >>> x[1] - 'one' + advanced indexing - >>> x['two'] - [1, 2] + Rather than using a scalar or slice as an index, an axis can be + indexed with an array, providing fine-grained selection. This is + known as + `advanced indexing `_ + or ``fancy indexing``. - Note that dictionaries are not stored in any specific order. Also, - most mutable (see *immutable* below) objects, such as lists, may not - be used as keys. - For more information on dictionaries, read the - `Python tutorial `_. + axis - field - In a :term:`structured data type`, each sub-type is called a `field`. + Another term for an array dimension. Axes are numbered left to right; + axis 0 is the first element in the shape tuple. + + In a two-dimensional vector, the elements of axis 0 are rows and the + elements of axis 1 are columns. + + In higher dimensions the picture changes. NumPy prints + higher-dimensional vectors as replications of row-by-column building + blocks, as in this three-dimensional vector: + + .. code:: python + + >>> a + array([[[ 0, 1, 2], + [ 3, 4, 5]], + + [[ 6, 7, 8], + [ 9, 10, 11]]]) + + >>> a.shape + (2, 2, 3) + + ``a`` is depicted as a two-element array whose elements are 2x3 vectors. + From this point of view, rows and columns are the final two axes, + respectively, in any shape. + + This rule helps you anticipate how a vector will be printed, and + conversely how to find the index of any of the printed elements. For + instance, in the example, the last two values of 8's index must be 0 and + 2. Since 8 appears in the second of the two 2x3's, the first index must + be 1: + + >>> a[1,0,2] + 8 + + A convenient way to count dimensions in a printed vector is to + count ``[`` symbols after the open-parenthesis. This is + useful in distinguishing, say, a (1,2,3) shape from a (2,3) shape: + + .. code:: python + + >>> a.shape + (2, 3) + >>> a + array([[0, 1, 2], + [3, 4, 5]]) + + >>> a.shape + (1, 2, 3) + >>> a + array([[[0, 1, 2], + [3, 4, 5]]]) + + + along an axis + + The result of an operation along an axis X is a vector in which X + disappears. This can surprise new users expecting the opposite. + + The operation can be visualized this way: + + Imagine a slice of array ``a`` where axis X has a fixed index + and the other dimensions are left full (``:``). + + >>> a.shape + (2,3,4) + >>> a[:,0,:].shape + (2,4) + + The slice has ``a``'s shape with the X dimension deleted. Saying an + operation ``op`` is ``performed along X`` means that ``op`` takes as its + operands slices having every value of X: + + >>> np.sum(a,axis=1) == a[:,0,:] + a[:,1,:] + a[:,2,:] + array([[ True, True, True, True], + [ True, True, True, True]]) + + + array + + Used synonymously in the NumPy docs with `ndarray`, NumPy's basic structure. + See :doc:`Array objects. ` + + + array_like + + Any `sequence `_ + that can be interpreted as an ndarray. In addition to ndarrays + this category includes lists (possibly nested and with + different element types), tuples, and scalars. Any argument accepted by + :doc:`numpy.array ` + is array_like. + + .. code:: + + >>> x = np.array([[1,2.0],[0,0],(1+1j,3.)]) + + >>> x + array([[1.+0.j, 2.+0.j], + [0.+0.j, 0.+0.j], + [1.+1.j, 3.+0.j]]) + + + array scalar + + For uniformity in handling operands, NumPy treats + a :doc:`scalar ` as an array of zero dimension. + + + `attribute `_ + \ + + + .base + + If an array does not own its memory, then its ``.base`` attribute + returns the object whose memory the array is referencing. That object + may may be borrowing the memory from still another object, so the + owning object may be ``a.base.base.base...``. Despite advice to the + contrary, testing ``base`` is not a surefire way to determine if two + arrays are `views. <#term-view>`_ + + + `big-endian `_ + \ + + + `BLAS `_ + \ + + + broadcast + + ``broadcasting`` is NumPy's ability to process ndarrays of + different sizes as if all were the same size. + + When NumPy operates on two arrays, it works element by + element -- for instance, ``c = a * b`` is + + .. code:: + + c[0,0,0] = a[0,0,0] * b[0,0,0] + c[0,0,1] = a[0,0,1] * b[0,0,1] + ... + + Ordinarily this means the shapes of a and b must be identical. But in + some cases, NumPy can fill "missing" axes or "too-short" dimensions + with duplicate data so shapes will match. The duplication costs + no memory or time. See :doc:`Broadcasting. ` + + + C order + + Same as `row-major. `_ + + + `column-major `_ + \ + + + copy + + See :term:`view`. + + + `decorator `_ + \ + + + `dictionary `_ + \ + + + dimension + + See :term:`axis`. + + + dtype + + The datatype describing the (identically typed) elements in an ndarray. + It can be changed to reinterpret the array contents. See + :doc:`Data type objects (dtype). ` + + + fancy indexing + + Another term for :term:`advanced indexing`. + + + field + + In a :term:`structured data type`, each subtype is called a `field`. The `field` has a name (a string), a type (any valid dtype), and - an optional `title`. See :ref:`arrays.dtypes` + an optional :term:`title`. See :ref:`arrays.dtypes`. + + + Fortran order - Fortran order - See `column-major` + Same as `column-major `_ - flattened - Collapsed to a one-dimensional array. See `numpy.ndarray.flatten` - for details. - homogeneous - Describes a block of memory comprised of blocks, each block comprised of - items and of the same size, and blocks are interpreted in exactly the - same way. In the simplest case each block contains a single item, for - instance int32 or float64. + flattened - immutable - An object that cannot be modified after execution is called - immutable. Two common examples are strings and tuples. + See :term:`ravel`. - instance - A class definition gives the blueprint for constructing an object:: - >>> class House: - ... wall_colour = 'white' + homogeneous - Yet, we have to *build* a house before it exists:: + All elements of a homogeneous array have the same type. ndarrays, in + contrast to Python lists, are homogeneous. The type can be complicated, + as in a :term:`structured array`, but all elements have that type. - >>> h = House() # build a house + NumPy `object arrays <#term-object-array>`_, which contain references to + Python objects, fill the role of heterogeneous arrays. - Now, ``h`` is called a ``House`` instance. An instance is therefore - a specific realisation of a class. - iterable - A sequence that allows "walking" (iterating) over items, typically - using a loop such as:: + `immutable `_ + \ - >>> x = [1, 2, 3] - >>> [item**2 for item in x] - [1, 4, 9] - It is often used in combination with ``enumerate``:: - >>> keys = ['a','b','c'] - >>> for n, k in enumerate(keys): - ... print("Key %d: %s" % (n, k)) - ... - Key 0: a - Key 1: b - Key 2: c + `iterable `_ + \ + + + itemsize - itemsize The size of the dtype element in bytes. - list - A Python container that can hold any number of objects or items. - The items do not have to be of the same type, and can even be - lists themselves:: - >>> x = [2, 2.0, "two", [2, 2.0]] + `list `_ + \ + - The list `x` contains 4 items, each which can be accessed individually:: + `little-endian `_ + \ - >>> x[2] # the string 'two' - 'two' - >>> x[3] # a list, containing an integer 2 and a float 2.0 - [2, 2.0] + mask - It is also possible to select more than one item at a time, - using *slicing*:: + The boolean array used to select elements in a :term:`masked array`. - >>> x[0:2] # or, equivalently, x[:2] - [2, 2.0] - In code, arrays are often conveniently expressed as nested lists:: + masked array + Bad or missing data can be cleanly ignored by putting it in a masked + array, which has an internal boolean array indicating invalid + entries. Operations with masked arrays ignore these entries. - >>> np.array([[1, 2], [3, 4]]) - array([[1, 2], - [3, 4]]) + >>> a = np.ma.masked_array([np.nan, 2, np.nan], [True, False, True]) + >>> a + masked_array(data = [-- 2.0 --], + mask = [ True False True], + fill_value = 1e+20) - For more information, read the section on lists in the `Python - tutorial `_. For a mapping - type (key-value), see *dictionary*. + >>> a + [1, 2, 3] + masked_array(data = [-- 4.0 --], + mask = [ True False True], + fill_value = 1e+20) - little-endian - When storing a multi-byte value in memory as a sequence of bytes, the - sequence addresses/sends/stores the least significant byte first (lowest - address) and the most significant byte last (highest address). Common in - x86 processors. + See :doc:`Masked arrays. ` - mask - A boolean array, used to select only certain elements for an operation:: - >>> x = np.arange(5) - >>> x - array([0, 1, 2, 3, 4]) + matrix - >>> mask = (x > 2) - >>> mask - array([False, False, False, True, True]) + NumPy's two-dimensional + :doc:`matrix class ` + should no longer be used; use regular ndarrays. - >>> x[mask] = -1 - >>> x - array([ 0, 1, 2, -1, -1]) - masked array - Array that suppressed values indicated by a mask:: + ndarray - >>> x = np.ma.masked_array([np.nan, 2, np.nan], [True, False, True]) - >>> x - masked_array(data=[--, 2.0, --], - mask=[ True, False, True], - fill_value=1e+20) + See :term:`array`. - >>> x + [1, 2, 3] - masked_array(data=[--, 4.0, --], - mask=[ True, False, True], - fill_value=1e+20) + object array - Masked arrays are often used when operating on arrays containing - missing or invalid entries. + An array whose dtype is ``object``; that is, it contains references to + Python objects. Indexing the array dereferences the Python objects, so + unlike other ndarrays, an object array has the ability to hold + heterogeneous objects. - matrix - A 2-dimensional ndarray that preserves its two-dimensional nature - throughout operations. It has certain special operations, such as ``*`` - (matrix multiplication) and ``**`` (matrix power), defined:: - >>> x = np.mat([[1, 2], [3, 4]]) - >>> x - matrix([[1, 2], - [3, 4]]) + ravel - >>> x**2 - matrix([[ 7, 10], - [15, 22]]) + :doc:`numpy.ravel \ + ` + and :doc:`numpy.flatten \ + ` + both flatten an ndarray. ``ravel`` will return a view if possible; + ``flatten`` always returns a copy. - method - A function associated with an object. For example, each ndarray has a - method called ``repeat``:: + Flattening collapses a multimdimensional array to a single dimension; + details of how this is done (for instance, whether ``a[n+1]`` should be + the next row or next column) are parameters. - >>> x = np.array([1, 2, 3]) - >>> x.repeat(2) - array([1, 1, 2, 2, 3, 3]) - ndarray - See *array*. + record array - record array - An :term:`ndarray` with :term:`structured data type` which has been - subclassed as ``np.recarray`` and whose dtype is of type ``np.record``, - making the fields of its data type to be accessible by attribute. + A :term:`structured array` with an additional way to access + fields -- ``a.field`` in addition to ``a['field']``. See + :doc:`numpy.recarray. ` - reference - If ``a`` is a reference to ``b``, then ``(a is b) == True``. Therefore, - ``a`` and ``b`` are different names for the same Python object. - row-major - A way to represent items in a N-dimensional array in the 1-dimensional - computer memory. In row-major order, the rightmost index "varies - the fastest": for example the array:: + `row-major `_ + \ - [[1, 2, 3], - [4, 5, 6]] + shape - is represented in the row-major order as:: + A tuple showing the length of each dimension of an ndarray. The + length of the tuple itself is the number of dimensions + (:doc:`numpy.ndim `). + The product of the tuple elements is the number of elements in the + array. - [1, 2, 3, 4, 5, 6] - Row-major order is also known as the C order, as the C programming - language uses it. New NumPy arrays are by default in row-major order. + :term:`slice <:>` + \ - self - Often seen in method signatures, ``self`` refers to the instance - of the associated class. For example: - >>> class Paintbrush: - ... color = 'blue' - ... - ... def paint(self): - ... print("Painting the city %s!" % self.color) - ... - >>> p = Paintbrush() - >>> p.color = 'red' - >>> p.paint() # self refers to 'p' - Painting the city red! + stride - slice - Used to select only certain elements from a sequence: + Tuple of bytes to step in each dimension when traversing an array. + The byte offset of element :math:`(i_0, i_1, ..., i_n)` in an array `a` is: :: - >>> x = range(5) - >>> x - [0, 1, 2, 3, 4] + offset = sum(np.array(i) * a.strides) - >>> x[1:3] # slice from 1 to 3 (excluding 3 itself) - [1, 2] + Stride is computed automatically from an array's dtype and + shape, but can be directly specified using + :doc:`as_strided. ` - >>> x[1:5:2] # slice from 1 to 5, but skipping every second element - [1, 3] + To see how striding underlies the power of NumPy views, see + `The NumPy array: a structure for efficient numerical computation. \ + `_ - >>> x[::-1] # slice a sequence in reverse - [4, 3, 2, 1, 0] - Arrays may have more than one dimension, each which can be sliced - individually: + structured array - >>> x = np.array([[1, 2], [3, 4]]) - >>> x - array([[1, 2], - [3, 4]]) + Array whose :term:`dtype` is a :term:`structured data type`. - >>> x[:, 1] - array([2, 4]) - structure - See :term:`structured data type` + structured data type - structured data type - A data type composed of other datatypes + Users can create arbitrarily complex :term:`dtypes ` + that can include other arrays and dtypes. These composite dtypes are called + :doc:`structured data types. ` - subarray data type - A :term:`structured data type` may contain a :term:`ndarray` with its - own dtype and shape: - >>> dt = np.dtype([('a', np.int32), ('b', np.float32, (3,))]) - >>> np.zeros(3, dtype=dt) - array([(0, [0., 0., 0.]), (0, [0., 0., 0.]), (0, [0., 0., 0.])], - dtype=[('a', '` which is an alias to the name and is - commonly used for plotting. + An array nested in a :term:`structured data type`: - tuple - A sequence that may contain a variable number of types of any - kind. A tuple is immutable, i.e., once constructed it cannot be - changed. Similar to a list, it can be indexed and sliced:: + .. code:: python - >>> x = (1, 'one', [1, 2]) - >>> x - (1, 'one', [1, 2]) + >>> dt = np.dtype([('a', np.int32), ('b', np.float32, (3,))]) + >>> np.zeros(3, dtype=dt) + array([(0, [0., 0., 0.]), (0, [0., 0., 0.]), (0, [0., 0., 0.])], + dtype=[('a', '>> x[0] - 1 - >>> x[:2] - (1, 'one') + subarray data type - A useful concept is "tuple unpacking", which allows variables to - be assigned to the contents of a tuple:: + An element of a strctured datatype that behaves like an ndarray. - >>> x, y = (1, 2) - >>> x, y = 1, 2 + .. + The entry is in numpy.doc.structured_arrays:51 and + so can't be deleted. - This is often used when a function returns multiple values: - >>> def return_many(): - ... return 1, 'alpha', None + title - >>> a, b, c = return_many() - >>> a, b, c - (1, 'alpha', None) + An alias for a field name in a structured datatype. - >>> a - 1 - >>> b - 'alpha' - - ufunc - Universal function. A fast element-wise, :term:`vectorized - ` array operation. Examples include ``add``, ``sin`` and - ``logical_or``. - - vectorization - Optimizing a looping block by specialized code. In a traditional sense, - vectorization performs the same operation on multiple elements with - fixed strides between them via specialized hardware. Compilers know how - to take advantage of well-constructed loops to implement such - optimizations. NumPy uses :ref:`vectorization ` - to mean any optimization via specialized code performing the same - operations on multiple elements, typically achieving speedups by - avoiding some of the overhead in looking up and converting the elements. - - view - An array that does not own its data, but refers to another array's - data instead. For example, we may create a view that only shows - every second element of another array:: - - >>> x = np.arange(5) - >>> x - array([0, 1, 2, 3, 4]) - - >>> y = x[::2] - >>> y - array([0, 2, 4]) - - >>> x[0] = 3 # changing x changes y as well, since y is a view on x - >>> y - array([3, 2, 4]) - - wrapper - Python is a high-level (highly abstracted, or English-like) language. - This abstraction comes at a price in execution speed, and sometimes - it becomes necessary to use lower level languages to do fast - computations. A wrapper is code that provides a bridge between - high and the low level languages, allowing, e.g., Python to execute - code written in C or Fortran. - - Examples include ctypes, SWIG and Cython (which wraps C and C++) - and f2py (which wraps Fortran). + .. + The entry is referenced in numpy.doc.structured_arrays:242 + and so can't be deleted. + + + `tuple `_ + \ + + + type + + In NumPy, a synonym for :term:`dtype`. For the more general Python + meaning, + `see here. `_ + + + ufunc + + NumPy's fast element-by-element computation (:term:`vectorization`) is + structured so as to leave the choice of function open. A function used + in vectorization is called a ``ufunc``, short for ``universal + function``. NumPy routines have built-in ufuncs, but users can also + :doc:`write their own. ` + + + vectorization + + NumPy hands off array processing to C, where looping and computation are + much faster than in Python. To exploit this, programmers using NumPy + eliminate Python loops in favor of array-to-array operations. + :term:`vectorization` can refer both to the C offloading and to + structuring NumPy code to leverage it. + + + view + + Without changing underlying data, NumPy can make one array masquerade as + any number of other arrays with different types, shapes, and even + content. This is much faster than creating those arrays. + + An array created this way is a ``view``, and the performance gain often + makes an array created as a view preferable to one created as a new + array. + But because a view shares data with the original array, a write in one + array can affect the other, even though they appear to be different + arrays. If this is an problem, a view can't be used; the second array + needs to be physically distinct -- a ``copy``. + Some NumPy routines always return views, some always return copies, some + may return one or the other, and for some the choice can be specified. + Responsiblity for managing views and copies falls to the programmer. + NumPy reports whether arrays share memory wih + :doc:`numpy.shares_memory `, + but an exact answer isn't always possible; see the link. From d7db94fa85be50a1ae840be5a6b2b5a50d56c557 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Mon, 3 Aug 2020 13:35:31 -0400 Subject: [PATCH 015/409] DOC: Wording improvement in #16996 More clearly describe `shares_memory` as being not always *feasible* rather than *possible* --- doc/source/glossary.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 8a7ebde827d1..4f264e0f3636 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -550,4 +550,4 @@ Glossary Responsiblity for managing views and copies falls to the programmer. NumPy reports whether arrays share memory wih :doc:`numpy.shares_memory `, - but an exact answer isn't always possible; see the link. + but an exact answer isn't always feasible; see the link. From d434a2e7332fa5cccc89a030268317a6225105aa Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Tue, 4 Aug 2020 12:30:04 -0400 Subject: [PATCH 016/409] DOC: Incorporate @BvB93's suggestions in PR #16996 Also reworded `stride` entry for clarity, moved 'axis' entry to correct alphabetical position. Also added anchors to reference/arrays.indexing for glossary references to point to. --- doc/source/glossary.rst | 177 ++++++++++++----------- doc/source/reference/arrays.indexing.rst | 3 + 2 files changed, 96 insertions(+), 84 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 4f264e0f3636..90e8ccd9164d 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -6,7 +6,7 @@ Glossary (`n`,) - A tuple with one element.The trailing comma distinguishes a one-element + A tuple with one element. The trailing comma distinguishes a one-element tuple from a parenthesized ``n``. @@ -48,8 +48,7 @@ Glossary IndexError: an index can only have a single ellipsis ('...') - For more, see the article on - :doc:`Indexing. ` + For details, see :doc:`Indexing. ` **In printouts**, NumPy substitutes ``...`` for the middle elements of large arrays. To see the entire array, use @@ -91,8 +90,8 @@ Glossary In contrast to Python, where slicing creates a copy, in NumPy slicing creates a :term:`view`. - For more, see - `Indexing. `_ + For details, see :ref:`Combining advanced and basic indexing `. + ``<`` @@ -114,13 +113,72 @@ Glossary advanced indexing - Rather than using a scalar or slice as an index, an axis can be - indexed with an array, providing fine-grained selection. This is - known as - `advanced indexing `_ + Rather than using a :doc:`scalar ` or slice as + an index, an axis can be indexed with an array, providing fine-grained + selection. This is known as :ref:`advanced indexing` or ``fancy indexing``. + along an axis + + The result of an operation along an :term:`axis` X is an array in which X + disappears. This can surprise new users expecting the opposite. + + The operation can be visualized this way: + + Imagine a slice of array ``a`` where axis X has a fixed index + and the other dimensions are left full (``:``). + + >>> a.shape + (2,3,4) + >>> a[:,0,:].shape + (2,4) + + The slice has ``a``'s shape with the X dimension deleted. Saying an + operation ``op`` is ``performed along X`` means that ``op`` takes as its + operands slices having every value of X: + + >>> np.sum(a,axis=1) == a[:,0,:] + a[:,1,:] + a[:,2,:] + array([[ True, True, True, True], + [ True, True, True, True]]) + + + array + + Used synonymously in the NumPy docs with + :doc:`ndarray `, NumPy's basic structure. + + + array_like + + Any :doc:`scalar ` or + `sequence `_ + that can be interpreted as an ndarray. In addition to ndarrays + and scalars this category includes lists (possibly nested and with + different element types) and tuples. Any argument accepted by + :doc:`numpy.array ` + is array_like. + + .. code:: + + >>> x = np.array([[1,2.0],[0,0],(1+1j,3.)]) + + >>> x + array([[1.+0.j, 2.+0.j], + [0.+0.j, 0.+0.j], + [1.+1.j, 3.+0.j]]) + + + array scalar + + For uniformity in handling operands, NumPy treats + a :doc:`scalar ` as an array of zero dimension. + + + `attribute `_ + \ + + axis Another term for an array dimension. Axes are numbered left to right; @@ -177,68 +235,10 @@ Glossary [3, 4, 5]]]) - along an axis - - The result of an operation along an axis X is a vector in which X - disappears. This can surprise new users expecting the opposite. - - The operation can be visualized this way: - - Imagine a slice of array ``a`` where axis X has a fixed index - and the other dimensions are left full (``:``). - - >>> a.shape - (2,3,4) - >>> a[:,0,:].shape - (2,4) - - The slice has ``a``'s shape with the X dimension deleted. Saying an - operation ``op`` is ``performed along X`` means that ``op`` takes as its - operands slices having every value of X: - - >>> np.sum(a,axis=1) == a[:,0,:] + a[:,1,:] + a[:,2,:] - array([[ True, True, True, True], - [ True, True, True, True]]) - - - array - - Used synonymously in the NumPy docs with `ndarray`, NumPy's basic structure. - See :doc:`Array objects. ` - - - array_like - - Any `sequence `_ - that can be interpreted as an ndarray. In addition to ndarrays - this category includes lists (possibly nested and with - different element types), tuples, and scalars. Any argument accepted by - :doc:`numpy.array ` - is array_like. - - .. code:: - - >>> x = np.array([[1,2.0],[0,0],(1+1j,3.)]) - - >>> x - array([[1.+0.j, 2.+0.j], - [0.+0.j, 0.+0.j], - [1.+1.j, 3.+0.j]]) - - - array scalar - - For uniformity in handling operands, NumPy treats - a :doc:`scalar ` as an array of zero dimension. - - - `attribute `_ - \ - - .base - If an array does not own its memory, then its ``.base`` attribute + If an array does not own its memory, then its + :doc:`base ` attribute returns the object whose memory the array is referencing. That object may may be borrowing the memory from still another object, so the owning object may be ``a.base.base.base...``. Despite advice to the @@ -271,7 +271,7 @@ Glossary Ordinarily this means the shapes of a and b must be identical. But in some cases, NumPy can fill "missing" axes or "too-short" dimensions with duplicate data so shapes will match. The duplication costs - no memory or time. See :doc:`Broadcasting. ` + no memory or time. For details, see :doc:`Broadcasting. ` C order @@ -304,7 +304,7 @@ Glossary dtype The datatype describing the (identically typed) elements in an ndarray. - It can be changed to reinterpret the array contents. See + It can be changed to reinterpret the array contents. For details, see :doc:`Data type objects (dtype). ` @@ -315,9 +315,10 @@ Glossary field - In a :term:`structured data type`, each subtype is called a `field`. - The `field` has a name (a string), a type (any valid dtype), and - an optional :term:`title`. See :ref:`arrays.dtypes`. + In a :term:`structured data type`, each subtype is called a + :doc:`field `. + A field has a name (a string), a type (any valid dtype), and + an optional :term:`title`. For details, see :ref:`arrays.dtypes`. Fortran order @@ -383,7 +384,7 @@ Glossary mask = [ True False True], fill_value = 1e+20) - See :doc:`Masked arrays. ` + For details, see :doc:`Masked arrays. ` matrix @@ -423,12 +424,15 @@ Glossary record array A :term:`structured array` with an additional way to access - fields -- ``a.field`` in addition to ``a['field']``. See + fields -- ``a.field`` in addition to ``a['field']``. For details, see :doc:`numpy.recarray. ` `row-major `_ - \ + \ + + :doc:`scalar ` + \ shape @@ -436,7 +440,8 @@ Glossary length of the tuple itself is the number of dimensions (:doc:`numpy.ndim `). The product of the tuple elements is the number of elements in the - array. + array. For details, see + :doc:`numpy.ndarray.shape `. :term:`slice <:>` @@ -445,14 +450,18 @@ Glossary stride - Tuple of bytes to step in each dimension when traversing an array. - The byte offset of element :math:`(i_0, i_1, ..., i_n)` in an array `a` is: :: - - offset = sum(np.array(i) * a.strides) + Physical memory is one-dimensional; ``stride`` maps an index in an + N-dimensional ndarray to an address in memory. For an N-dimensional + array, stride is an N-element tuple; advancing from index ``i`` to index + ``i+1`` on axis ``n`` means adding ``a.strides[n]`` bytes to the + address. Stride is computed automatically from an array's dtype and shape, but can be directly specified using - :doc:`as_strided. ` + :doc:`as_strided. ` + + For details, see + :doc:`numpy.ndarray.strides `. To see how striding underlies the power of NumPy views, see `The NumPy array: a structure for efficient numerical computation. \ @@ -548,6 +557,6 @@ Glossary Some NumPy routines always return views, some always return copies, some may return one or the other, and for some the choice can be specified. Responsiblity for managing views and copies falls to the programmer. - NumPy reports whether arrays share memory wih + NumPy reports whether arrays share memory :doc:`numpy.shares_memory `, but an exact answer isn't always feasible; see the link. diff --git a/doc/source/reference/arrays.indexing.rst b/doc/source/reference/arrays.indexing.rst index 3e600b7c456e..180a79dae18c 100644 --- a/doc/source/reference/arrays.indexing.rst +++ b/doc/source/reference/arrays.indexing.rst @@ -198,6 +198,7 @@ concepts to remember include: create an axis of length one. :const:`newaxis` is an alias for 'None', and 'None' can be used in place of this with the same result. +.. _advanced-indexing: Advanced Indexing ----------------- @@ -304,6 +305,8 @@ understood with an example. most important thing to remember about indexing with multiple advanced indexes. +.. _combining-advanced-and-basic-indexing: + Combining advanced and basic indexing """"""""""""""""""""""""""""""""""""" From ef3b71e7c2bc6ae42c41d31a9bd83f57f604b496 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Tue, 4 Aug 2020 13:54:01 -0400 Subject: [PATCH 017/409] DOC: Avoid unwanted doctest on PR #14979 Failing CI because it's running doctest on the example strings. Also, deleting glossary.py, source file of the original glossary. --- doc/source/glossary.rst | 117 +++++++++++++++++----------------------- 1 file changed, 48 insertions(+), 69 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 90e8ccd9164d..ea2a9a6f25bc 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -19,34 +19,32 @@ Glossary ``...`` **When indexing an array**, shorthand that the missing axes, if they - exist, are full slices. + exist, are full slices. :: - .. code:: python - - >>> a.shape - (2, 3, 4) + >>> a.shape + (2, 3, 4) - >>> a[...].shape - (2, 3, 4) + >>> a[...].shape + (2, 3, 4) - >>> a[...,0].shape - (2, 3) + >>> a[...,0].shape + (2, 3) - >>> a[0,...].shape - (3, 4) + >>> a[0,...].shape + (3, 4) - >>> a[0,...,0].shape - (3,) + >>> a[0,...,0].shape + (3,) - It can be used at most once: :: + It can be used at most once: :: - >>> a[0,...,0,...].shape - --------------------------------------------------------------------------- - IndexError Traceback (most recent call last) - in - ----> 1 a[0,...,0,...].shape + >>> a[0,...,0,...].shape + --------------------------------------------------------------------------- + IndexError Traceback (most recent call last) + in + ----> 1 a[0,...,0,...].shape - IndexError: an index can only have a single ellipsis ('...') + IndexError: an index can only have a single ellipsis ('...') For details, see :doc:`Indexing. ` @@ -60,9 +58,7 @@ Glossary The Python `slice `_ operator. In ndarrays, slicing can be applied to every - axis: - - .. code:: python + axis: :: >>> a = np.arange(24).reshape(2,3,4) a @@ -78,9 +74,7 @@ Glossary array([[[16, 17, 18], [20, 21, 22]]]) - Trailing slices can be omitted: - - .. code:: python + Trailing slices can be omitted: :: >>> a[1] == a[1,:,:] array([[ True, True, True, True], @@ -127,7 +121,7 @@ Glossary The operation can be visualized this way: Imagine a slice of array ``a`` where axis X has a fixed index - and the other dimensions are left full (``:``). + and the other dimensions are left full (``:``). :: >>> a.shape (2,3,4) @@ -136,7 +130,7 @@ Glossary The slice has ``a``'s shape with the X dimension deleted. Saying an operation ``op`` is ``performed along X`` means that ``op`` takes as its - operands slices having every value of X: + operands slices having every value of X: :: >>> np.sum(a,axis=1) == a[:,0,:] + a[:,1,:] + a[:,2,:] array([[ True, True, True, True], @@ -157,9 +151,7 @@ Glossary and scalars this category includes lists (possibly nested and with different element types) and tuples. Any argument accepted by :doc:`numpy.array ` - is array_like. - - .. code:: + is array_like. :: >>> x = np.array([[1,2.0],[0,0],(1+1j,3.)]) @@ -172,7 +164,8 @@ Glossary array scalar For uniformity in handling operands, NumPy treats - a :doc:`scalar ` as an array of zero dimension. + a :doc:`scalar ` as an array of zero + dimension. `attribute `_ @@ -189,19 +182,17 @@ Glossary In higher dimensions the picture changes. NumPy prints higher-dimensional vectors as replications of row-by-column building - blocks, as in this three-dimensional vector: - - .. code:: python + blocks, as in this three-dimensional vector: :: - >>> a - array([[[ 0, 1, 2], - [ 3, 4, 5]], + >>> a + array([[[ 0, 1, 2], + [ 3, 4, 5]], - [[ 6, 7, 8], - [ 9, 10, 11]]]) + [[ 6, 7, 8], + [ 9, 10, 11]]]) - >>> a.shape - (2, 2, 3) + >>> a.shape + (2, 2, 3) ``a`` is depicted as a two-element array whose elements are 2x3 vectors. From this point of view, rows and columns are the final two axes, @@ -218,21 +209,19 @@ Glossary A convenient way to count dimensions in a printed vector is to count ``[`` symbols after the open-parenthesis. This is - useful in distinguishing, say, a (1,2,3) shape from a (2,3) shape: + useful in distinguishing, say, a (1,2,3) shape from a (2,3) shape: :: - .. code:: python - - >>> a.shape - (2, 3) - >>> a - array([[0, 1, 2], - [3, 4, 5]]) + >>> a.shape + (2, 3) + >>> a + array([[0, 1, 2], + [3, 4, 5]]) - >>> a.shape - (1, 2, 3) - >>> a - array([[[0, 1, 2], - [3, 4, 5]]]) + >>> a.shape + (1, 2, 3) + >>> a + array([[[0, 1, 2], + [3, 4, 5]]]) .base @@ -243,7 +232,7 @@ Glossary may may be borrowing the memory from still another object, so the owning object may be ``a.base.base.base...``. Despite advice to the contrary, testing ``base`` is not a surefire way to determine if two - arrays are `views. <#term-view>`_ + arrays are :term:`view`\ s. `big-endian `_ @@ -260,9 +249,7 @@ Glossary different sizes as if all were the same size. When NumPy operates on two arrays, it works element by - element -- for instance, ``c = a * b`` is - - .. code:: + element -- for instance, ``c = a * b`` is :: c[0,0,0] = a[0,0,0] * b[0,0,0] c[0,0,1] = a[0,0,1] * b[0,0,1] @@ -371,7 +358,7 @@ Glossary Bad or missing data can be cleanly ignored by putting it in a masked array, which has an internal boolean array indicating invalid - entries. Operations with masked arrays ignore these entries. + entries. Operations with masked arrays ignore these entries. :: >>> a = np.ma.masked_array([np.nan, 2, np.nan], [True, False, True]) >>> a @@ -482,9 +469,7 @@ Glossary subarray - An array nested in a :term:`structured data type`: - - .. code:: python + An array nested in a :term:`structured data type`: :: >>> dt = np.dtype([('a', np.int32), ('b', np.float32, (3,))]) >>> np.zeros(3, dtype=dt) @@ -497,18 +482,12 @@ Glossary An element of a strctured datatype that behaves like an ndarray. .. - The entry is in numpy.doc.structured_arrays:51 and - so can't be deleted. title An alias for a field name in a structured datatype. - .. - The entry is referenced in numpy.doc.structured_arrays:242 - and so can't be deleted. - `tuple `_ \ From 348a32ef50d11ab206c0d58764728f8c7792b68a Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Wed, 5 Aug 2020 06:44:26 -0400 Subject: [PATCH 018/409] DOC: 2nd try: Avoid unwanted doctest on PR #14979 --- doc/source/glossary.rst | 72 +++++++++++++++++++++++++++-------------- 1 file changed, 48 insertions(+), 24 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index ea2a9a6f25bc..ce6781ce9db8 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -19,9 +19,11 @@ Glossary ``...`` **When indexing an array**, shorthand that the missing axes, if they - exist, are full slices. :: + exist, are full slices. + + .. doctest:: + :skipif: True - >>> a.shape (2, 3, 4) >>> a[...].shape @@ -36,7 +38,10 @@ Glossary >>> a[0,...,0].shape (3,) - It can be used at most once: :: + It can be used at most once: + + .. doctest:: + :skipif: True >>> a[0,...,0,...].shape --------------------------------------------------------------------------- @@ -121,20 +126,27 @@ Glossary The operation can be visualized this way: Imagine a slice of array ``a`` where axis X has a fixed index - and the other dimensions are left full (``:``). :: + and the other dimensions are left full (``:``). - >>> a.shape - (2,3,4) - >>> a[:,0,:].shape - (2,4) + .. doctest:: + :skipif: True + + >>> a.shape + >>> a.shape + (2,3,4) + >>> a[:,0,:].shape + (2,4) The slice has ``a``'s shape with the X dimension deleted. Saying an operation ``op`` is ``performed along X`` means that ``op`` takes as its - operands slices having every value of X: :: + operands slices having every value of X: - >>> np.sum(a,axis=1) == a[:,0,:] + a[:,1,:] + a[:,2,:] - array([[ True, True, True, True], - [ True, True, True, True]]) + .. doctest:: + :skipif: True + + >>> np.sum(a,axis=1) == a[:,0,:] + a[:,1,:] + a[:,2,:] + array([[ True, True, True, True], + [ True, True, True, True]]) array @@ -153,9 +165,9 @@ Glossary :doc:`numpy.array ` is array_like. :: - >>> x = np.array([[1,2.0],[0,0],(1+1j,3.)]) + >>> a = np.array([[1,2.0],[0,0],(1+1j,3.)]) - >>> x + >>> a array([[1.+0.j, 2.+0.j], [0.+0.j, 0.+0.j], [1.+1.j, 3.+0.j]]) @@ -182,17 +194,20 @@ Glossary In higher dimensions the picture changes. NumPy prints higher-dimensional vectors as replications of row-by-column building - blocks, as in this three-dimensional vector: :: + blocks, as in this three-dimensional vector: - >>> a - array([[[ 0, 1, 2], - [ 3, 4, 5]], + .. doctest:: + :skipif: True - [[ 6, 7, 8], - [ 9, 10, 11]]]) + >>> a + array([[[ 0, 1, 2], + [ 3, 4, 5]], - >>> a.shape - (2, 2, 3) + [[ 6, 7, 8], + [ 9, 10, 11]]]) + + >>> a.shape + (2, 2, 3) ``a`` is depicted as a two-element array whose elements are 2x3 vectors. From this point of view, rows and columns are the final two axes, @@ -204,12 +219,18 @@ Glossary 2. Since 8 appears in the second of the two 2x3's, the first index must be 1: + .. doctest:: + :skipif: True + >>> a[1,0,2] 8 A convenient way to count dimensions in a printed vector is to count ``[`` symbols after the open-parenthesis. This is - useful in distinguishing, say, a (1,2,3) shape from a (2,3) shape: :: + useful in distinguishing, say, a (1,2,3) shape from a (2,3) shape: + + .. doctest:: + :skipif: True >>> a.shape (2, 3) @@ -249,7 +270,10 @@ Glossary different sizes as if all were the same size. When NumPy operates on two arrays, it works element by - element -- for instance, ``c = a * b`` is :: + element -- for instance, ``c = a * b`` is + + .. doctest:: + :skipif: True c[0,0,0] = a[0,0,0] * b[0,0,0] c[0,0,1] = a[0,0,1] * b[0,0,1] From 4d4ba091492cbe149210447c580039c744e4d5b9 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Wed, 5 Aug 2020 07:12:18 -0400 Subject: [PATCH 019/409] DOC: 3rd try: Avoid unwanted doctest on PR #14979 --- doc/source/glossary.rst | 48 ++++++++++++++--------------------------- 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index ce6781ce9db8..776701814ae3 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -21,22 +21,19 @@ Glossary **When indexing an array**, shorthand that the missing axes, if they exist, are full slices. - .. doctest:: - :skipif: True - - (2, 3, 4) + >>> a = np.arange(24).reshape(2,3,4) - >>> a[...].shape - (2, 3, 4) + >>> a[...].shape + (2, 3, 4) - >>> a[...,0].shape - (2, 3) + >>> a[...,0].shape + (2, 3) - >>> a[0,...].shape - (3, 4) + >>> a[0,...].shape + (3, 4) - >>> a[0,...,0].shape - (3,) + >>> a[0,...,0].shape + (3,) It can be used at most once: @@ -63,7 +60,7 @@ Glossary The Python `slice `_ operator. In ndarrays, slicing can be applied to every - axis: :: + axis: >>> a = np.arange(24).reshape(2,3,4) a @@ -128,12 +125,11 @@ Glossary Imagine a slice of array ``a`` where axis X has a fixed index and the other dimensions are left full (``:``). - .. doctest:: - :skipif: True + >>> a = np.arange(24).reshape(2,3,4) - >>> a.shape >>> a.shape (2,3,4) + >>> a[:,0,:].shape (2,4) @@ -141,9 +137,6 @@ Glossary operation ``op`` is ``performed along X`` means that ``op`` takes as its operands slices having every value of X: - .. doctest:: - :skipif: True - >>> np.sum(a,axis=1) == a[:,0,:] + a[:,1,:] + a[:,2,:] array([[ True, True, True, True], [ True, True, True, True]]) @@ -196,9 +189,7 @@ Glossary higher-dimensional vectors as replications of row-by-column building blocks, as in this three-dimensional vector: - .. doctest:: - :skipif: True - + >>> a = np.arange(12).reshape(2,2,3) >>> a array([[[ 0, 1, 2], [ 3, 4, 5]], @@ -219,9 +210,6 @@ Glossary 2. Since 8 appears in the second of the two 2x3's, the first index must be 1: - .. doctest:: - :skipif: True - >>> a[1,0,2] 8 @@ -229,15 +217,14 @@ Glossary count ``[`` symbols after the open-parenthesis. This is useful in distinguishing, say, a (1,2,3) shape from a (2,3) shape: - .. doctest:: - :skipif: True - + >>> a = np.arange(6).reshape(2,3) >>> a.shape (2, 3) >>> a array([[0, 1, 2], [3, 4, 5]]) + >>> a = np.arange(6).reshape(1,2,3) >>> a.shape (1, 2, 3) >>> a @@ -270,10 +257,7 @@ Glossary different sizes as if all were the same size. When NumPy operates on two arrays, it works element by - element -- for instance, ``c = a * b`` is - - .. doctest:: - :skipif: True + element -- for instance, ``c = a * b`` is :: c[0,0,0] = a[0,0,0] * b[0,0,0] c[0,0,1] = a[0,0,1] * b[0,0,1] From 181446848f1d26bd387e4606de588980144ab91c Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Wed, 5 Aug 2020 07:31:27 -0400 Subject: [PATCH 020/409] DOC: 4th try: Avoid unwanted doctest on PR #14979 --- doc/source/glossary.rst | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 776701814ae3..a1822b6e793e 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -37,16 +37,13 @@ Glossary It can be used at most once: - .. doctest:: - :skipif: True + >>> a[0,...,0,...].shape + --------------------------------------------------------------------------- + IndexError Traceback (most recent call last) + in + ----> 1 a[0,...,0,...].shape - >>> a[0,...,0,...].shape - --------------------------------------------------------------------------- - IndexError Traceback (most recent call last) - in - ----> 1 a[0,...,0,...].shape - - IndexError: an index can only have a single ellipsis ('...') + IndexError: an index can only have a single ellipsis ('...') For details, see :doc:`Indexing. ` @@ -63,7 +60,7 @@ Glossary axis: >>> a = np.arange(24).reshape(2,3,4) - a + >>> a array([[[ 0, 1, 2, 3], [ 4, 5, 6, 7], [ 8, 9, 10, 11]], @@ -189,7 +186,7 @@ Glossary higher-dimensional vectors as replications of row-by-column building blocks, as in this three-dimensional vector: - >>> a = np.arange(12).reshape(2,2,3) + >>> a = np.arange(12).reshape(2,2,3) >>> a array([[[ 0, 1, 2], [ 3, 4, 5]], @@ -368,16 +365,17 @@ Glossary array, which has an internal boolean array indicating invalid entries. Operations with masked arrays ignore these entries. :: - >>> a = np.ma.masked_array([np.nan, 2, np.nan], [True, False, True]) - >>> a - masked_array(data = [-- 2.0 --], - mask = [ True False True], - fill_value = 1e+20) + >>> a = np.ma.masked_array([np.nan, 2, np.nan], [True, False, True]) + >>> a + masked_array(data=[--, 2.0, --], + mask=[ True, False, True], + fill_value=1e+20) + >>> a + [1, 2, 3] - masked_array(data = [-- 4.0 --], - mask = [ True False True], - fill_value = 1e+20) + masked_array(data=[--, 4.0, --], + mask=[ True, False, True], + fill_value=1e+20) For details, see :doc:`Masked arrays. ` From c26d56933627cf08b3cfffcaa320ef77fdd348b3 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Wed, 5 Aug 2020 07:47:50 -0400 Subject: [PATCH 021/409] DOC: 5th try: Avoid unwanted doctest on PR #14979 --- doc/source/glossary.rst | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index a1822b6e793e..0af59c021449 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -42,7 +42,7 @@ Glossary IndexError Traceback (most recent call last) in ----> 1 a[0,...,0,...].shape - + IndexError: an index can only have a single ellipsis ('...') For details, see :doc:`Indexing. ` @@ -64,7 +64,7 @@ Glossary array([[[ 0, 1, 2, 3], [ 4, 5, 6, 7], [ 8, 9, 10, 11]], - + [[12, 13, 14, 15], [16, 17, 18, 19], [20, 21, 22, 23]]]) @@ -190,13 +190,10 @@ Glossary >>> a array([[[ 0, 1, 2], [ 3, 4, 5]], - + [[ 6, 7, 8], [ 9, 10, 11]]]) - >>> a.shape - (2, 2, 3) - ``a`` is depicted as a two-element array whose elements are 2x3 vectors. From this point of view, rows and columns are the final two axes, respectively, in any shape. @@ -371,7 +368,6 @@ Glossary mask=[ True, False, True], fill_value=1e+20) - >>> a + [1, 2, 3] masked_array(data=[--, 4.0, --], mask=[ True, False, True], From 03e116da14d8413ebc9eda1ed7aa115a70d5feef Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Wed, 5 Aug 2020 08:05:28 -0400 Subject: [PATCH 022/409] DOC: 6th try: Fix doctest on PR #16996 --- doc/source/glossary.rst | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 0af59c021449..7c1ba64ee569 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -35,17 +35,7 @@ Glossary >>> a[0,...,0].shape (3,) - It can be used at most once: - - >>> a[0,...,0,...].shape - --------------------------------------------------------------------------- - IndexError Traceback (most recent call last) - in - ----> 1 a[0,...,0,...].shape - - IndexError: an index can only have a single ellipsis ('...') - - For details, see :doc:`Indexing. ` + It can be used at most once; ``a[...,0,...]`` raises an ``IndexError``. **In printouts**, NumPy substitutes ``...`` for the middle elements of large arrays. To see the entire array, use From 15292c9186a8b139ceaaa56aa4cafca42e787fef Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Wed, 5 Aug 2020 10:03:50 -0400 Subject: [PATCH 023/409] DOC: Fix typos in #16996 strctured -> structured, caught by @WarrenWeckesser. Fixed missing language in `view` entry. --- doc/source/glossary.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 7c1ba64ee569..c20d98b67eec 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -471,7 +471,7 @@ Glossary subarray data type - An element of a strctured datatype that behaves like an ndarray. + An element of a structured datatype that behaves like an ndarray. .. @@ -528,6 +528,7 @@ Glossary Some NumPy routines always return views, some always return copies, some may return one or the other, and for some the choice can be specified. Responsiblity for managing views and copies falls to the programmer. - NumPy reports whether arrays share memory - :doc:`numpy.shares_memory `, - but an exact answer isn't always feasible; see the link. + :doc:`numpy.shares_memory ` + will check whether ``b`` is a view of ``a``, + but an exact answer isn't always feasible, as the documentation page + explains. From 55cd824ac52b573273eab0f85712aead8f9a5588 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Wed, 5 Aug 2020 10:12:26 -0400 Subject: [PATCH 024/409] DOC: Fix repeated word in PR #16996 Also adding s to some examples for clarity. --- doc/source/glossary.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index c20d98b67eec..9bfd0d76c020 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -58,7 +58,7 @@ Glossary [[12, 13, 14, 15], [16, 17, 18, 19], [20, 21, 22, 23]]]) - + >>> a[1:,-2:,:-1] array([[[16, 17, 18], [20, 21, 22]]]) @@ -113,10 +113,10 @@ Glossary and the other dimensions are left full (``:``). >>> a = np.arange(24).reshape(2,3,4) - + >>> a.shape (2,3,4) - + >>> a[:,0,:].shape (2,4) @@ -221,7 +221,7 @@ Glossary If an array does not own its memory, then its :doc:`base ` attribute returns the object whose memory the array is referencing. That object - may may be borrowing the memory from still another object, so the + may be borrowing the memory from still another object, so the owning object may be ``a.base.base.base...``. Despite advice to the contrary, testing ``base`` is not a surefire way to determine if two arrays are :term:`view`\ s. From cef9b98cc1fd9ccd981a6d4295a7c0607669baba Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Wed, 5 Aug 2020 12:28:12 -0400 Subject: [PATCH 025/409] DOC: Fix example formatting in PR #16996 --- doc/source/glossary.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 9bfd0d76c020..8bf7bc5feb03 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -115,10 +115,10 @@ Glossary >>> a = np.arange(24).reshape(2,3,4) >>> a.shape - (2,3,4) + (2, 3, 4) >>> a[:,0,:].shape - (2,4) + (2, 4) The slice has ``a``'s shape with the X dimension deleted. Saying an operation ``op`` is ``performed along X`` means that ``op`` takes as its From 0dd96fba418b6ffab1df813ff2ba02a2e99709b8 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Wed, 5 Aug 2020 12:18:20 -0700 Subject: [PATCH 026/409] DOC: Use intersphinx links. --- doc/source/glossary.rst | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 8bf7bc5feb03..22b1e38ffc80 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -44,8 +44,7 @@ Glossary ``:`` - The Python - `slice `_ + The Python :term:`python:slice` operator. In ndarrays, slicing can be applied to every axis: @@ -138,7 +137,7 @@ Glossary array_like Any :doc:`scalar ` or - `sequence `_ + :term:`python:sequence` that can be interpreted as an ndarray. In addition to ndarrays and scalars this category includes lists (possibly nested and with different element types) and tuples. Any argument accepted by @@ -488,8 +487,7 @@ Glossary type In NumPy, a synonym for :term:`dtype`. For the more general Python - meaning, - `see here. `_ + meaning, :term:`see here. `. ufunc From 5e0159cc322b84ea73c10c149267e3d6b810a95a Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Wed, 5 Aug 2020 12:39:59 -0700 Subject: [PATCH 027/409] DOC: intersphinx standalone glossary entries --- doc/source/glossary.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 22b1e38ffc80..725a132fb75b 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -159,7 +159,7 @@ Glossary dimension. - `attribute `_ + :term:`attribute ` \ @@ -266,11 +266,11 @@ Glossary See :term:`view`. - `decorator `_ + :term:`decorator ` \ - `dictionary `_ + :term:`dictionary ` \ @@ -319,11 +319,11 @@ Glossary Python objects, fill the role of heterogeneous arrays. - `immutable `_ + :term:`immutable ` \ - `iterable `_ + :term:`iterable ` \ @@ -332,7 +332,7 @@ Glossary The size of the dtype element in bytes. - `list `_ + :term:`list ` \ From 40dc5982f2df98aa153d3834a4a39e3df78b33c8 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Wed, 5 Aug 2020 12:40:43 -0700 Subject: [PATCH 028/409] DOC: rm tuple from glossary --- doc/source/glossary.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 725a132fb75b..11439187a01b 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -480,10 +480,6 @@ Glossary An alias for a field name in a structured datatype. - `tuple `_ - \ - - type In NumPy, a synonym for :term:`dtype`. For the more general Python From 66de9eb30017c2950cecf046c0626fa983e347b6 Mon Sep 17 00:00:00 2001 From: MelissaWM Date: Tue, 8 Sep 2020 19:41:52 -0300 Subject: [PATCH 029/409] Added post-install issue template and responses to PR comments. --- .github/ISSUE_TEMPLATE.md | 27 ----------------------- .github/ISSUE_TEMPLATE/bug-report.md | 2 +- .github/ISSUE_TEMPLATE/feature-request.md | 4 ++-- .github/ISSUE_TEMPLATE/post-install.md | 20 +++++++++++++++++ .github/PULL_REQUEST_TEMPLATE.md | 2 +- 5 files changed, 24 insertions(+), 31 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/ISSUE_TEMPLATE/post-install.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index 3a25eeb1e9a3..000000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,27 +0,0 @@ - - - -### Reproducing code example: - - - -```python -import numpy as np -<< your code here >> -``` - - - -### Error message: - - - - - -### Numpy/Python version information: - - - diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index 123498224627..d2df08689da5 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -24,7 +24,7 @@ https://github.com/numpy/numpy/blob/master/doc/source/dev/development_environmen -### Numpy/Python version information: +### NumPy/Python version information: diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md index 38d64710bbb2..a6a3ca4234af 100644 --- a/.github/ISSUE_TEMPLATE/feature-request.md +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -10,7 +10,7 @@ If you're looking to request a new feature or change in functionality, including adding or changing the meaning of arguments to an existing function, please post your idea on the [numpy-discussion mailing list] (https://mail.python.org/mailman/listinfo/numpy-discussion) to explain your -reasoning before you open an issue or pull request. You can also check out our -[Contributor Guide] +reasoning in addition to opening an issue or pull request. You can also check +out our [Contributor Guide] (https://github.com/numpy/numpy/blob/master/doc/source/dev/index.rst) if you need more information. diff --git a/.github/ISSUE_TEMPLATE/post-install.md b/.github/ISSUE_TEMPLATE/post-install.md new file mode 100644 index 000000000000..fb8b3507678d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/post-install.md @@ -0,0 +1,20 @@ +--- +name: "Post-install/importing issue" +about: If you have trouble importing or using NumPy after installation + +--- + + + +### Steps to reproduce: + + + +### Error message: + + + + + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 1f313fd75027..528580a8e6df 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -6,6 +6,6 @@ http://www.numpy.org/devdocs/dev/development_workflow.html http://www.numpy.org/devdocs/dev/development_workflow.html#writing-the-commit-message --> - From f25a376cbd89512516f7fc458815f0c4159d7ff9 Mon Sep 17 00:00:00 2001 From: MelissaWM Date: Wed, 9 Sep 2020 16:32:42 -0300 Subject: [PATCH 030/409] Added labels to doc and post-install templates. --- .github/ISSUE_TEMPLATE/config.yml | 1 - .github/ISSUE_TEMPLATE/documentation.md | 1 + .github/ISSUE_TEMPLATE/post-install.md | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 3027d66506a7..adfff81bd004 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,4 +1,3 @@ -blank_issues_enabled: false contact_links: - name: Question/Help/Support url: https://numpy.org/gethelp/ diff --git a/.github/ISSUE_TEMPLATE/documentation.md b/.github/ISSUE_TEMPLATE/documentation.md index b0435a5654d5..67e04c8af0cf 100644 --- a/.github/ISSUE_TEMPLATE/documentation.md +++ b/.github/ISSUE_TEMPLATE/documentation.md @@ -1,6 +1,7 @@ --- name: "Documentation" about: Report an issue related to the NumPy documentation +labels: 04 - Documentation --- diff --git a/.github/ISSUE_TEMPLATE/post-install.md b/.github/ISSUE_TEMPLATE/post-install.md index fb8b3507678d..c0ec7896a40d 100644 --- a/.github/ISSUE_TEMPLATE/post-install.md +++ b/.github/ISSUE_TEMPLATE/post-install.md @@ -1,6 +1,7 @@ --- name: "Post-install/importing issue" about: If you have trouble importing or using NumPy after installation +labels: 32 - Installation --- From 2e3de29722cc42970a31fe6843c5aa0dbcf0ee7d Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 10 Sep 2020 16:02:55 -0500 Subject: [PATCH 031/409] MAINT: Simplify ufunc pickling This also allows at least in principle numba dynamically generated ufuncs to be pickled (with some hacking), see: https://github.com/dask/distributed/issues/3450 If the name of the ufunc is set to a qualname, using this method, pickle should be able to unpickle the ufunc correctly. We may want to allow setting the module and qualname explicitly on the ufunc object to remove the need for the custom pickler completely. --- numpy/core/__init__.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/numpy/core/__init__.py b/numpy/core/__init__.py index c77885954ef1..a0769cc89112 100644 --- a/numpy/core/__init__.py +++ b/numpy/core/__init__.py @@ -113,10 +113,9 @@ __all__ += shape_base.__all__ __all__ += einsumfunc.__all__ -# Make it possible so that ufuncs can be pickled -# Here are the loading and unloading functions -# The name numpy.core._ufunc_reconstruct must be -# available for unpickling to work. +# We used to use `np.core._ufunc_reconstruct` to unpickle. This is unnecessary, +# but old pickles saved before 1.20 will be using it, and there is no reason +# to break loading them. def _ufunc_reconstruct(module, name): # The `fromlist` kwarg is required to ensure that `mod` points to the # inner-most module rather than the parent package when module name is @@ -126,14 +125,17 @@ def _ufunc_reconstruct(module, name): return getattr(mod, name) def _ufunc_reduce(func): - from pickle import whichmodule - name = func.__name__ - return _ufunc_reconstruct, (whichmodule(func, name), name) + # Report the `__name__`. pickle will try to find the module. Note that + # pickle supports for this `__name__` to be a `__qualname__`. It may + # make sense to add a `__qualname__` to ufuncs, to allow this more + # explicitly (Numba has ufuncs as attributes). + # See also: https://github.com/dask/distributed/issues/3450 + return func.__name__ import copyreg -copyreg.pickle(ufunc, _ufunc_reduce, _ufunc_reconstruct) +copyreg.pickle(ufunc, _ufunc_reduce) # Unclutter namespace (must keep _ufunc_reconstruct for unpickling) del copyreg del _ufunc_reduce From 2ec7ea418e6f6e054e9aad2b18adfd9827c90249 Mon Sep 17 00:00:00 2001 From: MelissaWM Date: Fri, 11 Sep 2020 10:19:30 -0300 Subject: [PATCH 032/409] Put feature request template as comments. --- .github/ISSUE_TEMPLATE/feature-request.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md index a6a3ca4234af..0be94f92893d 100644 --- a/.github/ISSUE_TEMPLATE/feature-request.md +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -6,11 +6,11 @@ about: Check instructions for submitting your idea on the mailing list first. ## Feature -If you're looking to request a new feature or change in functionality, including + From 96be6959f970f56fda1c4dfe38db3eabc807f7a6 Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Thu, 10 Sep 2020 15:31:27 +0200 Subject: [PATCH 033/409] ENH: Pass optimizations arguments to asv build This patch allows passing `-j`, `--cpu-baseline`, `--cpu-dispatch` and `--disable-optimization` to ASV build when argument `--bench-compare` is used. --- .gitignore | 1 + benchmarks/asv_compare.conf.json.tpl | 95 ++++++++++++++++++++++++ runtests.py | 105 ++++++++++++++++++++++++++- 3 files changed, 198 insertions(+), 3 deletions(-) create mode 100644 benchmarks/asv_compare.conf.json.tpl diff --git a/.gitignore b/.gitignore index c58b0e62feb9..85318467e849 100644 --- a/.gitignore +++ b/.gitignore @@ -182,6 +182,7 @@ benchmarks/results benchmarks/html benchmarks/env benchmarks/numpy +benchmarks/_asv_compare.conf.json # cythonized files cythonize.dat numpy/random/_mtrand/_mtrand.c diff --git a/benchmarks/asv_compare.conf.json.tpl b/benchmarks/asv_compare.conf.json.tpl new file mode 100644 index 000000000000..1f339077c66d --- /dev/null +++ b/benchmarks/asv_compare.conf.json.tpl @@ -0,0 +1,95 @@ +// This config file is almost similar to 'asv.conf.json' except it contains +// custom tokens that can be substituted by 'runtests.py' and ASV, +// due to the necessity to add custom build options when `--bench-compare` +// is used. +{ + // The version of the config file format. Do not change, unless + // you know what you are doing. + "version": 1, + + // The name of the project being benchmarked + "project": "numpy", + + // The project's homepage + "project_url": "https://www.numpy.org/", + + // The URL or local path of the source code repository for the + // project being benchmarked + "repo": "..", + + // List of branches to benchmark. If not provided, defaults to "master" + // (for git) or "tip" (for mercurial). + "branches": ["HEAD"], + + // The DVCS being used. If not set, it will be automatically + // determined from "repo" by looking at the protocol in the URL + // (if remote), or by looking for special directories, such as + // ".git" (if local). + "dvcs": "git", + + // The tool to use to create environments. May be "conda", + // "virtualenv" or other value depending on the plugins in use. + // If missing or the empty string, the tool will be automatically + // determined by looking for tools on the PATH environment + // variable. + "environment_type": "virtualenv", + + // the base URL to show a commit for the project. + "show_commit_url": "https://github.com/numpy/numpy/commit/", + + // The Pythons you'd like to test against. If not provided, defaults + // to the current version of Python used to run `asv`. + "pythons": ["3.7"], + + // The matrix of dependencies to test. Each key is the name of a + // package (in PyPI) and the values are version numbers. An empty + // list indicates to just test against the default (latest) + // version. + "matrix": { + "Cython": [], + }, + + // The directory (relative to the current directory) that benchmarks are + // stored in. If not provided, defaults to "benchmarks" + "benchmark_dir": "benchmarks", + + // The directory (relative to the current directory) to cache the Python + // environments in. If not provided, defaults to "env" + // NOTE: changes dir name will requires update `generate_asv_config()` in + // runtests.py + "env_dir": "env", + + + // The directory (relative to the current directory) that raw benchmark + // results are stored in. If not provided, defaults to "results". + "results_dir": "results", + + // The directory (relative to the current directory) that the html tree + // should be written to. If not provided, defaults to "html". + "html_dir": "html", + + // The number of characters to retain in the commit hashes. + // "hash_length": 8, + + // `asv` will cache wheels of the recent builds in each + // environment, making them faster to install next time. This is + // number of builds to keep, per environment. + "build_cache_size": 8, + + "build_command" : [ + "python setup.py build {numpy_build_options}", + "PIP_NO_BUILD_ISOLATION=false python -mpip wheel --no-deps --no-index -w {build_cache_dir} {build_dir}" + ], + // The commits after which the regression search in `asv publish` + // should start looking for regressions. Dictionary whose keys are + // regexps matching to benchmark names, and values corresponding to + // the commit (exclusive) after which to start looking for + // regressions. The default is to start from the first commit + // with results. If the commit is `null`, regression detection is + // skipped for the matching benchmark. + // + // "regressions_first_commits": { + // "some_benchmark": "352cdf", // Consider regressions only after this commit + // "another_benchmark": null, // Skip regression detection altogether + // } +} diff --git a/runtests.py b/runtests.py index 2f07749f8147..f8b70d9360df 100755 --- a/runtests.py +++ b/runtests.py @@ -52,7 +52,7 @@ import sys -import os +import os, glob # In case we are run from the source directory, we don't want to import the # project from there: @@ -310,8 +310,16 @@ def main(argv): out = subprocess.check_output(['git', 'rev-parse', commit_a]) commit_a = out.strip().decode('ascii') + # generate config file with the required build options + asv_cfpath = [ + '--config', asv_compare_config( + os.path.join(ROOT_DIR, 'benchmarks'), args, + # to clear the cache if the user changed build options + (commit_a, commit_b) + ) + ] cmd = ['asv', 'continuous', '-e', '-f', '1.05', - commit_a, commit_b] + bench_args + commit_a, commit_b] + asv_cfpath + bench_args ret = subprocess.call(cmd, cwd=os.path.join(ROOT_DIR, 'benchmarks')) sys.exit(ret) @@ -361,7 +369,6 @@ def main(argv): else: sys.exit(1) - def build_project(args): """ Build a dev version of the project. @@ -491,6 +498,98 @@ def build_project(args): return site_dir, site_dir_noarch +def asv_compare_config(bench_path, args, h_commits): + """ + Fill the required build options through custom variable + 'numpy_build_options' and return the generated config path. + """ + conf_path = os.path.join(bench_path, "asv_compare.conf.json.tpl") + nconf_path = os.path.join(bench_path, "_asv_compare.conf.json") + + # add custom build + build = [] + if args.parallel > 1: + build += ["-j", str(args.parallel)] + if args.cpu_baseline: + build += ["--cpu-baseline", args.cpu_baseline] + if args.cpu_dispatch: + build += ["--cpu-dispatch", args.cpu_dispatch] + if args.disable_optimization: + build += ["--disable-optimization"] + + is_cached = asv_substitute_config(conf_path, nconf_path, + numpy_build_options = ' '.join([f'\\"{v}\\"' for v in build]), + ) + if not is_cached: + asv_clear_cache(bench_path, h_commits) + return nconf_path + +def asv_clear_cache(bench_path, h_commits, env_dir="env"): + """ + Force ASV to clear the cache according to specified commit hashes. + """ + # FIXME: only clear the cache from the current environment dir + asv_build_pattern = os.path.join(bench_path, env_dir, "*", "asv-build-cache") + for asv_build_cache in glob.glob(asv_build_pattern, recursive=True): + for c in h_commits: + try: shutil.rmtree(os.path.join(asv_build_cache, c)) + except OSError: pass + +def asv_substitute_config(in_config, out_config, **custom_vars): + """ + A workaround to allow substituting custom tokens within + ASV configuration file since there's no official way to add custom + variables(e.g. env vars). + + Parameters + ---------- + in_config : str + The path of ASV configuration file, e.g. '/path/to/asv.conf.json' + out_config : str + The path of generated configuration file, + e.g. '/path/to/asv_substituted.conf.json'. + + The other keyword arguments represent the custom variables. + + Returns + ------- + True(is cached) if 'out_config' is already generated with + the same '**custom_vars' and updated with latest 'in_config', + False otherwise. + + Examples + -------- + See asv_compare_config(). + """ + assert in_config != out_config + assert len(custom_vars) > 0 + + def sdbm_hash(*factors): + chash = 0 + for f in factors: + for char in str(f): + chash = ord(char) + (chash << 6) + (chash << 16) - chash + chash &= 0xFFFFFFFF + return chash + + vars_hash = sdbm_hash(custom_vars, os.path.getmtime(in_config)) + try: + with open(out_config, "r") as wfd: + hash_line = wfd.readline().split('hash:') + if len(hash_line) > 1 and int(hash_line[1]) == vars_hash: + return True + except IOError: + pass + + custom_vars = {f'{{{k}}}':v for k, v in custom_vars.items()} + with open(in_config, "r") as rfd, open(out_config, "w") as wfd: + wfd.write(f"// hash:{vars_hash}\n") + wfd.write("// This file is automatically generated by runtests.py\n") + for line in rfd: + for key, val in custom_vars.items(): + line = line.replace(key, val) + wfd.write(line) + return False # # GCOV support From c70fbb6d7206155f8f9d75af585f32c7b057051a Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Sat, 12 Sep 2020 16:39:50 -0500 Subject: [PATCH 034/409] MAINT: (dtype-transfer) make copyswapn and legacy cast wrapper method This is just a small preparation to make the code slightly more modular, since I definitely will need to retain the wrapping for legacy dtype `dtype->f->castfuncs` and the `dtype->f->copyswapn`, but want more freedom to see whether I can split out the specialized casts we use internally (since NumPy has those cast functions, but normally never uses them). Splitting these two out should make it easier to write a function to support current "legacy" dtype cast functions without worrying as much about the casts used internally for NumPy dtypes. --- numpy/core/src/multiarray/dtype_transfer.c | 321 +++++++++++---------- 1 file changed, 176 insertions(+), 145 deletions(-) diff --git a/numpy/core/src/multiarray/dtype_transfer.c b/numpy/core/src/multiarray/dtype_transfer.c index 42c66ee7f6e7..af4e6c22e601 100644 --- a/numpy/core/src/multiarray/dtype_transfer.c +++ b/numpy/core/src/multiarray/dtype_transfer.c @@ -1322,95 +1322,21 @@ get_unicode_to_datetime_transfer_function(int aligned, return NPY_SUCCEED; } + static int -get_nbo_cast_transfer_function(int aligned, - npy_intp src_stride, npy_intp dst_stride, - PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, - int move_references, - PyArray_StridedUnaryOp **out_stransfer, - NpyAuxData **out_transferdata, - int *out_needs_api, - int *out_needs_wrap) +get_legacy_dtype_cast_function( + int aligned, npy_intp src_stride, npy_intp dst_stride, + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + int move_references, + PyArray_StridedUnaryOp **out_stransfer, NpyAuxData **out_transferdata, + int *out_needs_api, int *out_needs_wrap) { _strided_cast_data *data; PyArray_VectorUnaryFunc *castfunc; PyArray_Descr *tmp_dtype; - npy_intp shape = 1, src_itemsize = src_dtype->elsize, - dst_itemsize = dst_dtype->elsize; - - if (PyTypeNum_ISNUMBER(src_dtype->type_num) && - PyTypeNum_ISNUMBER(dst_dtype->type_num)) { - *out_needs_wrap = !PyArray_ISNBO(src_dtype->byteorder) || - !PyArray_ISNBO(dst_dtype->byteorder); - return get_nbo_cast_numeric_transfer_function(aligned, - src_stride, dst_stride, - src_dtype->type_num, dst_dtype->type_num, - out_stransfer, out_transferdata); - } - - if (src_dtype->type_num == NPY_DATETIME || - src_dtype->type_num == NPY_TIMEDELTA || - dst_dtype->type_num == NPY_DATETIME || - dst_dtype->type_num == NPY_TIMEDELTA) { - /* A parameterized type, datetime->datetime sometimes needs casting */ - if ((src_dtype->type_num == NPY_DATETIME && - dst_dtype->type_num == NPY_DATETIME) || - (src_dtype->type_num == NPY_TIMEDELTA && - dst_dtype->type_num == NPY_TIMEDELTA)) { - *out_needs_wrap = !PyArray_ISNBO(src_dtype->byteorder) || - !PyArray_ISNBO(dst_dtype->byteorder); - return get_nbo_cast_datetime_transfer_function(aligned, - src_stride, dst_stride, - src_dtype, dst_dtype, - out_stransfer, out_transferdata); - } - - /* - * Datetime <-> string conversions can be handled specially. - * The functions may raise an error if the strings have no - * space, or can't be parsed properly. - */ - if (src_dtype->type_num == NPY_DATETIME) { - switch (dst_dtype->type_num) { - case NPY_STRING: - *out_needs_api = 1; - *out_needs_wrap = !PyArray_ISNBO(src_dtype->byteorder); - return get_nbo_datetime_to_string_transfer_function( - aligned, - src_stride, dst_stride, - src_dtype, dst_dtype, - out_stransfer, out_transferdata); - - case NPY_UNICODE: - return get_datetime_to_unicode_transfer_function( - aligned, - src_stride, dst_stride, - src_dtype, dst_dtype, - out_stransfer, out_transferdata, - out_needs_api); - } - } - else if (dst_dtype->type_num == NPY_DATETIME) { - switch (src_dtype->type_num) { - case NPY_STRING: - *out_needs_api = 1; - *out_needs_wrap = !PyArray_ISNBO(dst_dtype->byteorder); - return get_nbo_string_to_datetime_transfer_function( - aligned, - src_stride, dst_stride, - src_dtype, dst_dtype, - out_stransfer, out_transferdata); - - case NPY_UNICODE: - return get_unicode_to_datetime_transfer_function( - aligned, - src_stride, dst_stride, - src_dtype, dst_dtype, - out_stransfer, out_transferdata, - out_needs_api); - } - } - } + npy_intp shape = 1; + npy_intp src_itemsize = src_dtype->elsize; + npy_intp dst_itemsize = dst_dtype->elsize; *out_needs_wrap = !aligned || !PyArray_ISNBO(src_dtype->byteorder) || @@ -1543,6 +1469,167 @@ get_nbo_cast_transfer_function(int aligned, return NPY_SUCCEED; } + +static int +get_nbo_cast_transfer_function(int aligned, + npy_intp src_stride, npy_intp dst_stride, + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + int move_references, + PyArray_StridedUnaryOp **out_stransfer, + NpyAuxData **out_transferdata, + int *out_needs_api, + int *out_needs_wrap) +{ + if (PyTypeNum_ISNUMBER(src_dtype->type_num) && + PyTypeNum_ISNUMBER(dst_dtype->type_num)) { + *out_needs_wrap = !PyArray_ISNBO(src_dtype->byteorder) || + !PyArray_ISNBO(dst_dtype->byteorder); + return get_nbo_cast_numeric_transfer_function(aligned, + src_stride, dst_stride, + src_dtype->type_num, dst_dtype->type_num, + out_stransfer, out_transferdata); + } + + if (src_dtype->type_num == NPY_DATETIME || + src_dtype->type_num == NPY_TIMEDELTA || + dst_dtype->type_num == NPY_DATETIME || + dst_dtype->type_num == NPY_TIMEDELTA) { + /* A parameterized type, datetime->datetime sometimes needs casting */ + if ((src_dtype->type_num == NPY_DATETIME && + dst_dtype->type_num == NPY_DATETIME) || + (src_dtype->type_num == NPY_TIMEDELTA && + dst_dtype->type_num == NPY_TIMEDELTA)) { + *out_needs_wrap = !PyArray_ISNBO(src_dtype->byteorder) || + !PyArray_ISNBO(dst_dtype->byteorder); + return get_nbo_cast_datetime_transfer_function(aligned, + src_stride, dst_stride, + src_dtype, dst_dtype, + out_stransfer, out_transferdata); + } + + /* + * Datetime <-> string conversions can be handled specially. + * The functions may raise an error if the strings have no + * space, or can't be parsed properly. + */ + if (src_dtype->type_num == NPY_DATETIME) { + switch (dst_dtype->type_num) { + case NPY_STRING: + *out_needs_api = 1; + *out_needs_wrap = !PyArray_ISNBO(src_dtype->byteorder); + return get_nbo_datetime_to_string_transfer_function( + aligned, + src_stride, dst_stride, + src_dtype, dst_dtype, + out_stransfer, out_transferdata); + + case NPY_UNICODE: + return get_datetime_to_unicode_transfer_function( + aligned, + src_stride, dst_stride, + src_dtype, dst_dtype, + out_stransfer, out_transferdata, + out_needs_api); + } + } + else if (dst_dtype->type_num == NPY_DATETIME) { + switch (src_dtype->type_num) { + case NPY_STRING: + *out_needs_api = 1; + *out_needs_wrap = !PyArray_ISNBO(dst_dtype->byteorder); + return get_nbo_string_to_datetime_transfer_function( + aligned, + src_stride, dst_stride, + src_dtype, dst_dtype, + out_stransfer, out_transferdata); + + case NPY_UNICODE: + return get_unicode_to_datetime_transfer_function( + aligned, + src_stride, dst_stride, + src_dtype, dst_dtype, + out_stransfer, out_transferdata, + out_needs_api); + } + } + } + + return get_legacy_dtype_cast_function( + aligned, src_stride, dst_stride, src_dtype, dst_dtype, + move_references, out_stransfer, out_transferdata, + out_needs_api, out_needs_wrap); +} + + +static int +wrap_aligned_contig_transfer_function_with_copyswapn( + int aligned, npy_intp src_stride, npy_intp dst_stride, + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + PyArray_StridedUnaryOp **out_stransfer, NpyAuxData **out_transferdata, + int *out_needs_api, + PyArray_StridedUnaryOp *caststransfer, NpyAuxData *castdata) +{ + NpyAuxData *todata = NULL, *fromdata = NULL; + PyArray_StridedUnaryOp *tobuffer, *frombuffer; + npy_intp src_itemsize = src_dtype->elsize; + npy_intp dst_itemsize = dst_dtype->elsize; + + /* Get the copy/swap operation from src */ + PyArray_GetDTypeCopySwapFn( + aligned, src_stride, src_itemsize, src_dtype, &tobuffer, &todata); + + if (!PyDataType_REFCHK(dst_dtype)) { + /* Copying from buffer is a simple copy/swap operation */ + PyArray_GetDTypeCopySwapFn( + aligned, dst_itemsize, dst_stride, dst_dtype, + &frombuffer, &fromdata); + } + else { + /* + * Since the buffer is initialized to NULL, need to move the + * references in order to DECREF the existing data. + */ + /* Object types cannot be byte swapped */ + assert(PyDataType_ISNOTSWAPPED(dst_dtype)); + /* The loop already needs the python api if this is reached */ + assert(*out_needs_api); + + if (PyArray_GetDTypeTransferFunction( + aligned, dst_itemsize, dst_stride, + dst_dtype, dst_dtype, 1, + &frombuffer, &fromdata, out_needs_api) != NPY_SUCCEED) { + return NPY_FAIL; + } + } + + if (frombuffer == NULL || tobuffer == NULL) { + NPY_AUXDATA_FREE(castdata); + NPY_AUXDATA_FREE(todata); + NPY_AUXDATA_FREE(fromdata); + return NPY_FAIL; + } + + *out_stransfer = caststransfer; + + /* Wrap it all up in a new transfer function + data */ + if (wrap_aligned_contig_transfer_function( + src_itemsize, dst_itemsize, + tobuffer, todata, + frombuffer, fromdata, + caststransfer, castdata, + PyDataType_FLAGCHK(dst_dtype, NPY_NEEDS_INIT), + *out_needs_api, + out_stransfer, out_transferdata) != NPY_SUCCEED) { + NPY_AUXDATA_FREE(castdata); + NPY_AUXDATA_FREE(todata); + NPY_AUXDATA_FREE(fromdata); + return NPY_FAIL; + } + + return NPY_SUCCEED; +} + + static int get_cast_transfer_function(int aligned, npy_intp src_stride, npy_intp dst_stride, @@ -1553,10 +1640,8 @@ get_cast_transfer_function(int aligned, int *out_needs_api) { PyArray_StridedUnaryOp *caststransfer; - NpyAuxData *castdata, *todata = NULL, *fromdata = NULL; + NpyAuxData *castdata; int needs_wrap = 0; - npy_intp src_itemsize = src_dtype->elsize, - dst_itemsize = dst_dtype->elsize; if (get_nbo_cast_transfer_function(aligned, src_stride, dst_stride, @@ -1581,64 +1666,10 @@ get_cast_transfer_function(int aligned, } /* Otherwise, we have to copy and/or swap to aligned temporaries */ else { - PyArray_StridedUnaryOp *tobuffer, *frombuffer; - - /* Get the copy/swap operation from src */ - PyArray_GetDTypeCopySwapFn(aligned, - src_stride, src_itemsize, - src_dtype, - &tobuffer, &todata); - - if (!PyDataType_REFCHK(dst_dtype)) { - /* Copying from buffer is a simple copy/swap operation */ - PyArray_GetDTypeCopySwapFn(aligned, - dst_itemsize, dst_stride, - dst_dtype, - &frombuffer, &fromdata); - } - else { - /* - * Since the buffer is initialized to NULL, need to move the - * references in order to DECREF the existing data. - */ - /* Object types cannot be byte swapped */ - assert(PyDataType_ISNOTSWAPPED(dst_dtype)); - /* The loop already needs the python api if this is reached */ - assert(*out_needs_api); - - if (PyArray_GetDTypeTransferFunction( - aligned, dst_itemsize, dst_stride, - dst_dtype, dst_dtype, 1, - &frombuffer, &fromdata, out_needs_api) != NPY_SUCCEED) { - return NPY_FAIL; - } - } - - if (frombuffer == NULL || tobuffer == NULL) { - NPY_AUXDATA_FREE(castdata); - NPY_AUXDATA_FREE(todata); - NPY_AUXDATA_FREE(fromdata); - return NPY_FAIL; - } - - *out_stransfer = caststransfer; - - /* Wrap it all up in a new transfer function + data */ - if (wrap_aligned_contig_transfer_function( - src_itemsize, dst_itemsize, - tobuffer, todata, - frombuffer, fromdata, - caststransfer, castdata, - PyDataType_FLAGCHK(dst_dtype, NPY_NEEDS_INIT), - *out_needs_api, - out_stransfer, out_transferdata) != NPY_SUCCEED) { - NPY_AUXDATA_FREE(castdata); - NPY_AUXDATA_FREE(todata); - NPY_AUXDATA_FREE(fromdata); - return NPY_FAIL; - } - - return NPY_SUCCEED; + return wrap_aligned_contig_transfer_function_with_copyswapn( + aligned, src_stride, dst_stride, src_dtype, dst_dtype, + out_stransfer, out_transferdata, out_needs_api, + caststransfer, castdata); } } From bae33831e42a6bf07b66284aec618cc49b13d030 Mon Sep 17 00:00:00 2001 From: mattip Date: Sun, 13 Sep 2020 16:12:02 +0300 Subject: [PATCH 035/409] WIP, BUILD: start to debug circleCI failures --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f4ffb5223bdf..4222895df651 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -28,10 +28,10 @@ jobs: name: build numpy command: | . venv/bin/activate - pip install --upgrade pip 'setuptools<49.2.0' - pip install -r test_requirements.txt + pip install --progress-bar=off --upgrade pip 'setuptools<49.2.0' + pip install --progress-bar=off -r test_requirements.txt pip install . - pip install -r doc_requirements.txt + pip install --progress-bar=off -r doc_requirements.txt - run: name: create release notes From de0cf43c44b315b70addd920722d1668a9a0c77c Mon Sep 17 00:00:00 2001 From: mattip Date: Sun, 13 Sep 2020 21:17:22 +0300 Subject: [PATCH 036/409] BUILD: try python 3.8.4 --- .circleci/config.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4222895df651..f4536cca5525 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ jobs: docker: # CircleCI maintains a library of pre-built images # documented at https://circleci.com/docs/2.0/circleci-images/ - - image: circleci/python:3.6.6 + - image: circleci/python:3.8.4 working_directory: ~/repo @@ -18,11 +18,10 @@ jobs: - run: name: create virtual environment, install dependencies command: | - python3 -m venv venv - ln -s $(which python3) venv/bin/python3.6 - . venv/bin/activate sudo apt-get update sudo apt-get install -y graphviz texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra texlive-generic-extra latexmk texlive-xetex + python3.8 -m venv venv + . venv/bin/activate - run: name: build numpy From 52746a756992a32ad46cc7ab69c0f1e5e6e88ed6 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2020 07:26:01 +0000 Subject: [PATCH 037/409] MAINT: Bump hypothesis from 5.33.0 to 5.35.1 Bumps [hypothesis](https://github.com/HypothesisWorks/hypothesis) from 5.33.0 to 5.35.1. - [Release notes](https://github.com/HypothesisWorks/hypothesis/releases) - [Commits](https://github.com/HypothesisWorks/hypothesis/compare/hypothesis-python-5.33.0...hypothesis-python-5.35.1) Signed-off-by: dependabot-preview[bot] --- test_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index 7ef91125c4e4..0349b711be58 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,7 +1,7 @@ cython==0.29.21 wheel setuptools<49.2.0 -hypothesis==5.33.0 +hypothesis==5.35.1 pytest==6.0.1 pytz==2020.1 pytest-cov==2.10.1 From f64d9afed7fb473914dbe88037c665c501e4bf68 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Mon, 14 Sep 2020 12:48:22 +0200 Subject: [PATCH 038/409] MAINT: Move the `fromnumeric` annotations to their own stub file --- numpy/__init__.pyi | 560 ++++++------------------------------- numpy/core/fromnumeric.pyi | 492 ++++++++++++++++++++++++++++++++ 2 files changed, 576 insertions(+), 476 deletions(-) create mode 100644 numpy/core/fromnumeric.pyi diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index bf54207a4637..d4eda6b3150f 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -66,10 +66,93 @@ from numpy.core.function_base import ( geomspace, ) +from numpy.core.fromnumeric import ( + take, + reshape, + choose, + repeat, + put, + swapaxes, + transpose, + partition, + argpartition, + sort, + argsort, + argmax, + argmin, + searchsorted, + resize, + squeeze, + diagonal, + trace, + ravel, + nonzero, + shape, + compress, + clip, + sum, + all, + any, + cumsum, + ptp, + amax, + amin, + prod, + cumprod, + ndim, + size, + around, + mean, + std, + var, +) + # Add an object to `__all__` if their stubs are defined in an external file; # their stubs will not be recognized otherwise. # NOTE: This is redundant for objects defined within this file. -__all__ = ["linspace", "logspace", "geomspace"] +__all__ = [ + "linspace", + "logspace", + "geomspace", + "take", + "reshape", + "choose", + "repeat", + "put", + "swapaxes", + "transpose", + "partition", + "argpartition", + "sort", + "argsort", + "argmax", + "argmin", + "searchsorted", + "resize", + "squeeze", + "diagonal", + "trace", + "ravel", + "nonzero", + "shape", + "compress", + "clip", + "sum", + "all", + "any", + "cumsum", + "ptp", + "amax", + "amin", + "prod", + "cumprod", + "ndim", + "size", + "around", + "mean", + "std", + "var", +] # TODO: remove when the full numpy namespace is defined def __getattr__(name: str) -> Any: ... @@ -998,481 +1081,6 @@ def find_common_type( array_types: Sequence[DtypeLike], scalar_types: Sequence[DtypeLike] ) -> dtype: ... -# Functions from np.core.fromnumeric -_Mode = Literal["raise", "wrap", "clip"] -_PartitionKind = Literal["introselect"] -_SortKind = Literal["quicksort", "mergesort", "heapsort", "stable"] -_Side = Literal["left", "right"] - -# Various annotations for scalars - -# While dt.datetime and dt.timedelta are not technically part of NumPy, -# they are one of the rare few builtin scalars which serve as valid return types. -# See https://github.com/numpy/numpy-stubs/pull/67#discussion_r412604113. -_ScalarNumpy = Union[generic, dt.datetime, dt.timedelta] -_ScalarBuiltin = Union[str, bytes, dt.date, dt.timedelta, bool, int, float, complex] -_Scalar = Union[_ScalarBuiltin, _ScalarNumpy] - -# Integers and booleans can generally be used interchangeably -_ScalarIntOrBool = TypeVar("_ScalarIntOrBool", bound=Union[integer, bool_]) -_ScalarGeneric = TypeVar("_ScalarGeneric", bound=generic) -_ScalarGenericDT = TypeVar( - "_ScalarGenericDT", bound=Union[dt.datetime, dt.timedelta, generic] -) - -_Number = TypeVar('_Number', bound=number) _NumberLike = Union[int, float, complex, number, bool_] - -# An array-like object consisting of integers _IntLike = Union[int, integer] _BoolLike = Union[bool, bool_] -_IntOrBool = Union[_IntLike, _BoolLike] -_ArrayLikeIntNested = ArrayLike # TODO: wait for support for recursive types -_ArrayLikeBoolNested = ArrayLike # TODO: wait for support for recursive types - -# Integers and booleans can generally be used interchangeably -_ArrayLikeIntOrBool = Union[ - _IntOrBool, - ndarray, - Sequence[_IntOrBool], - Sequence[_ArrayLikeIntNested], - Sequence[_ArrayLikeBoolNested], -] -_ArrayLikeBool = Union[ - _BoolLike, - Sequence[_BoolLike], - ndarray -] - -# The signature of take() follows a common theme with its overloads: -# 1. A generic comes in; the same generic comes out -# 2. A scalar comes in; a generic comes out -# 3. An array-like object comes in; some keyword ensures that a generic comes out -# 4. An array-like object comes in; an ndarray or generic comes out -@overload -def take( - a: _ScalarGenericDT, - indices: int, - axis: Optional[int] = ..., - out: Optional[ndarray] = ..., - mode: _Mode = ..., -) -> _ScalarGenericDT: ... -@overload -def take( - a: _Scalar, - indices: int, - axis: Optional[int] = ..., - out: Optional[ndarray] = ..., - mode: _Mode = ..., -) -> _ScalarNumpy: ... -@overload -def take( - a: ArrayLike, - indices: int, - axis: Optional[int] = ..., - out: Optional[ndarray] = ..., - mode: _Mode = ..., -) -> _ScalarNumpy: ... -@overload -def take( - a: ArrayLike, - indices: _ArrayLikeIntOrBool, - axis: Optional[int] = ..., - out: Optional[ndarray] = ..., - mode: _Mode = ..., -) -> Union[_ScalarNumpy, ndarray]: ... -def reshape(a: ArrayLike, newshape: _ShapeLike, order: _OrderACF = ...) -> ndarray: ... -@overload -def choose( - a: _ScalarIntOrBool, - choices: ArrayLike, - out: Optional[ndarray] = ..., - mode: _Mode = ..., -) -> _ScalarIntOrBool: ... -@overload -def choose( - a: _IntOrBool, choices: ArrayLike, out: Optional[ndarray] = ..., mode: _Mode = ... -) -> Union[integer, bool_]: ... -@overload -def choose( - a: _ArrayLikeIntOrBool, - choices: ArrayLike, - out: Optional[ndarray] = ..., - mode: _Mode = ..., -) -> ndarray: ... -def repeat( - a: ArrayLike, repeats: _ArrayLikeIntOrBool, axis: Optional[int] = ... -) -> ndarray: ... -def put( - a: ndarray, ind: _ArrayLikeIntOrBool, v: ArrayLike, mode: _Mode = ... -) -> None: ... -def swapaxes(a: ArrayLike, axis1: int, axis2: int) -> ndarray: ... -def transpose( - a: ArrayLike, axes: Union[None, Sequence[int], ndarray] = ... -) -> ndarray: ... -def partition( - a: ArrayLike, - kth: _ArrayLikeIntOrBool, - axis: Optional[int] = ..., - kind: _PartitionKind = ..., - order: Union[None, str, Sequence[str]] = ..., -) -> ndarray: ... -@overload -def argpartition( - a: generic, - kth: _ArrayLikeIntOrBool, - axis: Optional[int] = ..., - kind: _PartitionKind = ..., - order: Union[None, str, Sequence[str]] = ..., -) -> integer: ... -@overload -def argpartition( - a: _ScalarBuiltin, - kth: _ArrayLikeIntOrBool, - axis: Optional[int] = ..., - kind: _PartitionKind = ..., - order: Union[None, str, Sequence[str]] = ..., -) -> ndarray: ... -@overload -def argpartition( - a: ArrayLike, - kth: _ArrayLikeIntOrBool, - axis: Optional[int] = ..., - kind: _PartitionKind = ..., - order: Union[None, str, Sequence[str]] = ..., -) -> ndarray: ... -def sort( - a: ArrayLike, - axis: Optional[int] = ..., - kind: Optional[_SortKind] = ..., - order: Union[None, str, Sequence[str]] = ..., -) -> ndarray: ... -def argsort( - a: ArrayLike, - axis: Optional[int] = ..., - kind: Optional[_SortKind] = ..., - order: Union[None, str, Sequence[str]] = ..., -) -> ndarray: ... -@overload -def argmax(a: ArrayLike, axis: None = ..., out: Optional[ndarray] = ...) -> integer: ... -@overload -def argmax( - a: ArrayLike, axis: int = ..., out: Optional[ndarray] = ... -) -> Union[integer, ndarray]: ... -@overload -def argmin(a: ArrayLike, axis: None = ..., out: Optional[ndarray] = ...) -> integer: ... -@overload -def argmin( - a: ArrayLike, axis: int = ..., out: Optional[ndarray] = ... -) -> Union[integer, ndarray]: ... -@overload -def searchsorted( - a: ArrayLike, - v: _Scalar, - side: _Side = ..., - sorter: Optional[_ArrayLikeIntOrBool] = ..., # 1D int array -) -> integer: ... -@overload -def searchsorted( - a: ArrayLike, - v: ArrayLike, - side: _Side = ..., - sorter: Optional[_ArrayLikeIntOrBool] = ..., # 1D int array -) -> ndarray: ... -def resize(a: ArrayLike, new_shape: _ShapeLike) -> ndarray: ... -@overload -def squeeze(a: _ScalarGeneric, axis: Optional[_ShapeLike] = ...) -> _ScalarGeneric: ... -@overload -def squeeze(a: ArrayLike, axis: Optional[_ShapeLike] = ...) -> ndarray: ... -def diagonal( - a: ArrayLike, offset: int = ..., axis1: int = ..., axis2: int = ... # >= 2D array -) -> ndarray: ... -def trace( - a: ArrayLike, # >= 2D array - offset: int = ..., - axis1: int = ..., - axis2: int = ..., - dtype: DtypeLike = ..., - out: Optional[ndarray] = ..., -) -> Union[number, ndarray]: ... -def ravel(a: ArrayLike, order: _OrderKACF = ...) -> ndarray: ... -def nonzero(a: ArrayLike) -> Tuple[ndarray, ...]: ... -def shape(a: ArrayLike) -> _Shape: ... -def compress( - condition: ArrayLike, # 1D bool array - a: ArrayLike, - axis: Optional[int] = ..., - out: Optional[ndarray] = ..., -) -> ndarray: ... -@overload -def clip( - a: _Number, - a_min: ArrayLike, - a_max: Optional[ArrayLike], - out: Optional[ndarray] = ..., - **kwargs: Any, -) -> _Number: ... -@overload -def clip( - a: _Number, - a_min: None, - a_max: ArrayLike, - out: Optional[ndarray] = ..., - **kwargs: Any, -) -> _Number: ... -@overload -def clip( - a: ArrayLike, - a_min: ArrayLike, - a_max: Optional[ArrayLike], - out: Optional[ndarray] = ..., - **kwargs: Any, -) -> Union[number, ndarray]: ... -@overload -def clip( - a: ArrayLike, - a_min: None, - a_max: ArrayLike, - out: Optional[ndarray] = ..., - **kwargs: Any, -) -> Union[number, ndarray]: ... -@overload -def sum( - a: _Number, - axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> _Number: ... -@overload -def sum( - a: ArrayLike, - axis: _ShapeLike = ..., - dtype: DtypeLike = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> Union[number, ndarray]: ... -@overload -def all( - a: ArrayLike, - axis: None = ..., - out: Optional[ndarray] = ..., - keepdims: Literal[False] = ..., -) -> bool_: ... -@overload -def all( - a: ArrayLike, - axis: Optional[_ShapeLike] = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., -) -> Union[bool_, ndarray]: ... -@overload -def any( - a: ArrayLike, - axis: None = ..., - out: Optional[ndarray] = ..., - keepdims: Literal[False] = ..., -) -> bool_: ... -@overload -def any( - a: ArrayLike, - axis: Optional[_ShapeLike] = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., -) -> Union[bool_, ndarray]: ... -def cumsum( - a: ArrayLike, - axis: Optional[int] = ..., - dtype: DtypeLike = ..., - out: Optional[ndarray] = ..., -) -> ndarray: ... -@overload -def ptp( - a: _Number, - axis: Optional[_ShapeLike] = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., -) -> _Number: ... -@overload -def ptp( - a: ArrayLike, - axis: None = ..., - out: Optional[ndarray] = ..., - keepdims: Literal[False] = ..., -) -> number: ... -@overload -def ptp( - a: ArrayLike, - axis: Optional[_ShapeLike] = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., -) -> Union[number, ndarray]: ... -@overload -def amax( - a: _Number, - axis: Optional[_ShapeLike] = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> _Number: ... -@overload -def amax( - a: ArrayLike, - axis: None = ..., - out: Optional[ndarray] = ..., - keepdims: Literal[False] = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> number: ... -@overload -def amax( - a: ArrayLike, - axis: Optional[_ShapeLike] = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> Union[number, ndarray]: ... -@overload -def amin( - a: _Number, - axis: Optional[_ShapeLike] = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> _Number: ... -@overload -def amin( - a: ArrayLike, - axis: None = ..., - out: Optional[ndarray] = ..., - keepdims: Literal[False] = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> number: ... -@overload -def amin( - a: ArrayLike, - axis: Optional[_ShapeLike] = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> Union[number, ndarray]: ... - -# TODO: `np.prod()``: For object arrays `initial` does not necessarily -# have to be a numerical scalar. -# The only requirement is that it is compatible -# with the `.__mul__()` method(s) of the passed array's elements. - -# Note that the same situation holds for all wrappers around -# `np.ufunc.reduce`, e.g. `np.sum()` (`.__add__()`). - -@overload -def prod( - a: _Number, - axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., - out: None = ..., - keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> _Number: ... -@overload -def prod( - a: ArrayLike, - axis: None = ..., - dtype: DtypeLike = ..., - out: None = ..., - keepdims: Literal[False] = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> number: ... -@overload -def prod( - a: ArrayLike, - axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> Union[number, ndarray]: ... -def cumprod( - a: ArrayLike, - axis: Optional[int] = ..., - dtype: DtypeLike = ..., - out: Optional[ndarray] = ..., -) -> ndarray: ... -def ndim(a: ArrayLike) -> int: ... -def size(a: ArrayLike, axis: Optional[int] = ...) -> int: ... -@overload -def around( - a: _Number, decimals: int = ..., out: Optional[ndarray] = ... -) -> _Number: ... -@overload -def around( - a: _NumberLike, decimals: int = ..., out: Optional[ndarray] = ... -) -> number: ... -@overload -def around( - a: ArrayLike, decimals: int = ..., out: Optional[ndarray] = ... -) -> ndarray: ... -@overload -def mean( - a: ArrayLike, - axis: None = ..., - dtype: DtypeLike = ..., - out: None = ..., - keepdims: Literal[False] = ..., -) -> number: ... -@overload -def mean( - a: ArrayLike, - axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., -) -> Union[number, ndarray]: ... -@overload -def std( - a: ArrayLike, - axis: None = ..., - dtype: DtypeLike = ..., - out: None = ..., - ddof: int = ..., - keepdims: Literal[False] = ..., -) -> number: ... -@overload -def std( - a: ArrayLike, - axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., - out: Optional[ndarray] = ..., - ddof: int = ..., - keepdims: bool = ..., -) -> Union[number, ndarray]: ... -@overload -def var( - a: ArrayLike, - axis: None = ..., - dtype: DtypeLike = ..., - out: None = ..., - ddof: int = ..., - keepdims: Literal[False] = ..., -) -> number: ... -@overload -def var( - a: ArrayLike, - axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., - out: Optional[ndarray] = ..., - ddof: int = ..., - keepdims: bool = ..., -) -> Union[number, ndarray]: ... diff --git a/numpy/core/fromnumeric.pyi b/numpy/core/fromnumeric.pyi new file mode 100644 index 000000000000..7ad772b07e54 --- /dev/null +++ b/numpy/core/fromnumeric.pyi @@ -0,0 +1,492 @@ +import sys +import datetime as dt +from typing import Optional, Union, Sequence, Tuple, Any, overload, TypeVar + +from numpy import ( + ndarray, + number, + integer, + bool_, + generic, + _OrderKACF, + _OrderACF, + _IntLike, + _BoolLike, + _NumberLike, +) +from numpy.typing import DtypeLike, ArrayLike, _ShapeLike, _Shape + +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + +_Mode = Literal["raise", "wrap", "clip"] +_PartitionKind = Literal["introselect"] +_SortKind = Literal["quicksort", "mergesort", "heapsort", "stable"] +_Side = Literal["left", "right"] + +# Various annotations for scalars + +# While dt.datetime and dt.timedelta are not technically part of NumPy, +# they are one of the rare few builtin scalars which serve as valid return types. +# See https://github.com/numpy/numpy-stubs/pull/67#discussion_r412604113. +_ScalarNumpy = Union[generic, dt.datetime, dt.timedelta] +_ScalarBuiltin = Union[str, bytes, dt.date, dt.timedelta, bool, int, float, complex] +_Scalar = Union[_ScalarBuiltin, _ScalarNumpy] + +# Integers and booleans can generally be used interchangeably +_ScalarIntOrBool = TypeVar("_ScalarIntOrBool", bound=Union[integer, bool_]) +_ScalarGeneric = TypeVar("_ScalarGeneric", bound=generic) +_ScalarGenericDT = TypeVar( + "_ScalarGenericDT", bound=Union[dt.datetime, dt.timedelta, generic] +) + +_Number = TypeVar("_Number", bound=number) + +# An array-like object consisting of integers +_IntOrBool = Union[_IntLike, _BoolLike] +_ArrayLikeIntNested = ArrayLike # TODO: wait for support for recursive types +_ArrayLikeBoolNested = ArrayLike # TODO: wait for support for recursive types + +# Integers and booleans can generally be used interchangeably +_ArrayLikeIntOrBool = Union[ + _IntOrBool, + ndarray, + Sequence[_IntOrBool], + Sequence[_ArrayLikeIntNested], + Sequence[_ArrayLikeBoolNested], +] +_ArrayLikeBool = Union[_BoolLike, Sequence[_BoolLike], ndarray] + +# The signature of take() follows a common theme with its overloads: +# 1. A generic comes in; the same generic comes out +# 2. A scalar comes in; a generic comes out +# 3. An array-like object comes in; some keyword ensures that a generic comes out +# 4. An array-like object comes in; an ndarray or generic comes out +@overload +def take( + a: _ScalarGenericDT, + indices: int, + axis: Optional[int] = ..., + out: Optional[ndarray] = ..., + mode: _Mode = ..., +) -> _ScalarGenericDT: ... +@overload +def take( + a: _Scalar, + indices: int, + axis: Optional[int] = ..., + out: Optional[ndarray] = ..., + mode: _Mode = ..., +) -> _ScalarNumpy: ... +@overload +def take( + a: ArrayLike, + indices: int, + axis: Optional[int] = ..., + out: Optional[ndarray] = ..., + mode: _Mode = ..., +) -> _ScalarNumpy: ... +@overload +def take( + a: ArrayLike, + indices: _ArrayLikeIntOrBool, + axis: Optional[int] = ..., + out: Optional[ndarray] = ..., + mode: _Mode = ..., +) -> Union[_ScalarNumpy, ndarray]: ... +def reshape(a: ArrayLike, newshape: _ShapeLike, order: _OrderACF = ...) -> ndarray: ... +@overload +def choose( + a: _ScalarIntOrBool, + choices: ArrayLike, + out: Optional[ndarray] = ..., + mode: _Mode = ..., +) -> _ScalarIntOrBool: ... +@overload +def choose( + a: _IntOrBool, choices: ArrayLike, out: Optional[ndarray] = ..., mode: _Mode = ... +) -> Union[integer, bool_]: ... +@overload +def choose( + a: _ArrayLikeIntOrBool, + choices: ArrayLike, + out: Optional[ndarray] = ..., + mode: _Mode = ..., +) -> ndarray: ... +def repeat( + a: ArrayLike, repeats: _ArrayLikeIntOrBool, axis: Optional[int] = ... +) -> ndarray: ... +def put( + a: ndarray, ind: _ArrayLikeIntOrBool, v: ArrayLike, mode: _Mode = ... +) -> None: ... +def swapaxes(a: ArrayLike, axis1: int, axis2: int) -> ndarray: ... +def transpose( + a: ArrayLike, axes: Union[None, Sequence[int], ndarray] = ... +) -> ndarray: ... +def partition( + a: ArrayLike, + kth: _ArrayLikeIntOrBool, + axis: Optional[int] = ..., + kind: _PartitionKind = ..., + order: Union[None, str, Sequence[str]] = ..., +) -> ndarray: ... +@overload +def argpartition( + a: generic, + kth: _ArrayLikeIntOrBool, + axis: Optional[int] = ..., + kind: _PartitionKind = ..., + order: Union[None, str, Sequence[str]] = ..., +) -> integer: ... +@overload +def argpartition( + a: _ScalarBuiltin, + kth: _ArrayLikeIntOrBool, + axis: Optional[int] = ..., + kind: _PartitionKind = ..., + order: Union[None, str, Sequence[str]] = ..., +) -> ndarray: ... +@overload +def argpartition( + a: ArrayLike, + kth: _ArrayLikeIntOrBool, + axis: Optional[int] = ..., + kind: _PartitionKind = ..., + order: Union[None, str, Sequence[str]] = ..., +) -> ndarray: ... +def sort( + a: ArrayLike, + axis: Optional[int] = ..., + kind: Optional[_SortKind] = ..., + order: Union[None, str, Sequence[str]] = ..., +) -> ndarray: ... +def argsort( + a: ArrayLike, + axis: Optional[int] = ..., + kind: Optional[_SortKind] = ..., + order: Union[None, str, Sequence[str]] = ..., +) -> ndarray: ... +@overload +def argmax(a: ArrayLike, axis: None = ..., out: Optional[ndarray] = ...) -> integer: ... +@overload +def argmax( + a: ArrayLike, axis: int = ..., out: Optional[ndarray] = ... +) -> Union[integer, ndarray]: ... +@overload +def argmin(a: ArrayLike, axis: None = ..., out: Optional[ndarray] = ...) -> integer: ... +@overload +def argmin( + a: ArrayLike, axis: int = ..., out: Optional[ndarray] = ... +) -> Union[integer, ndarray]: ... +@overload +def searchsorted( + a: ArrayLike, + v: _Scalar, + side: _Side = ..., + sorter: Optional[_ArrayLikeIntOrBool] = ..., # 1D int array +) -> integer: ... +@overload +def searchsorted( + a: ArrayLike, + v: ArrayLike, + side: _Side = ..., + sorter: Optional[_ArrayLikeIntOrBool] = ..., # 1D int array +) -> ndarray: ... +def resize(a: ArrayLike, new_shape: _ShapeLike) -> ndarray: ... +@overload +def squeeze(a: _ScalarGeneric, axis: Optional[_ShapeLike] = ...) -> _ScalarGeneric: ... +@overload +def squeeze(a: ArrayLike, axis: Optional[_ShapeLike] = ...) -> ndarray: ... +def diagonal( + a: ArrayLike, offset: int = ..., axis1: int = ..., axis2: int = ... # >= 2D array +) -> ndarray: ... +def trace( + a: ArrayLike, # >= 2D array + offset: int = ..., + axis1: int = ..., + axis2: int = ..., + dtype: DtypeLike = ..., + out: Optional[ndarray] = ..., +) -> Union[number, ndarray]: ... +def ravel(a: ArrayLike, order: _OrderKACF = ...) -> ndarray: ... +def nonzero(a: ArrayLike) -> Tuple[ndarray, ...]: ... +def shape(a: ArrayLike) -> _Shape: ... +def compress( + condition: ArrayLike, # 1D bool array + a: ArrayLike, + axis: Optional[int] = ..., + out: Optional[ndarray] = ..., +) -> ndarray: ... +@overload +def clip( + a: _Number, + a_min: ArrayLike, + a_max: Optional[ArrayLike], + out: Optional[ndarray] = ..., + **kwargs: Any, +) -> _Number: ... +@overload +def clip( + a: _Number, + a_min: None, + a_max: ArrayLike, + out: Optional[ndarray] = ..., + **kwargs: Any, +) -> _Number: ... +@overload +def clip( + a: ArrayLike, + a_min: ArrayLike, + a_max: Optional[ArrayLike], + out: Optional[ndarray] = ..., + **kwargs: Any, +) -> Union[number, ndarray]: ... +@overload +def clip( + a: ArrayLike, + a_min: None, + a_max: ArrayLike, + out: Optional[ndarray] = ..., + **kwargs: Any, +) -> Union[number, ndarray]: ... +@overload +def sum( + a: _Number, + axis: Optional[_ShapeLike] = ..., + dtype: DtypeLike = ..., + out: Optional[ndarray] = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., +) -> _Number: ... +@overload +def sum( + a: ArrayLike, + axis: _ShapeLike = ..., + dtype: DtypeLike = ..., + out: Optional[ndarray] = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., +) -> Union[number, ndarray]: ... +@overload +def all( + a: ArrayLike, + axis: None = ..., + out: Optional[ndarray] = ..., + keepdims: Literal[False] = ..., +) -> bool_: ... +@overload +def all( + a: ArrayLike, + axis: Optional[_ShapeLike] = ..., + out: Optional[ndarray] = ..., + keepdims: bool = ..., +) -> Union[bool_, ndarray]: ... +@overload +def any( + a: ArrayLike, + axis: None = ..., + out: Optional[ndarray] = ..., + keepdims: Literal[False] = ..., +) -> bool_: ... +@overload +def any( + a: ArrayLike, + axis: Optional[_ShapeLike] = ..., + out: Optional[ndarray] = ..., + keepdims: bool = ..., +) -> Union[bool_, ndarray]: ... +def cumsum( + a: ArrayLike, + axis: Optional[int] = ..., + dtype: DtypeLike = ..., + out: Optional[ndarray] = ..., +) -> ndarray: ... +@overload +def ptp( + a: _Number, + axis: Optional[_ShapeLike] = ..., + out: Optional[ndarray] = ..., + keepdims: bool = ..., +) -> _Number: ... +@overload +def ptp( + a: ArrayLike, + axis: None = ..., + out: Optional[ndarray] = ..., + keepdims: Literal[False] = ..., +) -> number: ... +@overload +def ptp( + a: ArrayLike, + axis: Optional[_ShapeLike] = ..., + out: Optional[ndarray] = ..., + keepdims: bool = ..., +) -> Union[number, ndarray]: ... +@overload +def amax( + a: _Number, + axis: Optional[_ShapeLike] = ..., + out: Optional[ndarray] = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., +) -> _Number: ... +@overload +def amax( + a: ArrayLike, + axis: None = ..., + out: Optional[ndarray] = ..., + keepdims: Literal[False] = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., +) -> number: ... +@overload +def amax( + a: ArrayLike, + axis: Optional[_ShapeLike] = ..., + out: Optional[ndarray] = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., +) -> Union[number, ndarray]: ... +@overload +def amin( + a: _Number, + axis: Optional[_ShapeLike] = ..., + out: Optional[ndarray] = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., +) -> _Number: ... +@overload +def amin( + a: ArrayLike, + axis: None = ..., + out: Optional[ndarray] = ..., + keepdims: Literal[False] = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., +) -> number: ... +@overload +def amin( + a: ArrayLike, + axis: Optional[_ShapeLike] = ..., + out: Optional[ndarray] = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., +) -> Union[number, ndarray]: ... + +# TODO: `np.prod()``: For object arrays `initial` does not necessarily +# have to be a numerical scalar. +# The only requirement is that it is compatible +# with the `.__mul__()` method(s) of the passed array's elements. + +# Note that the same situation holds for all wrappers around +# `np.ufunc.reduce`, e.g. `np.sum()` (`.__add__()`). +@overload +def prod( + a: _Number, + axis: Optional[_ShapeLike] = ..., + dtype: DtypeLike = ..., + out: None = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., +) -> _Number: ... +@overload +def prod( + a: ArrayLike, + axis: None = ..., + dtype: DtypeLike = ..., + out: None = ..., + keepdims: Literal[False] = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., +) -> number: ... +@overload +def prod( + a: ArrayLike, + axis: Optional[_ShapeLike] = ..., + dtype: DtypeLike = ..., + out: Optional[ndarray] = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., +) -> Union[number, ndarray]: ... +def cumprod( + a: ArrayLike, + axis: Optional[int] = ..., + dtype: DtypeLike = ..., + out: Optional[ndarray] = ..., +) -> ndarray: ... +def ndim(a: ArrayLike) -> int: ... +def size(a: ArrayLike, axis: Optional[int] = ...) -> int: ... +@overload +def around( + a: _Number, decimals: int = ..., out: Optional[ndarray] = ... +) -> _Number: ... +@overload +def around( + a: _NumberLike, decimals: int = ..., out: Optional[ndarray] = ... +) -> number: ... +@overload +def around( + a: ArrayLike, decimals: int = ..., out: Optional[ndarray] = ... +) -> ndarray: ... +@overload +def mean( + a: ArrayLike, + axis: None = ..., + dtype: DtypeLike = ..., + out: None = ..., + keepdims: Literal[False] = ..., +) -> number: ... +@overload +def mean( + a: ArrayLike, + axis: Optional[_ShapeLike] = ..., + dtype: DtypeLike = ..., + out: Optional[ndarray] = ..., + keepdims: bool = ..., +) -> Union[number, ndarray]: ... +@overload +def std( + a: ArrayLike, + axis: None = ..., + dtype: DtypeLike = ..., + out: None = ..., + ddof: int = ..., + keepdims: Literal[False] = ..., +) -> number: ... +@overload +def std( + a: ArrayLike, + axis: Optional[_ShapeLike] = ..., + dtype: DtypeLike = ..., + out: Optional[ndarray] = ..., + ddof: int = ..., + keepdims: bool = ..., +) -> Union[number, ndarray]: ... +@overload +def var( + a: ArrayLike, + axis: None = ..., + dtype: DtypeLike = ..., + out: None = ..., + ddof: int = ..., + keepdims: Literal[False] = ..., +) -> number: ... +@overload +def var( + a: ArrayLike, + axis: Optional[_ShapeLike] = ..., + dtype: DtypeLike = ..., + out: Optional[ndarray] = ..., + ddof: int = ..., + keepdims: bool = ..., +) -> Union[number, ndarray]: ... From 34719a111253e36f463aa4629eb75716e1c5c13c Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Mon, 14 Sep 2020 15:14:58 +0100 Subject: [PATCH 039/409] MAINT: Syntax-highlight .src files on github --- .gitattributes | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitattributes b/.gitattributes index dad6dde37cd0..81b8037d4d71 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,3 +3,8 @@ numpy/lib/tests/data/*.npy binary # Release notes, reduce number of conflicts. doc/release/*.rst merge=union + +# Highlight our custom templating language as C, since it's hopefully better +# than nothing. This also affects repo statistics. +*.c.src linguist-language=C +*.h.src linguist-language=C From 5abb267691a44e908bc4084d1b83ad7d73c34796 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2020 14:22:24 +0000 Subject: [PATCH 040/409] MAINT: Bump pytest from 6.0.1 to 6.0.2 Bumps [pytest](https://github.com/pytest-dev/pytest) from 6.0.1 to 6.0.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/6.0.1...6.0.2) Signed-off-by: dependabot-preview[bot] --- test_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index 0349b711be58..ca39de622bc0 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -2,7 +2,7 @@ cython==0.29.21 wheel setuptools<49.2.0 hypothesis==5.35.1 -pytest==6.0.1 +pytest==6.0.2 pytz==2020.1 pytest-cov==2.10.1 pickle5; python_version == '3.7' From 4cdd3606160de923fb4054cf93f4ea02a356def0 Mon Sep 17 00:00:00 2001 From: Matti Picus Date: Mon, 14 Sep 2020 18:55:12 +0300 Subject: [PATCH 041/409] DOC: add new glossary terms (#17263) * DOC: add new glossary terms * DOC: link to python Ellipsis * DOC: fixes from review * DOC: fixes from review * DOC: remove glossary items that belong to python --- doc/source/glossary.rst | 279 +++++++++++++++++++++-- doc/source/reference/arrays.indexing.rst | 3 + 2 files changed, 262 insertions(+), 20 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index d375349603df..4a59c990bdd4 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -2,10 +2,98 @@ Glossary ******** -.. toctree:: - .. glossary:: + + (`n`,) + A tuple with one element. The trailing comma distinguishes a one-element + tuple from a parenthesized ``n``. + + + -1 + Used as a dimension entry, ``-1`` instructs NumPy to choose the length + that will keep the total number of elements the same. + + + ``...`` + An :py:data:`Ellipsis` + + **When indexing an array**, shorthand that the missing axes, if they + exist, are full slices. + + >>> a = np.arange(24).reshape(2,3,4) + + >>> a[...].shape + (2, 3, 4) + + >>> a[...,0].shape + (2, 3) + + >>> a[0,...].shape + (3, 4) + + >>> a[0,...,0].shape + (3,) + + It can be used at most once; ``a[...,0,...]`` raises an :exc:`IndexError`. + + **In printouts**, NumPy substitutes ``...`` for the middle elements of + large arrays. To see the entire array, use `numpy.printoptions` + + + ``:`` + The Python :term:`python:slice` + operator. In ndarrays, slicing can be applied to every + axis: + + >>> a = np.arange(24).reshape(2,3,4) + >>> a + array([[[ 0, 1, 2, 3], + [ 4, 5, 6, 7], + [ 8, 9, 10, 11]], + + [[12, 13, 14, 15], + [16, 17, 18, 19], + [20, 21, 22, 23]]]) + + >>> a[1:,-2:,:-1] + array([[[16, 17, 18], + [20, 21, 22]]]) + + Trailing slices can be omitted: :: + + >>> a[1] == a[1,:,:] + array([[ True, True, True, True], + [ True, True, True, True], + [ True, True, True, True]]) + + In contrast to Python, where slicing creates a copy, in NumPy slicing + creates a :term:`view`. + + For details, see :ref:`combining-advanced-and-basic-indexing`. + + + ``<`` + In a dtype declaration, indicates that the data is + :term:`little-endian` (the bracket is big on the right). :: + + >>> dt = np.dtype('`` + In a dtype declaration, indicates that the data is + :term:`big-endian` (the bracket is big on the left). :: + + >>> dt = np.dtype('>H') # big-endian unsigned short + + + advanced indexing + Rather than using a :doc:`scalar ` or slice as + an index, an axis can be indexed with an array, providing fine-grained + selection. This is known as :ref:`advanced indexing` + or "fancy indexing". + + along an axis Axes are defined for arrays with more than one dimension. A 2-dimensional array has two corresponding axes: the first running @@ -26,6 +114,7 @@ Glossary >>> x.sum(axis=1) array([ 6, 22, 38]) + array A homogeneous container of numerical elements. Each element in the array occupies a fixed amount of memory (hence homogeneous), and @@ -50,19 +139,92 @@ Glossary Fast element-wise operations, called a :term:`ufunc`, operate on arrays. + array_like Any sequence that can be interpreted as an ndarray. This includes nested lists, tuples, scalars and existing arrays. + + array scalar + For uniformity in handling operands, NumPy treats + a :doc:`scalar ` as an array of zero + dimension. + + + axis + + Another term for an array dimension. Axes are numbered left to right; + axis 0 is the first element in the shape tuple. + + In a two-dimensional vector, the elements of axis 0 are rows and the + elements of axis 1 are columns. + + In higher dimensions, the picture changes. NumPy prints + higher-dimensional vectors as replications of row-by-column building + blocks, as in this three-dimensional vector: + + >>> a = np.arange(12).reshape(2,2,3) + >>> a + array([[[ 0, 1, 2], + [ 3, 4, 5]], + + [[ 6, 7, 8], + [ 9, 10, 11]]]) + + ``a`` is depicted as a two-element array whose elements are 2x3 vectors. + From this point of view, rows and columns are the final two axes, + respectively, in any shape. + + This rule helps you anticipate how a vector will be printed, and + conversely how to find the index of any of the printed elements. For + instance, in the example, the last two values of 8's index must be 0 and + 2. Since 8 appears in the second of the two 2x3's, the first index must + be 1: + + >>> a[1,0,2] + 8 + + A convenient way to count dimensions in a printed vector is to + count ``[`` symbols after the open-parenthesis. This is + useful in distinguishing, say, a (1,2,3) shape from a (2,3) shape: + + >>> a = np.arange(6).reshape(2,3) + >>> a.ndim + 2 + >>> a + array([[0, 1, 2], + [3, 4, 5]]) + + >>> a = np.arange(6).reshape(1,2,3) + >>> a.ndim + 3 + >>> a + array([[[0, 1, 2], + [3, 4, 5]]]) + + + .base + + If an array does not own its memory, then its + :doc:`base ` attribute + returns the object whose memory the array is referencing. That object + may be borrowing the memory from still another object, so the + owning object may be ``a.base.base.base...``. Despite advice to the + contrary, testing ``base`` is not a surefire way to determine if two + arrays are :term:`view`\ s. + + big-endian When storing a multi-byte value in memory as a sequence of bytes, the sequence addresses/sends/stores the most significant byte first (lowest address) and the least significant byte last (highest address). Common in micro-processors and used for transmission of data over network protocols. + BLAS `Basic Linear Algebra Subprograms `_ + broadcast NumPy can do operations on arrays whose shapes are mismatched:: @@ -82,9 +244,11 @@ Glossary See `basics.broadcasting` for more information. + C order See `row-major` + column-major A way to represent items in a N-dimensional array in the 1-dimensional computer memory. In column-major order, the leftmost index "varies the @@ -100,6 +264,11 @@ Glossary Column-major order is also known as the Fortran order, as the Fortran programming language uses it. + copy + + See :term:`view`. + + decorator An operator that transforms a function. For example, a ``log`` decorator may be defined to print debugging information upon @@ -124,6 +293,7 @@ Glossary Logging call with parameters: (1, 2) {} 3 + dictionary Resembling a language dictionary, which provides a mapping between words and descriptions thereof, a Python dictionary is a mapping @@ -149,48 +319,55 @@ Glossary For more information on dictionaries, read the `Python tutorial `_. + + dimension + + See :term:`axis`. + + + dtype + + The datatype describing the (identically typed) elements in an ndarray. + It can be changed to reinterpret the array contents. For details, see + :doc:`Data type objects (dtype). ` + + + fancy indexing + + Another term for :term:`advanced indexing`. + + field In a :term:`structured data type`, each sub-type is called a `field`. The `field` has a name (a string), a type (any valid dtype), and an optional `title`. See :ref:`arrays.dtypes` + Fortran order See `column-major` + flattened Collapsed to a one-dimensional array. See `numpy.ndarray.flatten` for details. + homogeneous - Describes a block of memory comprised of blocks, each block comprised of + Describes a block of memory comprised of blocks, each block comprised of items and of the same size, and blocks are interpreted in exactly the same way. In the simplest case each block contains a single item, for instance int32 or float64. + immutable An object that cannot be modified after execution is called immutable. Two common examples are strings and tuples. - iterable - A sequence that allows "walking" (iterating) over items, typically - using a loop such as:: - - >>> x = [1, 2, 3] - >>> [item**2 for item in x] - [1, 4, 9] - - It is often used in combination with ``enumerate``:: - >>> keys = ['a','b','c'] - >>> for n, k in enumerate(keys): - ... print("Key %d: %s" % (n, k)) - ... - Key 0: a - Key 1: b - Key 2: c itemsize The size of the dtype element in bytes. + list A Python container that can hold any number of objects or items. The items do not have to be of the same type, and can even be @@ -223,12 +400,14 @@ Glossary tutorial `_. For a mapping type (key-value), see *dictionary*. + little-endian When storing a multi-byte value in memory as a sequence of bytes, the sequence addresses/sends/stores the least significant byte first (lowest address) and the most significant byte last (highest address). Common in x86 processors. + mask A boolean array, used to select only certain elements for an operation:: @@ -244,6 +423,7 @@ Glossary >>> x array([ 0, 1, 2, -1, -1]) + masked array Array that suppressed values indicated by a mask:: @@ -262,6 +442,7 @@ Glossary Masked arrays are often used when operating on arrays containing missing or invalid entries. + matrix A 2-dimensional ndarray that preserves its two-dimensional nature throughout operations. It has certain special operations, such as ``*`` @@ -276,18 +457,40 @@ Glossary matrix([[ 7, 10], [15, 22]]) + ndarray See *array*. + + object array + + An array whose dtype is ``object``; that is, it contains references to + Python objects. Indexing the array dereferences the Python objects, so + unlike other ndarrays, an object array has the ability to hold + heterogeneous objects. + + + ravel + + `numpy.ravel` and `numpy.ndarray.flatten` both flatten an ndarray. ``ravel`` + will return a view if possible; ``flatten`` always returns a copy. + + Flattening collapses a multi-dimensional array to a single dimension; + details of how this is done (for instance, whether ``a[n+1]`` should be + the next row or next column) are parameters. + + record array An :term:`ndarray` with :term:`structured data type` which has been subclassed as ``np.recarray`` and whose dtype is of type ``np.record``, making the fields of its data type to be accessible by attribute. + reference If ``a`` is a reference to ``b``, then ``(a is b) == True``. Therefore, ``a`` and ``b`` are different names for the same Python object. + row-major A way to represent items in a N-dimensional array in the 1-dimensional computer memory. In row-major order, the rightmost index "varies @@ -303,6 +506,7 @@ Glossary Row-major order is also known as the C order, as the C programming language uses it. New NumPy arrays are by default in row-major order. + slice Used to select only certain elements from a sequence: @@ -330,9 +534,40 @@ Glossary >>> x[:, 1] array([2, 4]) + + stride + + Physical memory is one-dimensional; strides provide a mechanism to map + a given index to an address in memory. For an N-dimensional array, its + ``strides`` attribute is an N-element tuple; advancing from index + ``i`` to index ``i+1`` on axis ``n`` means adding ``a.strides[n]`` bytes + to the address. + + Strides are computed automatically from an array's dtype and + shape, but can be directly specified using + :doc:`as_strided. ` + + For details, see + :doc:`numpy.ndarray.strides `. + + To see how striding underlies the power of NumPy views, see + `The NumPy array: a structure for efficient numerical computation. \ + `_ + + + structure + See :term:`structured data type` + + + structured array + + Array whose :term:`dtype` is a :term:`structured data type`. + + structured data type A data type composed of other datatypes + subarray data type A :term:`structured data type` may contain a :term:`ndarray` with its own dtype and shape: @@ -342,16 +577,19 @@ Glossary array([(0, [0., 0., 0.]), (0, [0., 0., 0.]), (0, [0., 0., 0.])], dtype=[('a', '` which is an alias to the name and is commonly used for plotting. + ufunc Universal function. A fast element-wise, :term:`vectorized ` array operation. Examples include ``add``, ``sin`` and ``logical_or``. + vectorization Optimizing a looping block by specialized code. In a traditional sense, vectorization performs the same operation on multiple elements with @@ -362,6 +600,7 @@ Glossary operations on multiple elements, typically achieving speedups by avoiding some of the overhead in looking up and converting the elements. + view An array that does not own its data, but refers to another array's data instead. For example, we may create a view that only shows @@ -379,6 +618,7 @@ Glossary >>> y array([3, 2, 4]) + wrapper Python is a high-level (highly abstracted, or English-like) language. This abstraction comes at a price in execution speed, and sometimes @@ -390,4 +630,3 @@ Glossary Examples include ctypes, SWIG and Cython (which wraps C and C++) and f2py (which wraps Fortran). - diff --git a/doc/source/reference/arrays.indexing.rst b/doc/source/reference/arrays.indexing.rst index 3e600b7c456e..180a79dae18c 100644 --- a/doc/source/reference/arrays.indexing.rst +++ b/doc/source/reference/arrays.indexing.rst @@ -198,6 +198,7 @@ concepts to remember include: create an axis of length one. :const:`newaxis` is an alias for 'None', and 'None' can be used in place of this with the same result. +.. _advanced-indexing: Advanced Indexing ----------------- @@ -304,6 +305,8 @@ understood with an example. most important thing to remember about indexing with multiple advanced indexes. +.. _combining-advanced-and-basic-indexing: + Combining advanced and basic indexing """"""""""""""""""""""""""""""""""""" From 365cd3dba3487802d5a2d4e046a5c16a31bd1590 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Mon, 14 Sep 2020 11:20:00 -0500 Subject: [PATCH 042/409] MAINT: Mark vendored/generated files in .gitattributes This marks some files as vendored or generated. There may be more files around. This should mostly change our language statistics on github, so it probably isn't worth much trouble, but maybe it makes the stats slightly more representative. --- .gitattributes | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.gitattributes b/.gitattributes index 81b8037d4d71..ad7d3b227108 100644 --- a/.gitattributes +++ b/.gitattributes @@ -8,3 +8,13 @@ doc/release/*.rst merge=union # than nothing. This also affects repo statistics. *.c.src linguist-language=C *.h.src linguist-language=C + +# Mark some files as vendored +numpy/linalg/lapack_lite/f2c.c linguist-vendored +numpy/linalg/lapack_lite/f2c.h linguist-vendored +tools/npy_tempita/* linguist-vendored + +# Mark some files as generated +numpy/linalg/lapack_lite/f2c_*.c linguist-generated +numpy/linalg/lapack_lite/lapack_lite_names.h linguist-generated + From 01996b8439892a12d511652db7d8882d11e2a09f Mon Sep 17 00:00:00 2001 From: mattip Date: Mon, 14 Sep 2020 22:56:46 +0300 Subject: [PATCH 043/409] BUILD: pin pygments to 2.6.1 --- doc_requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc_requirements.txt b/doc_requirements.txt index e2694ba12808..815aac3079ec 100644 --- a/doc_requirements.txt +++ b/doc_requirements.txt @@ -1,3 +1,4 @@ +pygments==2.6.1 sphinx>=2.2.0,<3.0 ipython scipy From ff44f113101814b6321906a312fe9f0db1b2f1e6 Mon Sep 17 00:00:00 2001 From: Chris Brown Date: Tue, 15 Sep 2020 14:35:14 +0100 Subject: [PATCH 044/409] BUG: Set deprecated fields to null in PyArray_InitArrFuncs Initializing the deprecated fields to null ensures that if a user sets them to their own function pointers, this can be detected and the warning about using deprecated fields can be printed. --- numpy/core/src/multiarray/usertypes.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/numpy/core/src/multiarray/usertypes.c b/numpy/core/src/multiarray/usertypes.c index 6b6c6bd9d057..d75fd313090e 100644 --- a/numpy/core/src/multiarray/usertypes.c +++ b/numpy/core/src/multiarray/usertypes.c @@ -127,6 +127,9 @@ PyArray_InitArrFuncs(PyArray_ArrFuncs *f) f->scalarkind = NULL; f->cancastscalarkindto = NULL; f->cancastto = NULL; + f->fastclip = NULL; + f->fastputmask = NULL; + f->fasttake = NULL; } From a50d3006b1c0c526dd91ffa6cbe36b1120683b65 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 15 Sep 2020 12:55:55 -0500 Subject: [PATCH 045/409] BUG: Relax registration of user dtypes with `NPY_ITEM_REFCOUNT` This also includes `NPY_ITEM_IS_POINTER`. A previous changed raised an error when this happened, but some downstream libraries use it to create a custom dtype with a single object field. It seems acceptable to create such a dtype if (and only if) that dtype hardcodes names and fields at creation time, so this change allows that, but otherwise keeps the error intact. This should work fine, although some care may be required --- numpy/core/src/multiarray/usertypes.c | 33 +++++++++++++++++++-------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/numpy/core/src/multiarray/usertypes.c b/numpy/core/src/multiarray/usertypes.c index d75fd313090e..a816a701d944 100644 --- a/numpy/core/src/multiarray/usertypes.c +++ b/numpy/core/src/multiarray/usertypes.c @@ -195,7 +195,7 @@ PyArray_RegisterDataType(PyArray_Descr *descr) } } typenum = NPY_USERDEF + NPY_NUMUSERTYPES; - descr->type_num = typenum; + descr->type_num = -1; if (PyDataType_ISUNSIZED(descr)) { PyErr_SetString(PyExc_ValueError, "cannot register a" \ "flexible data-type"); @@ -214,18 +214,31 @@ PyArray_RegisterDataType(PyArray_Descr *descr) " is missing."); return -1; } - if (descr->flags & (NPY_ITEM_IS_POINTER | NPY_ITEM_REFCOUNT)) { - PyErr_SetString(PyExc_ValueError, - "Legacy user dtypes referencing python objects or generally " - "allocated memory are unsupported. " - "If you see this error in an existing, working code base, " - "please contact the NumPy developers."); - return -1; - } if (descr->typeobj == NULL) { PyErr_SetString(PyExc_ValueError, "missing typeobject"); return -1; } + if (descr->flags & (NPY_ITEM_IS_POINTER | NPY_ITEM_REFCOUNT)) { + /* + * User dtype can't actually do reference counting, however, there + * are existing hacks (e.g. xpress), which use a structured one: + * dtype((xpress.var, [('variable', 'O')])) + * so we have to support this. But such a structure must be constant + * (i.e. fixed at registration time, this is the case for `xpress`). + */ + if (descr->names == NULL || descr->fields == NULL || + !PyDict_CheckExact(descr->fields)) { + PyErr_Format(PyExc_ValueError, + "Failed to register dtype for %S: Legacy user dtypes " + "using `NPY_ITEM_IS_POINTER` or `NPY_ITEM_REFCOUNT` are" + "unsupported. It is possible to create such a dtype only " + "if it is a structured dtype with names and fields " + "hardcoded at registration time.\n" + "Please contact the NumPy developers if this used to work " + "but now fails.", descr->typeobj); + return -1; + } + } if (test_deprecated_arrfuncs_members(f) < 0) { return -1; @@ -242,7 +255,7 @@ PyArray_RegisterDataType(PyArray_Descr *descr) if (dtypemeta_wrap_legacy_descriptor(descr) < 0) { return -1; } - + descr->type_num = typenum; return typenum; } From 4e9c37d0555fe4d58632d47fb5de98a40a427050 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 15 Sep 2020 13:01:57 -0500 Subject: [PATCH 046/409] ENH: Make dtype registration time error messages more clear --- numpy/core/src/multiarray/dtypemeta.c | 6 +++++- numpy/core/src/multiarray/usertypes.c | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/numpy/core/src/multiarray/dtypemeta.c b/numpy/core/src/multiarray/dtypemeta.c index d07dc700dccd..c483f993781a 100644 --- a/numpy/core/src/multiarray/dtypemeta.c +++ b/numpy/core/src/multiarray/dtypemeta.c @@ -315,7 +315,11 @@ dtypemeta_wrap_legacy_descriptor(PyArray_Descr *descr) if (Py_TYPE(descr) != &PyArrayDescr_Type) { PyErr_Format(PyExc_RuntimeError, "During creation/wrapping of legacy DType, the original class " - "was not PyArrayDescr_Type (it is replaced in this step)."); + "was not of PyArrayDescr_Type (it is replaced in this step). " + "The extension creating a custom DType for type %S must be " + "modified to ensure `Py_TYPE(descr) == &PyArrayDescr_Type` at " + "registration time.", + descr->typeobj); return -1; } diff --git a/numpy/core/src/multiarray/usertypes.c b/numpy/core/src/multiarray/usertypes.c index a816a701d944..4174e9c0891d 100644 --- a/numpy/core/src/multiarray/usertypes.c +++ b/numpy/core/src/multiarray/usertypes.c @@ -315,7 +315,7 @@ PyArray_RegisterCanCast(PyArray_Descr *descr, int totype, if (!PyTypeNum_ISUSERDEF(descr->type_num) && !PyTypeNum_ISUSERDEF(totype)) { PyErr_SetString(PyExc_ValueError, - "At least one of the types provided to" + "At least one of the types provided to " "RegisterCanCast must be user-defined."); return -1; } From fc0297e3311de5f47a46767f6d6c5181c9a7b07d Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 15 Sep 2020 13:43:14 -0500 Subject: [PATCH 047/409] MAINT: Relax check that a correct type is already set This allows running `xpress` on current master. `xpress` copies the type from `np.dtype(object)` rather than using `&PyArrayDescr_Type`. That seems overall fine, we can just replace it. The only reason for this check is to ensure that the user does not override our updated `type(user_dtype)` and thus corrupting it. --- numpy/core/src/multiarray/dtypemeta.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/numpy/core/src/multiarray/dtypemeta.c b/numpy/core/src/multiarray/dtypemeta.c index c483f993781a..ae1dfdd81a4b 100644 --- a/numpy/core/src/multiarray/dtypemeta.c +++ b/numpy/core/src/multiarray/dtypemeta.c @@ -312,14 +312,28 @@ string_known_scalar_types( NPY_NO_EXPORT int dtypemeta_wrap_legacy_descriptor(PyArray_Descr *descr) { - if (Py_TYPE(descr) != &PyArrayDescr_Type) { + int has_type_set = Py_TYPE(descr) == &PyArrayDescr_Type; + + if (!has_type_set) { + /* Accept if the type was filled in from an existing builtin dtype */ + for (int i = 0; i < NPY_NTYPES; i++) { + PyArray_Descr *builtin = PyArray_DescrFromType(i); + has_type_set = Py_TYPE(descr) == Py_TYPE(builtin); + Py_DECREF(builtin); + if (has_type_set) { + break; + } + } + } + if (!has_type_set) { PyErr_Format(PyExc_RuntimeError, "During creation/wrapping of legacy DType, the original class " "was not of PyArrayDescr_Type (it is replaced in this step). " "The extension creating a custom DType for type %S must be " - "modified to ensure `Py_TYPE(descr) == &PyArrayDescr_Type` at " - "registration time.", - descr->typeobj); + "modified to ensure `Py_TYPE(descr) == &PyArrayDescr_Type` or " + "that of an existing dtype (with the assumption it is just " + "copied over and can be replaced).", + descr->typeobj, Py_TYPE(descr)); return -1; } From bd263912e11f7f79310c5516fab6bd0a7b7c8eb9 Mon Sep 17 00:00:00 2001 From: Ryan C Cooper Date: Tue, 15 Sep 2020 16:35:06 -0400 Subject: [PATCH 048/409] DOC: Update numpy4matlab (#17159) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update doc/source/user/numpy-for-matlab-users.rst Co-authored-by: Eric Wieser Co-authored-by: Ben Nathanson Co-authored-by: Melissa Weber Mendonça --- doc/source/user/numpy-for-matlab-users.rst | 737 ++++++++++++--------- 1 file changed, 420 insertions(+), 317 deletions(-) diff --git a/doc/source/user/numpy-for-matlab-users.rst b/doc/source/user/numpy-for-matlab-users.rst index 547d5b2a0767..eb011de633c4 100644 --- a/doc/source/user/numpy-for-matlab-users.rst +++ b/doc/source/user/numpy-for-matlab-users.rst @@ -7,12 +7,9 @@ NumPy for MATLAB users Introduction ============ -MATLAB® and NumPy/SciPy have a lot in common. But there are many -differences. NumPy and SciPy were created to do numerical and scientific -computing in the most natural way with Python, not to be MATLAB clones. -This page is intended to be a place to collect wisdom about the -differences, mostly for the purpose of helping proficient MATLAB users -become proficient NumPy and SciPy users. +MATLAB® and NumPy have a lot in common, but NumPy was created to work with +Python, not to be a MATLAB clone. This guide will help MATLAB users get started +with NumPy. .. raw:: html @@ -20,234 +17,184 @@ become proficient NumPy and SciPy users. table.docutils td { border: solid 1px #ccc; } -Some Key Differences +Some key differences ==================== .. list-table:: - - * - In MATLAB, the basic data type is a multidimensional array of - double precision floating point numbers. Most expressions take such - arrays and return such arrays. Operations on the 2-D instances of - these arrays are designed to act more or less like matrix operations - in linear algebra. - - In NumPy the basic type is a multidimensional ``array``. Operations - on these arrays in all dimensionalities including 2D are element-wise - operations. One needs to use specific functions for linear algebra - (though for matrix multiplication, one can use the ``@`` operator - in python 3.5 and above). - - * - MATLAB uses 1 (one) based indexing. The initial element of a - sequence is found using a(1). + :class: docutils + + * - In MATLAB, the basic type, even for scalars, is a + multidimensional array. Array assignments in MATLAB are stored as + 2D arrays of double precision floating point numbers, unless you + specify the number of dimensions and type. Operations on the 2D + instances of these arrays are modeled on matrix operations in + linear algebra. + + - In NumPy, the basic type is a multidimensional ``array``. Array + assignments in NumPy are usually stored as :ref:`n-dimensional arrays` with the + minimum type required to hold the objects in sequence, unless you + specify the number of dimensions and type. NumPy performs + operations element-by-element, so multiplying 2D arrays with + ``*`` is not a matrix multiplication -- it's an + element-by-element multiplication. (The ``@`` operator, available + since Python 3.5, can be used for conventional matrix + multiplication.) + + * - MATLAB numbers indices from 1; ``a(1)`` is the first element. :ref:`See note INDEXING ` - - Python uses 0 (zero) based indexing. The initial element of a - sequence is found using a[0]. - - * - MATLAB's scripting language was created for doing linear algebra. - The syntax for basic matrix operations is nice and clean, but the API - for adding GUIs and making full-fledged applications is more or less - an afterthought. - - NumPy is based on Python, which was designed from the outset to be - an excellent general-purpose programming language. While MATLAB's - syntax for some array manipulations is more compact than - NumPy's, NumPy (by virtue of being an add-on to Python) can do many - things that MATLAB just cannot, for instance dealing properly with - stacks of matrices. - - * - In MATLAB, arrays have pass-by-value semantics, with a lazy - copy-on-write scheme to prevent actually creating copies until they - are actually needed. Slice operations copy parts of the array. - - In NumPy arrays have pass-by-reference semantics. Slice operations - are views into an array. - - -'array' or 'matrix'? Which should I use? -======================================== - -Historically, NumPy has provided a special matrix type, `np.matrix`, which -is a subclass of ndarray which makes binary operations linear algebra -operations. You may see it used in some existing code instead of `np.array`. -So, which one to use? - -Short answer ------------- - -**Use arrays**. - -- They are the standard vector/matrix/tensor type of numpy. Many numpy - functions return arrays, not matrices. -- There is a clear distinction between element-wise operations and - linear algebra operations. -- You can have standard vectors or row/column vectors if you like. - -Until Python 3.5 the only disadvantage of using the array type was that you -had to use ``dot`` instead of ``*`` to multiply (reduce) two tensors -(scalar product, matrix vector multiplication etc.). Since Python 3.5 you -can use the matrix multiplication ``@`` operator. - -Given the above, we intend to deprecate ``matrix`` eventually. - -Long answer ------------ - -NumPy contains both an ``array`` class and a ``matrix`` class. The -``array`` class is intended to be a general-purpose n-dimensional array -for many kinds of numerical computing, while ``matrix`` is intended to -facilitate linear algebra computations specifically. In practice there -are only a handful of key differences between the two. - -- Operators ``*`` and ``@``, functions ``dot()``, and ``multiply()``: - - - For ``array``, **``*`` means element-wise multiplication**, while - **``@`` means matrix multiplication**; they have associated functions - ``multiply()`` and ``dot()``. (Before python 3.5, ``@`` did not exist - and one had to use ``dot()`` for matrix multiplication). - - For ``matrix``, **``*`` means matrix multiplication**, and for - element-wise multiplication one has to use the ``multiply()`` function. - -- Handling of vectors (one-dimensional arrays) - - - For ``array``, the **vector shapes 1xN, Nx1, and N are all different - things**. Operations like ``A[:,1]`` return a one-dimensional array of - shape N, not a two-dimensional array of shape Nx1. Transpose on a - one-dimensional ``array`` does nothing. - - For ``matrix``, **one-dimensional arrays are always upconverted to 1xN - or Nx1 matrices** (row or column vectors). ``A[:,1]`` returns a - two-dimensional matrix of shape Nx1. - -- Handling of higher-dimensional arrays (ndim > 2) - - - ``array`` objects **can have number of dimensions > 2**; - - ``matrix`` objects **always have exactly two dimensions**. - -- Convenience attributes - - - ``array`` **has a .T attribute**, which returns the transpose of - the data. - - ``matrix`` **also has .H, .I, and .A attributes**, which return - the conjugate transpose, inverse, and ``asarray()`` of the matrix, - respectively. - -- Convenience constructor + - NumPy, like Python, numbers indices from 0; ``a[0]`` is the first + element. - - The ``array`` constructor **takes (nested) Python sequences as - initializers**. As in, ``array([[1,2,3],[4,5,6]])``. - - The ``matrix`` constructor additionally **takes a convenient - string initializer**. As in ``matrix("[1 2 3; 4 5 6]")``. - -There are pros and cons to using both: - -- ``array`` - - - ``:)`` Element-wise multiplication is easy: ``A*B``. - - ``:(`` You have to remember that matrix multiplication has its own - operator, ``@``. - - ``:)`` You can treat one-dimensional arrays as *either* row or column - vectors. ``A @ v`` treats ``v`` as a column vector, while - ``v @ A`` treats ``v`` as a row vector. This can save you having to - type a lot of transposes. - - ``:)`` ``array`` is the "default" NumPy type, so it gets the most - testing, and is the type most likely to be returned by 3rd party - code that uses NumPy. - - ``:)`` Is quite at home handling data of any number of dimensions. - - ``:)`` Closer in semantics to tensor algebra, if you are familiar - with that. - - ``:)`` *All* operations (``*``, ``/``, ``+``, ``-`` etc.) are - element-wise. - - ``:(`` Sparse matrices from ``scipy.sparse`` do not interact as well - with arrays. - -- ``matrix`` - - - ``:\\`` Behavior is more like that of MATLAB matrices. - - ``<:(`` Maximum of two-dimensional. To hold three-dimensional data you - need ``array`` or perhaps a Python list of ``matrix``. - - ``<:(`` Minimum of two-dimensional. You cannot have vectors. They must be - cast as single-column or single-row matrices. - - ``<:(`` Since ``array`` is the default in NumPy, some functions may - return an ``array`` even if you give them a ``matrix`` as an - argument. This shouldn't happen with NumPy functions (if it does - it's a bug), but 3rd party code based on NumPy may not honor type - preservation like NumPy does. - - ``:)`` ``A*B`` is matrix multiplication, so it looks just like you write - it in linear algebra (For Python >= 3.5 plain arrays have the same - convenience with the ``@`` operator). - - ``<:(`` Element-wise multiplication requires calling a function, - ``multiply(A,B)``. - - ``<:(`` The use of operator overloading is a bit illogical: ``*`` - does not work element-wise but ``/`` does. - - Interaction with ``scipy.sparse`` is a bit cleaner. - -The ``array`` is thus much more advisable to use. Indeed, we intend to -deprecate ``matrix`` eventually. - -Table of Rough MATLAB-NumPy Equivalents + * - MATLAB's scripting language was created for linear algebra so the + syntax for some array manipulations is more compact than + NumPy's. On the other hand, the API for adding GUIs and creating + full-fledged applications is more or less an afterthought. + - NumPy is based on Python, a + general-purpose language. The advantage to NumPy + is access to Python libraries including: `SciPy + `_, `Matplotlib `_, + `Pandas `_, `OpenCV `_, + and more. In addition, Python is often `embedded as a scripting language + `_ + in other software, allowing NumPy to be used there too. + + * - MATLAB array slicing uses pass-by-value semantics, with a lazy + copy-on-write scheme to prevent creating copies until they are + needed. Slicing operations copy parts of the array. + - NumPy array slicing uses pass-by-reference, that does not copy + the arguments. Slicing operations are views into an array. + + +Rough equivalents ======================================= The table below gives rough equivalents for some common MATLAB -expressions. **These are not exact equivalents**, but rather should be -taken as hints to get you going in the right direction. For more detail -read the built-in documentation on the NumPy functions. +expressions. These are similar expressions, not equivalents. For +details, see the :ref:`documentation`. In the table below, it is assumed that you have executed the following commands in Python: :: - from numpy import * - import scipy.linalg + import numpy as np + from scipy import io, integrate, linalg, signal + from scipy.sparse.linalg import eigs Also assume below that if the Notes talk about "matrix" that the arguments are two-dimensional entities. -General Purpose Equivalents +General purpose equivalents --------------------------- .. list-table:: :header-rows: 1 - * - **MATLAB** - - **numpy** - - **Notes** + * - MATLAB + - NumPy + - Notes * - ``help func`` - - ``info(func)`` or ``help(func)`` or ``func?`` (in Ipython) + - ``info(func)`` or ``help(func)`` or ``func?`` (in IPython) - get help on the function *func* * - ``which func`` - - `see note HELP `__ + - :ref:`see note HELP ` - find out where *func* is defined * - ``type func`` - - ``source(func)`` or ``func??`` (in Ipython) + - ``np.source(func)`` or ``func??`` (in IPython) - print source for *func* (if not a native function) + * - ``% comment`` + - ``# comment`` + - comment a line of code with the text ``comment`` + + * - :: + + for i=1:3 + fprintf('%i\n',i) + end + + - :: + + for i in range(1, 4): + print(i) + + - use a for-loop to print the numbers 1, 2, and 3 using :py:class:`range ` + * - ``a && b`` - ``a and b`` - - short-circuiting logical AND operator (Python native operator); + - short-circuiting logical AND operator (:ref:`Python native operator `); scalar arguments only * - ``a || b`` - ``a or b`` - - short-circuiting logical OR operator (Python native operator); + - short-circuiting logical OR operator (:ref:`Python native operator `); scalar arguments only + * - .. code:: matlab + + >> 4 == 4 + ans = 1 + >> 4 == 5 + ans = 0 + + - :: + + >>> 4 == 4 + True + >>> 4 == 5 + False + + - The :ref:`boolean objects ` + in Python are ``True`` and ``False``, as opposed to MATLAB + logical types of ``1`` and ``0``. + + * - .. code:: matlab + + a=4 + if a==4 + fprintf('a = 4\n') + elseif a==5 + fprintf('a = 5\n') + end + + - :: + + a = 4 + if a == 4: + print('a = 4') + elif a == 5: + print('a = 5') + + - create an if-else statement to check if ``a`` is 4 or 5 and print result + * - ``1*i``, ``1*j``, ``1i``, ``1j`` - ``1j`` - complex numbers * - ``eps`` - - ``np.spacing(1)`` - - Distance between 1 and the nearest floating point number. + - ``np.finfo(float).eps`` or ``np.spacing(1)`` + - Upper bound to relative error due to rounding in 64-bit floating point + arithmetic. + + * - ``load data.mat`` + - ``io.loadmat('data.mat')`` + - Load MATLAB variables saved to the file ``data.mat``. (Note: When saving arrays to + ``data.mat`` in MATLAB/Octave, use a recent binary format. :func:`scipy.io.loadmat` + will create a dictionary with the saved arrays and further information.) * - ``ode45`` - - ``scipy.integrate.solve_ivp(f)`` + - ``integrate.solve_ivp(f)`` - integrate an ODE with Runge-Kutta 4,5 * - ``ode15s`` - - ``scipy.integrate.solve_ivp(f, method='BDF')`` + - ``integrate.solve_ivp(f, method='BDF')`` - integrate an ODE with BDF method -Linear Algebra Equivalents + +Linear algebra equivalents -------------------------- .. list-table:: @@ -258,16 +205,16 @@ Linear Algebra Equivalents - Notes * - ``ndims(a)`` - - ``ndim(a)`` or ``a.ndim`` - - get the number of dimensions of an array + - ``np.ndim(a)`` or ``a.ndim`` + - number of dimensions of array ``a`` * - ``numel(a)`` - - ``size(a)`` or ``a.size`` - - get the number of elements of an array + - ``np.size(a)`` or ``a.size`` + - number of elements of array ``a`` * - ``size(a)`` - - ``shape(a)`` or ``a.shape`` - - get the "size" of the matrix + - ``np.shape(a)`` or ``a.shape`` + - "size" of array ``a`` * - ``size(a,n)`` - ``a.shape[n-1]`` @@ -276,45 +223,45 @@ Linear Algebra Equivalents See note :ref:`INDEXING `) * - ``[ 1 2 3; 4 5 6 ]`` - - ``array([[1.,2.,3.], [4.,5.,6.]])`` - - 2x3 matrix literal + - ``np.array([[1. ,2. ,3.], [4. ,5. ,6.]])`` + - define a 2x3 2D array * - ``[ a b; c d ]`` - - ``block([[a,b], [c,d]])`` + - ``np.block([[a, b], [c, d]])`` - construct a matrix from blocks ``a``, ``b``, ``c``, and ``d`` * - ``a(end)`` - ``a[-1]`` - - access last element in the 1xn matrix ``a`` + - access last element in MATLAB vector (1xn or nx1) or 1D NumPy array + ``a`` (length n) * - ``a(2,5)`` - - ``a[1,4]`` - - access element in second row, fifth column + - ``a[1, 4]`` + - access element in second row, fifth column in 2D array ``a`` * - ``a(2,:)`` - - ``a[1]`` or ``a[1,:]`` - - entire second row of ``a`` + - ``a[1]`` or ``a[1, :]`` + - entire second row of 2D array ``a`` * - ``a(1:5,:)`` - - ``a[0:5]`` or ``a[:5]`` or ``a[0:5,:]`` - - the first five rows of ``a`` + - ``a[0:5]`` or ``a[:5]`` or ``a[0:5, :]`` + - first 5 rows of 2D array ``a`` * - ``a(end-4:end,:)`` - ``a[-5:]`` - - the last five rows of ``a`` + - last 5 rows of 2D array ``a`` * - ``a(1:3,5:9)`` - - ``a[0:3][:,4:9]`` - - rows one to three and columns five to nine of ``a``. This gives - read-only access. + - ``a[0:3, 4:9]`` + - The first through third rows and fifth through ninth columns of a 2D array, ``a``. * - ``a([2,4,5],[1,3])`` - - ``a[ix_([1,3,4],[0,2])]`` + - ``a[np.ix_([1, 3, 4], [0, 2])]`` - rows 2,4 and 5 and columns 1 and 3. This allows the matrix to be modified, and doesn't require a regular slice. * - ``a(3:2:21,:)`` - - ``a[ 2:21:2,:]`` + - ``a[2:21:2,:]`` - every other row of ``a``, starting with the third and going to the twenty-first @@ -323,11 +270,11 @@ Linear Algebra Equivalents - every other row of ``a``, starting with the first * - ``a(end:-1:1,:)`` or ``flipud(a)`` - - ``a[ ::-1,:]`` + - ``a[::-1,:]`` - ``a`` with rows in reverse order * - ``a([1:end 1],:)`` - - ``a[r_[:len(a),0]]`` + - ``a[np.r_[:len(a),0]]`` - ``a`` with copy of the first row appended to the end * - ``a.'`` @@ -354,30 +301,30 @@ Linear Algebra Equivalents - ``a**3`` - element-wise exponentiation - * - ``(a>0.5)`` - - ``(a>0.5)`` + * - ``(a > 0.5)`` + - ``(a > 0.5)`` - matrix whose i,jth element is (a_ij > 0.5). The MATLAB result is an - array of 0s and 1s. The NumPy result is an array of the boolean + array of logical values 0 and 1. The NumPy result is an array of the boolean values ``False`` and ``True``. - * - ``find(a>0.5)`` - - ``nonzero(a>0.5)`` + * - ``find(a > 0.5)`` + - ``np.nonzero(a > 0.5)`` - find the indices where (``a`` > 0.5) - * - ``a(:,find(v>0.5))`` - - ``a[:,nonzero(v>0.5)[0]]`` + * - ``a(:,find(v > 0.5))`` + - ``a[:,np.nonzero(v > 0.5)[0]]`` - extract the columms of ``a`` where vector v > 0.5 * - ``a(:,find(v>0.5))`` - - ``a[:,v.T>0.5]`` + - ``a[:, v.T > 0.5]`` - extract the columms of ``a`` where column vector v > 0.5 * - ``a(a<0.5)=0`` - - ``a[a<0.5]=0`` + - ``a[a < 0.5]=0`` - ``a`` with elements less than 0.5 zeroed out * - ``a .* (a>0.5)`` - - ``a * (a>0.5)`` + - ``a * (a > 0.5)`` - ``a`` with elements less than 0.5 zeroed out * - ``a(:) = 3`` @@ -386,11 +333,11 @@ Linear Algebra Equivalents * - ``y=x`` - ``y = x.copy()`` - - numpy assigns by reference + - NumPy assigns by reference * - ``y=x(2,:)`` - - ``y = x[1,:].copy()`` - - numpy slices are by reference + - ``y = x[1, :].copy()`` + - NumPy slices are by reference * - ``y=x(:)`` - ``y = x.flatten()`` @@ -398,62 +345,74 @@ Linear Algebra Equivalents same data ordering as in MATLAB, use ``x.flatten('F')``. * - ``1:10`` - - ``arange(1.,11.)`` or ``r_[1.:11.]`` or ``r_[1:10:10j]`` + - ``np.arange(1., 11.)`` or ``np.r_[1.:11.]`` or ``np.r_[1:10:10j]`` - create an increasing vector (see note :ref:`RANGES `) * - ``0:9`` - - ``arange(10.)`` or ``r_[:10.]`` or ``r_[:9:10j]`` + - ``np.arange(10.)`` or ``np.r_[:10.]`` or ``np.r_[:9:10j]`` - create an increasing vector (see note :ref:`RANGES `) * - ``[1:10]'`` - - ``arange(1.,11.)[:, newaxis]`` + - ``np.arange(1.,11.)[:, np.newaxis]`` - create a column vector * - ``zeros(3,4)`` - - ``zeros((3,4))`` + - ``np.zeros((3, 4))`` - 3x4 two-dimensional array full of 64-bit floating point zeros * - ``zeros(3,4,5)`` - - ``zeros((3,4,5))`` + - ``np.zeros((3, 4, 5))`` - 3x4x5 three-dimensional array full of 64-bit floating point zeros * - ``ones(3,4)`` - - ``ones((3,4))`` + - ``np.ones((3, 4))`` - 3x4 two-dimensional array full of 64-bit floating point ones * - ``eye(3)`` - - ``eye(3)`` + - ``np.eye(3)`` - 3x3 identity matrix * - ``diag(a)`` - - ``diag(a)`` - - vector of diagonal elements of ``a`` + - ``np.diag(a)`` + - returns a vector of the diagonal elements of 2D array, ``a`` + + * - ``diag(v,0)`` + - ``np.diag(v, 0)`` + - returns a square diagonal matrix whose nonzero values are the elements of + vector, ``v`` - * - ``diag(a,0)`` - - ``diag(a,0)`` - - square diagonal matrix whose nonzero values are the elements of - ``a`` + * - .. code:: matlab + + rng(42,'twister') + rand(3,4) - * - ``rand(3,4)`` - - ``random.rand(3,4)`` or ``random.random_sample((3, 4))`` - - random 3x4 matrix + - :: + + from numpy.random import default_rng + rng = default_rng(42) + rng.random(3, 4) + + or older version: ``random.rand((3, 4))`` + + - generate a random 3x4 array with default random number generator and + seed = 42 * - ``linspace(1,3,4)`` - - ``linspace(1,3,4)`` + - ``np.linspace(1,3,4)`` - 4 equally spaced samples between 1 and 3, inclusive * - ``[x,y]=meshgrid(0:8,0:5)`` - - ``mgrid[0:9.,0:6.]`` or ``meshgrid(r_[0:9.],r_[0:6.]`` + - ``np.mgrid[0:9.,0:6.]`` or ``np.meshgrid(r_[0:9.],r_[0:6.]`` - two 2D arrays: one of x values, the other of y values * - - - ``ogrid[0:9.,0:6.]`` or ``ix_(r_[0:9.],r_[0:6.]`` + - ``ogrid[0:9.,0:6.]`` or ``np.ix_(np.r_[0:9.],np.r_[0:6.]`` - the best way to eval functions on a grid * - ``[x,y]=meshgrid([1,2,4],[2,4,5])`` - - ``meshgrid([1,2,4],[2,4,5])`` + - ``np.meshgrid([1,2,4],[2,4,5])`` - * - @@ -461,37 +420,38 @@ Linear Algebra Equivalents - the best way to eval functions on a grid * - ``repmat(a, m, n)`` - - ``tile(a, (m, n))`` + - ``np.tile(a, (m, n))`` - create m by n copies of ``a`` * - ``[a b]`` - - ``concatenate((a,b),1)`` or ``hstack((a,b))`` or - ``column_stack((a,b))`` or ``c_[a,b]`` + - ``np.concatenate((a,b),1)`` or ``np.hstack((a,b))`` or + ``np.column_stack((a,b))`` or ``np.c_[a,b]`` - concatenate columns of ``a`` and ``b`` * - ``[a; b]`` - - ``concatenate((a,b))`` or ``vstack((a,b))`` or ``r_[a,b]`` + - ``np.concatenate((a,b))`` or ``np.vstack((a,b))`` or ``np.r_[a,b]`` - concatenate rows of ``a`` and ``b`` * - ``max(max(a))`` - - ``a.max()`` - - maximum element of ``a`` (with ndims(a)<=2 for MATLAB) + - ``a.max()`` or ``np.nanmax(a)`` + - maximum element of ``a`` (with ndims(a)<=2 for MATLAB, if there are + NaN's, ``nanmax`` will ignore these and return largest value) * - ``max(a)`` - ``a.max(0)`` - - maximum element of each column of matrix ``a`` + - maximum element of each column of array ``a`` * - ``max(a,[],2)`` - ``a.max(1)`` - - maximum element of each row of matrix ``a`` + - maximum element of each row of array ``a`` * - ``max(a,b)`` - - ``maximum(a, b)`` + - ``np.maximum(a, b)`` - compares ``a`` and ``b`` element-wise, and returns the maximum value from each pair * - ``norm(v)`` - - ``sqrt(v @ v)`` or ``np.linalg.norm(v)`` + - ``np.sqrt(v @ v)`` or ``np.linalg.norm(v)`` - L2 norm of vector ``v`` * - ``a & b`` @@ -500,7 +460,7 @@ Linear Algebra Equivalents LOGICOPS ` * - ``a | b`` - - ``logical_or(a,b)`` + - ``np.logical_or(a,b)`` - element-by-element OR operator (NumPy ufunc) :ref:`See note LOGICOPS ` @@ -514,90 +474,99 @@ Linear Algebra Equivalents * - ``inv(a)`` - ``linalg.inv(a)`` - - inverse of square matrix ``a`` + - inverse of square 2D array ``a`` * - ``pinv(a)`` - ``linalg.pinv(a)`` - - pseudo-inverse of matrix ``a`` + - pseudo-inverse of 2D array ``a`` * - ``rank(a)`` - ``linalg.matrix_rank(a)`` - - matrix rank of a 2D array / matrix ``a`` + - matrix rank of a 2D array ``a`` * - ``a\b`` - - ``linalg.solve(a,b)`` if ``a`` is square; ``linalg.lstsq(a,b)`` + - ``linalg.solve(a, b)`` if ``a`` is square; ``linalg.lstsq(a, b)`` otherwise - solution of a x = b for x * - ``b/a`` - - Solve a.T x.T = b.T instead + - Solve ``a.T x.T = b.T`` instead - solution of x a = b for x * - ``[U,S,V]=svd(a)`` - ``U, S, Vh = linalg.svd(a), V = Vh.T`` - singular value decomposition of ``a`` - * - ``chol(a)`` - - ``linalg.cholesky(a).T`` - - cholesky factorization of a matrix (``chol(a)`` in MATLAB returns an - upper triangular matrix, but ``linalg.cholesky(a)`` returns a lower - triangular matrix) + * - ``c=chol(a)`` where ``a==c'*c`` + - ``c = linalg.cholesky(a)`` where ``a == c@c.T`` + - Cholesky factorization of a 2D array (``chol(a)`` in MATLAB returns an + upper triangular 2D array, but :func:`~scipy.linalg.cholesky` returns a lower + triangular 2D array) * - ``[V,D]=eig(a)`` - ``D,V = linalg.eig(a)`` - - eigenvalues and eigenvectors of ``a`` + - eigenvalues :math:`\lambda` and eigenvectors :math:`\bar{v}` of ``a``, + where :math:`\lambda\bar{v}=\mathbf{a}\bar{v}` * - ``[V,D]=eig(a,b)`` - - ``D,V = scipy.linalg.eig(a,b)`` - - eigenvalues and eigenvectors of ``a``, ``b`` + - ``D,V = linalg.eig(a, b)`` + - eigenvalues :math:`\lambda` and eigenvectors :math:`\bar{v}` of + ``a``, ``b`` + where :math:`\lambda\mathbf{b}\bar{v}=\mathbf{a}\bar{v}` - * - ``[V,D]=eigs(a,k)`` - - - - find the ``k`` largest eigenvalues and eigenvectors of ``a`` + * - ``[V,D]=eigs(a,3)`` + - ``D,V = eigs(a, k = 3)`` + - find the ``k=3`` largest eigenvalues and eigenvectors of 2D array, ``a`` * - ``[Q,R,P]=qr(a,0)`` - - ``Q,R = scipy.linalg.qr(a)`` + - ``Q,R = linalg.qr(a)`` - QR decomposition - * - ``[L,U,P]=lu(a)`` - - ``L,U = scipy.linalg.lu(a)`` or ``LU,P=scipy.linalg.lu_factor(a)`` - - LU decomposition (note: P(MATLAB) == transpose(P(numpy)) ) + * - ``[L,U,P]=lu(a)`` where ``a==P'*L*U`` + - ``P,L,U = linalg.lu(a)`` where ``a == P@L@U`` + - LU decomposition (note: P(MATLAB) == transpose(P(NumPy))) * - ``conjgrad`` - - ``scipy.sparse.linalg.cg`` + - ``cg`` - Conjugate gradients solver * - ``fft(a)`` - - ``fft(a)`` + - ``np.fft(a)`` - Fourier transform of ``a`` * - ``ifft(a)`` - - ``ifft(a)`` + - ``np.ifft(a)`` - inverse Fourier transform of ``a`` * - ``sort(a)`` - - ``sort(a)`` or ``a.sort()`` - - sort the matrix + - ``np.sort(a)`` or ``a.sort(axis=0)`` + - sort each column of a 2D array, ``a`` - * - ``[b,I] = sortrows(a,i)`` - - ``I = argsort(a[:,i]), b=a[I,:]`` - - sort the rows of the matrix + * - ``sort(a, 2)`` + - ``np.sort(a, axis = 1)`` or ``a.sort(axis = 1)`` + - sort the each row of 2D array, ``a`` - * - ``regress(y,X)`` - - ``linalg.lstsq(X,y)`` - - multilinear regression + * - ``[b,I]=sortrows(a,1)`` + - ``I = np.argsort(a[:, 0]); b = a[I,:]`` + - save the array ``a`` as array ``b`` with rows sorted by the first column + + * - ``x = Z\y`` + - ``x = linalg.lstsq(Z, y)`` + - perform a linear regression of the form :math:`\mathbf{Zx}=\mathbf{y}` * - ``decimate(x, q)`` - - ``scipy.signal.resample(x, len(x)/q)`` + - ``signal.resample(x, np.ceil(len(x)/q))`` - downsample with low-pass filtering * - ``unique(a)`` - - ``unique(a)`` - - + - ``np.unique(a)`` + - a vector of unique values in array ``a`` * - ``squeeze(a)`` - ``a.squeeze()`` - - + - remove singleton dimensions of array ``a``. Note that MATLAB will always + return arrays of 2D or higher while NumPy will return arrays of 0D or + higher .. _numpy-for-matlab-users.notes: @@ -605,11 +574,11 @@ Notes ===== \ **Submatrix**: Assignment to a submatrix can be done with lists of -indexes using the ``ix_`` command. E.g., for 2d array ``a``, one might -do: ``ind=[1,3]; a[np.ix_(ind,ind)]+=100``. +indices using the ``ix_`` command. E.g., for 2D array ``a``, one might +do: ``ind=[1, 3]; a[np.ix_(ind, ind)] += 100``. \ **HELP**: There is no direct equivalent of MATLAB's ``which`` command, -but the commands ``help`` and ``source`` will usually list the filename +but the commands :func:`help`` and :func:`numpy.source` will usually list the filename where the function is located. Python also has an ``inspect`` module (do ``import inspect``) which provides a ``getfile`` that often works. @@ -627,35 +596,35 @@ Dijkstra ; MATLAB's is the reverse. + operators like ``<`` and ``>``; MATLAB's is the reverse. If you know you have boolean arguments, you can get away with using -NumPy's bitwise operators, but be careful with parentheses, like this: z -= (x > 1) & (x < 2). The absence of NumPy operator forms of logical\_and -and logical\_or is an unfortunate consequence of Python's design. +NumPy's bitwise operators, but be careful with parentheses, like this: ``z += (x > 1) & (x < 2)``. The absence of NumPy operator forms of ``logical_and`` +and ``logical_or`` is an unfortunate consequence of Python's design. **RESHAPE and LINEAR INDEXING**: MATLAB always allows multi-dimensional arrays to be accessed using scalar or linear indices, NumPy does not. -Linear indices are common in MATLAB programs, e.g. find() on a matrix +Linear indices are common in MATLAB programs, e.g. ``find()`` on a matrix returns them, whereas NumPy's find behaves differently. When converting MATLAB code it might be necessary to first reshape a matrix to a linear sequence, perform some indexing operations and then reshape back. As @@ -664,11 +633,132 @@ possible to do this fairly efficiently. Note that the scan order used by reshape in NumPy defaults to the 'C' order, whereas MATLAB uses the Fortran order. If you are simply converting to a linear sequence and back this doesn't matter. But if you are converting reshapes from MATLAB -code which relies on the scan order, then this MATLAB code: z = -reshape(x,3,4); should become z = x.reshape(3,4,order='F').copy() in +code which relies on the scan order, then this MATLAB code: ``z = +reshape(x,3,4);`` should become ``z = x.reshape(3,4,order='F').copy()`` in NumPy. -Customizing Your Environment +'array' or 'matrix'? Which should I use? +======================================== + +Historically, NumPy has provided a special matrix type, `np.matrix`, which +is a subclass of ndarray which makes binary operations linear algebra +operations. You may see it used in some existing code instead of `np.array`. +So, which one to use? + +Short answer +------------ + +**Use arrays**. + +- They support multidimensional array algebra that is supported in MATLAB +- They are the standard vector/matrix/tensor type of NumPy. Many NumPy + functions return arrays, not matrices. +- There is a clear distinction between element-wise operations and + linear algebra operations. +- You can have standard vectors or row/column vectors if you like. + +Until Python 3.5 the only disadvantage of using the array type was that you +had to use ``dot`` instead of ``*`` to multiply (reduce) two tensors +(scalar product, matrix vector multiplication etc.). Since Python 3.5 you +can use the matrix multiplication ``@`` operator. + +Given the above, we intend to deprecate ``matrix`` eventually. + +Long answer +----------- + +NumPy contains both an ``array`` class and a ``matrix`` class. The +``array`` class is intended to be a general-purpose n-dimensional array +for many kinds of numerical computing, while ``matrix`` is intended to +facilitate linear algebra computations specifically. In practice there +are only a handful of key differences between the two. + +- Operators ``*`` and ``@``, functions ``dot()``, and ``multiply()``: + + - For ``array``, **``*`` means element-wise multiplication**, while + **``@`` means matrix multiplication**; they have associated functions + ``multiply()`` and ``dot()``. (Before Python 3.5, ``@`` did not exist + and one had to use ``dot()`` for matrix multiplication). + - For ``matrix``, **``*`` means matrix multiplication**, and for + element-wise multiplication one has to use the ``multiply()`` function. + +- Handling of vectors (one-dimensional arrays) + + - For ``array``, the **vector shapes 1xN, Nx1, and N are all different + things**. Operations like ``A[:,1]`` return a one-dimensional array of + shape N, not a two-dimensional array of shape Nx1. Transpose on a + one-dimensional ``array`` does nothing. + - For ``matrix``, **one-dimensional arrays are always upconverted to 1xN + or Nx1 matrices** (row or column vectors). ``A[:,1]`` returns a + two-dimensional matrix of shape Nx1. + +- Handling of higher-dimensional arrays (ndim > 2) + + - ``array`` objects **can have number of dimensions > 2**; + - ``matrix`` objects **always have exactly two dimensions**. + +- Convenience attributes + + - ``array`` **has a .T attribute**, which returns the transpose of + the data. + - ``matrix`` **also has .H, .I, and .A attributes**, which return + the conjugate transpose, inverse, and ``asarray()`` of the matrix, + respectively. + +- Convenience constructor + + - The ``array`` constructor **takes (nested) Python sequences as + initializers**. As in, ``array([[1,2,3],[4,5,6]])``. + - The ``matrix`` constructor additionally **takes a convenient + string initializer**. As in ``matrix("[1 2 3; 4 5 6]")``. + +There are pros and cons to using both: + +- ``array`` + + - ``:)`` Element-wise multiplication is easy: ``A*B``. + - ``:(`` You have to remember that matrix multiplication has its own + operator, ``@``. + - ``:)`` You can treat one-dimensional arrays as *either* row or column + vectors. ``A @ v`` treats ``v`` as a column vector, while + ``v @ A`` treats ``v`` as a row vector. This can save you having to + type a lot of transposes. + - ``:)`` ``array`` is the "default" NumPy type, so it gets the most + testing, and is the type most likely to be returned by 3rd party + code that uses NumPy. + - ``:)`` Is quite at home handling data of any number of dimensions. + - ``:)`` Closer in semantics to tensor algebra, if you are familiar + with that. + - ``:)`` *All* operations (``*``, ``/``, ``+``, ``-`` etc.) are + element-wise. + - ``:(`` Sparse matrices from ``scipy.sparse`` do not interact as well + with arrays. + +- ``matrix`` + + - ``:\\`` Behavior is more like that of MATLAB matrices. + - ``<:(`` Maximum of two-dimensional. To hold three-dimensional data you + need ``array`` or perhaps a Python list of ``matrix``. + - ``<:(`` Minimum of two-dimensional. You cannot have vectors. They must be + cast as single-column or single-row matrices. + - ``<:(`` Since ``array`` is the default in NumPy, some functions may + return an ``array`` even if you give them a ``matrix`` as an + argument. This shouldn't happen with NumPy functions (if it does + it's a bug), but 3rd party code based on NumPy may not honor type + preservation like NumPy does. + - ``:)`` ``A*B`` is matrix multiplication, so it looks just like you write + it in linear algebra (For Python >= 3.5 plain arrays have the same + convenience with the ``@`` operator). + - ``<:(`` Element-wise multiplication requires calling a function, + ``multiply(A,B)``. + - ``<:(`` The use of operator overloading is a bit illogical: ``*`` + does not work element-wise but ``/`` does. + - Interaction with ``scipy.sparse`` is a bit cleaner. + +The ``array`` is thus much more advisable to use. Indeed, we intend to +deprecate ``matrix`` eventually. + +Customizing your environment ============================ In MATLAB the main tool available to you for customizing the @@ -696,26 +786,39 @@ this is just an example, not a statement of "best practices"): # Make all numpy available via shorter 'np' prefix import numpy as np - # Make all matlib functions accessible at the top level via M.func() - import numpy.matlib as M - # Make some matlib functions accessible directly at the top level via, e.g. rand(3,3) - from numpy.matlib import rand,zeros,ones,empty,eye + # + # Make the SciPy linear algebra functions available as linalg.func() + # e.g. linalg.lu, linalg.eig (for general l*B@u==A@u solution) + from scipy import linalg + # # Define a Hermitian function def hermitian(A, **kwargs): - return np.transpose(A,**kwargs).conj() - # Make some shortcuts for transpose,hermitian: - # np.transpose(A) --> T(A) + return np.conj(A,**kwargs).T + # Make a shortcut for hermitian: # hermitian(A) --> H(A) - T = np.transpose H = hermitian +To use the deprecated `matrix` and other `matlib` functions: + +:: + + # Make all matlib functions accessible at the top level via M.func() + import numpy.matlib as M + # Make some matlib functions accessible directly at the top level via, e.g. rand(3,3) + from numpy.matlib import matrix,rand,zeros,ones,empty,eye + Links ===== -See http://mathesaurus.sf.net/ for another MATLAB/NumPy -cross-reference. +Another somewhat outdated MATLAB/NumPy cross-reference can be found at +http://mathesaurus.sf.net/ -An extensive list of tools for scientific work with python can be +An extensive list of tools for scientific work with Python can be found in the `topical software page `__. -MATLAB® and SimuLink® are registered trademarks of The MathWorks. +See +`List of Python software: scripting +`_ +for a list of softwares that use Python as a scripting language + +MATLAB® and SimuLink® are registered trademarks of The MathWorks, Inc. From d5718145fff9bc70d3e3143a85ef42bd910209ce Mon Sep 17 00:00:00 2001 From: mattip Date: Wed, 16 Sep 2020 11:25:15 +0300 Subject: [PATCH 049/409] adjust whitespace to minimize the diff --- doc/source/glossary.rst | 602 +++++++++++++++++++--------------------- 1 file changed, 288 insertions(+), 314 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 11439187a01b..978589996a5b 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -2,463 +2,443 @@ Glossary ******** -.. glossary:: +.. glossary:: - (`n`,) + (`n`,) + A tuple with one element. The trailing comma distinguishes a one-element + tuple from a parenthesized ``n``. - A tuple with one element. The trailing comma distinguishes a one-element - tuple from a parenthesized ``n``. + -1 + Used as a dimension entry, ``-1`` instructs NumPy to choose the length + that will keep the total number of elements the same. - -1 - Used as a dimension entry, ``-1`` instructs NumPy to choose the length - that will keep the total number of elements the same. + ``...`` + **When indexing an array**, shorthand that the missing axes, if they + exist, are full slices. - ``...`` + >>> a = np.arange(24).reshape(2,3,4) - **When indexing an array**, shorthand that the missing axes, if they - exist, are full slices. + >>> a[...].shape + (2, 3, 4) - >>> a = np.arange(24).reshape(2,3,4) + >>> a[...,0].shape + (2, 3) - >>> a[...].shape - (2, 3, 4) + >>> a[0,...].shape + (3, 4) - >>> a[...,0].shape - (2, 3) + >>> a[0,...,0].shape + (3,) - >>> a[0,...].shape - (3, 4) + It can be used at most once; ``a[...,0,...]`` raises an ``IndexError``. - >>> a[0,...,0].shape - (3,) + **In printouts**, NumPy substitutes ``...`` for the middle elements of + large arrays. To see the entire array, use + :doc:`numpy.printoptions. ` - It can be used at most once; ``a[...,0,...]`` raises an ``IndexError``. - **In printouts**, NumPy substitutes ``...`` for the middle elements of - large arrays. To see the entire array, use - :doc:`numpy.printoptions. ` + ``:`` + The Python :term:`python:slice` + operator. In ndarrays, slicing can be applied to every + axis: - ``:`` + >>> a = np.arange(24).reshape(2,3,4) + >>> a + array([[[ 0, 1, 2, 3], + [ 4, 5, 6, 7], + [ 8, 9, 10, 11]], + + [[12, 13, 14, 15], + [16, 17, 18, 19], + [20, 21, 22, 23]]]) + + >>> a[1:,-2:,:-1] + array([[[16, 17, 18], + [20, 21, 22]]]) - The Python :term:`python:slice` - operator. In ndarrays, slicing can be applied to every - axis: + Trailing slices can be omitted: :: - >>> a = np.arange(24).reshape(2,3,4) - >>> a - array([[[ 0, 1, 2, 3], - [ 4, 5, 6, 7], - [ 8, 9, 10, 11]], - - [[12, 13, 14, 15], - [16, 17, 18, 19], - [20, 21, 22, 23]]]) - - >>> a[1:,-2:,:-1] - array([[[16, 17, 18], - [20, 21, 22]]]) + >>> a[1] == a[1,:,:] + array([[ True, True, True, True], + [ True, True, True, True], + [ True, True, True, True]]) - Trailing slices can be omitted: :: + In contrast to Python, where slicing creates a copy, in NumPy slicing + creates a :term:`view`. - >>> a[1] == a[1,:,:] - array([[ True, True, True, True], - [ True, True, True, True], - [ True, True, True, True]]) + For details, see :ref:`Combining advanced and basic indexing `. - In contrast to Python, where slicing creates a copy, in NumPy slicing - creates a :term:`view`. - For details, see :ref:`Combining advanced and basic indexing `. + ``<`` + In a dtype declaration, indicates that the data is + `little-endian `_ + (the bracket is big on the right). :: - ``<`` + >>> dt = np.dtype('`_ - (the bracket is big on the right). :: - >>> dt = np.dtype('`` + In a dtype declaration, indicates that the data is + `big-endian `_ + (the bracket is big on the left). :: - ``>`` + >>> dt = np.dtype('>H') # big-endian unsigned short - In a dtype declaration, indicates that the data is - `big-endian `_ - (the bracket is big on the left). :: - >>> dt = np.dtype('>H') # big-endian unsigned short + advanced indexing + Rather than using a :doc:`scalar ` or slice as + an index, an axis can be indexed with an array, providing fine-grained + selection. This is known as :ref:`advanced indexing` + or ``fancy indexing``. - advanced indexing - Rather than using a :doc:`scalar ` or slice as - an index, an axis can be indexed with an array, providing fine-grained - selection. This is known as :ref:`advanced indexing` - or ``fancy indexing``. + along an axis + The result of an operation along an :term:`axis` X is an array in which X + disappears. This can surprise new users expecting the opposite. - along an axis + The operation can be visualized this way: - The result of an operation along an :term:`axis` X is an array in which X - disappears. This can surprise new users expecting the opposite. + Imagine a slice of array ``a`` where axis X has a fixed index + and the other dimensions are left full (``:``). - The operation can be visualized this way: + >>> a = np.arange(24).reshape(2,3,4) + + >>> a.shape + (2, 3, 4) + + >>> a[:,0,:].shape + (2, 4) - Imagine a slice of array ``a`` where axis X has a fixed index - and the other dimensions are left full (``:``). + The slice has ``a``'s shape with the X dimension deleted. Saying an + operation ``op`` is ``performed along X`` means that ``op`` takes as its + operands slices having every value of X: - >>> a = np.arange(24).reshape(2,3,4) - - >>> a.shape - (2, 3, 4) - - >>> a[:,0,:].shape - (2, 4) + >>> np.sum(a,axis=1) == a[:,0,:] + a[:,1,:] + a[:,2,:] + array([[ True, True, True, True], + [ True, True, True, True]]) - The slice has ``a``'s shape with the X dimension deleted. Saying an - operation ``op`` is ``performed along X`` means that ``op`` takes as its - operands slices having every value of X: - >>> np.sum(a,axis=1) == a[:,0,:] + a[:,1,:] + a[:,2,:] - array([[ True, True, True, True], - [ True, True, True, True]]) - - - array + array Used synonymously in the NumPy docs with :doc:`ndarray `, NumPy's basic structure. - array_like - - Any :doc:`scalar ` or - :term:`python:sequence` - that can be interpreted as an ndarray. In addition to ndarrays - and scalars this category includes lists (possibly nested and with - different element types) and tuples. Any argument accepted by - :doc:`numpy.array ` - is array_like. :: - - >>> a = np.array([[1,2.0],[0,0],(1+1j,3.)]) + array_like + Any :doc:`scalar ` or + :term:`python:sequence` + that can be interpreted as an ndarray. In addition to ndarrays + and scalars this category includes lists (possibly nested and with + different element types) and tuples. Any argument accepted by + :doc:`numpy.array ` + is array_like. :: - >>> a - array([[1.+0.j, 2.+0.j], - [0.+0.j, 0.+0.j], - [1.+1.j, 3.+0.j]]) + >>> a = np.array([[1,2.0],[0,0],(1+1j,3.)]) + >>> a + array([[1.+0.j, 2.+0.j], + [0.+0.j, 0.+0.j], + [1.+1.j, 3.+0.j]]) - array scalar - - For uniformity in handling operands, NumPy treats - a :doc:`scalar ` as an array of zero - dimension. - - - :term:`attribute ` - \ + array scalar + For uniformity in handling operands, NumPy treats + a :doc:`scalar ` as an array of zero + dimension. - axis - Another term for an array dimension. Axes are numbered left to right; - axis 0 is the first element in the shape tuple. + axis + Another term for an array dimension. Axes are numbered left to right; + axis 0 is the first element in the shape tuple. - In a two-dimensional vector, the elements of axis 0 are rows and the - elements of axis 1 are columns. + In a two-dimensional vector, the elements of axis 0 are rows and the + elements of axis 1 are columns. - In higher dimensions the picture changes. NumPy prints - higher-dimensional vectors as replications of row-by-column building - blocks, as in this three-dimensional vector: + In higher dimensions the picture changes. NumPy prints + higher-dimensional vectors as replications of row-by-column building + blocks, as in this three-dimensional vector: - >>> a = np.arange(12).reshape(2,2,3) - >>> a - array([[[ 0, 1, 2], - [ 3, 4, 5]], - - [[ 6, 7, 8], - [ 9, 10, 11]]]) + >>> a = np.arange(12).reshape(2,2,3) + >>> a + array([[[ 0, 1, 2], + [ 3, 4, 5]], + + [[ 6, 7, 8], + [ 9, 10, 11]]]) - ``a`` is depicted as a two-element array whose elements are 2x3 vectors. - From this point of view, rows and columns are the final two axes, - respectively, in any shape. + ``a`` is depicted as a two-element array whose elements are 2x3 vectors. + From this point of view, rows and columns are the final two axes, + respectively, in any shape. - This rule helps you anticipate how a vector will be printed, and - conversely how to find the index of any of the printed elements. For - instance, in the example, the last two values of 8's index must be 0 and - 2. Since 8 appears in the second of the two 2x3's, the first index must - be 1: + This rule helps you anticipate how a vector will be printed, and + conversely how to find the index of any of the printed elements. For + instance, in the example, the last two values of 8's index must be 0 and + 2. Since 8 appears in the second of the two 2x3's, the first index must + be 1: - >>> a[1,0,2] - 8 + >>> a[1,0,2] + 8 - A convenient way to count dimensions in a printed vector is to - count ``[`` symbols after the open-parenthesis. This is - useful in distinguishing, say, a (1,2,3) shape from a (2,3) shape: + A convenient way to count dimensions in a printed vector is to + count ``[`` symbols after the open-parenthesis. This is + useful in distinguishing, say, a (1,2,3) shape from a (2,3) shape: - >>> a = np.arange(6).reshape(2,3) - >>> a.shape - (2, 3) - >>> a - array([[0, 1, 2], - [3, 4, 5]]) + >>> a = np.arange(6).reshape(2,3) + >>> a.shape + (2, 3) + >>> a + array([[0, 1, 2], + [3, 4, 5]]) - >>> a = np.arange(6).reshape(1,2,3) - >>> a.shape - (1, 2, 3) - >>> a - array([[[0, 1, 2], - [3, 4, 5]]]) + >>> a = np.arange(6).reshape(1,2,3) + >>> a.shape + (1, 2, 3) + >>> a + array([[[0, 1, 2], + [3, 4, 5]]]) - .base + .base - If an array does not own its memory, then its - :doc:`base ` attribute - returns the object whose memory the array is referencing. That object - may be borrowing the memory from still another object, so the - owning object may be ``a.base.base.base...``. Despite advice to the - contrary, testing ``base`` is not a surefire way to determine if two - arrays are :term:`view`\ s. + If an array does not own its memory, then its + :doc:`base ` attribute + returns the object whose memory the array is referencing. That object + may be borrowing the memory from still another object, so the + owning object may be ``a.base.base.base...``. Despite advice to the + contrary, testing ``base`` is not a surefire way to determine if two + arrays are :term:`view`\ s. - `big-endian `_ + `big-endian `_ \ - `BLAS `_ + `BLAS `_ \ - broadcast + broadcast + ``broadcasting`` is NumPy's ability to process ndarrays of + different sizes as if all were the same size. - ``broadcasting`` is NumPy's ability to process ndarrays of - different sizes as if all were the same size. + When NumPy operates on two arrays, it works element by + element -- for instance, ``c = a * b`` is :: - When NumPy operates on two arrays, it works element by - element -- for instance, ``c = a * b`` is :: - - c[0,0,0] = a[0,0,0] * b[0,0,0] - c[0,0,1] = a[0,0,1] * b[0,0,1] + c[0,0,0] = a[0,0,0] * b[0,0,0] + c[0,0,1] = a[0,0,1] * b[0,0,1] ... - Ordinarily this means the shapes of a and b must be identical. But in - some cases, NumPy can fill "missing" axes or "too-short" dimensions - with duplicate data so shapes will match. The duplication costs - no memory or time. For details, see :doc:`Broadcasting. ` - + Ordinarily this means the shapes of a and b must be identical. But in + some cases, NumPy can fill "missing" axes or "too-short" dimensions + with duplicate data so shapes will match. The duplication costs + no memory or time. For details, see :doc:`Broadcasting. ` - C order - Same as `row-major. `_ + C order + Same as `row-major. `_ - `column-major `_ - \ + `column-major `_ + \ - copy + copy - See :term:`view`. + See :term:`view`. - :term:`decorator ` + :term:`decorator ` \ - :term:`dictionary ` + :term:`dictionary ` \ - dimension + dimension - See :term:`axis`. + See :term:`axis`. - dtype + dtype - The datatype describing the (identically typed) elements in an ndarray. - It can be changed to reinterpret the array contents. For details, see - :doc:`Data type objects (dtype). ` + The datatype describing the (identically typed) elements in an ndarray. + It can be changed to reinterpret the array contents. For details, see + :doc:`Data type objects (dtype). ` - fancy indexing + fancy indexing Another term for :term:`advanced indexing`. - field - + field In a :term:`structured data type`, each subtype is called a :doc:`field `. A field has a name (a string), a type (any valid dtype), and an optional :term:`title`. For details, see :ref:`arrays.dtypes`. - Fortran order - + Fortran order Same as `column-major `_ - flattened - + flattened See :term:`ravel`. - homogeneous - - All elements of a homogeneous array have the same type. ndarrays, in - contrast to Python lists, are homogeneous. The type can be complicated, - as in a :term:`structured array`, but all elements have that type. + homogeneous + All elements of a homogeneous array have the same type. ndarrays, in + contrast to Python lists, are homogeneous. The type can be complicated, + as in a :term:`structured array`, but all elements have that type. NumPy `object arrays <#term-object-array>`_, which contain references to Python objects, fill the role of heterogeneous arrays. - :term:`immutable ` + :term:`immutable ` \ - :term:`iterable ` + :term:`iterable ` \ - itemsize - - The size of the dtype element in bytes. + itemsize + The size of the dtype element in bytes. - :term:`list ` + :term:`list ` \ - `little-endian `_ + `little-endian `_ \ - mask - + mask The boolean array used to select elements in a :term:`masked array`. - masked array - - Bad or missing data can be cleanly ignored by putting it in a masked - array, which has an internal boolean array indicating invalid - entries. Operations with masked arrays ignore these entries. :: + masked array + Bad or missing data can be cleanly ignored by putting it in a masked + array, which has an internal boolean array indicating invalid + entries. Operations with masked arrays ignore these entries. :: - >>> a = np.ma.masked_array([np.nan, 2, np.nan], [True, False, True]) - >>> a - masked_array(data=[--, 2.0, --], - mask=[ True, False, True], - fill_value=1e+20) + >>> a = np.ma.masked_array([np.nan, 2, np.nan], [True, False, True]) + >>> a + masked_array(data=[--, 2.0, --], + mask=[ True, False, True], + fill_value=1e+20) >>> a + [1, 2, 3] masked_array(data=[--, 4.0, --], mask=[ True, False, True], fill_value=1e+20) - For details, see :doc:`Masked arrays. ` - + For details, see :doc:`Masked arrays. ` - matrix - NumPy's two-dimensional - :doc:`matrix class ` - should no longer be used; use regular ndarrays. + matrix + NumPy's two-dimensional + :doc:`matrix class ` + should no longer be used; use regular ndarrays. - ndarray + ndarray See :term:`array`. - object array + object array - An array whose dtype is ``object``; that is, it contains references to - Python objects. Indexing the array dereferences the Python objects, so - unlike other ndarrays, an object array has the ability to hold - heterogeneous objects. + An array whose dtype is ``object``; that is, it contains references to + Python objects. Indexing the array dereferences the Python objects, so + unlike other ndarrays, an object array has the ability to hold + heterogeneous objects. - ravel + ravel - :doc:`numpy.ravel \ - ` - and :doc:`numpy.flatten \ - ` - both flatten an ndarray. ``ravel`` will return a view if possible; - ``flatten`` always returns a copy. + :doc:`numpy.ravel \ + ` + and :doc:`numpy.flatten \ + ` + both flatten an ndarray. ``ravel`` will return a view if possible; + ``flatten`` always returns a copy. - Flattening collapses a multimdimensional array to a single dimension; - details of how this is done (for instance, whether ``a[n+1]`` should be - the next row or next column) are parameters. + Flattening collapses a multidimensional array to a single dimension; + details of how this is done (for instance, whether ``a[n+1]`` should be + the next row or next column) are parameters. - record array - + record array A :term:`structured array` with an additional way to access fields -- ``a.field`` in addition to ``a['field']``. For details, see :doc:`numpy.recarray. ` - `row-major `_ + `row-major `_ \ - :doc:`scalar ` + :doc:`scalar ` \ - shape + shape - A tuple showing the length of each dimension of an ndarray. The - length of the tuple itself is the number of dimensions - (:doc:`numpy.ndim `). - The product of the tuple elements is the number of elements in the - array. For details, see - :doc:`numpy.ndarray.shape `. + A tuple showing the length of each dimension of an ndarray. The + length of the tuple itself is the number of dimensions + (:doc:`numpy.ndim `). + The product of the tuple elements is the number of elements in the + array. For details, see + :doc:`numpy.ndarray.shape `. - :term:`slice <:>` + :term:`slice <:>` \ - stride + stride - Physical memory is one-dimensional; ``stride`` maps an index in an - N-dimensional ndarray to an address in memory. For an N-dimensional - array, stride is an N-element tuple; advancing from index ``i`` to index - ``i+1`` on axis ``n`` means adding ``a.strides[n]`` bytes to the - address. + Physical memory is one-dimensional; ``stride`` maps an index in an + N-dimensional ndarray to an address in memory. For an N-dimensional + array, stride is an N-element tuple; advancing from index ``i`` to index + ``i+1`` on axis ``n`` means adding ``a.strides[n]`` bytes to the + address. - Stride is computed automatically from an array's dtype and - shape, but can be directly specified using - :doc:`as_strided. ` + Stride is computed automatically from an array's dtype and + shape, but can be directly specified using + :doc:`as_strided. ` - For details, see - :doc:`numpy.ndarray.strides `. + For details, see + :doc:`numpy.ndarray.strides `. - To see how striding underlies the power of NumPy views, see - `The NumPy array: a structure for efficient numerical computation. \ - `_ + To see how striding underlies the power of NumPy views, see + `The NumPy array: a structure for efficient numerical computation. \ + `_ - structured array - Array whose :term:`dtype` is a :term:`structured data type`. + structured array + Array whose :term:`dtype` is a :term:`structured data type`. - structured data type - Users can create arbitrarily complex :term:`dtypes ` - that can include other arrays and dtypes. These composite dtypes are called - :doc:`structured data types. ` + structured data type + Users can create arbitrarily complex :term:`dtypes ` + that can include other arrays and dtypes. These composite dtypes are called + :doc:`structured data types. ` - subarray + subarray An array nested in a :term:`structured data type`: :: @@ -468,61 +448,55 @@ Glossary dtype=[('a', '`. - + title + An alias for a field name in a structured datatype. - ufunc - NumPy's fast element-by-element computation (:term:`vectorization`) is - structured so as to leave the choice of function open. A function used - in vectorization is called a ``ufunc``, short for ``universal - function``. NumPy routines have built-in ufuncs, but users can also - :doc:`write their own. ` + type + In NumPy, a synonym for :term:`dtype`. For the more general Python + meaning, :term:`see here. ` - vectorization + ufunc + NumPy's fast element-by-element computation (:term:`vectorization`) is + structured so as to leave the choice of function open. A function used + in vectorization is called a ``ufunc``, short for ``universal + function``. NumPy routines have built-in ufuncs, but users can also + :doc:`write their own. ` - NumPy hands off array processing to C, where looping and computation are - much faster than in Python. To exploit this, programmers using NumPy - eliminate Python loops in favor of array-to-array operations. - :term:`vectorization` can refer both to the C offloading and to - structuring NumPy code to leverage it. + vectorization + NumPy hands off array processing to C, where looping and computation are + much faster than in Python. To exploit this, programmers using NumPy + eliminate Python loops in favor of array-to-array operations. + :term:`vectorization` can refer both to the C offloading and to + structuring NumPy code to leverage it. - view - Without changing underlying data, NumPy can make one array masquerade as - any number of other arrays with different types, shapes, and even - content. This is much faster than creating those arrays. + view + Without changing underlying data, NumPy can make one array masquerade as + any number of other arrays with different types, shapes, and even + content. This is much faster than creating those arrays. - An array created this way is a ``view``, and the performance gain often - makes an array created as a view preferable to one created as a new - array. + An array created this way is a ``view``, and the performance gain often + makes an array created as a view preferable to one created as a new + array. - But because a view shares data with the original array, a write in one - array can affect the other, even though they appear to be different - arrays. If this is an problem, a view can't be used; the second array - needs to be physically distinct -- a ``copy``. + But because a view shares data with the original array, a write in one + array can affect the other, even though they appear to be different + arrays. If this is an problem, a view can't be used; the second array + needs to be physically distinct -- a ``copy``. - Some NumPy routines always return views, some always return copies, some - may return one or the other, and for some the choice can be specified. - Responsiblity for managing views and copies falls to the programmer. - :doc:`numpy.shares_memory ` - will check whether ``b`` is a view of ``a``, - but an exact answer isn't always feasible, as the documentation page - explains. + Some NumPy routines always return views, some always return copies, some + may return one or the other, and for some the choice can be specified. + Responsiblity for managing views and copies falls to the programmer. + :doc:`numpy.shares_memory ` + will check whether ``b`` is a view of ``a``, + but an exact answer isn't always feasible, as the documentation page + explains. From 23d69b316edf60a87f6b8a782b85d8f0f3f33e6f Mon Sep 17 00:00:00 2001 From: mattip Date: Wed, 16 Sep 2020 11:52:49 +0300 Subject: [PATCH 050/409] DOC: manually merge some changes done in the other PRs --- doc/source/glossary.rst | 101 +++++++++++++++++++++++----------------- 1 file changed, 57 insertions(+), 44 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 978589996a5b..b32c2ef2986d 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -4,6 +4,7 @@ Glossary .. glossary:: + (`n`,) A tuple with one element. The trailing comma distinguishes a one-element tuple from a parenthesized ``n``. @@ -15,6 +16,7 @@ Glossary ``...`` + An :py:data:`Ellipsis` **When indexing an array**, shorthand that the missing axes, if they exist, are full slices. @@ -33,15 +35,13 @@ Glossary >>> a[0,...,0].shape (3,) - It can be used at most once; ``a[...,0,...]`` raises an ``IndexError``. + It can be used at most once; ``a[...,0,...]`` raises an :exc:`IndexError`. **In printouts**, NumPy substitutes ``...`` for the middle elements of - large arrays. To see the entire array, use - :doc:`numpy.printoptions. ` + large arrays. To see the entire array, use `numpy.printoptions` ``:`` - The Python :term:`python:slice` operator. In ndarrays, slicing can be applied to every axis: @@ -70,33 +70,28 @@ Glossary In contrast to Python, where slicing creates a copy, in NumPy slicing creates a :term:`view`. - For details, see :ref:`Combining advanced and basic indexing `. + For details, see :ref:`combining-advanced-and-basic-indexing`. ``<`` - In a dtype declaration, indicates that the data is - `little-endian `_ - (the bracket is big on the right). :: + :term:`little-endian` (the bracket is big on the right). :: - >>> dt = np.dtype('>> dt = np.dtype('`` - In a dtype declaration, indicates that the data is - `big-endian `_ - (the bracket is big on the left). :: + :term:`big-endian` (the bracket is big on the left). :: - >>> dt = np.dtype('>H') # big-endian unsigned short + >>> dt = np.dtype('>H') # big-endian unsigned short advanced indexing - Rather than using a :doc:`scalar ` or slice as an index, an axis can be indexed with an array, providing fine-grained selection. This is known as :ref:`advanced indexing` - or ``fancy indexing``. + or "fancy indexing". along an axis @@ -162,7 +157,7 @@ Glossary In a two-dimensional vector, the elements of axis 0 are rows and the elements of axis 1 are columns. - In higher dimensions the picture changes. NumPy prints + In higher dimensions, the picture changes. NumPy prints higher-dimensional vectors as replications of row-by-column building blocks, as in this three-dimensional vector: @@ -192,15 +187,15 @@ Glossary useful in distinguishing, say, a (1,2,3) shape from a (2,3) shape: >>> a = np.arange(6).reshape(2,3) - >>> a.shape - (2, 3) + >>> a.ndim + 2 >>> a array([[0, 1, 2], [3, 4, 5]]) >>> a = np.arange(6).reshape(1,2,3) - >>> a.shape - (1, 2, 3) + >>> a.ndim + 3 >>> a array([[[0, 1, 2], [3, 4, 5]]]) @@ -217,12 +212,15 @@ Glossary arrays are :term:`view`\ s. - `big-endian `_ - \ + big-endian + When storing a multi-byte value in memory as a sequence of bytes, the + sequence addresses/sends/stores the most significant byte first (lowest + address) and the least significant byte last (highest address). Common in + micro-processors and used for transmission of data over network protocols. - `BLAS `_ - \ + BLAS + `Basic Linear Algebra Subprograms `_ broadcast @@ -243,7 +241,16 @@ Glossary C order - Same as `row-major. `_ + See `row-major` + + + column-major + A way to represent items in a N-dimensional array in the 1-dimensional + computer memory. In column-major order, the leftmost index "varies the + fastest": for example the array:: + + [[1, 2, 3], + [4, 5, 6]] `column-major `_ @@ -281,14 +288,13 @@ Glossary field - In a :term:`structured data type`, each subtype is called a - :doc:`field `. - A field has a name (a string), a type (any valid dtype), and - an optional :term:`title`. For details, see :ref:`arrays.dtypes`. + In a :term:`structured data type`, each sub-type is called a `field`. + The `field` has a name (a string), a type (any valid dtype), and + an optional `title`. See :ref:`arrays.dtypes` Fortran order - Same as `column-major `_ + Same as `column-major` flattened @@ -321,7 +327,10 @@ Glossary `little-endian `_ - \ + When storing a multi-byte value in memory as a sequence of bytes, the + sequence addresses/sends/stores the least significant byte first (lowest + address) and the most significant byte last (highest address). Common in + x86 processors. mask @@ -348,7 +357,6 @@ Glossary matrix - NumPy's two-dimensional :doc:`matrix class ` should no longer be used; use regular ndarrays. @@ -368,12 +376,8 @@ Glossary ravel - :doc:`numpy.ravel \ - ` - and :doc:`numpy.flatten \ - ` - both flatten an ndarray. ``ravel`` will return a view if possible; - ``flatten`` always returns a copy. + `numpy.ravel` and `numpy.ndarray.flatten` both flatten an ndarray. ``ravel`` + will return a view if possible; ``flatten`` always returns a copy. Flattening collapses a multidimensional array to a single dimension; details of how this is done (for instance, whether ``a[n+1]`` should be @@ -390,7 +394,20 @@ Glossary \ :doc:`scalar ` - \ + A way to represent items in a N-dimensional array in the 1-dimensional + computer memory. In row-major order, the rightmost index "varies + the fastest": for example the array:: + + [[1, 2, 3], + [4, 5, 6]] + + is represented in the row-major order as:: + + [1, 2, 3, 4, 5, 6] + + Row-major order is also known as the C order, as the C programming + language uses it. New NumPy arrays are by default in row-major order. + shape @@ -414,7 +431,7 @@ Glossary ``i+1`` on axis ``n`` means adding ``a.strides[n]`` bytes to the address. - Stride is computed automatically from an array's dtype and + Strides are computed automatically from an array's dtype and shape, but can be directly specified using :doc:`as_strided. ` @@ -458,10 +475,6 @@ Glossary An alias for a field name in a structured datatype. - type - In NumPy, a synonym for :term:`dtype`. For the more general Python - meaning, :term:`see here. ` - ufunc NumPy's fast element-by-element computation (:term:`vectorization`) is From a182f32c77e557cadb0cd70f788e17c14671d171 Mon Sep 17 00:00:00 2001 From: mattip Date: Wed, 16 Sep 2020 16:14:14 +0300 Subject: [PATCH 051/409] DOC, BLD: fix templated C highlighting --- doc/source/conf.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index e34be7f5c12a..032e828ce4a2 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -367,15 +367,15 @@ def linkcode_resolve(domain, info): numpy.__version__, fn, linespec) from pygments.lexers import CLexer -import copy +from pygments.lexer import inherit, bygroups +from pygments.token import Comment class NumPyLexer(CLexer): name = 'NUMPYLEXER' - tokens = copy.deepcopy(CLexer.tokens) - # Extend the regex for valid identifiers with @ - for k, val in tokens.items(): - for i, v in enumerate(val): - if isinstance(v, tuple): - if isinstance(v[0], str): - val[i] = (v[0].replace('a-zA-Z', 'a-zA-Z@'),) + v[1:] + tokens = { + 'statements': [ + (r'@[a-zA-Z_]*@', Comment.Preproc, 'macro'), + inherit, + ], + } From 11d67d46931887f4f4e05da080cf99414bff329d Mon Sep 17 00:00:00 2001 From: mattip Date: Wed, 16 Sep 2020 16:15:03 +0300 Subject: [PATCH 052/409] DOC, BLD: update doc build dependencies, remove as numpydoc submodule --- doc/source/docs/howto_document.rst | 26 ++------------------------ doc_requirements.txt | 2 +- 2 files changed, 3 insertions(+), 25 deletions(-) diff --git a/doc/source/docs/howto_document.rst b/doc/source/docs/howto_document.rst index cf86b7e99870..9f9068ab32a7 100644 --- a/doc/source/docs/howto_document.rst +++ b/doc/source/docs/howto_document.rst @@ -40,29 +40,7 @@ after which you may use it:: np.fft.fft2(...) -.. rubric:: - **For convenience the** `formatting standard`_ **is included below with an - example** - -.. include:: ../../sphinxext/doc/format.rst - -.. _example: - -Example Source -============== - -.. literalinclude:: ../../sphinxext/doc/example.py - - - -Example Rendered -================ - -.. ifconfig:: python_version_major < '3' - - The example is rendered only when sphinx is run with python3 and above - -.. automodule:: doc.example - :members: +Please use the numpydoc `formatting standard`_ as shown in their example_ .. _`formatting standard`: https://numpydoc.readthedocs.io/en/latest/format.html +.. _example: https://numpydoc.readthedocs.io/en/latest/example.html diff --git a/doc_requirements.txt b/doc_requirements.txt index 815aac3079ec..36b651c6465f 100644 --- a/doc_requirements.txt +++ b/doc_requirements.txt @@ -1,5 +1,5 @@ -pygments==2.6.1 sphinx>=2.2.0,<3.0 +numpydoc==1.1.0 ipython scipy matplotlib From 3a06142f5e26c6294ffd5fa24cd450addef45e13 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Wed, 16 Sep 2020 12:49:41 +0100 Subject: [PATCH 053/409] DOC: Add more examples to the `issubdtype` docstring --- numpy/core/numerictypes.py | 38 +++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/numpy/core/numerictypes.py b/numpy/core/numerictypes.py index 2a015f48fc98..632f2b8cd17c 100644 --- a/numpy/core/numerictypes.py +++ b/numpy/core/numerictypes.py @@ -358,13 +358,15 @@ def issubsctype(arg1, arg2): @set_module('numpy') def issubdtype(arg1, arg2): - """ + r""" Returns True if first argument is a typecode lower/equal in type hierarchy. + This is like the builtin :func:`issubclass`, but for `dtype`\ s. + Parameters ---------- arg1, arg2 : dtype_like - dtype or string representing a typecode. + `dtype` or object coercible to one Returns ------- @@ -377,10 +379,40 @@ def issubdtype(arg1, arg2): Examples -------- - >>> np.issubdtype('S1', np.string_) + `issubdtype` can be used to check the type of arrays: + + >>> ints = np.array([1, 2, 3], dtype=np.int32) + >>> np.issubdtype(ints.dtype, np.integer) + True + >>> np.issubdtype(ints.dtype, np.floating) + False + + >>> floats = np.array([1, 2, 3], dtype=np.float32) + >>> np.issubdtype(floats.dtype, np.integer) + False + >>> np.issubdtype(floats.dtype, np.floating) True + + Similar types of different sizes are not subdtypes of each other: + >>> np.issubdtype(np.float64, np.float32) False + >>> np.issubdtype(np.float32, np.float64) + False + + but both are subtypes of `floating`: + + >>> np.issubdtype(np.float64, np.floating) + True + >>> np.issubdtype(np.float32, np.floating) + True + + For convenience, dtype-like objects are allowed too: + + >>> np.issubdtype('S1', np.string_) + True + >>> np.issubdtype('i4', np.signedinteger) + True """ if not issubclass_(arg1, generic): From 5c3f10061f39a7858a5f2c6cd48773abe3252e48 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Wed, 16 Sep 2020 13:08:11 +0100 Subject: [PATCH 054/409] DOC: Fix broken references and make the tables a little clearer `types.BooleanType` and similar names were python2 things that no longer exist `unicode` also no longer exists, and is now `str` --- doc/source/reference/arrays.scalars.rst | 55 +++++++++++++------------ numpy/core/numerictypes.py | 2 +- 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/doc/source/reference/arrays.scalars.rst b/doc/source/reference/arrays.scalars.rst index f57a117244aa..46d2bb8faeef 100644 --- a/doc/source/reference/arrays.scalars.rst +++ b/doc/source/reference/arrays.scalars.rst @@ -29,7 +29,7 @@ an array scalar object. Alternatively, what kind of array scalar is present can be determined using other members of the data type hierarchy. Thus, for example ``isinstance(val, np.complexfloating)`` will return :py:data:`True` if *val* is a complex valued type, while -:const:`isinstance(val, np.flexible)` will return true if *val* is one +``isinstance(val, np.flexible)`` will return true if *val* is one of the flexible itemsize array types (:class:`string`, :class:`unicode`, :class:`void`). @@ -65,19 +65,22 @@ Some of the scalar types are essentially equivalent to fundamental Python types and therefore inherit from them as well as from the generic array scalar type: -==================== ================================ -Array scalar type Related Python type -==================== ================================ -:class:`int_` :class:`IntType` (Python 2 only) -:class:`float_` :class:`FloatType` -:class:`complex_` :class:`ComplexType` -:class:`bytes_` :class:`BytesType` -:class:`unicode_` :class:`UnicodeType` -==================== ================================ +==================== =========================== ============= +Array scalar type Related Python type Inherits? +==================== =========================== ============= +:class:`int_` :class:`int` Python 2 only +:class:`float_` :class:`float` yes +:class:`complex_` :class:`complex` yes +:class:`bytes_` :class:`bytes` yes +:class:`str_` :class:`str` yes +:class:`bool_` :class:`bool` no +:class:`datetime64` :class:`datetime.datetime` no +:class:`timedelta64` :class:`datetime.timedelta` no +==================== =========================== ============= The :class:`bool_` data type is very similar to the Python -:class:`BooleanType` but does not inherit from it because Python's -:class:`BooleanType` does not allow itself to be inherited from, and +:class:`bool` but does not inherit from it because Python's +:class:`bool` does not allow itself to be inherited from, and on the C-level the size of the actual bool data is not the same as a Python Boolean scalar. @@ -86,7 +89,7 @@ Python Boolean scalar. The :class:`bool_` type is not a subclass of the :class:`int_` type (the :class:`bool_` is not even a number type). This is different than Python's default implementation of :class:`bool` as a - sub-class of int. + sub-class of :class:`int`. .. warning:: @@ -113,11 +116,11 @@ Type Remarks Character code Integers: =================== ============================= =============== -:class:`byte` compatible: C char ``'b'`` -:class:`short` compatible: C short ``'h'`` -:class:`intc` compatible: C int ``'i'`` -:class:`int_` compatible: Python int ``'l'`` -:class:`longlong` compatible: C long long ``'q'`` +:class:`byte` compatible: C ``char`` ``'b'`` +:class:`short` compatible: C ``short`` ``'h'`` +:class:`intc` compatible: C ``int`` ``'i'`` +:class:`int_` compatible: C ``long`` ``'l'`` +:class:`longlong` compatible: C ``long long`` ``'q'`` :class:`intp` large enough to fit a pointer ``'p'`` :class:`int8` 8 bits :class:`int16` 16 bits @@ -127,18 +130,18 @@ Integers: Unsigned integers: -=================== ============================= =============== -:class:`ubyte` compatible: C unsigned char ``'B'`` -:class:`ushort` compatible: C unsigned short ``'H'`` -:class:`uintc` compatible: C unsigned int ``'I'`` -:class:`uint` compatible: Python int ``'L'`` -:class:`ulonglong` compatible: C long long ``'Q'`` -:class:`uintp` large enough to fit a pointer ``'P'`` +=================== ================================= =============== +:class:`ubyte` compatible: C ``unsigned char`` ``'B'`` +:class:`ushort` compatible: C ``unsigned short`` ``'H'`` +:class:`uintc` compatible: C ``unsigned int`` ``'I'`` +:class:`uint` compatible: C ``long`` ``'L'`` +:class:`ulonglong` compatible: C ``long long`` ``'Q'`` +:class:`uintp` large enough to fit a pointer ``'P'`` :class:`uint8` 8 bits :class:`uint16` 16 bits :class:`uint32` 32 bits :class:`uint64` 64 bits -=================== ============================= =============== +=================== ================================= =============== Floating-point numbers: diff --git a/numpy/core/numerictypes.py b/numpy/core/numerictypes.py index 632f2b8cd17c..e705dd3ea855 100644 --- a/numpy/core/numerictypes.py +++ b/numpy/core/numerictypes.py @@ -374,8 +374,8 @@ def issubdtype(arg1, arg2): See Also -------- + :ref:`arrays.scalars` : Overview of the numpy type hierarchy. issubsctype, issubclass_ - numpy.core.numerictypes : Overview of numpy type hierarchy. Examples -------- From 9c0febeec5e6464d873e03ec74900e08989d75b7 Mon Sep 17 00:00:00 2001 From: mattip Date: Wed, 16 Sep 2020 17:29:02 +0300 Subject: [PATCH 055/409] BLD: remove submodule doc/sphinxext that pointed to numpydocs --- .gitmodules | 3 --- doc/sphinxext | 1 - 2 files changed, 4 deletions(-) delete mode 160000 doc/sphinxext diff --git a/.gitmodules b/.gitmodules index b1e13c3bc120..e69de29bb2d1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "doc/sphinxext"] - path = doc/sphinxext - url = https://github.com/numpy/numpydoc.git diff --git a/doc/sphinxext b/doc/sphinxext deleted file mode 160000 index b4c5fd17e2b8..000000000000 --- a/doc/sphinxext +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b4c5fd17e2b85c2416a5e586933eee353b58bf7c From 260f2f144018f1c4c3de171160dc53b5c642b205 Mon Sep 17 00:00:00 2001 From: Bijesh Mohan Date: Wed, 16 Sep 2020 20:04:09 +0530 Subject: [PATCH 056/409] MAINT: Chaining exceptions in npyio.py --- numpy/lib/npyio.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/numpy/lib/npyio.py b/numpy/lib/npyio.py index 90e16643c013..805e59bc1ff4 100644 --- a/numpy/lib/npyio.py +++ b/numpy/lib/npyio.py @@ -86,7 +86,7 @@ def __getattribute__(self, key): try: return object.__getattribute__(self, '_obj')[key] except KeyError: - raise AttributeError(key) + raise AttributeError(key) from None def __dir__(self): """ @@ -446,9 +446,9 @@ def load(file, mmap_mode=None, allow_pickle=False, fix_imports=True, "when allow_pickle=False") try: return pickle.load(fid, **pickle_kwargs) - except Exception: + except Exception as e: raise IOError( - "Failed to interpret file %s as a pickle" % repr(file)) + "Failed to interpret file %s as a pickle" % repr(file)) from e def _save_dispatcher(file, arr, allow_pickle=None, fix_imports=None): @@ -1435,10 +1435,10 @@ def first_write(self, v): for row in X: try: v = format % tuple(row) + newline - except TypeError: + except TypeError as e: raise TypeError("Mismatch between array dtype ('%s') and " "format specifier ('%s')" - % (str(X.dtype), format)) + % (str(X.dtype), format)) from e fh.write(v) if len(footer) > 0: From 26a407f701782f45d3165f92861bd6555d042596 Mon Sep 17 00:00:00 2001 From: Matti Picus Date: Wed, 16 Sep 2020 18:01:45 +0300 Subject: [PATCH 057/409] Update doc/source/conf.py Co-authored-by: Eric Wieser --- doc/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 032e828ce4a2..fe7ea096740d 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -376,6 +376,6 @@ class NumPyLexer(CLexer): tokens = { 'statements': [ (r'@[a-zA-Z_]*@', Comment.Preproc, 'macro'), - inherit, + inherit, ], } From 60a1e10c4593736b188b38e7d7c51aefb213af6a Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 17 Sep 2020 03:06:41 -0500 Subject: [PATCH 058/409] DOC: Fix syntax errors in docstrings for versionchanged, versionadded (#17338) * DOC: Fix typos in versionchanged. --- numpy/core/multiarray.py | 4 ++-- numpy/lib/function_base.py | 2 +- numpy/lib/recfunctions.py | 2 +- numpy/ma/extras.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/numpy/core/multiarray.py b/numpy/core/multiarray.py index 225c9554cec2..540d1ea9b5f6 100644 --- a/numpy/core/multiarray.py +++ b/numpy/core/multiarray.py @@ -163,12 +163,12 @@ def concatenate(arrays, axis=None, out=None, *, dtype=None, casting=None): If provided, the destination array will have this dtype. Cannot be provided together with `out`. - ..versionadded:: 1.20.0 + .. versionadded:: 1.20.0 casting : {'no', 'equiv', 'safe', 'same_kind', 'unsafe'}, optional Controls what kind of data casting may occur. Defaults to 'same_kind'. - ..versionadded:: 1.20.0 + .. versionadded:: 1.20.0 Returns ------- diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index c43b2fb531f3..c7ddbdb8de67 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -1450,7 +1450,7 @@ def angle(z, deg=False): The counterclockwise angle from the positive real axis on the complex plane in the range ``(-pi, pi]``, with dtype as numpy.float64. - ..versionchanged:: 1.16.0 + .. versionchanged:: 1.16.0 This function works on subclasses of ndarray like `ma.array`. See Also diff --git a/numpy/lib/recfunctions.py b/numpy/lib/recfunctions.py index cfc5dc9cae99..fbfbca73d442 100644 --- a/numpy/lib/recfunctions.py +++ b/numpy/lib/recfunctions.py @@ -513,7 +513,7 @@ def drop_fields(base, drop_names, usemask=True, asrecarray=False): Nested fields are supported. - ..versionchanged: 1.18.0 + .. versionchanged:: 1.18.0 `drop_fields` returns an array with 0 fields if all fields are dropped, rather than returning ``None`` as it did previously. diff --git a/numpy/ma/extras.py b/numpy/ma/extras.py index 8ede29da18af..613bcb550dfd 100644 --- a/numpy/ma/extras.py +++ b/numpy/ma/extras.py @@ -1641,7 +1641,7 @@ def flatnotmasked_contiguous(a): slice_list : list A sorted sequence of `slice` objects (start index, end index). - ..versionchanged:: 1.15.0 + .. versionchanged:: 1.15.0 Now returns an empty list instead of None for a fully masked array See Also From 233c63a56974de22b846ac989cef1fabe45e7296 Mon Sep 17 00:00:00 2001 From: "Nick R. Papior" Date: Thu, 17 Sep 2020 12:29:38 +0200 Subject: [PATCH 059/409] BLD: enabled negation of library choices in NPY_*_ORDER (#17219) BLD: enabled negation of library choices in NPY_*_ORDER When users build for a particular order it may be beneficial to disallow certain libraries. In particular a user may not care about which accelerated BLAS library is used, so long as the NetLIB or ATLAS library isn't used. This is now possible with: NPY_BLAS_ORDER='^blas,atlas' or NPY_BLAS_ORDER='!blas,atlas' Since we may envision more BLAS/LAPACK libraries to the pool, this will provide greater flexibility as they enter. A new (local) method is added in system_info.py which removes duplicate code and allows for easier usage across libraries. --- .../upcoming_changes/17219.new_feature.rst | 12 ++ doc/source/user/building.rst | 21 ++++ numpy/distutils/system_info.py | 119 +++++++++++++----- numpy/distutils/tests/test_system_info.py | 27 ++++ 4 files changed, 145 insertions(+), 34 deletions(-) create mode 100644 doc/release/upcoming_changes/17219.new_feature.rst diff --git a/doc/release/upcoming_changes/17219.new_feature.rst b/doc/release/upcoming_changes/17219.new_feature.rst new file mode 100644 index 000000000000..a6985ef0d2ad --- /dev/null +++ b/doc/release/upcoming_changes/17219.new_feature.rst @@ -0,0 +1,12 @@ +Negation of user-defined BLAS/LAPACK detection order +---------------------------------------------------- +`distutils` allows negation of libraries when determining BLAS/LAPACK +libraries. +This may be used to remove an item from the library resolution phase, i.e. +to disallow NetLIB libraries one could do:: + +.. code:: bash + + NPY_BLAS_ORDER='^blas' NPY_LAPACK_ORDER='^lapack' python setup.py build + +which will use any of the accelerated libraries instead. diff --git a/doc/source/user/building.rst b/doc/source/user/building.rst index 54ece3da3749..47399139e6e8 100644 --- a/doc/source/user/building.rst +++ b/doc/source/user/building.rst @@ -142,6 +142,16 @@ will prefer to use ATLAS, then BLIS, then OpenBLAS and as a last resort MKL. If neither of these exists the build will fail (names are compared lower case). +Alternatively one may use ``!`` or ``^`` to negate all items:: + + NPY_BLAS_ORDER='^blas,atlas' python setup.py build + +will allow using anything **but** NetLIB BLAS and ATLAS libraries, the order of the above +list is retained. + +One cannot mix negation and positives, nor have multiple negations, such cases will +raise an error. + LAPACK ~~~~~~ @@ -165,6 +175,17 @@ will prefer to use ATLAS, then OpenBLAS and as a last resort MKL. If neither of these exists the build will fail (names are compared lower case). +Alternatively one may use ``!`` or ``^`` to negate all items:: + + NPY_LAPACK_ORDER='^lapack' python setup.py build + +will allow using anything **but** the NetLIB LAPACK library, the order of the above +list is retained. + +One cannot mix negation and positives, nor have multiple negations, such cases will +raise an error. + + .. deprecated:: 1.20 The native libraries on macOS, provided by Accelerate, are not fit for use in NumPy since they have bugs that cause wrong output under easily reproducible diff --git a/numpy/distutils/system_info.py b/numpy/distutils/system_info.py index 19f7482f2a86..b4513825d04d 100644 --- a/numpy/distutils/system_info.py +++ b/numpy/distutils/system_info.py @@ -415,6 +415,83 @@ def get_standard_file(fname): return filenames +def _parse_env_order(base_order, env): + """ Parse an environment variable `env` by splitting with "," and only returning elements from `base_order` + + This method will sequence the environment variable and check for their invidual elements in `base_order`. + + The items in the environment variable may be negated via '^item' or '!itema,itemb'. + It must start with ^/! to negate all options. + + Raises + ------ + ValueError: for mixed negated and non-negated orders or multiple negated orders + + Parameters + ---------- + base_order : list of str + the base list of orders + env : str + the environment variable to be parsed, if none is found, `base_order` is returned + + Returns + ------- + allow_order : list of str + allowed orders in lower-case + unknown_order : list of str + for values not overlapping with `base_order` + """ + order_str = os.environ.get(env, None) + + # ensure all base-orders are lower-case (for easier comparison) + base_order = [order.lower() for order in base_order] + if order_str is None: + return base_order, [] + + neg = order_str.startswith('^') or order_str.startswith('!') + # Check format + order_str_l = list(order_str) + sum_neg = order_str_l.count('^') + order_str_l.count('!') + if neg: + if sum_neg > 1: + raise ValueError(f"Environment variable '{env}' may only contain a single (prefixed) negation: {order_str}") + # remove prefix + order_str = order_str[1:] + elif sum_neg > 0: + raise ValueError(f"Environment variable '{env}' may not mix negated an non-negated items: {order_str}") + + # Split and lower case + orders = order_str.lower().split(',') + + # to inform callee about non-overlapping elements + unknown_order = [] + + # if negated, we have to remove from the order + if neg: + allow_order = base_order.copy() + + for order in orders: + if order not in base_order: + unknown_order.append(order) + continue + + if order in allow_order: + allow_order.remove(order) + + else: + allow_order = [] + + for order in orders: + if order not in base_order: + unknown_order.append(order) + continue + + if order not in allow_order: + allow_order.append(order) + + return allow_order, unknown_order + + def get_info(name, notfound_action=0): """ notfound_action: @@ -1766,24 +1843,11 @@ def _calc_info(self, name): return getattr(self, '_calc_info_{}'.format(name))() def calc_info(self): - user_order = os.environ.get(self.order_env_var_name, None) - if user_order is None: - lapack_order = self.lapack_order - else: - # the user has requested the order of the - # check they are all in the available list, a COMMA SEPARATED list - user_order = user_order.lower().split(',') - non_existing = [] - lapack_order = [] - for order in user_order: - if order in self.lapack_order: - lapack_order.append(order) - elif len(order) > 0: - non_existing.append(order) - if len(non_existing) > 0: - raise ValueError("lapack_opt_info user defined " - "LAPACK order has unacceptable " - "values: {}".format(non_existing)) + lapack_order, unknown_order = _parse_env_order(self.lapack_order, self.order_env_var_name) + if len(unknown_order) > 0: + raise ValueError("lapack_opt_info user defined " + "LAPACK order has unacceptable " + "values: {}".format(unknown_order)) for lapack in lapack_order: if self._calc_info(lapack): @@ -1911,22 +1975,9 @@ def _calc_info(self, name): return getattr(self, '_calc_info_{}'.format(name))() def calc_info(self): - user_order = os.environ.get(self.order_env_var_name, None) - if user_order is None: - blas_order = self.blas_order - else: - # the user has requested the order of the - # check they are all in the available list - user_order = user_order.lower().split(',') - non_existing = [] - blas_order = [] - for order in user_order: - if order in self.blas_order: - blas_order.append(order) - elif len(order) > 0: - non_existing.append(order) - if len(non_existing) > 0: - raise ValueError("blas_opt_info user defined BLAS order has unacceptable values: {}".format(non_existing)) + blas_order, unknown_order = _parse_env_order(self.blas_order, self.order_env_var_name) + if len(unknown_order) > 0: + raise ValueError("blas_opt_info user defined BLAS order has unacceptable values: {}".format(unknown_order)) for blas in blas_order: if self._calc_info(blas): diff --git a/numpy/distutils/tests/test_system_info.py b/numpy/distutils/tests/test_system_info.py index 0768ffdde55a..46ad9b1033e0 100644 --- a/numpy/distutils/tests/test_system_info.py +++ b/numpy/distutils/tests/test_system_info.py @@ -285,3 +285,30 @@ def test_overrides(self): finally: os.chdir(previousDir) + +def test_distutils_parse_env_order(monkeypatch): + from numpy.distutils.system_info import _parse_env_order + env = 'NPY_TESTS_DISTUTILS_PARSE_ENV_ORDER' + + base_order = list('abcdef') + + monkeypatch.setenv(env, 'b,i,e,f') + order, unknown = _parse_env_order(base_order, env) + assert len(order) == 3 + assert order == list('bef') + assert len(unknown) == 1 + + for prefix in '^!': + monkeypatch.setenv(env, f'{prefix}b,i,e') + order, unknown = _parse_env_order(base_order, env) + assert len(order) == 4 + assert order == list('acdf') + assert len(unknown) == 1 + + with pytest.raises(ValueError): + monkeypatch.setenv(env, 'b,^e,i') + _parse_env_order(base_order, env) + + with pytest.raises(ValueError): + monkeypatch.setenv(env, '!b,^e,i') + _parse_env_order(base_order, env) From 58b5b3dce56c91489308f0c631ee067b4aa26ae8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Melissa=20Weber=20Mendon=C3=A7a?= Date: Thu, 17 Sep 2020 12:02:20 -0300 Subject: [PATCH 060/409] Fix typos Co-authored-by: Bradley Dice --- .github/ISSUE_TEMPLATE/documentation.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/documentation.md b/.github/ISSUE_TEMPLATE/documentation.md index 67e04c8af0cf..cdb7cde2ee2f 100644 --- a/.github/ISSUE_TEMPLATE/documentation.md +++ b/.github/ISSUE_TEMPLATE/documentation.md @@ -7,8 +7,8 @@ labels: 04 - Documentation ## Documentation - - From e5f2ce3c6465222f3dfdc186c930b3e049a4d597 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 17 Sep 2020 11:47:31 -0500 Subject: [PATCH 061/409] TST: Add test for creating an xpress like user-dtype This includes a few error paths. Note that the test creates a new dtype ever time it is run, and dtypes in NumPy are persistent so that this leaks references. --- .../src/multiarray/_multiarray_tests.c.src | 68 +++++++++++++++++++ numpy/core/tests/test_dtype.py | 37 ++++++++++ 2 files changed, 105 insertions(+) diff --git a/numpy/core/src/multiarray/_multiarray_tests.c.src b/numpy/core/src/multiarray/_multiarray_tests.c.src index ea04c82bdf2e..cbfb1e17954d 100644 --- a/numpy/core/src/multiarray/_multiarray_tests.c.src +++ b/numpy/core/src/multiarray/_multiarray_tests.c.src @@ -610,6 +610,71 @@ fromstring_null_term_c_api(PyObject *dummy, PyObject *byte_obj) } +/* + * Create a custom field dtype from an existing void one (and test some errors). + * The dtypes created by this function may be not be usable (or even crash + * while using). + */ +static PyObject * +create_custom_field_dtype(PyObject *NPY_UNUSED(mod), PyObject *args) +{ + PyArray_Descr *dtype; + PyTypeObject *scalar_type; + PyTypeObject *original_type = NULL; + int error_path; + + if (!PyArg_ParseTuple(args, "O!O!i", + &PyArrayDescr_Type, &dtype, + &PyType_Type, &scalar_type, + &error_path)) { + return NULL; + } + /* check that the result should be more or less valid */ + if (dtype->type_num != NPY_VOID || dtype->fields == NULL || + !PyDict_CheckExact(dtype->fields) || + PyTuple_Size(dtype->names) != 1 || + !PyDataType_REFCHK(dtype) || + dtype->elsize != sizeof(PyObject *)) { + PyErr_SetString(PyExc_ValueError, + "Bad dtype passed to test function, must be an object " + "containing void with a single field."); + return NULL; + } + + /* Copy and then appropriate this dtype */ + original_type = Py_TYPE(dtype); + dtype = PyArray_DescrNew(dtype); + if (dtype == NULL) { + return NULL; + } + + Py_INCREF(scalar_type); + Py_SETREF(dtype->typeobj, scalar_type); + if (error_path == 1) { + /* Test that we reject this, if fields was not already set */ + Py_SETREF(dtype->fields, NULL); + } + else if (error_path == 2) { + /* + * Test that we reject this if the type is not set to something that + * we are pretty sure can be safely replaced. + */ + Py_SET_TYPE(dtype, scalar_type); + } + else if (error_path != 0) { + PyErr_SetString(PyExc_ValueError, + "invalid error argument to test function."); + } + if (PyArray_RegisterDataType(dtype) < 0) { + /* Fix original type in the error_path == 2 case. */ + Py_SET_TYPE(dtype, original_type); + return NULL; + } + Py_INCREF(dtype); + return (PyObject *)dtype; +} + + /* check no elison for avoided increfs */ static PyObject * incref_elide(PyObject *dummy, PyObject *args) @@ -2081,6 +2146,9 @@ static PyMethodDef Multiarray_TestsMethods[] = { {"fromstring_null_term_c_api", fromstring_null_term_c_api, METH_O, NULL}, + {"create_custom_field_dtype", + create_custom_field_dtype, + METH_VARARGS, NULL}, {"incref_elide", incref_elide, METH_VARARGS, NULL}, diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index 2e2b0dbe2a34..898ceebcd45b 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -6,6 +6,7 @@ import numpy as np from numpy.core._rational_tests import rational +from numpy.core._multiarray_tests import create_custom_field_dtype from numpy.testing import ( assert_, assert_equal, assert_array_equal, assert_raises, HAS_REFCOUNT) from numpy.compat import pickle @@ -1338,3 +1339,39 @@ def test_pairs(self, pair): pair_type = np.dtype('{},{}'.format(*pair)) expected = np.dtype([('f0', pair[0]), ('f1', pair[1])]) assert_equal(pair_type, expected) + + +class TestUserDType: + @pytest.mark.leaks_references(reason="dynamically creates custom dtype.") + def test_custom_structured_dtype(self): + class mytype: + pass + + blueprint = np.dtype([("field", object)]) + dt = create_custom_field_dtype(blueprint, mytype, 0) + assert dt.type == mytype + # We cannot (currently) *create* this dtype with `np.dtype` because + # mytype does not inherit from `np.generic`. This seems like an + # unnecessary restriction, but one that has been around forever: + assert np.dtype(mytype) == np.dtype("O") + + with pytest.raises(RuntimeError): + # Registering a second time should fail + create_custom_field_dtype(blueprint, mytype, 0) + + def test_custom_structured_dtype_errors(self): + class mytype: + pass + + blueprint = np.dtype([("field", object)]) + + with pytest.raises(ValueError): + # Tests what happens if fields are unset during creation + # which is currently rejected due to the containing object + # (see PyArray_RegisterDataType). + create_custom_field_dtype(blueprint, mytype, 1) + + with pytest.raises(RuntimeError): + # Tests that a dtype must have its type field set up to np.dtype + # or in this case a builtin instance. + create_custom_field_dtype(blueprint, mytype, 2) From c6853c7c27bb8352ab498848439d4fee9eb79a33 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Thu, 17 Sep 2020 12:20:56 -0700 Subject: [PATCH 062/409] NEP: Regenerate table in NEP 29 (add numpy 1.18 and 1.19 to list) (#17337) * Regenerate Table for NEP 29. Numpy 1.18 and 1.19 have been released, which add two new recommended stop date for recommended numpy support in 2021 and 2022. * Infer next version as max-known-minor-version+1 Do not print the first 4 entries to be dropped; as they are pre-nep-29 --- doc/neps/nep-0029-deprecation_policy.rst | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/doc/neps/nep-0029-deprecation_policy.rst b/doc/neps/nep-0029-deprecation_policy.rst index dbead1b9b550..4674d24ec663 100644 --- a/doc/neps/nep-0029-deprecation_policy.rst +++ b/doc/neps/nep-0029-deprecation_policy.rst @@ -111,8 +111,10 @@ Jun 23, 2020 3.7+ 1.15+ Jul 23, 2020 3.7+ 1.16+ Jan 13, 2021 3.7+ 1.17+ Jul 26, 2021 3.7+ 1.18+ -Dec 26, 2021 3.8+ 1.18+ -Apr 14, 2023 3.9+ 1.18+ +Dec 22, 2021 3.7+ 1.19+ +Dec 26, 2021 3.8+ 1.19+ +Jun 21, 2022 3.8+ 1.20+ +Apr 14, 2023 3.9+ 1.20+ ============ ====== ===== @@ -127,7 +129,9 @@ Drop Schedule On Jul 23, 2020 drop support for Numpy 1.15 (initially released on Jul 23, 2018) On Jan 13, 2021 drop support for Numpy 1.16 (initially released on Jan 13, 2019) On Jul 26, 2021 drop support for Numpy 1.17 (initially released on Jul 26, 2019) + On Dec 22, 2021 drop support for Numpy 1.18 (initially released on Dec 22, 2019) On Dec 26, 2021 drop support for Python 3.7 (initially released on Jun 27, 2018) + On Jun 21, 2022 drop support for Numpy 1.19 (initially released on Jun 20, 2020) On Apr 14, 2023 drop support for Python 3.8 (initially released on Oct 14, 2019) @@ -255,6 +259,8 @@ Code to generate support and drop schedule tables :: Jan 13, 2019: Numpy 1.16 Jul 26, 2019: Numpy 1.17 Oct 14, 2019: Python 3.8 + Dec 22, 2019: Numpy 1.18 + Jun 20, 2020: Numpy 1.19 """ releases = [] @@ -274,8 +280,12 @@ Code to generate support and drop schedule tables :: releases = sorted(releases, key=lambda x: x[0]) - minpy = '3.9+' - minnum = '1.18+' + + py_major,py_minor = sorted([int(x) for x in r[2].split('.')] for r in releases if r[1] == 'Python')[-1] + minpy = f"{py_major}.{py_minor+1}+" + + num_major,num_minor = sorted([int(x) for x in r[2].split('.')] for r in releases if r[1] == 'Numpy')[-1] + minnum = f"{num_major}.{num_minor+1}+" toprint_drop_dates = [''] toprint_support_table = [] @@ -289,14 +299,14 @@ Code to generate support and drop schedule tables :: minnum = v+'+' else: minpy = v+'+' - - for e in toprint_drop_dates[::-1]: + print("On next release, drop support for Python 3.5 (initially released on Sep 13, 2015)") + for e in toprint_drop_dates[-4::-1]: print(e) print('============ ====== =====') print('Date Python NumPy') print('------------ ------ -----') - for e in toprint_support_table[::-1]: + for e in toprint_support_table[-4::-1]: print(e) print('============ ====== =====') From 511d7694075fc06aee1091a67b830a7338f8c9a5 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Thu, 17 Sep 2020 15:54:17 -0400 Subject: [PATCH 063/409] DOC: Updated for reviewer comments Also added "contiguous," which is referenced in docs but hadn't been defined. --- doc/source/glossary.rst | 220 ++++++++++++---------------------------- 1 file changed, 67 insertions(+), 153 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 6decf76f9de7..e6c8bbd17270 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -14,12 +14,16 @@ Glossary Used as a dimension entry, ``-1`` instructs NumPy to choose the length that will keep the total number of elements the same. + >>> np.arange(12).reshape(4,-1).shape + (4, 3) + + ``...`` - An :py:data:`Ellipsis` + An :py:data:`Ellipsis`. - **When indexing an array**, shorthand that the missing axes, if they - exist, are full slices. + - **When indexing an array**, shorthand that the missing axes, if they + exist, are full slices. >>> a = np.arange(24).reshape(2,3,4) @@ -37,8 +41,8 @@ Glossary It can be used at most once; ``a[...,0,...]`` raises an :exc:`IndexError`. - **In printouts**, NumPy substitutes ``...`` for the middle elements of - large arrays. To see the entire array, use `numpy.printoptions` + - **In printouts**, NumPy substitutes ``...`` for the middle elements of + large arrays. To see the entire array, use `numpy.printoptions` ``:`` @@ -95,40 +99,47 @@ Glossary along an axis + Reversing along axis 0 (the row axis) below reverses the columns, and + reversing along axis 1 (the column axis) reverses the rows. This might + be the opposite of what a new user would expect. - The result of an operation along an :term:`axis` X is an array in which X - disappears. This can surprise new users expecting the opposite. - - The operation can be visualized this way: - - Imagine a slice of array ``a`` where axis X has a fixed index - and the other dimensions are left full (``:``). + >>> a = np.arange(12).reshape(3,4) + >>> a + array([[ 0, 1, 2, 3], + [ 4, 5, 6, 7], + [ 8, 9, 10, 11]]) + + >>> np.flip(a,axis=0) + array([[ 8, 9, 10, 11], + [ 4, 5, 6, 7], + [ 0, 1, 2, 3]]) + + >>> np.flip(a,axis=1) + array([[ 3, 2, 1, 0], + [ 7, 6, 5, 4], + [11, 10, 9, 8]]) - >>> a = np.arange(24).reshape(2,3,4) - - >>> a.shape - (2, 3, 4) - - >>> a[:,0,:].shape - (2, 4) + An operation "along axis 0" behaves as if the argument were an array of + ``a`` slices taking successive indexes of axis 0: - The slice has ``a``'s shape with the X dimension deleted. Saying an - operation ``op`` is ``performed along X`` means that ``op`` takes as its - operands slices having every value of X: + >>> np.array((a[0,:], a[1,:], a[2,:])) + array([[ 0, 1, 2, 3], + [ 4, 5, 6, 7], + [ 8, 9, 10, 11]]) - >>> np.sum(a,axis=1) == a[:,0,:] + a[:,1,:] + a[:,2,:] - array([[ True, True, True, True], - [ True, True, True, True]]) + Reversing the slices results in ``np.flip(a,axis=0)``: + >>> np.array((a[2,:],a[1,:],a[0,:])) + array([[ 8, 9, 10, 11], + [ 4, 5, 6, 7], + [ 0, 1, 2, 3]]) array - Used synonymously in the NumPy docs with :doc:`ndarray `, NumPy's basic structure. - array_like Any :doc:`scalar ` or :term:`python:sequence` @@ -138,7 +149,7 @@ Glossary :doc:`numpy.array ` is array_like. :: - >>> a = np.array([[1, 2.0],[0, 0],(1+1j, 3.)]) + >>> a = np.array([[1, 2.0], [0, 0], (1+1j, 3.)]) >>> a array([[1.+0.j, 2.+0.j], @@ -214,11 +225,8 @@ Glossary arrays are :term:`view`\ s. - big-endian - When storing a multi-byte value in memory as a sequence of bytes, the - sequence addresses/sends/stores the most significant byte first (lowest - address) and the least significant byte last (highest address). Common in - micro-processors and used for transmission of data over network protocols. + `big-endian `_ + \ BLAS @@ -242,69 +250,33 @@ Glossary no memory or time. For details, see :doc:`Broadcasting. ` - C order - See `row-major` - - - column-major - A way to represent items in a N-dimensional array in the 1-dimensional - computer memory. In column-major order, the leftmost index "varies the - fastest": for example the array:: - - [[1, 2, 3], - [4, 5, 6]] + See :term:`row-major`. `column-major `_ \ - copy - - See :term:`view`. - - - :term:`decorator ` - \ - - - :term:`dictionary ` - \ + contiguous + An array is contiguous if it occupies a single unbroken block of memory. - dimension - - See :term:`axis`. - - - dtype - - The datatype describing the (identically typed) elements in an ndarray. - It can be changed to reinterpret the array contents. For details, see - :doc:`Data type objects (dtype). ` - - - fancy indexing - - Another term for :term:`advanced indexing`. - + copy + See :term:`view`. dimension - See :term:`axis`. dtype - The datatype describing the (identically typed) elements in an ndarray. It can be changed to reinterpret the array contents. For details, see :doc:`Data type objects (dtype). ` fancy indexing - Another term for :term:`advanced indexing`. @@ -315,15 +287,13 @@ Glossary Fortran order - Same as `column-major` - + Same as `column-major `_ flattened See :term:`ravel`. - homogeneous All elements of a homogeneous array have the same type. ndarrays, in contrast to Python lists, are homogeneous. The type can be complicated, @@ -333,34 +303,18 @@ Glossary Python objects, fill the role of heterogeneous arrays. - :term:`immutable ` - \ - - - :term:`iterable ` - \ - - itemsize The size of the dtype element in bytes. - :term:`list ` - \ - - `little-endian `_ - When storing a multi-byte value in memory as a sequence of bytes, the - sequence addresses/sends/stores the least significant byte first (lowest - address) and the most significant byte last (highest address). Common in - x86 processors. + \ mask The boolean array used to select elements in a :term:`masked array`. - masked array Bad or missing data can be cleanly ignored by putting it in a masked array, which has an internal boolean array indicating invalid @@ -391,26 +345,6 @@ Glossary object array - - An array whose dtype is ``object``; that is, it contains references to - Python objects. Indexing the array dereferences the Python objects, so - unlike other ndarrays, an object array has the ability to hold - heterogeneous objects. - - - ravel - - `numpy.ravel` and `numpy.ndarray.flatten` both flatten an ndarray. ``ravel`` - will return a view if possible; ``flatten`` always returns a copy. - - Flattening collapses a multidimensional array to a single dimension; - details of how this is done (for instance, whether ``a[n+1]`` should be - the next row or next column) are parameters. - - - - object array - An array whose dtype is ``object``; that is, it contains references to Python objects. Indexing the array dereferences the Python objects, so unlike other ndarrays, an object array has the ability to hold @@ -418,11 +352,14 @@ Glossary ravel - - `numpy.ravel` and `numpy.ndarray.flatten` both flatten an ndarray. ``ravel`` - will return a view if possible; ``flatten`` always returns a copy. - - Flattening collapses a multi-dimensional array to a single dimension; + :doc:`numpy.ravel \ + ` + and :doc:`numpy.flatten \ + ` + both flatten an ndarray. ``ravel`` will return a view if possible; + ``flatten`` always returns a copy. + + Flattening collapses a multimdimensional array to a single dimension; details of how this is done (for instance, whether ``a[n+1]`` should be the next row or next column) are parameters. @@ -434,23 +371,16 @@ Glossary row-major - A way to represent items in a N-dimensional array in the 1-dimensional - computer memory. In row-major order, the rightmost index "varies - the fastest": for example the array:: + `row-major `_ + order is also known as C order, as the C programming language uses it. + New NumPy arrays are by default in row-major order. - [[1, 2, 3], - [4, 5, 6]] - is represented in the row-major order as:: - - [1, 2, 3, 4, 5, 6] - - Row-major order is also known as the C order, as the C programming - language uses it. New NumPy arrays are by default in row-major order. + :doc:`scalar ` + \ shape - A tuple showing the length of each dimension of an ndarray. The length of the tuple itself is the number of dimensions (:doc:`numpy.ndim `). @@ -464,7 +394,6 @@ Glossary stride - Physical memory is one-dimensional; strides provide a mechanism to map a given index to an address in memory. For an N-dimensional array, its ``strides`` attribute is an N-element tuple; advancing from index @@ -485,7 +414,6 @@ Glossary structured array - Array whose :term:`dtype` is a :term:`structured data type`. @@ -496,7 +424,6 @@ Glossary subarray - An array nested in a :term:`structured data type`: :: >>> dt = np.dtype([('a', np.int32), ('b', np.float32, (3,))]) @@ -505,18 +432,19 @@ Glossary dtype=[('a', '` + + ufunc NumPy's fast element-by-element computation (:term:`vectorization`) is structured so as to leave the choice of function open. A function used @@ -525,7 +453,6 @@ Glossary :doc:`write their own. ` - vectorization NumPy hands off array processing to C, where looping and computation are much faster than in Python. To exploit this, programmers using NumPy @@ -533,14 +460,12 @@ Glossary :term:`vectorization` can refer both to the C offloading and to structuring NumPy code to leverage it. - - view Without changing underlying data, NumPy can make one array masquerade as any number of other arrays with different types, shapes, and even content. This is much faster than creating those arrays. - An array created this way is a ``view``, and the performance gain often + An array created this way is a `view`, and the performance gain often makes an array created as a view preferable to one created as a new array. @@ -569,14 +494,3 @@ Glossary >>> y array([3, 2, 4]) - - wrapper - Python is a high-level (highly abstracted, or English-like) language. - This abstraction comes at a price in execution speed, and sometimes - it becomes necessary to use lower level languages to do fast - computations. A wrapper is code that provides a bridge between - high and the low level languages, allowing, e.g., Python to execute - code written in C or Fortran. - - Examples include ctypes, SWIG and Cython (which wraps C and C++) - and f2py (which wraps Fortran). From 5bec8df317ffbda8f97486195b758f2261495749 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Fri, 18 Sep 2020 02:09:02 +0200 Subject: [PATCH 064/409] BLD: Fix a macOS build failure when `NPY_BLAS_ORDER=""` --- numpy/distutils/system_info.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/numpy/distutils/system_info.py b/numpy/distutils/system_info.py index b4513825d04d..c3bd6347c172 100644 --- a/numpy/distutils/system_info.py +++ b/numpy/distutils/system_info.py @@ -471,6 +471,9 @@ def _parse_env_order(base_order, env): allow_order = base_order.copy() for order in orders: + if not order: + continue + if order not in base_order: unknown_order.append(order) continue @@ -482,6 +485,9 @@ def _parse_env_order(base_order, env): allow_order = [] for order in orders: + if not order: + continue + if order not in base_order: unknown_order.append(order) continue From 6d3a9d7f97c1033de02a3a611ee3e8111f81738c Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Fri, 18 Sep 2020 11:38:08 +0200 Subject: [PATCH 065/409] TST: Expanded the `_parse_env_order` tests Relevant for when LAPACK/BLAS optimization is disabled. Addresses https://github.com/numpy/numpy/pull/17346#issuecomment-694744389. --- numpy/distutils/tests/test_system_info.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/numpy/distutils/tests/test_system_info.py b/numpy/distutils/tests/test_system_info.py index 46ad9b1033e0..ec15126f7f7b 100644 --- a/numpy/distutils/tests/test_system_info.py +++ b/numpy/distutils/tests/test_system_info.py @@ -284,7 +284,7 @@ def test_overrides(self): assert info.get_lib_dirs() == lib_dirs finally: os.chdir(previousDir) - + def test_distutils_parse_env_order(monkeypatch): from numpy.distutils.system_info import _parse_env_order @@ -298,6 +298,12 @@ def test_distutils_parse_env_order(monkeypatch): assert order == list('bef') assert len(unknown) == 1 + # For when LAPACK/BLAS optimization is disabled + monkeypatch.setenv(env, '') + order, unknown = _parse_env_order(base_order, env) + assert len(order) == 0 + assert len(unknown) == 0 + for prefix in '^!': monkeypatch.setenv(env, f'{prefix}b,i,e') order, unknown = _parse_env_order(base_order, env) From 90111e2dd99b7f29e2134c4e326ac68d86cda393 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Fri, 18 Sep 2020 10:02:54 -0400 Subject: [PATCH 066/409] DOC: 'Along an axis' rewrite and other reviewer changes --- doc/source/glossary.rst | 168 ++++++++++++++++++++++++---------------- 1 file changed, 100 insertions(+), 68 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index e6c8bbd17270..20d3227d2a6b 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -6,20 +6,23 @@ Glossary (`n`,) - A tuple with one element. The trailing comma distinguishes a one-element - tuple from a parenthesized ``n``. + A parenthesized number followed by a comma denotes a tuple with one + element. The trailing comma distinguishes a one-element tuple from a + parenthesized ``n``. -1 - Used as a dimension entry, ``-1`` instructs NumPy to choose the length - that will keep the total number of elements the same. + - **In a dimension entry**, instructs NumPy to choose the length + that will keep the total number of array elements the same. - >>> np.arange(12).reshape(4,-1).shape + >>> np.arange(12).reshape(4, -1).shape (4, 3) + - **In an index**, any negative value + `denotes `_ + indexing from the right. - - ``...`` + . . . An :py:data:`Ellipsis`. - **When indexing an array**, shorthand that the missing axes, if they @@ -39,13 +42,13 @@ Glossary >>> a[0,...,0].shape (3,) - It can be used at most once; ``a[...,0,...]`` raises an :exc:`IndexError`. + It can be used at most once; ``a[...,0,...]`` raises an :exc:`IndexError`. - **In printouts**, NumPy substitutes ``...`` for the middle elements of large arrays. To see the entire array, use `numpy.printoptions` - ``:`` + : The Python :term:`python:slice` operator. In ndarrays, slicing can be applied to every axis: @@ -77,14 +80,14 @@ Glossary For details, see :ref:`combining-advanced-and-basic-indexing`. - ``<`` + < In a dtype declaration, indicates that the data is :term:`little-endian` (the bracket is big on the right). :: >>> dt = np.dtype('`` + > In a dtype declaration, indicates that the data is :term:`big-endian` (the bracket is big on the left). :: @@ -99,35 +102,41 @@ Glossary along an axis - Reversing along axis 0 (the row axis) below reverses the columns, and - reversing along axis 1 (the column axis) reverses the rows. This might - be the opposite of what a new user would expect. + An operation `along axis n` of array ``a`` behaves as if its argument + were an array of slices of ``a`` where each slice has a successive + index of axis `n`. + + For example, if ``a`` is a 3 x `N` array, an operation along axis 0 + behaves as if its argument were an array containing slices of each row: + + >>> np.array((a[0,:], a[1,:], a[2,:])) + + To make it concrete, we can pick the operation to be the array-reversal + function :func:`numpy.flip`, which accepts an ``axis`` argument. We + construct a 3 x 4 array ``a``: >>> a = np.arange(12).reshape(3,4) >>> a array([[ 0, 1, 2, 3], [ 4, 5, 6, 7], [ 8, 9, 10, 11]]) - + + Reversing along axis 0 (the row axis) yields + >>> np.flip(a,axis=0) array([[ 8, 9, 10, 11], [ 4, 5, 6, 7], [ 0, 1, 2, 3]]) - - >>> np.flip(a,axis=1) - array([[ 3, 2, 1, 0], - [ 7, 6, 5, 4], - [11, 10, 9, 8]]) - An operation "along axis 0" behaves as if the argument were an array of - ``a`` slices taking successive indexes of axis 0: + Recalling the definition of `along an axis`, ``flip`` along axis 0 is + treating its argument as if it were >>> np.array((a[0,:], a[1,:], a[2,:])) array([[ 0, 1, 2, 3], [ 4, 5, 6, 7], [ 8, 9, 10, 11]]) - Reversing the slices results in ``np.flip(a,axis=0)``: + and the result of ``np.flip(a,axis=0)`` is to reverse the slices: >>> np.array((a[2,:],a[1,:],a[0,:])) array([[ 8, 9, 10, 11], @@ -136,8 +145,7 @@ Glossary array - Used synonymously in the NumPy docs with - :doc:`ndarray `, NumPy's basic structure. + Used synonymously in the NumPy docs with :term:`ndarray`. array_like @@ -178,7 +186,6 @@ Glossary >>> a array([[[ 0, 1, 2], [ 3, 4, 5]], - [[ 6, 7, 8], [ 9, 10, 11]]]) @@ -217,12 +224,12 @@ Glossary .base If an array does not own its memory, then its - :doc:`base ` attribute - returns the object whose memory the array is referencing. That object - may be borrowing the memory from still another object, so the - owning object may be ``a.base.base.base...``. Despite advice to the - contrary, testing ``base`` is not a surefire way to determine if two - arrays are :term:`view`\ s. + :doc:`base ` attribute returns + the object whose memory the array is referencing. That object may be + referencing the memory from still another object, so the owning object + may be ``a.base.base.base...``. Some writers erroneously claim that + testing ``base`` determines if arrays are :term:`view`\ s. For the + correct way, see :func:`numpy.shares_memory`. `big-endian `_ @@ -237,17 +244,30 @@ Glossary ``broadcasting`` is NumPy's ability to process ndarrays of different sizes as if all were the same size. - When NumPy operates on two arrays, it works element by - element -- for instance, ``c = a * b`` is :: + It permits an elegant do-what-I-mean behavior where, for instance, + adding a scalar to a vector adds the scalar value to every element. + + >>> a = np.arange(3) + >>> a + array([0, 1, 2]) + + >>> a + [3, 3, 3] + array([3, 4, 5]) + + >>> a + 3 + array([3, 4, 5]) + + Ordinarly, vector operands must all be the same size, because NumPy + works element by element -- for instance, ``c = a * b`` is :: c[0,0,0] = a[0,0,0] * b[0,0,0] c[0,0,1] = a[0,0,1] * b[0,0,1] ... - Ordinarily this means the shapes of a and b must be identical. But in - some cases, NumPy can fill "missing" axes or "too-short" dimensions - with duplicate data so shapes will match. The duplication costs - no memory or time. For details, see :doc:`Broadcasting. ` + But in certain useful cases, NumPy can duplicate data along "missing" + axes or "too-short" dimensions so shapes will match. The duplication + costs no memory or time. For details, see + :doc:`Broadcasting. ` C order @@ -259,7 +279,10 @@ Glossary contiguous - An array is contiguous if it occupies a single unbroken block of memory. + An array is contiguous if + * it occupies an unbroken block of memory, and + * array elements with higher indexes occupy higher addresses (that + is, the :term:`stride` is not negative). copy @@ -299,12 +322,12 @@ Glossary contrast to Python lists, are homogeneous. The type can be complicated, as in a :term:`structured array`, but all elements have that type. - NumPy `object arrays <#term-object-array>`_, which contain references to - Python objects, fill the role of heterogeneous arrays. + NumPy `object arrays <#term-object-array>`_, which contain references to + Python objects, fill the role of heterogeneous arrays. itemsize - The size of the dtype element in bytes. + The size of the dtype element in bytes. `little-endian `_ @@ -312,7 +335,19 @@ Glossary mask - The boolean array used to select elements in a :term:`masked array`. + A boolean array used to select only certain elements for an operation: + + >>> x = np.arange(5) + >>> x + array([0, 1, 2, 3, 4]) + + >>> mask = (x > 2) + >>> mask + array([False, False, False, True, True]) + + >>> x[mask] = -1 + >>> x + array([ 0, 1, 2, -1, -1]) masked array @@ -341,7 +376,7 @@ Glossary ndarray - See :term:`array`. + :doc:`NumPy's basic structure `. object array @@ -365,15 +400,15 @@ Glossary record array - A :term:`structured array` with an additional way to access - fields -- ``a.field`` in addition to ``a['field']``. For details, see + A :term:`structured array` with allowing access in an attribute style + (``a.field``) in addition to ``a['field']``. For details, see :doc:`numpy.recarray. ` row-major - `row-major `_ - order is also known as C order, as the C programming language uses it. - New NumPy arrays are by default in row-major order. + `row-major `_ + order is also known as C order, as the C programming language uses it. + New NumPy arrays are by default in row-major order. :doc:`scalar ` @@ -390,7 +425,7 @@ Glossary :term:`slice <:>` - \ + \ stride @@ -424,7 +459,7 @@ Glossary subarray - An array nested in a :term:`structured data type`: :: + An array nested in a :term:`structured data type`, as ``b`` is here: >>> dt = np.dtype([('a', np.int32), ('b', np.float32, (3,))]) >>> np.zeros(3, dtype=dt) @@ -433,7 +468,7 @@ Glossary subarray data type - An element of a structured datatype that behaves like an ndarray. + An element of a structured datatype that behaves like an ndarray. title @@ -446,10 +481,10 @@ Glossary ufunc - NumPy's fast element-by-element computation (:term:`vectorization`) is - structured so as to leave the choice of function open. A function used - in vectorization is called a ``ufunc``, short for ``universal - function``. NumPy routines have built-in ufuncs, but users can also + NumPy's fast element-by-element computation (:term:`vectorization`) + gives a choice which function gets applied. The general term for the + function is ``ufunc``, short for ``universal function``. NumPy routines + have built-in ufuncs, but users can also :doc:`write their own. ` @@ -461,18 +496,15 @@ Glossary structuring NumPy code to leverage it. view - Without changing underlying data, NumPy can make one array masquerade as - any number of other arrays with different types, shapes, and even - content. This is much faster than creating those arrays. - - An array created this way is a `view`, and the performance gain often - makes an array created as a view preferable to one created as a new - array. - - But because a view shares data with the original array, a write in one - array can affect the other, even though they appear to be different - arrays. If this is an problem, a view can't be used; the second array - needs to be physically distinct -- a ``copy``. + Without touching underlying data, NumPy can make one array appear + to change its datatype and shape. + + An array created this way is a `view`, and NumPy often exploits the + performance gain of using a view versus making a new array. + + A potential drawback is that writing to a view can alter the original + as well. If this is a problem, the NumPy instead needs to create a + physically distinct array -- a `copy`. Some NumPy routines always return views, some always return copies, some may return one or the other, and for some the choice can be specified. From 563f5c27ab365204d72b436654a2dad54af0658d Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Fri, 18 Sep 2020 10:55:34 -0400 Subject: [PATCH 067/409] DOC: Replace linked headings with explicit 'See' --- doc/source/glossary.rst | 39 ++++++++++++++++----------------------- 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 20d3227d2a6b..0ca3becb161d 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -109,7 +109,7 @@ Glossary For example, if ``a`` is a 3 x `N` array, an operation along axis 0 behaves as if its argument were an array containing slices of each row: - >>> np.array((a[0,:], a[1,:], a[2,:])) + >>> np.array((a[0,:], a[1,:], a[2,:])) #doctest: +SKIP To make it concrete, we can pick the operation to be the array-reversal function :func:`numpy.flip`, which accepts an ``axis`` argument. We @@ -232,8 +232,8 @@ Glossary correct way, see :func:`numpy.shares_memory`. - `big-endian `_ - \ + big-endian + See `Endianness `_. BLAS @@ -271,11 +271,11 @@ Glossary C order - See :term:`row-major`. + Same as :term:`row-major`. - `column-major `_ - \ + column-major + See `Row- and column-major order `_. contiguous @@ -310,7 +310,7 @@ Glossary Fortran order - Same as `column-major `_ + Same as :term:`column-major`. flattened @@ -330,8 +330,8 @@ Glossary The size of the dtype element in bytes. - `little-endian `_ - \ + little-endian + See `Endianness `_. mask @@ -406,13 +406,12 @@ Glossary row-major - `row-major `_ - order is also known as C order, as the C programming language uses it. - New NumPy arrays are by default in row-major order. + See `Row- and column-major order `_. + NumPy creates arrays in row-major order by default. - :doc:`scalar ` - \ + scalar + In NumPy, usually a synonym for :term:`array scalar`. shape @@ -424,10 +423,6 @@ Glossary :doc:`numpy.ndarray.shape `. - :term:`slice <:>` - \ - - stride Physical memory is one-dimensional; strides provide a mechanism to map a given index to an address in memory. For an N-dimensional array, its @@ -447,7 +442,6 @@ Glossary `_ - structured array Array whose :term:`dtype` is a :term:`structured data type`. @@ -509,10 +503,9 @@ Glossary Some NumPy routines always return views, some always return copies, some may return one or the other, and for some the choice can be specified. Responsiblity for managing views and copies falls to the programmer. - :doc:`numpy.shares_memory ` - will check whether ``b`` is a view of ``a``, - but an exact answer isn't always feasible, as the documentation page - explains. + :func:`numpy.shares_memory` will check whether ``b`` is a view of + ``a``, but an exact answer isn't always feasible, as the documentation + page explains. >>> x = np.arange(5) >>> x From 85f364a0c9dd5ac91dfae6d9ce05193a5f2f7eb0 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Fri, 18 Sep 2020 12:08:09 -0400 Subject: [PATCH 068/409] DOC: Fix minor typos --- doc/source/glossary.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 0ca3becb161d..d4db87723d3d 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -304,9 +304,9 @@ Glossary field - In a :term:`structured data type`, each sub-type is called a `field`. + In a :term:`structured data type`, each subtype is called a `field`. The `field` has a name (a string), a type (any valid dtype), and - an optional `title`. See :ref:`arrays.dtypes` + an optional `title`. See :ref:`arrays.dtypes`. Fortran order @@ -497,7 +497,7 @@ Glossary performance gain of using a view versus making a new array. A potential drawback is that writing to a view can alter the original - as well. If this is a problem, the NumPy instead needs to create a + as well. If this is a problem, NumPy instead needs to create a physically distinct array -- a `copy`. Some NumPy routines always return views, some always return copies, some From a368e00114df2e5e3b74f050d4cf7aa04e58a228 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Wed, 16 Sep 2020 12:53:26 +0100 Subject: [PATCH 069/409] DOC: Fix incorrect `.. deprecated::` syntax that led to this note being invisible. --- doc/source/reference/arrays.dtypes.rst | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/doc/source/reference/arrays.dtypes.rst b/doc/source/reference/arrays.dtypes.rst index 575984707ee3..301e26c55cb3 100644 --- a/doc/source/reference/arrays.dtypes.rst +++ b/doc/source/reference/arrays.dtypes.rst @@ -152,14 +152,6 @@ Array-scalar types >>> dt = np.dtype(np.complex128) # 128-bit complex floating-point number Generic types - .. deprecated NumPy 1.19:: - - The use of generic types is deprecated. This is because it can be - unexpected in a context such as ``arr.astype(dtype=np.floating)``. - ``arr.astype(dtype=np.floating)`` which casts an array of ``float32`` - to an array of ``float64``, even though ``float32`` is a subdtype of - ``np.floating``. - The generic hierarchical type objects convert to corresponding type objects according to the associations: @@ -172,6 +164,15 @@ Generic types :class:`generic`, :class:`flexible` :class:`void` ===================================================== =============== + .. deprecated:: 1.19 + + This conversion of generic scalar types is deprecated. + This is because it can be unexpected in a context such as + ``arr.astype(dtype=np.floating)``, which casts an array of ``float32`` + to an array of ``float64``, even though ``float32`` is a subdtype of + ``np.floating``. + + Built-in Python types Several python types are equivalent to a corresponding array scalar when used to generate a :class:`dtype` object: From 93867a956e6b9c8db036c1f03eeeced9ea460a1b Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Fri, 18 Sep 2020 18:37:24 -0400 Subject: [PATCH 070/409] DOC: How-to guide for drafting how-tos Migrated from numpy-tutorials --- doc/source/user/how-to-how-to.rst | 115 ++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 doc/source/user/how-to-how-to.rst diff --git a/doc/source/user/how-to-how-to.rst b/doc/source/user/how-to-how-to.rst new file mode 100644 index 000000000000..0a037f23728d --- /dev/null +++ b/doc/source/user/how-to-how-to.rst @@ -0,0 +1,115 @@ +:orphan: + +.source/_templates/autosummary/attribute.rst. _how-to-how-to: + +############################################################################## +How to write a NumPy how-to +############################################################################## + +How-tos get straight to the point -- they + + - answer a focused question, or + - narrow a broad question into focused questions that the user can + choose among. + +****************************************************************************** +A stranger has asked for directions... +****************************************************************************** + +**"I need to fill my tank."** + +****************************************************************************** +Give a brief but explicit answer +****************************************************************************** + + - `"Three miles, take a right at Hayseed Road, it's on your left."` + +Add helpful details for newcomers ("Hayseed Road", even though it's the only +turnoff at three miles). But not irrelevant ones: + + - Don't also give directions from Route 7. + - Don't explain why the town has only one gas station. + +If there's related background (tutorial, explanation, reference, alternative +approach), bring it to the user's attention with a link ("Directions from Route 7," +"Why so few gas stations?"). + + +****************************************************************************** +Delegate +****************************************************************************** + + - `"Three miles, take a right at Hayseed Road, follow the signs."` + +If the information is already documented and succinct enough for a how-to, +just link to it, possibly after an introduction ("Three miles, take a right"). + +****************************************************************************** +If the question is broad, narrow and redirect it +****************************************************************************** + + **"I want to see the sights"** + +The `See the sights` how-to should link to a set of narrower how-tos: + +- Find historic buildings +- Find scenic lookouts +- Find the town center + +and these might in turn link to still narrower how-tos -- so the town center +page might link to + + - Find the court house + - Find city hall + +By organizing how-tos this way, you not only display the options for people +who need to narrow their question, you also have provided answers for users +who start with narrower questions ("I want to see historic buildings," "Which +way to city hall?"). + +****************************************************************************** +Break it up +****************************************************************************** + +If a how-to has many steps: + + - Consider breaking a step out into an individual how-to and linking to it. + - Include subheadings. They help readers grasp what's coming and return + where they left off. + +****************************************************************************** +Why write how-tos when there's Stack Overflow, Reddit, Gitter...? +****************************************************************************** + + - We have authoritative answers. + - How-tos make the site less forbidding to non-experts. + - How-tos bring people into the site and help them discover other information + that's here . + - Creating how-tos helps us see NumPy usability through new eyes. + +****************************************************************************** +Aren't how-tos and tutorials the same thing? +****************************************************************************** + +People use them interchangeably, but we draw a distinction, following `Daniele +Procida `_. Documentation needs to meet +users where they are. `How-tos` offer get-it-done information; the user wants +steps to copy and doesn't necessarily want to understand NumPy. `Tutorials` +are warm-fuzzy information; the user wants a feel for some aspect of NumPy +(and again, may or may not care about deeper knowledge). + +We distinguish both tutorials and how-tos from `Explanations`, which are +deep dives intended to give understanding rather than immediate assistance, +and `References`, which give complete, autoritative data on some concrete +part of NumPy (like its API) but aren't obligated to paint a broader picture. + +For more on tutorials, see +`the tutorial how-to `. + + +****************************************************************************** +Is this page an example of a how-to? +****************************************************************************** + +Yes -- until the sections with question-mark headings; they explain rather +than giving directions. In a how-to, those would be links. \ No newline at end of file From e34a30e9736e8f966c1eac1af299dfd193db6dd3 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Fri, 18 Sep 2020 18:37:24 -0400 Subject: [PATCH 071/409] DOC: How-to guide for drafting how-tos Migrated from numpy-tutorials --- doc/source/user/how-to-how-to.rst | 116 ++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 doc/source/user/how-to-how-to.rst diff --git a/doc/source/user/how-to-how-to.rst b/doc/source/user/how-to-how-to.rst new file mode 100644 index 000000000000..59a51efda0ab --- /dev/null +++ b/doc/source/user/how-to-how-to.rst @@ -0,0 +1,116 @@ +:orphan: + +.. _how-to-how-to: + +############################################################################## +How to write a NumPy how-to +############################################################################## + +How-tos get straight to the point -- they + + - answer a focused question, or + - narrow a broad question into focused questions that the user can + choose among. + +****************************************************************************** +A stranger has asked for directions... +****************************************************************************** + +**"I need to fill my tank."** + +****************************************************************************** +Give a brief but explicit answer +****************************************************************************** + + - `"Three miles, take a right at Hayseed Road, it's on your left."` + +Add helpful details for newcomers ("Hayseed Road", even though it's the only +turnoff at three miles). But not irrelevant ones: + + - Don't also give directions from Route 7. + - Don't explain why the town has only one gas station. + +If there's related background (tutorial, explanation, reference, alternative +approach), bring it to the user's attention with a link ("Directions from Route 7," +"Why so few gas stations?"). + + +****************************************************************************** +Delegate +****************************************************************************** + + - `"Three miles, take a right at Hayseed Road, follow the signs."` + +If the information is already documented and succinct enough for a how-to, +just link to it, possibly after an introduction ("Three miles, take a right"). + +****************************************************************************** +If the question is broad, narrow and redirect it +****************************************************************************** + + **"I want to see the sights"** + +The `See the sights` how-to should link to a set of narrower how-tos: + +- Find historic buildings +- Find scenic lookouts +- Find the town center + +and these might in turn link to still narrower how-tos -- so the town center +page might link to + + - Find the court house + - Find city hall + +By organizing how-tos this way, you not only display the options for people +who need to narrow their question, you also have provided answers for users +who start with narrower questions ("I want to see historic buildings," "Which +way to city hall?"). + +****************************************************************************** +If there are many steps, break them up +****************************************************************************** + +If a how-to has many steps: + + - Consider breaking a step out into an individual how-to and linking to it. + - Include subheadings. They help readers grasp what's coming and return + where they left off. + +****************************************************************************** +Why write how-tos when there's Stack Overflow, Reddit, Gitter...? +****************************************************************************** + + - We have authoritative answers. + - How-tos make the site less forbidding to non-experts. + - How-tos bring people into the site and help them discover other information + that's here . + - Creating how-tos helps us see NumPy usability through new eyes. + +****************************************************************************** +Aren't how-tos and tutorials the same thing? +****************************************************************************** + +People use the terms "how-to" and "tutorial" interchangeably, but we draw a +distinction, following `Daniele Procida `_. +Documentation needs to meet users where they are. `How-tos` offer get-it-done +information; the user wants steps to copy and doesn't necessarily want to +understand NumPy. `Tutorials` are warm-fuzzy information; the user wants a +feel for some aspect of NumPy (and again, may or may not care about deeper +knowledge). + +We distinguish both tutorials and how-tos from `Explanations`, which are +deep dives intended to give understanding rather than immediate assistance, +and `References`, which give complete, autoritative data on some concrete +part of NumPy (like its API) but aren't obligated to paint a broader picture. + +For more on tutorials, see +`the tutorial how-to `. + + +****************************************************************************** +Is this page an example of a how-to? +****************************************************************************** + +Yes -- until the sections with question-mark headings; they explain rather +than giving directions. In a how-to, those would be links. \ No newline at end of file From a9b4bff3613a0081703f46523d37f81982170a8c Mon Sep 17 00:00:00 2001 From: Stefan Appelhoff Date: Sat, 19 Sep 2020 11:48:11 +0200 Subject: [PATCH 072/409] DOC: clarify residuals return param Specify that the sum of *squared* residuals are returned. Use @ operator instead of * in code example. --- numpy/linalg/linalg.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/numpy/linalg/linalg.py b/numpy/linalg/linalg.py index 92f93d671a12..b6d860dfabad 100644 --- a/numpy/linalg/linalg.py +++ b/numpy/linalg/linalg.py @@ -2206,8 +2206,8 @@ def lstsq(a, b, rcond="warn"): Least-squares solution. If `b` is two-dimensional, the solutions are in the `K` columns of `x`. residuals : {(1,), (K,), (0,)} ndarray - Sums of residuals; squared Euclidean 2-norm for each column in - ``b - a*x``. + Sums of squared residuals: Squared Euclidean 2-norm for each column in + ``b - a @ x``. If the rank of `a` is < N or M <= N, this is an empty array. If `b` is 1-dimensional, this is a (1,) shape array. Otherwise the shape is (K,). @@ -2558,7 +2558,7 @@ def norm(x, ord=None, axis=None, keepdims=False): # special case for speedup s = (x.conj() * x).real return sqrt(add.reduce(s, axis=axis, keepdims=keepdims)) - # None of the str-type keywords for ord ('fro', 'nuc') + # None of the str-type keywords for ord ('fro', 'nuc') # are valid for vectors elif isinstance(ord, str): raise ValueError(f"Invalid norm order '{ord}' for vectors") From fa1fa77264e977ace57afc2520f4d12d39757cf8 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2020 07:16:13 +0000 Subject: [PATCH 073/409] MAINT: Bump hypothesis from 5.35.1 to 5.35.3 Bumps [hypothesis](https://github.com/HypothesisWorks/hypothesis) from 5.35.1 to 5.35.3. - [Release notes](https://github.com/HypothesisWorks/hypothesis/releases) - [Commits](https://github.com/HypothesisWorks/hypothesis/compare/hypothesis-python-5.35.1...hypothesis-python-5.35.3) Signed-off-by: dependabot-preview[bot] --- test_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index ca39de622bc0..036e9861fa4e 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,7 +1,7 @@ cython==0.29.21 wheel setuptools<49.2.0 -hypothesis==5.35.1 +hypothesis==5.35.3 pytest==6.0.2 pytz==2020.1 pytest-cov==2.10.1 From f983885f4cafddfe5b5c1e7e19e8d3a04d735fee Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Sat, 19 Sep 2020 18:37:23 -0600 Subject: [PATCH 074/409] ENH: Add PyLong_AsInt function. This compliments the PyLong_As* functions provided by Python. It is copied from the Python private function _PyLong_AsInt. It allows replacing some incorrect uses of PyInt_Check. --- numpy/core/include/numpy/npy_3kcompat.h | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/numpy/core/include/numpy/npy_3kcompat.h b/numpy/core/include/numpy/npy_3kcompat.h index 4bc06fc9695e..191cd244f875 100644 --- a/numpy/core/include/numpy/npy_3kcompat.h +++ b/numpy/core/include/numpy/npy_3kcompat.h @@ -28,6 +28,30 @@ extern "C" { * PyInt -> PyLong */ + +/* + * This is a renamed copy of the Python non-limited API function _PyLong_AsInt. It is + * included here because it is missing from the PyPy API. It completes the PyLong_As* + * group of functions and can be useful in replacing PyInt_Check. + */ +static NPY_INLINE int +Npy__PyLong_AsInt(PyObject *obj) +{ + int overflow; + long result = PyLong_AsLongAndOverflow(obj, &overflow); + + /* INT_MAX and INT_MIN are defined in Python.h */ + if (overflow || result > INT_MAX || result < INT_MIN) { + /* XXX: could be cute and give a different + message for overflow == -1 */ + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C int"); + return -1; + } + return (int)result; +} + + #if defined(NPY_PY3K) /* Return True only if the long fits in a C long */ static NPY_INLINE int PyInt_Check(PyObject *op) { @@ -39,6 +63,7 @@ static NPY_INLINE int PyInt_Check(PyObject *op) { return (overflow == 0); } + #define PyInt_FromLong PyLong_FromLong #define PyInt_AsLong PyLong_AsLong #define PyInt_AS_LONG PyLong_AsLong From ec565addec836ea0fe88f8766eec6168db3211f7 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Fri, 11 Sep 2020 10:55:01 -0600 Subject: [PATCH 075/409] MAINT: Cleanup f2py/cfuncs.py - Replace npy_3kcompat macro PyInt_AsLong by appropriate definition - Replace npy_3kcompat macro PyInt_AS_LONG by appropriate definition. - Reformat code for readability. The code in extensively reformated in this PR which may hide the most important parts of the diff, but I needed to do it to make the code readable. --- numpy/f2py/cfuncs.py | 250 ++++++++++++++++++++++++++++--------------- 1 file changed, 164 insertions(+), 86 deletions(-) diff --git a/numpy/f2py/cfuncs.py b/numpy/f2py/cfuncs.py index 94867b3093ec..26b43e7e6964 100644 --- a/numpy/f2py/cfuncs.py +++ b/numpy/f2py/cfuncs.py @@ -629,7 +629,9 @@ """ needs['string_from_pyobj'] = ['string', 'STRINGMALLOC', 'STRINGCOPYN'] cfuncs['string_from_pyobj'] = """\ -static int string_from_pyobj(string *str,int *len,const string inistr,PyObject *obj,const char *errmess) { +static int +string_from_pyobj(string *str,int *len,const string inistr,PyObject *obj,const char *errmess) +{ PyArrayObject *arr = NULL; PyObject *tmp = NULL; #ifdef DEBUGCFUNCS @@ -684,127 +686,165 @@ Py_XDECREF(tmp); { PyObject* err = PyErr_Occurred(); - if (err==NULL) err = #modulename#_error; - PyErr_SetString(err,errmess); + if (err == NULL) { + err = #modulename#_error; + } + PyErr_SetString(err, errmess); } return 0; } """ + + needs['char_from_pyobj'] = ['int_from_pyobj'] cfuncs['char_from_pyobj'] = """\ -static int char_from_pyobj(char* v,PyObject *obj,const char *errmess) { - int i=0; - if (int_from_pyobj(&i,obj,errmess)) { +static int +char_from_pyobj(char* v, PyObject *obj, const char *errmess) { + int i = 0; + if (int_from_pyobj(&i, obj, errmess)) { *v = (char)i; return 1; } return 0; } """ + + needs['signed_char_from_pyobj'] = ['int_from_pyobj', 'signed_char'] cfuncs['signed_char_from_pyobj'] = """\ -static int signed_char_from_pyobj(signed_char* v,PyObject *obj,const char *errmess) { - int i=0; - if (int_from_pyobj(&i,obj,errmess)) { +static int +signed_char_from_pyobj(signed_char* v, PyObject *obj, const char *errmess) { + int i = 0; + if (int_from_pyobj(&i, obj, errmess)) { *v = (signed_char)i; return 1; } return 0; } """ + + needs['short_from_pyobj'] = ['int_from_pyobj'] cfuncs['short_from_pyobj'] = """\ -static int short_from_pyobj(short* v,PyObject *obj,const char *errmess) { - int i=0; - if (int_from_pyobj(&i,obj,errmess)) { +static int +short_from_pyobj(short* v, PyObject *obj, const char *errmess) { + int i = 0; + if (int_from_pyobj(&i, obj, errmess)) { *v = (short)i; return 1; } return 0; } """ + + cfuncs['int_from_pyobj'] = """\ -static int int_from_pyobj(int* v,PyObject *obj,const char *errmess) { +static int +int_from_pyobj(int* v, PyObject *obj, const char *errmess) +{ PyObject* tmp = NULL; - if (PyInt_Check(obj)) { - *v = (int)PyInt_AS_LONG(obj); - return 1; + + if (PyLong_Check(obj)) { + *v = Npy__PyLong_AsInt(obj); + return !(*v == -1 && PyErr_Occurred()); } + tmp = PyNumber_Long(obj); if (tmp) { - *v = PyInt_AS_LONG(tmp); + *v = Npy__PyLong_AsInt(tmp); Py_DECREF(tmp); - return 1; + return !(*v == -1 && PyErr_Occurred()); } + if (PyComplex_Check(obj)) tmp = PyObject_GetAttrString(obj,\"real\"); else if (PyBytes_Check(obj) || PyUnicode_Check(obj)) /*pass*/; else if (PySequence_Check(obj)) - tmp = PySequence_GetItem(obj,0); + tmp = PySequence_GetItem(obj, 0); if (tmp) { PyErr_Clear(); - if (int_from_pyobj(v,tmp,errmess)) {Py_DECREF(tmp); return 1;} + if (int_from_pyobj(v, tmp, errmess)) { + Py_DECREF(tmp); + return 1; + } Py_DECREF(tmp); } { PyObject* err = PyErr_Occurred(); - if (err==NULL) err = #modulename#_error; - PyErr_SetString(err,errmess); + if (err == NULL) { + err = #modulename#_error; + } + PyErr_SetString(err, errmess); } return 0; } """ + + cfuncs['long_from_pyobj'] = """\ -static int long_from_pyobj(long* v,PyObject *obj,const char *errmess) { +static int +long_from_pyobj(long* v, PyObject *obj, const char *errmess) { PyObject* tmp = NULL; - if (PyInt_Check(obj)) { - *v = PyInt_AS_LONG(obj); - return 1; + + if (PyLong_Check(obj)) { + *v = PyLong_AsLong(obj); + return !(*v == -1 && PyErr_Occurred()); } + tmp = PyNumber_Long(obj); if (tmp) { - *v = PyInt_AS_LONG(tmp); + *v = PyLong_AsLong(tmp); Py_DECREF(tmp); - return 1; + return !(*v == -1 && PyErr_Occurred()); } + if (PyComplex_Check(obj)) tmp = PyObject_GetAttrString(obj,\"real\"); else if (PyBytes_Check(obj) || PyUnicode_Check(obj)) /*pass*/; else if (PySequence_Check(obj)) tmp = PySequence_GetItem(obj,0); + if (tmp) { PyErr_Clear(); - if (long_from_pyobj(v,tmp,errmess)) {Py_DECREF(tmp); return 1;} + if (long_from_pyobj(v, tmp, errmess)) { + Py_DECREF(tmp); + return 1; + } Py_DECREF(tmp); } { PyObject* err = PyErr_Occurred(); - if (err==NULL) err = #modulename#_error; - PyErr_SetString(err,errmess); + if (err == NULL) { + err = #modulename#_error; + } + PyErr_SetString(err, errmess); } return 0; } """ + + needs['long_long_from_pyobj'] = ['long_long'] cfuncs['long_long_from_pyobj'] = """\ -static int long_long_from_pyobj(long_long* v,PyObject *obj,const char *errmess) { +static int +long_long_from_pyobj(long_long* v, PyObject *obj, const char *errmess) +{ PyObject* tmp = NULL; + if (PyLong_Check(obj)) { *v = PyLong_AsLongLong(obj); - return (!PyErr_Occurred()); - } - if (PyInt_Check(obj)) { - *v = (long_long)PyInt_AS_LONG(obj); - return 1; + return !(*v == -1 && PyErr_Occurred()); } + tmp = PyNumber_Long(obj); if (tmp) { *v = PyLong_AsLongLong(tmp); Py_DECREF(tmp); - return (!PyErr_Occurred()); + return !(*v == -1 && PyErr_Occurred()); } + if (PyComplex_Check(obj)) tmp = PyObject_GetAttrString(obj,\"real\"); else if (PyBytes_Check(obj) || PyUnicode_Check(obj)) @@ -813,58 +853,64 @@ tmp = PySequence_GetItem(obj,0); if (tmp) { PyErr_Clear(); - if (long_long_from_pyobj(v,tmp,errmess)) {Py_DECREF(tmp); return 1;} + if (long_long_from_pyobj(v, tmp, errmess)) { + Py_DECREF(tmp); + return 1; + } Py_DECREF(tmp); } { PyObject* err = PyErr_Occurred(); - if (err==NULL) err = #modulename#_error; + if (err == NULL) { + err = #modulename#_error; + } PyErr_SetString(err,errmess); } return 0; } """ + + needs['long_double_from_pyobj'] = ['double_from_pyobj', 'long_double'] cfuncs['long_double_from_pyobj'] = """\ -static int long_double_from_pyobj(long_double* v,PyObject *obj,const char *errmess) { +static int +long_double_from_pyobj(long_double* v, PyObject *obj, const char *errmess) +{ double d=0; if (PyArray_CheckScalar(obj)){ if PyArray_IsScalar(obj, LongDouble) { PyArray_ScalarAsCtype(obj, v); return 1; } - else if (PyArray_Check(obj) && PyArray_TYPE(obj)==NPY_LONGDOUBLE) { + else if (PyArray_Check(obj) && PyArray_TYPE(obj) == NPY_LONGDOUBLE) { (*v) = *((npy_longdouble *)PyArray_DATA(obj)); return 1; } } - if (double_from_pyobj(&d,obj,errmess)) { + if (double_from_pyobj(&d, obj, errmess)) { *v = (long_double)d; return 1; } return 0; } """ + + cfuncs['double_from_pyobj'] = """\ -static int double_from_pyobj(double* v,PyObject *obj,const char *errmess) { +static int +double_from_pyobj(double* v, PyObject *obj, const char *errmess) +{ PyObject* tmp = NULL; if (PyFloat_Check(obj)) { -#ifdef __sgi *v = PyFloat_AsDouble(obj); -#else - *v = PyFloat_AS_DOUBLE(obj); -#endif - return 1; + return !(*v == -1.0 && PyErr_Occurred()); } + tmp = PyNumber_Float(obj); if (tmp) { -#ifdef __sgi *v = PyFloat_AsDouble(tmp); -#else - *v = PyFloat_AS_DOUBLE(tmp); -#endif Py_DECREF(tmp); - return 1; + return !(*v == -1.0 && PyErr_Occurred()); } if (PyComplex_Check(obj)) tmp = PyObject_GetAttrString(obj,\"real\"); @@ -885,9 +931,13 @@ return 0; } """ + + needs['float_from_pyobj'] = ['double_from_pyobj'] cfuncs['float_from_pyobj'] = """\ -static int float_from_pyobj(float* v,PyObject *obj,const char *errmess) { +static int +float_from_pyobj(float* v, PyObject *obj, const char *errmess) +{ double d=0.0; if (double_from_pyobj(&d,obj,errmess)) { *v = (float)d; @@ -896,11 +946,15 @@ return 0; } """ + + needs['complex_long_double_from_pyobj'] = ['complex_long_double', 'long_double', 'complex_double_from_pyobj'] cfuncs['complex_long_double_from_pyobj'] = """\ -static int complex_long_double_from_pyobj(complex_long_double* v,PyObject *obj,const char *errmess) { - complex_double cd={0.0,0.0}; +static int +complex_long_double_from_pyobj(complex_long_double* v, PyObject *obj, const char *errmess) +{ + complex_double cd = {0.0,0.0}; if (PyArray_CheckScalar(obj)){ if PyArray_IsScalar(obj, CLongDouble) { PyArray_ScalarAsCtype(obj, v); @@ -920,13 +974,17 @@ return 0; } """ + + needs['complex_double_from_pyobj'] = ['complex_double'] cfuncs['complex_double_from_pyobj'] = """\ -static int complex_double_from_pyobj(complex_double* v,PyObject *obj,const char *errmess) { +static int +complex_double_from_pyobj(complex_double* v, PyObject *obj, const char *errmess) { Py_complex c; if (PyComplex_Check(obj)) { - c=PyComplex_AsCComplex(obj); - (*v).r=c.real, (*v).i=c.imag; + c = PyComplex_AsCComplex(obj); + (*v).r = c.real; + (*v).i = c.imag; return 1; } if (PyArray_IsScalar(obj, ComplexFloating)) { @@ -955,28 +1013,22 @@ else { arr = PyArray_FromScalar(obj, PyArray_DescrFromType(NPY_CDOUBLE)); } - if (arr==NULL) return 0; + if (arr == NULL) { + return 0; + } (*v).r = ((npy_cdouble *)PyArray_DATA(arr))->real; (*v).i = ((npy_cdouble *)PyArray_DATA(arr))->imag; return 1; } /* Python does not provide PyNumber_Complex function :-( */ - (*v).i=0.0; + (*v).i = 0.0; if (PyFloat_Check(obj)) { -#ifdef __sgi (*v).r = PyFloat_AsDouble(obj); -#else - (*v).r = PyFloat_AS_DOUBLE(obj); -#endif - return 1; - } - if (PyInt_Check(obj)) { - (*v).r = (double)PyInt_AS_LONG(obj); - return 1; + return !((*v).r == -1.0 && PyErr_Occurred()); } if (PyLong_Check(obj)) { (*v).r = PyLong_AsDouble(obj); - return (!PyErr_Occurred()); + return !((*v).r == -1.0 && PyErr_Occurred()); } if (PySequence_Check(obj) && !(PyBytes_Check(obj) || PyUnicode_Check(obj))) { PyObject *tmp = PySequence_GetItem(obj,0); @@ -997,10 +1049,14 @@ return 0; } """ + + needs['complex_float_from_pyobj'] = [ 'complex_float', 'complex_double_from_pyobj'] cfuncs['complex_float_from_pyobj'] = """\ -static int complex_float_from_pyobj(complex_float* v,PyObject *obj,const char *errmess) { +static int +complex_float_from_pyobj(complex_float* v,PyObject *obj,const char *errmess) +{ complex_double cd={0.0,0.0}; if (complex_double_from_pyobj(&cd,obj,errmess)) { (*v).r = (float)cd.r; @@ -1010,6 +1066,8 @@ return 0; } """ + + needs['try_pyarr_from_char'] = ['pyobj_from_char1', 'TRYPYARRAYTEMPLATE'] cfuncs[ 'try_pyarr_from_char'] = 'static int try_pyarr_from_char(PyObject* obj,char* v) {\n TRYPYARRAYTEMPLATE(char,\'c\');\n}\n' @@ -1047,14 +1105,18 @@ cfuncs[ 'try_pyarr_from_complex_double'] = 'static int try_pyarr_from_complex_double(PyObject* obj,complex_double* v) {\n TRYCOMPLEXPYARRAYTEMPLATE(double,\'D\');\n}\n' -needs['create_cb_arglist'] = ['CFUNCSMESS', 'PRINTPYOBJERR', 'MINMAX'] +needs['create_cb_arglist'] = ['CFUNCSMESS', 'PRINTPYOBJERR', 'MINMAX'] # create the list of arguments to be used when calling back to python cfuncs['create_cb_arglist'] = """\ -static int create_cb_arglist(PyObject* fun,PyTupleObject* xa,const int maxnofargs,const int nofoptargs,int *nofargs,PyTupleObject **args,const char *errmess) { +static int +create_cb_arglist(PyObject* fun, PyTupleObject* xa , const int maxnofargs, + const int nofoptargs, int *nofargs, PyTupleObject **args, + const char *errmess) +{ PyObject *tmp = NULL; PyObject *tmp_fun = NULL; - int tot,opt,ext,siz,i,di=0; + Py_ssize_t tot, opt, ext, siz, i, di = 0; CFUNCSMESS(\"create_cb_arglist\\n\"); tot=opt=ext=siz=0; /* Get the total number of arguments */ @@ -1103,10 +1165,15 @@ Py_INCREF(tmp_fun); } } -if (tmp_fun==NULL) { -fprintf(stderr,\"Call-back argument must be function|instance|instance.__call__|f2py-function but got %s.\\n\",(fun==NULL?\"NULL\":Py_TYPE(fun)->tp_name)); -goto capi_fail; -} + + if (tmp_fun == NULL) { + fprintf(stderr, + \"Call-back argument must be function|instance|instance.__call__|f2py-function \" + \"but got %s.\\n\", + ((fun == NULL) ? \"NULL\" : Py_TYPE(fun)->tp_name)); + goto capi_fail; + } + if (PyObject_HasAttrString(tmp_fun,\"__code__\")) { if (PyObject_HasAttrString(tmp = PyObject_GetAttrString(tmp_fun,\"__code__\"),\"co_argcount\")) { PyObject *tmp_argcount = PyObject_GetAttrString(tmp,\"co_argcount\"); @@ -1114,7 +1181,7 @@ if (tmp_argcount == NULL) { goto capi_fail; } - tot = PyInt_AsLong(tmp_argcount) - di; + tot = PyLong_AsSsize_t(tmp_argcount) - di; Py_DECREF(tmp_argcount); } } @@ -1130,13 +1197,23 @@ /* Calculate the size of call-backs argument list */ siz = MIN(maxnofargs+ext,tot); *nofargs = MAX(0,siz-ext); + #ifdef DEBUGCFUNCS - fprintf(stderr,\"debug-capi:create_cb_arglist:maxnofargs(-nofoptargs),tot,opt,ext,siz,nofargs=%d(-%d),%d,%d,%d,%d,%d\\n\",maxnofargs,nofoptargs,tot,opt,ext,siz,*nofargs); + fprintf(stderr, + \"debug-capi:create_cb_arglist:maxnofargs(-nofoptargs),\" + \"tot,opt,ext,siz,nofargs = %d(-%d), %zd, %zd, %zd, %zd, %d\\n\", + maxnofargs, nofoptargs, tot, opt, ext, siz, *nofargs); #endif - if (siz Date: Mon, 21 Sep 2020 16:32:14 -0400 Subject: [PATCH 076/409] DOC: Add how-to index entry for how-to guide PR #17352 --- doc/source/user/how-to-how-to.rst | 2 -- doc/source/user/howtos_index.rst | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/source/user/how-to-how-to.rst b/doc/source/user/how-to-how-to.rst index 59a51efda0ab..48feae6db92d 100644 --- a/doc/source/user/how-to-how-to.rst +++ b/doc/source/user/how-to-how-to.rst @@ -1,5 +1,3 @@ -:orphan: - .. _how-to-how-to: ############################################################################## diff --git a/doc/source/user/howtos_index.rst b/doc/source/user/howtos_index.rst index c052286b9be8..45e013e6fb6a 100644 --- a/doc/source/user/howtos_index.rst +++ b/doc/source/user/howtos_index.rst @@ -11,4 +11,5 @@ the package, see the :ref:`API reference `. .. toctree:: :maxdepth: 1 + how-to-how-to ionumpy From e1c109910e6f78fce17099c5cbdebd08867b1d8b Mon Sep 17 00:00:00 2001 From: Daniel Vanzo Date: Fri, 18 Sep 2020 10:31:23 +0200 Subject: [PATCH 077/409] ENH: Add support for the NVIDIA HPC SDK fortran compiler --- numpy/distutils/fcompiler/__init__.py | 2 +- numpy/distutils/fcompiler/nv.py | 48 +++++++++++++++++++++++++++ numpy/tests/test_public_api.py | 1 + 3 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 numpy/distutils/fcompiler/nv.py diff --git a/numpy/distutils/fcompiler/__init__.py b/numpy/distutils/fcompiler/__init__.py index 1c3069363f84..31c992d8fb1d 100644 --- a/numpy/distutils/fcompiler/__init__.py +++ b/numpy/distutils/fcompiler/__init__.py @@ -745,7 +745,7 @@ def wrap_unlinkable_objects(self, objects, output_dir, extra_dll_dir): ('win32', ('gnu', 'intelv', 'absoft', 'compaqv', 'intelev', 'gnu95', 'g95', 'intelvem', 'intelem', 'flang')), ('cygwin.*', ('gnu', 'intelv', 'absoft', 'compaqv', 'intelev', 'gnu95', 'g95')), - ('linux.*', ('gnu95', 'intel', 'lahey', 'pg', 'absoft', 'nag', 'vast', 'compaq', + ('linux.*', ('gnu95', 'intel', 'lahey', 'pg', 'nv', 'absoft', 'nag', 'vast', 'compaq', 'intele', 'intelem', 'gnu', 'g95', 'pathf95', 'nagfor')), ('darwin.*', ('gnu95', 'nag', 'absoft', 'ibm', 'intel', 'gnu', 'g95', 'pg')), ('sunos.*', ('sun', 'gnu', 'gnu95', 'g95')), diff --git a/numpy/distutils/fcompiler/nv.py b/numpy/distutils/fcompiler/nv.py new file mode 100644 index 000000000000..60d117c80166 --- /dev/null +++ b/numpy/distutils/fcompiler/nv.py @@ -0,0 +1,48 @@ +# https://developer.nvidia.com/hpc-sdk +import sys + +from numpy.distutils.fcompiler import FCompiler + +compilers = ['NVHPCFCompiler'] + +class NVHPCFCompiler(FCompiler): + + compiler_type = 'nv' + description = 'NVIDIA HPC SDK' + version_pattern = r'\s*(nvfortran|(pg(f77|f90|fortran)) \(aka nvfortran\)) (?P[\d.-]+).*' + + executables = { + 'version_cmd': ["", "-V"], + 'compiler_f77': ["nvfortran"], + 'compiler_fix': ["nvfortran", "-Mfixed"], + 'compiler_f90': ["nvfortran"], + 'linker_so': [""], + 'archiver': ["ar", "-cr"], + 'ranlib': ["ranlib"] + } + pic_flags = ['-fpic'] + + module_dir_switch = '-module ' + module_include_switch = '-I' + + def get_flags(self): + opt = ['-Minform=inform', '-Mnosecond_underscore'] + return self.pic_flags + opt + + def get_flags_opt(self): + return ['-fast'] + + def get_flags_debug(self): + return ['-g'] + + def get_flags_linker_so(self): + return ["-shared", '-fpic'] + + def runtime_library_dir_option(self, dir): + return '-R%s' % dir + +if __name__ == '__main__': + from distutils import log + log.set_verbosity(2) + from numpy.distutils import customized_fcompiler + print(customized_fcompiler(compiler='nv').get_version()) diff --git a/numpy/tests/test_public_api.py b/numpy/tests/test_public_api.py index a9d6da01c7db..ae00d1fc9e35 100644 --- a/numpy/tests/test_public_api.py +++ b/numpy/tests/test_public_api.py @@ -251,6 +251,7 @@ def test_NPY_NO_EXPORT(): "distutils.fcompiler.none", "distutils.fcompiler.pathf95", "distutils.fcompiler.pg", + "distutils.fcompiler.nv", "distutils.fcompiler.sun", "distutils.fcompiler.vast", "distutils.from_template", From 0a93d77580656c7451d7f3dc31eaa9c1fcb76881 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Tue, 22 Sep 2020 08:49:44 -0400 Subject: [PATCH 078/409] DOC: Revise #17352 with @mattip suggestions --- doc/source/user/how-to-how-to.rst | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/doc/source/user/how-to-how-to.rst b/doc/source/user/how-to-how-to.rst index 48feae6db92d..de8afc28ae9e 100644 --- a/doc/source/user/how-to-how-to.rst +++ b/doc/source/user/how-to-how-to.rst @@ -14,39 +14,39 @@ How-tos get straight to the point -- they A stranger has asked for directions... ****************************************************************************** -**"I need to fill my tank."** +**"I need to refuel my car."** ****************************************************************************** Give a brief but explicit answer ****************************************************************************** - - `"Three miles, take a right at Hayseed Road, it's on your left."` + - `"Three kilometers/miles, take a right at Hayseed Road, it's on your left."` Add helpful details for newcomers ("Hayseed Road", even though it's the only -turnoff at three miles). But not irrelevant ones: +turnoff at three km/mi). But not irrelevant ones: - Don't also give directions from Route 7. - - Don't explain why the town has only one gas station. + - Don't explain why the town has only one filling station. If there's related background (tutorial, explanation, reference, alternative approach), bring it to the user's attention with a link ("Directions from Route 7," -"Why so few gas stations?"). +"Why so few filling stations?"). ****************************************************************************** Delegate ****************************************************************************** - - `"Three miles, take a right at Hayseed Road, follow the signs."` + - `"Three km/mi, take a right at Hayseed Road, follow the signs."` If the information is already documented and succinct enough for a how-to, -just link to it, possibly after an introduction ("Three miles, take a right"). +just link to it, possibly after an introduction ("Three km/mi, take a right"). ****************************************************************************** If the question is broad, narrow and redirect it ****************************************************************************** - **"I want to see the sights"** + **"I want to see the sights."** The `See the sights` how-to should link to a set of narrower how-tos: @@ -90,7 +90,10 @@ Aren't how-tos and tutorials the same thing? ****************************************************************************** People use the terms "how-to" and "tutorial" interchangeably, but we draw a -distinction, following `Daniele Procida `_. +distinction, following Daniele Procida's `taxonomy of documentation`_. + + .. _`taxonomy of documentation`: https://documentation.divio.com/ + Documentation needs to meet users where they are. `How-tos` offer get-it-done information; the user wants steps to copy and doesn't necessarily want to understand NumPy. `Tutorials` are warm-fuzzy information; the user wants a @@ -102,8 +105,9 @@ deep dives intended to give understanding rather than immediate assistance, and `References`, which give complete, autoritative data on some concrete part of NumPy (like its API) but aren't obligated to paint a broader picture. -For more on tutorials, see -`the tutorial how-to `. +For more on tutorials, see the `tutorial how-to`_. + +.. _`tutorial how-to`: https://github.com/numpy/numpy-tutorials/blob/master/tutorial_style.ipynb ****************************************************************************** From 155248f763db534abde85ce382716cf0ab3347b9 Mon Sep 17 00:00:00 2001 From: Daniel Vanzo Date: Tue, 22 Sep 2020 15:08:21 +0200 Subject: [PATCH 079/409] ENH: Add comments about Nvidia HPC SDK and PGI --- numpy/distutils/fcompiler/nv.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/numpy/distutils/fcompiler/nv.py b/numpy/distutils/fcompiler/nv.py index 60d117c80166..4cf30f5ef100 100644 --- a/numpy/distutils/fcompiler/nv.py +++ b/numpy/distutils/fcompiler/nv.py @@ -5,6 +5,10 @@ compilers = ['NVHPCFCompiler'] +""" +Since august 2020 the NVIDIA HPC SDK includes the compilers formely known as The Portland Group compilers. +https://www.pgroup.com/index.htm +""" class NVHPCFCompiler(FCompiler): compiler_type = 'nv' From 7afa1d0a5566e56c7f444ac6729784f9730bb559 Mon Sep 17 00:00:00 2001 From: Gerry Manoim Date: Fri, 18 Sep 2020 13:27:07 -0400 Subject: [PATCH 080/409] DEV: add pr prefix labeler and prefix mapping --- .github/pr-prefix-labeler.yml | 12 ++++++++++++ .github/workflows/labeler.yml | 13 +++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 .github/pr-prefix-labeler.yml create mode 100644 .github/workflows/labeler.yml diff --git a/.github/pr-prefix-labeler.yml b/.github/pr-prefix-labeler.yml new file mode 100644 index 000000000000..b50def97e930 --- /dev/null +++ b/.github/pr-prefix-labeler.yml @@ -0,0 +1,12 @@ +"API": "30 - API" +"BENCH": "28 - Benchmark" +"BUG": "00 - Bug" +"DEP": "07 - Deprecation" +"DEV": "16 - Development" +"DOC": "04 - Documentation" +"ENH": "01 - Enhancement" +"MAINT": "03 - Maintenance" +"REV": "34 - Reversion" +"TST": "05 - Testing" +"REL": "14 - Release" +"WIP": "25 - WIP" diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml new file mode 100644 index 000000000000..99db967b383b --- /dev/null +++ b/.github/workflows/labeler.yml @@ -0,0 +1,13 @@ +name: "Pull Request Labeler" +on: + pull_request_target: + types: [opened, synchronize, reopened, edited] + +jobs: + pr-labeler: + runs-on: ubuntu-latest + steps: + - name: Label the PR + uses: gerrymanoim/pr-prefix-labeler@v3 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From c46d1822b56b2f40c79665fb045f6860d6540038 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Tue, 15 Sep 2020 17:42:50 +0200 Subject: [PATCH 081/409] ENH: Add annotations for non-magic ndarray methods --- numpy/__init__.pyi | 466 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 430 insertions(+), 36 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index d4eda6b3150f..eb4b15d61987 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -409,6 +409,8 @@ class _ArrayOrScalarCommon( ) -> _ArraySelf: ... def swapaxes(self: _ArraySelf, axis1: int, axis2: int) -> _ArraySelf: ... def tobytes(self, order: _OrderKACF = ...) -> bytes: ... + # NOTE: `tostring()` is deprecated and therefore excluded + # def tostring(self, order=...): ... def tofile( self, fid: Union[IO[bytes], str], sep: str = ..., format: str = ... ) -> None: ... @@ -437,42 +439,434 @@ class _ArrayOrScalarCommon( def __array_struct__(self): ... def __array_wrap__(array, context=...): ... def __setstate__(self, __state): ... - def all(self, axis=..., out=..., keepdims=...): ... - def any(self, axis=..., out=..., keepdims=...): ... - def argmax(self, axis=..., out=...): ... - def argmin(self, axis=..., out=...): ... - def argpartition(self, kth, axis=..., kind=..., order=...): ... - def argsort(self, axis=..., kind=..., order=...): ... - def choose(self, choices, out=..., mode=...): ... - def clip(self, min=..., max=..., out=..., **kwargs): ... - def compress(self, condition, axis=..., out=...): ... - def conj(self): ... - def conjugate(self): ... - def cumprod(self, axis=..., dtype=..., out=...): ... - def cumsum(self, axis=..., dtype=..., out=...): ... - def diagonal(self, offset=..., axis1=..., axis2=...): ... - def dot(self, b, out=...): ... - def max(self, axis=..., out=..., keepdims=..., initial=..., where=...): ... - def mean(self, axis=..., dtype=..., out=..., keepdims=...): ... - def min(self, axis=..., out=..., keepdims=..., initial=..., where=...): ... - def newbyteorder(self, new_order=...): ... - def nonzero(self): ... - def partition(self, kth, axis=..., kind=..., order=...): ... - def prod(self, axis=..., dtype=..., out=..., keepdims=..., initial=..., where=...): ... - def ptp(self, axis=..., out=..., keepdims=...): ... - def put(self, indices, values, mode=...): ... - def repeat(self, repeats, axis=...): ... - def round(self, decimals=..., out=...): ... - def searchsorted(self, v, side=..., sorter=...): ... - def setfield(self, val, dtype, offset=...): ... - def sort(self, axis=..., kind=..., order=...): ... - def std(self, axis=..., dtype=..., out=..., ddof=..., keepdims=...): ... - def sum(self, axis=..., dtype=..., out=..., keepdims=..., initial=..., where=...): ... - def take(self, indices, axis=..., out=..., mode=...): ... - # NOTE: `tostring()` is deprecated and therefore excluded - # def tostring(self, order=...): ... - def trace(self, offset=..., axis1=..., axis2=..., dtype=..., out=...): ... - def var(self, axis=..., dtype=..., out=..., ddof=..., keepdims=...): ... + # a `bool_` is returned when `keepdims=True` and `self` is a 0d array + @overload + def all( + self, axis: None = ..., out: None = ..., keepdims: Literal[False] = ... + ) -> bool_: ... + @overload + def all( + self, axis: Optional[_ShapeLike] = ..., out: None = ..., keepdims: bool = ... + ) -> Union[bool_, ndarray]: ... + @overload + def all( + self, + axis: Optional[_ShapeLike] = ..., + out: _NdArraySubClass = ..., + keepdims: bool = ..., + ) -> _NdArraySubClass: ... + @overload + def any( + self, axis: None = ..., out: None = ..., keepdims: Literal[False] = ... + ) -> bool_: ... + @overload + def any( + self, axis: Optional[_ShapeLike] = ..., out: None = ..., keepdims: bool = ... + ) -> Union[bool_, ndarray]: ... + @overload + def any( + self, + axis: Optional[_ShapeLike] = ..., + out: _NdArraySubClass = ..., + keepdims: bool = ..., + ) -> _NdArraySubClass: ... + @overload + def argmax(self, axis: None = ..., out: None = ...) -> signedinteger: ... + @overload + def argmax( + self, axis: _ShapeLike = ..., out: None = ... + ) -> Union[signedinteger, ndarray]: ... + @overload + def argmax( + self, axis: Optional[_ShapeLike] = ..., out: _NdArraySubClass = ... + ) -> _NdArraySubClass: ... + @overload + def argmin(self, axis: None = ..., out: None = ...) -> signedinteger: ... + @overload + def argmin( + self, axis: _ShapeLike = ..., out: None = ... + ) -> Union[signedinteger, ndarray]: ... + @overload + def argmin( + self, axis: Optional[_ShapeLike] = ..., out: _NdArraySubClass = ... + ) -> _NdArraySubClass: ... + def argpartition( + self, + kth: _ArrayLikeIntOrBool, + axis: Optional[int] = ..., + kind: _PartitionKind = ..., + order: Union[None, str, Sequence[str]] = ..., + ) -> ndarray: ... + def argsort( + self, + axis: Optional[int] = ..., + kind: Optional[_SortKind] = ..., + order: Union[None, str, Sequence[str]] = ..., + ) -> ndarray: ... + @overload + def choose( + self, choices: ArrayLike, out: None = ..., mode: _Mode = ..., + ) -> ndarray: ... + @overload + def choose( + self, choices: ArrayLike, out: _NdArraySubClass = ..., mode: _Mode = ..., + ) -> _NdArraySubClass: ... + @overload + def clip( + self: _ArraySelf, + min: ArrayLike = ..., + max: Optional[ArrayLike] = ..., + out: None = ..., + **kwargs: Any, + ) -> Union[_ArraySelf, number]: ... + @overload + def clip( + self: _ArraySelf, + min: None = ..., + max: ArrayLike = ..., + out: None = ..., + **kwargs: Any, + ) -> Union[_ArraySelf, number]: ... + @overload + def clip( + self, + min: ArrayLike = ..., + max: Optional[ArrayLike] = ..., + out: _NdArraySubClass = ..., + **kwargs: Any, + ) -> _NdArraySubClass: ... + @overload + def clip( + self, + min: None = ..., + max: ArrayLike = ..., + out: _NdArraySubClass = ..., + **kwargs: Any, + ) -> _NdArraySubClass: ... + @overload + def compress( + self: _ArraySelf, a: ArrayLike, axis: Optional[int] = ..., out: None = ..., + ) -> _ArraySelf: ... + @overload + def compress( + self, a: ArrayLike, axis: Optional[int] = ..., out: _NdArraySubClass = ..., + ) -> _NdArraySubClass: ... + def conj(self: _ArraySelf) -> _ArraySelf: ... + def conjugate(self: _ArraySelf) -> _ArraySelf: ... + @overload + def cumprod( + self, axis: Optional[int] = ..., dtype: DtypeLike = ..., out: None = ..., + ) -> ndarray: ... + @overload + def cumprod( + self, + axis: Optional[int] = ..., + dtype: DtypeLike = ..., + out: _NdArraySubClass = ..., + ) -> _NdArraySubClass: ... + @overload + def cumsum( + self, axis: Optional[int] = ..., dtype: DtypeLike = ..., out: None = ..., + ) -> ndarray: ... + @overload + def cumsum( + self, + axis: Optional[int] = ..., + dtype: DtypeLike = ..., + out: _NdArraySubClass = ..., + ) -> _NdArraySubClass: ... + def diagonal( + self: _ArraySelf, offset: int = ..., axis1: int = ..., axis2: int = ... + ) -> _ArraySelf: ... + @overload + def dot(self, b: ArrayLike, out: None = ...) -> Union[number, ndarray]: ... + @overload + def dot(self, b: ArrayLike, out: _NdArraySubClass = ...) -> _NdArraySubClass: ... + @overload + def max( + self, + axis: None = ..., + out: None = ..., + keepdims: Literal[False] = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., + ) -> number: ... + @overload + def max( + self, + axis: Optional[_ShapeLike] = ..., + out: None = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., + ) -> Union[number, ndarray]: ... + @overload + def max( + self, + axis: Optional[_ShapeLike] = ..., + out: _NdArraySubClass = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., + ) -> _NdArraySubClass: ... + @overload + def mean( + self, + axis: None = ..., + dtype: DtypeLike = ..., + out: None = ..., + keepdims: Literal[False] = ..., + ) -> number: ... + @overload + def mean( + self, + axis: Optional[_ShapeLike] = ..., + dtype: DtypeLike = ..., + out: None = ..., + keepdims: bool = ..., + ) -> Union[number, ndarray]: ... + @overload + def mean( + self, + axis: Optional[_ShapeLike] = ..., + dtype: DtypeLike = ..., + out: _NdArraySubClass = ..., + keepdims: bool = ..., + ) -> _NdArraySubClass: ... + @overload + def min( + self, + axis: None = ..., + out: None = ..., + keepdims: Literal[False] = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., + ) -> number: ... + @overload + def min( + self, + axis: Optional[_ShapeLike] = ..., + out: None = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., + ) -> Union[number, ndarray]: ... + @overload + def min( + self, + axis: Optional[_ShapeLike] = ..., + out: _NdArraySubClass = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., + ) -> _NdArraySubClass: ... + def newbyteorder(self: _ArraySelf, __new_order: _ByteOrder = ...) -> _ArraySelf: ... + def nonzero(self) -> Tuple[ndarray, ...]: ... + def partition( + self, + kth: _ArrayLikeIntOrBool, + axis: int = ..., + kind: _PartitionKind = ..., + order: Union[None, str, Sequence[str]] = ..., + ) -> None: ... + @overload + def prod( + self, + axis: None = ..., + dtype: DtypeLike = ..., + out: None = ..., + keepdims: Literal[False] = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., + ) -> number: ... + @overload + def prod( + self, + axis: Optional[_ShapeLike] = ..., + dtype: DtypeLike = ..., + out: None = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., + ) -> Union[number, ndarray]: ... + @overload + def prod( + self, + axis: Optional[_ShapeLike] = ..., + dtype: DtypeLike = ..., + out: _NdArraySubClass = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., + ) -> _NdArraySubClass: ... + @overload + def ptp( + self, axis: None = ..., out: None = ..., keepdims: Literal[False] = ..., + ) -> number: ... + @overload + def ptp( + self, axis: Optional[_ShapeLike] = ..., out: None = ..., keepdims: bool = ..., + ) -> Union[number, ndarray]: ... + @overload + def ptp( + self, + axis: Optional[_ShapeLike] = ..., + out: _NdArraySubClass = ..., + keepdims: bool = ..., + ) -> _NdArraySubClass: ... + def put( + self, ind: _ArrayLikeIntOrBool, v: ArrayLike, mode: _Mode = ... + ) -> None: ... + def repeat( + self, repeats: _ArrayLikeIntOrBool, axis: Optional[int] = ... + ) -> ndarray: ... + @overload + def round(self: _ArraySelf, decimals: int = ..., out: None = ...) -> _ArraySelf: ... + @overload + def round( + self, decimals: int = ..., out: _NdArraySubClass = ... + ) -> _NdArraySubClass: ... + def searchsorted( + self, + v: ArrayLike, + side: _Side = ..., + sorter: Optional[_ArrayLikeIntOrBool] = ..., # 1D int array + ) -> ndarray: ... + def setfield( + self, val: ArithmeticError, dtype: DtypeLike, offset: int = ... + ) -> None: ... + def sort( + self, + axis: Optional[int] = ..., + kind: Optional[_SortKind] = ..., + order: Union[None, str, Sequence[str]] = ..., + ) -> None: ... + @overload + def std( + self, + axis: None = ..., + dtype: DtypeLike = ..., + out: None = ..., + ddof: int = ..., + keepdims: Literal[False] = ..., + ) -> number: ... + @overload + def std( + self, + axis: Optional[_ShapeLike] = ..., + dtype: DtypeLike = ..., + out: None = ..., + ddof: int = ..., + keepdims: bool = ..., + ) -> Union[number, ndarray]: ... + @overload + def std( + self, + axis: Optional[_ShapeLike] = ..., + dtype: DtypeLike = ..., + out: _NdArraySubClass = ..., + ddof: int = ..., + keepdims: bool = ..., + ) -> _NdArraySubClass: ... + @overload + def sum( + self, + axis: None = ..., + dtype: DtypeLike = ..., + out: None = ..., + keepdims: Literal[False] = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., + ) -> number: ... + @overload + def sum( + self, + axis: Optional[_ShapeLike] = ..., + dtype: DtypeLike = ..., + out: None = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., + ) -> Union[number, ndarray]: ... + @overload + def sum( + self, + axis: Optional[_ShapeLike] = ..., + dtype: DtypeLike = ..., + out: _NdArraySubClass = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., + ) -> _NdArraySubClass: ... + @overload + def take( + self, + indices: Union[_IntLike, _BoolLike], + axis: Optional[int] = ..., + out: None = ..., + mode: _Mode = ..., + ) -> generic: ... + @overload + def take( + self, + indices: _ArrayLikeIntOrBool, + axis: Optional[int] = ..., + out: None = ..., + mode: _Mode = ..., + ) -> ndarray: ... + @overload + def take( + self, + indices: _ArrayLikeIntOrBool, + axis: Optional[int] = ..., + out: _NdArraySubClass = ..., + mode: _Mode = ..., + ) -> _NdArraySubClass: ... + @overload + def trace( + self, # >= 2D array + offset: int = ..., + axis1: int = ..., + axis2: int = ..., + dtype: DtypeLike = ..., + out: None = ..., + ) -> Union[number, ndarray]: ... + @overload + def trace( + self, # >= 2D array + offset: int = ..., + axis1: int = ..., + axis2: int = ..., + dtype: DtypeLike = ..., + out: _NdArraySubClass = ..., + ) -> _NdArraySubClass: ... + @overload + def var( + self, + axis: None = ..., + dtype: DtypeLike = ..., + out: None = ..., + ddof: int = ..., + keepdims: Literal[False] = ..., + ) -> number: ... + @overload + def var( + self, + axis: Optional[_ShapeLike] = ..., + dtype: DtypeLike = ..., + out: None = ..., + ddof: int = ..., + keepdims: bool = ..., + ) -> Union[number, ndarray]: ... + @overload + def var( + self, + axis: Optional[_ShapeLike] = ..., + dtype: DtypeLike = ..., + out: _NdArraySubClass = ..., + ddof: int = ..., + keepdims: bool = ..., + ) -> _NdArraySubClass: ... _BufferType = Union[ndarray, bytes, bytearray, memoryview] _Casting = Literal["no", "equiv", "safe", "same_kind", "unsafe"] From 40cce4f69f6e4dfbda45e76294238db1f3223a97 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Tue, 15 Sep 2020 19:27:45 +0200 Subject: [PATCH 082/409] MAINT: Moved a number of methods from `_ArrayOrScalarCommon` to `ndarray` A number of methods require a >= 1D array or are completelly absent from `generic`. Move them to `ndarray`. --- numpy/__init__.pyi | 123 +++++++++++++++++++++++---------------------- 1 file changed, 63 insertions(+), 60 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index eb4b15d61987..480732c15e67 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -490,13 +490,6 @@ class _ArrayOrScalarCommon( def argmin( self, axis: Optional[_ShapeLike] = ..., out: _NdArraySubClass = ... ) -> _NdArraySubClass: ... - def argpartition( - self, - kth: _ArrayLikeIntOrBool, - axis: Optional[int] = ..., - kind: _PartitionKind = ..., - order: Union[None, str, Sequence[str]] = ..., - ) -> ndarray: ... def argsort( self, axis: Optional[int] = ..., @@ -545,8 +538,8 @@ class _ArrayOrScalarCommon( ) -> _NdArraySubClass: ... @overload def compress( - self: _ArraySelf, a: ArrayLike, axis: Optional[int] = ..., out: None = ..., - ) -> _ArraySelf: ... + self, a: ArrayLike, axis: Optional[int] = ..., out: None = ..., + ) -> ndarray: ... @overload def compress( self, a: ArrayLike, axis: Optional[int] = ..., out: _NdArraySubClass = ..., @@ -575,13 +568,6 @@ class _ArrayOrScalarCommon( dtype: DtypeLike = ..., out: _NdArraySubClass = ..., ) -> _NdArraySubClass: ... - def diagonal( - self: _ArraySelf, offset: int = ..., axis1: int = ..., axis2: int = ... - ) -> _ArraySelf: ... - @overload - def dot(self, b: ArrayLike, out: None = ...) -> Union[number, ndarray]: ... - @overload - def dot(self, b: ArrayLike, out: _NdArraySubClass = ...) -> _NdArraySubClass: ... @overload def max( self, @@ -661,14 +647,6 @@ class _ArrayOrScalarCommon( where: _ArrayLikeBool = ..., ) -> _NdArraySubClass: ... def newbyteorder(self: _ArraySelf, __new_order: _ByteOrder = ...) -> _ArraySelf: ... - def nonzero(self) -> Tuple[ndarray, ...]: ... - def partition( - self, - kth: _ArrayLikeIntOrBool, - axis: int = ..., - kind: _PartitionKind = ..., - order: Union[None, str, Sequence[str]] = ..., - ) -> None: ... @overload def prod( self, @@ -714,9 +692,6 @@ class _ArrayOrScalarCommon( out: _NdArraySubClass = ..., keepdims: bool = ..., ) -> _NdArraySubClass: ... - def put( - self, ind: _ArrayLikeIntOrBool, v: ArrayLike, mode: _Mode = ... - ) -> None: ... def repeat( self, repeats: _ArrayLikeIntOrBool, axis: Optional[int] = ... ) -> ndarray: ... @@ -726,21 +701,6 @@ class _ArrayOrScalarCommon( def round( self, decimals: int = ..., out: _NdArraySubClass = ... ) -> _NdArraySubClass: ... - def searchsorted( - self, - v: ArrayLike, - side: _Side = ..., - sorter: Optional[_ArrayLikeIntOrBool] = ..., # 1D int array - ) -> ndarray: ... - def setfield( - self, val: ArithmeticError, dtype: DtypeLike, offset: int = ... - ) -> None: ... - def sort( - self, - axis: Optional[int] = ..., - kind: Optional[_SortKind] = ..., - order: Union[None, str, Sequence[str]] = ..., - ) -> None: ... @overload def std( self, @@ -823,24 +783,6 @@ class _ArrayOrScalarCommon( mode: _Mode = ..., ) -> _NdArraySubClass: ... @overload - def trace( - self, # >= 2D array - offset: int = ..., - axis1: int = ..., - axis2: int = ..., - dtype: DtypeLike = ..., - out: None = ..., - ) -> Union[number, ndarray]: ... - @overload - def trace( - self, # >= 2D array - offset: int = ..., - axis1: int = ..., - axis2: int = ..., - dtype: DtypeLike = ..., - out: _NdArraySubClass = ..., - ) -> _NdArraySubClass: ... - @overload def var( self, axis: None = ..., @@ -901,6 +843,67 @@ class ndarray(_ArrayOrScalarCommon, Iterable, Sized, Container): def strides(self) -> _Shape: ... @strides.setter def strides(self, value: _ShapeLike): ... + def argpartition( + self, + kth: _ArrayLikeIntOrBool, + axis: Optional[int] = ..., + kind: _PartitionKind = ..., + order: Union[None, str, Sequence[str]] = ..., + ) -> ndarray: ... + def diagonal( + self: _ArraySelf, offset: int = ..., axis1: int = ..., axis2: int = ... + ) -> _ArraySelf: ... + @overload + def dot(self, b: ArrayLike, out: None = ...) -> Union[number, ndarray]: ... + @overload + def dot(self, b: ArrayLike, out: _NdArraySubClass = ...) -> _NdArraySubClass: ... + # `nonzero()` is deprecated for 0d arrays/generics + def nonzero(self) -> Tuple[ndarray, ...]: ... + def partition( + self, + kth: _ArrayLikeIntOrBool, + axis: int = ..., + kind: _PartitionKind = ..., + order: Union[None, str, Sequence[str]] = ..., + ) -> None: ... + # `put` is technically available to `generic`, + # but is pointless as `generic`s are immutable + def put( + self, ind: _ArrayLikeIntOrBool, v: ArrayLike, mode: _Mode = ... + ) -> None: ... + def searchsorted( + self, # >= 1D array + v: ArrayLike, + side: _Side = ..., + sorter: Optional[_ArrayLikeIntOrBool] = ..., # 1D int array + ) -> ndarray: ... + def setfield( + self, val: ArrayLike, dtype: DtypeLike, offset: int = ... + ) -> None: ... + def sort( + self, + axis: int = ..., + kind: Optional[_SortKind] = ..., + order: Union[None, str, Sequence[str]] = ..., + ) -> None: ... + @overload + def trace( + self, # >= 2D array + offset: int = ..., + axis1: int = ..., + axis2: int = ..., + dtype: DtypeLike = ..., + out: None = ..., + ) -> Union[number, ndarray]: ... + @overload + def trace( + self, # >= 2D array + offset: int = ..., + axis1: int = ..., + axis2: int = ..., + dtype: DtypeLike = ..., + out: _NdArraySubClass = ..., + ) -> _NdArraySubClass: ... # Many of these special methods are irrelevant currently, since protocols # aren't supported yet. That said, I'm adding them for completeness. # https://docs.python.org/3/reference/datamodel.html From 932d4e8852cc2d1fd8efead9ca68de33fbf1ce67 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Thu, 17 Sep 2020 12:37:33 +0200 Subject: [PATCH 083/409] MAINT: Moved a few constants back to `__init__.pyi`; cleaned up their names --- numpy/__init__.pyi | 29 ++++++++++++++++++------ numpy/core/fromnumeric.pyi | 46 +++++++++++++------------------------- 2 files changed, 38 insertions(+), 37 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 480732c15e67..fd7731133f3b 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -287,6 +287,21 @@ _OrderKACF = Optional[Literal["K", "A", "C", "F"]] _OrderACF = Optional[Literal["A", "C", "F"]] _OrderCF = Optional[Literal["C", "F"]] +_ModeKind = Literal["raise", "wrap", "clip"] +_PartitionKind = Literal["introselect"] +_SortKind = Literal["quicksort", "mergesort", "heapsort", "stable"] +_SortSide = Literal["left", "right"] + +_ArrayLikeBool = Union[_BoolLike, Sequence[_BoolLike], ndarray] +_ArrayLikeIntOrBool = Union[ + _IntLike, + _BoolLike, + ndarray, + Sequence[_IntLike], + Sequence[_BoolLike], + Sequence[Sequence[Any]], # TODO: wait for support for recursive types +] + _ArraySelf = TypeVar("_ArraySelf", bound=_ArrayOrScalarCommon) class _ArrayOrScalarCommon( @@ -498,11 +513,11 @@ class _ArrayOrScalarCommon( ) -> ndarray: ... @overload def choose( - self, choices: ArrayLike, out: None = ..., mode: _Mode = ..., + self, choices: ArrayLike, out: None = ..., mode: _ModeKind = ..., ) -> ndarray: ... @overload def choose( - self, choices: ArrayLike, out: _NdArraySubClass = ..., mode: _Mode = ..., + self, choices: ArrayLike, out: _NdArraySubClass = ..., mode: _ModeKind = ..., ) -> _NdArraySubClass: ... @overload def clip( @@ -764,7 +779,7 @@ class _ArrayOrScalarCommon( indices: Union[_IntLike, _BoolLike], axis: Optional[int] = ..., out: None = ..., - mode: _Mode = ..., + mode: _ModeKind = ..., ) -> generic: ... @overload def take( @@ -772,7 +787,7 @@ class _ArrayOrScalarCommon( indices: _ArrayLikeIntOrBool, axis: Optional[int] = ..., out: None = ..., - mode: _Mode = ..., + mode: _ModeKind = ..., ) -> ndarray: ... @overload def take( @@ -780,7 +795,7 @@ class _ArrayOrScalarCommon( indices: _ArrayLikeIntOrBool, axis: Optional[int] = ..., out: _NdArraySubClass = ..., - mode: _Mode = ..., + mode: _ModeKind = ..., ) -> _NdArraySubClass: ... @overload def var( @@ -869,12 +884,12 @@ class ndarray(_ArrayOrScalarCommon, Iterable, Sized, Container): # `put` is technically available to `generic`, # but is pointless as `generic`s are immutable def put( - self, ind: _ArrayLikeIntOrBool, v: ArrayLike, mode: _Mode = ... + self, ind: _ArrayLikeIntOrBool, v: ArrayLike, mode: _ModeKind = ... ) -> None: ... def searchsorted( self, # >= 1D array v: ArrayLike, - side: _Side = ..., + side: _SortSide = ..., sorter: Optional[_ArrayLikeIntOrBool] = ..., # 1D int array ) -> ndarray: ... def setfield( diff --git a/numpy/core/fromnumeric.pyi b/numpy/core/fromnumeric.pyi index 7ad772b07e54..6b3d2268f35e 100644 --- a/numpy/core/fromnumeric.pyi +++ b/numpy/core/fromnumeric.pyi @@ -13,6 +13,12 @@ from numpy import ( _IntLike, _BoolLike, _NumberLike, + _ArrayLikeBool, + _ArrayLikeIntOrBool, + _ModeKind, + _PartitionKind, + _SortKind, + _SortSide, ) from numpy.typing import DtypeLike, ArrayLike, _ShapeLike, _Shape @@ -21,11 +27,6 @@ if sys.version_info >= (3, 8): else: from typing_extensions import Literal -_Mode = Literal["raise", "wrap", "clip"] -_PartitionKind = Literal["introselect"] -_SortKind = Literal["quicksort", "mergesort", "heapsort", "stable"] -_Side = Literal["left", "right"] - # Various annotations for scalars # While dt.datetime and dt.timedelta are not technically part of NumPy, @@ -44,21 +45,6 @@ _ScalarGenericDT = TypeVar( _Number = TypeVar("_Number", bound=number) -# An array-like object consisting of integers -_IntOrBool = Union[_IntLike, _BoolLike] -_ArrayLikeIntNested = ArrayLike # TODO: wait for support for recursive types -_ArrayLikeBoolNested = ArrayLike # TODO: wait for support for recursive types - -# Integers and booleans can generally be used interchangeably -_ArrayLikeIntOrBool = Union[ - _IntOrBool, - ndarray, - Sequence[_IntOrBool], - Sequence[_ArrayLikeIntNested], - Sequence[_ArrayLikeBoolNested], -] -_ArrayLikeBool = Union[_BoolLike, Sequence[_BoolLike], ndarray] - # The signature of take() follows a common theme with its overloads: # 1. A generic comes in; the same generic comes out # 2. A scalar comes in; a generic comes out @@ -70,7 +56,7 @@ def take( indices: int, axis: Optional[int] = ..., out: Optional[ndarray] = ..., - mode: _Mode = ..., + mode: _ModeKind = ..., ) -> _ScalarGenericDT: ... @overload def take( @@ -78,7 +64,7 @@ def take( indices: int, axis: Optional[int] = ..., out: Optional[ndarray] = ..., - mode: _Mode = ..., + mode: _ModeKind = ..., ) -> _ScalarNumpy: ... @overload def take( @@ -86,7 +72,7 @@ def take( indices: int, axis: Optional[int] = ..., out: Optional[ndarray] = ..., - mode: _Mode = ..., + mode: _ModeKind = ..., ) -> _ScalarNumpy: ... @overload def take( @@ -94,7 +80,7 @@ def take( indices: _ArrayLikeIntOrBool, axis: Optional[int] = ..., out: Optional[ndarray] = ..., - mode: _Mode = ..., + mode: _ModeKind = ..., ) -> Union[_ScalarNumpy, ndarray]: ... def reshape(a: ArrayLike, newshape: _ShapeLike, order: _OrderACF = ...) -> ndarray: ... @overload @@ -102,24 +88,24 @@ def choose( a: _ScalarIntOrBool, choices: ArrayLike, out: Optional[ndarray] = ..., - mode: _Mode = ..., + mode: _ModeKind = ..., ) -> _ScalarIntOrBool: ... @overload def choose( - a: _IntOrBool, choices: ArrayLike, out: Optional[ndarray] = ..., mode: _Mode = ... + a: Union[_IntLike, _BoolLike], choices: ArrayLike, out: Optional[ndarray] = ..., mode: _ModeKind = ... ) -> Union[integer, bool_]: ... @overload def choose( a: _ArrayLikeIntOrBool, choices: ArrayLike, out: Optional[ndarray] = ..., - mode: _Mode = ..., + mode: _ModeKind = ..., ) -> ndarray: ... def repeat( a: ArrayLike, repeats: _ArrayLikeIntOrBool, axis: Optional[int] = ... ) -> ndarray: ... def put( - a: ndarray, ind: _ArrayLikeIntOrBool, v: ArrayLike, mode: _Mode = ... + a: ndarray, ind: _ArrayLikeIntOrBool, v: ArrayLike, mode: _ModeKind = ... ) -> None: ... def swapaxes(a: ArrayLike, axis1: int, axis2: int) -> ndarray: ... def transpose( @@ -184,14 +170,14 @@ def argmin( def searchsorted( a: ArrayLike, v: _Scalar, - side: _Side = ..., + side: _SortSide = ..., sorter: Optional[_ArrayLikeIntOrBool] = ..., # 1D int array ) -> integer: ... @overload def searchsorted( a: ArrayLike, v: ArrayLike, - side: _Side = ..., + side: _SortSide = ..., sorter: Optional[_ArrayLikeIntOrBool] = ..., # 1D int array ) -> ndarray: ... def resize(a: ArrayLike, new_shape: _ShapeLike) -> ndarray: ... From 5c07541e6380f4bd0f5dee73e20deac21d40c529 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Fri, 18 Sep 2020 17:23:15 +0200 Subject: [PATCH 084/409] TST: Added new `ndarray`/`generic` typing tests --- numpy/__init__.pyi | 11 +- numpy/typing/tests/data/fail/ndarray_misc.py | 21 +++ numpy/typing/tests/data/pass/ndarray_misc.py | 158 ++++++++++++++++++ .../typing/tests/data/reveal/ndarray_misc.py | 150 +++++++++++++++++ 4 files changed, 336 insertions(+), 4 deletions(-) create mode 100644 numpy/typing/tests/data/fail/ndarray_misc.py create mode 100644 numpy/typing/tests/data/pass/ndarray_misc.py create mode 100644 numpy/typing/tests/data/reveal/ndarray_misc.py diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index fd7731133f3b..633aa0ca393c 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -521,20 +521,20 @@ class _ArrayOrScalarCommon( ) -> _NdArraySubClass: ... @overload def clip( - self: _ArraySelf, + self, min: ArrayLike = ..., max: Optional[ArrayLike] = ..., out: None = ..., **kwargs: Any, - ) -> Union[_ArraySelf, number]: ... + ) -> Union[number, ndarray]: ... @overload def clip( - self: _ArraySelf, + self, min: None = ..., max: ArrayLike = ..., out: None = ..., **kwargs: Any, - ) -> Union[_ArraySelf, number]: ... + ) -> Union[number, ndarray]: ... @overload def clip( self, @@ -1088,6 +1088,9 @@ class void(flexible): def real(self: _ArraySelf) -> _ArraySelf: ... @property def imag(self: _ArraySelf) -> _ArraySelf: ... + def setfield( + self, val: ArrayLike, dtype: DtypeLike, offset: int = ... + ) -> None: ... class character(flexible): ... # type: ignore diff --git a/numpy/typing/tests/data/fail/ndarray_misc.py b/numpy/typing/tests/data/fail/ndarray_misc.py new file mode 100644 index 000000000000..1e1496bfecca --- /dev/null +++ b/numpy/typing/tests/data/fail/ndarray_misc.py @@ -0,0 +1,21 @@ +""" +Tests for miscellaneous (non-magic) ``np.ndarray``/``np.generic`` methods. + +More extensive tests are performed for the methods' +function-based counterpart in `../from_numeric.py`. + +""" + +import numpy as np + +f8: np.float64 + +f8.argpartition(0) # E: has no attribute +f8.diagonal() # E: has no attribute +f8.dot(1) # E: has no attribute +f8.nonzero() # E: has no attribute +f8.partition(0) # E: has no attribute +f8.put(0, 2) # E: has no attribute +f8.setfield(2, np.float64) # E: has no attribute +f8.sort() # E: has no attribute +f8.trace() # E: has no attribute diff --git a/numpy/typing/tests/data/pass/ndarray_misc.py b/numpy/typing/tests/data/pass/ndarray_misc.py new file mode 100644 index 000000000000..1871aebba402 --- /dev/null +++ b/numpy/typing/tests/data/pass/ndarray_misc.py @@ -0,0 +1,158 @@ +""" +Tests for miscellaneous (non-magic) ``np.ndarray``/``np.generic`` methods. + +More extensive tests are performed for the methods' +function-based counterpart in `../from_numeric.py`. + +""" + +from typing import cast +import numpy as np + +class SubClass(np.ndarray): ... + +i8 = np.int64(1) +A = np.array([[1]], dtype=np.int64) +B0 = np.empty((), dtype=np.int64).view(SubClass) +B1 = np.empty((1,), dtype=np.int64).view(SubClass) +B2 = np.empty((1, 1), dtype=np.int64).view(SubClass) +C = np.array([0, 1, 2], dtype=np.int64) +D = np.empty(3).view(SubClass) + +i8.all() +A.all() +A.all(axis=0) +A.all(keepdims=True) +A.all(out=B0) + +i8.any() +A.any() +A.any(axis=0) +A.any(keepdims=True) +A.any(out=B0) + +i8.argmax() +A.argmax() +A.argmax(axis=0) +A.argmax(out=B0) + +i8.argmin() +A.argmin() +A.argmin(axis=0) +A.argmin(out=B0) + +i8.argsort() +A.argsort() + +i8.choose([()]) +C.choose([[0, 1, 2], [3, 4, 5], [6, 7, 8]]) +C.choose([[0, 1, 2], [3, 4, 5], [6, 7, 8]], out=D) + +i8.clip(1) +A.clip(1) +A.clip(None, 1) +A.clip(1, out=B2) +A.clip(None, 1, out=B2) + +i8.compress([1]) +A.compress([1]) +A.compress([1], out=B1) + +i8.conj() +A.conj() +B0.conj() + +i8.conjugate() +A.conjugate() +B0.conjugate() + +i8.cumprod() +A.cumprod() +A.cumprod(out=B1) + +i8.cumsum() +A.cumsum() +A.cumsum(out=B1) + +i8.max() +A.max() +A.max(axis=0) +A.max(keepdims=True) +A.max(out=B0) + +i8.mean() +A.mean() +A.mean(axis=0) +A.mean(keepdims=True) +A.mean(out=B0) + +i8.min() +A.min() +A.min(axis=0) +A.min(keepdims=True) +A.min(out=B0) + +i8.newbyteorder() +A.newbyteorder() +B0.newbyteorder('|') + +i8.prod() +A.prod() +A.prod(axis=0) +A.prod(keepdims=True) +A.prod(out=B0) + +i8.ptp() +A.ptp() +A.ptp(axis=0) +A.ptp(keepdims=True) +A.astype(int).ptp(out=B0) + +i8.round() +A.round() +A.round(out=B2) + +i8.repeat(1) +A.repeat(1) +B0.repeat(1) + +i8.std() +A.std() +A.std(axis=0) +A.std(keepdims=True) +A.std(out=B0.astype(np.float64)) + +i8.sum() +A.sum() +A.sum(axis=0) +A.sum(keepdims=True) +A.sum(out=B0) + +i8.take(0) +A.take(0) +A.take([0]) +A.take(0, out=B0) +A.take([0], out=B1) + +i8.var() +A.var() +A.var(axis=0) +A.var(keepdims=True) +A.var(out=B0) + +A.argpartition([0]) + +A.diagonal() + +A.dot(1) +A.dot(1, out=B0) + +A.nonzero() + +C.searchsorted(1) + +A.trace() +A.trace(out=B0) + +void = cast(np.void, np.array(1, dtype=[("f", np.float64)]).take(0)) +void.setfield(10, np.float64) diff --git a/numpy/typing/tests/data/reveal/ndarray_misc.py b/numpy/typing/tests/data/reveal/ndarray_misc.py new file mode 100644 index 000000000000..22a4564df1b7 --- /dev/null +++ b/numpy/typing/tests/data/reveal/ndarray_misc.py @@ -0,0 +1,150 @@ +""" +Tests for miscellaneous (non-magic) ``np.ndarray``/``np.generic`` methods. + +More extensive tests are performed for the methods' +function-based counterpart in `../from_numeric.py`. + +""" + +import numpy as np + +class SubClass(np.ndarray): ... + +f8: np.float64 +A: np.ndarray +B: SubClass + +reveal_type(f8.all()) # E: numpy.bool_ +reveal_type(A.all()) # E: numpy.bool_ +reveal_type(A.all(axis=0)) # E: Union[numpy.bool_, numpy.ndarray] +reveal_type(A.all(keepdims=True)) # E: Union[numpy.bool_, numpy.ndarray] +reveal_type(A.all(out=B)) # E: SubClass + +reveal_type(f8.any()) # E: numpy.bool_ +reveal_type(A.any()) # E: numpy.bool_ +reveal_type(A.any(axis=0)) # E: Union[numpy.bool_, numpy.ndarray] +reveal_type(A.any(keepdims=True)) # E: Union[numpy.bool_, numpy.ndarray] +reveal_type(A.any(out=B)) # E: SubClass + +reveal_type(f8.argmax()) # E: numpy.signedinteger +reveal_type(A.argmax()) # E: numpy.signedinteger +reveal_type(A.argmax(axis=0)) # E: Union[numpy.signedinteger, numpy.ndarray] +reveal_type(A.argmax(out=B)) # E: SubClass + +reveal_type(f8.argmin()) # E: numpy.signedinteger +reveal_type(A.argmin()) # E: numpy.signedinteger +reveal_type(A.argmin(axis=0)) # E: Union[numpy.signedinteger, numpy.ndarray] +reveal_type(A.argmin(out=B)) # E: SubClass + +reveal_type(f8.argsort()) # E: numpy.ndarray +reveal_type(A.argsort()) # E: numpy.ndarray + +reveal_type(f8.astype(np.int64).choose([()])) # E: numpy.ndarray +reveal_type(A.choose([0])) # E: numpy.ndarray +reveal_type(A.choose([0], out=B)) # E: SubClass + +reveal_type(f8.clip(1)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(A.clip(1)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(A.clip(None, 1)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(A.clip(1, out=B)) # E: SubClass +reveal_type(A.clip(None, 1, out=B)) # E: SubClass + +reveal_type(f8.compress([0])) # E: numpy.ndarray +reveal_type(A.compress([0])) # E: numpy.ndarray +reveal_type(A.compress([0], out=B)) # E: SubClass + +reveal_type(f8.conj()) # E: numpy.float64 +reveal_type(A.conj()) # E: numpy.ndarray +reveal_type(B.conj()) # E: SubClass + +reveal_type(f8.conjugate()) # E: numpy.float64 +reveal_type(A.conjugate()) # E: numpy.ndarray +reveal_type(B.conjugate()) # E: SubClass + +reveal_type(f8.cumprod()) # E: numpy.ndarray +reveal_type(A.cumprod()) # E: numpy.ndarray +reveal_type(A.cumprod(out=B)) # E: SubClass + +reveal_type(f8.cumsum()) # E: numpy.ndarray +reveal_type(A.cumsum()) # E: numpy.ndarray +reveal_type(A.cumsum(out=B)) # E: SubClass + +reveal_type(f8.max()) # E: numpy.number +reveal_type(A.max()) # E: numpy.number +reveal_type(A.max(axis=0)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(A.max(keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(A.max(out=B)) # E: SubClass + +reveal_type(f8.mean()) # E: numpy.number +reveal_type(A.mean()) # E: numpy.number +reveal_type(A.mean(axis=0)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(A.mean(keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(A.mean(out=B)) # E: SubClass + +reveal_type(f8.min()) # E: numpy.number +reveal_type(A.min()) # E: numpy.number +reveal_type(A.min(axis=0)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(A.min(keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(A.min(out=B)) # E: SubClass + +reveal_type(f8.newbyteorder()) # E: numpy.float64 +reveal_type(A.newbyteorder()) # E: numpy.ndarray +reveal_type(B.newbyteorder('|')) # E: SubClass + +reveal_type(f8.prod()) # E: numpy.number +reveal_type(A.prod()) # E: numpy.number +reveal_type(A.prod(axis=0)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(A.prod(keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(A.prod(out=B)) # E: SubClass + +reveal_type(f8.ptp()) # E: numpy.number +reveal_type(A.ptp()) # E: numpy.number +reveal_type(A.ptp(axis=0)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(A.ptp(keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(A.ptp(out=B)) # E: SubClass + +reveal_type(f8.round()) # E: numpy.float64 +reveal_type(A.round()) # E: numpy.ndarray +reveal_type(A.round(out=B)) # E: SubClass + +reveal_type(f8.repeat(1)) # E: numpy.ndarray +reveal_type(A.repeat(1)) # E: numpy.ndarray +reveal_type(B.repeat(1)) # E: numpy.ndarray + +reveal_type(f8.std()) # E: numpy.number +reveal_type(A.std()) # E: numpy.number +reveal_type(A.std(axis=0)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(A.std(keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(A.std(out=B)) # E: SubClass + +reveal_type(f8.sum()) # E: numpy.number +reveal_type(A.sum()) # E: numpy.number +reveal_type(A.sum(axis=0)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(A.sum(keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(A.sum(out=B)) # E: SubClass + +reveal_type(f8.take(0)) # E: numpy.generic +reveal_type(A.take(0)) # E: numpy.generic +reveal_type(A.take([0])) # E: numpy.ndarray +reveal_type(A.take(0, out=B)) # E: SubClass +reveal_type(A.take([0], out=B)) # E: SubClass + +reveal_type(f8.var()) # E: numpy.number +reveal_type(A.var()) # E: numpy.number +reveal_type(A.var(axis=0)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(A.var(keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(A.var(out=B)) # E: SubClass + +reveal_type(A.argpartition([0])) # E: numpy.ndarray + +reveal_type(A.diagonal()) # E: numpy.ndarray + +reveal_type(A.dot(1)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(A.dot(1, out=B)) # E: SubClass + +reveal_type(A.nonzero()) # E: tuple[numpy.ndarray] + +reveal_type(A.searchsorted([1])) # E: numpy.ndarray + +reveal_type(A.trace()) # E: Union[numpy.number, numpy.ndarray] +reveal_type(A.trace(out=B)) # E: SubClass From fe7c3614a491d087ddc9102bfcff63bed7f8cea1 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Wed, 23 Sep 2020 08:39:45 -0400 Subject: [PATCH 085/409] DOC: History cleanup for #17353 --- doc/source/conf.py | 2 + doc/source/user/how-to-io.rst | 322 +++++++++++++++++++++++++++++++ doc/source/user/howtos_index.rst | 2 +- doc/source/user/ionumpy.rst | 20 -- 4 files changed, 325 insertions(+), 21 deletions(-) create mode 100644 doc/source/user/how-to-io.rst delete mode 100644 doc/source/user/ionumpy.rst diff --git a/doc/source/conf.py b/doc/source/conf.py index fe7ea096740d..66eb948561e6 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -227,6 +227,8 @@ def setup(app): 'matplotlib': ('https://matplotlib.org', None), 'imageio': ('https://imageio.readthedocs.io/en/stable', None), 'skimage': ('https://scikit-image.org/docs/stable', None), + 'pandas': ('https://pandas.pydata.org/pandas-docs/stable', None), + 'scipy-lecture-notes': ('https://scipy-lectures.org', None), } diff --git a/doc/source/user/how-to-io.rst b/doc/source/user/how-to-io.rst new file mode 100644 index 000000000000..49c0583c3b41 --- /dev/null +++ b/doc/source/user/how-to-io.rst @@ -0,0 +1,322 @@ +.. _how-to-io: + +############################################################################## +Reading and writing files +############################################################################## + +This page tackles common applications; for the full collection of I/O +routines, see :ref:`routines.io`. + + +****************************************************************************** +Reading text and CSV_ files +****************************************************************************** + +.. _CSV: https://en.wikipedia.org/wiki/Comma-separated_values + +With no missing values +============================================================================== + +Use :func:`numpy.loadtxt`. + +With missing values +============================================================================== + +Use :func:`numpy.genfromtxt`. + +:func:`numpy.genfromtxt` will either + + - return a :ref:`masked array` + **masking out missing values** (if ``usemask=True``), or + + - **fill in the missing value** with the value specified in + ``filling_values`` (default is ``np.nan`` for float, -1 for int). + +With non-whitespace delimiters +------------------------------------------------------------------------------ +:: + + $ cat csv.txt # doctest: +SKIP + 1, 2, 3 + 4,, 6 + 7, 8, 9 + + +Masked-array output +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: + + >>> np.genfromtxt("csv.txt", delimiter=",", usemask=True) # doctest: +SKIP + masked_array( + data=[[1.0, 2.0, 3.0], + [4.0, --, 6.0], + [7.0, 8.0, 9.0]], + mask=[[False, False, False], + [False, True, False], + [False, False, False]], + fill_value=1e+20) + +Array output +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: + + >>> np.genfromtxt("csv.txt", delimiter=",") # doctest: +SKIP + array([[ 1., 2., 3.], + [ 4., nan, 6.], + [ 7., 8., 9.]]) + +Array output, specified fill-in value +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: + + >>> np.genfromtxt("csv.txt", delimiter=",", dtype=np.int8, filling_values=99) # doctest: +SKIP + array([[ 1, 2, 3], + [ 4, 99, 6], + [ 7, 8, 9]], dtype=int8) + +Whitespace-delimited +------------------------------------------------------------------------------- + +:func:`numpy.genfromtxt` can also parse whitespace-delimited data files +that have missing values if + +* **Each field has a fixed width**: Use the width as the `delimiter` argument. + :: + + # File with width=4. The data does not have to be justified (for example, the + # 2 in row 1), the last column can be less than width (for example, the 6 in + # row 2), and no delimiting character is required (for instance 8888 and 9 in row 3) + + $cat fixedwidth.txt # doctest: +SKIP + 1 2 3 + 44 6 + 7 88889 + + # Showing spaces as '^' + $ tr ' ' '^' < fixedwidth.txt # doctest: +SKIP + 1^^^2^^^^^^3 + 44^^^^^^6 + 7^^^88889 + + >>> np.genfromtxt("fixedwidth.txt", delimiter=4) # doctest: +SKIP + array([[1.000e+00, 2.000e+00, 3.000e+00], + [4.400e+01, nan, 6.000e+00], + [7.000e+00, 8.888e+03, 9.000e+00]]) + +* **A special value (e.g. "x") indicates a missing field**: Use it as the + `missing_values` argument. + :: + + $ cat nan.txt # doctest: +SKIP + 1 2 3 + 44 x 6 + 7 8888 9 + + >>> np.genfromtxt("nan.txt", missing_values='x') # doctest: +SKIP + array([[1.000e+00, 2.000e+00, 3.000e+00], + [4.400e+01, nan, 6.000e+00], + [7.000e+00, 8.888e+03, 9.000e+00]]) + +* **You want to skip the rows with missing values**: Set + `invalid_raise=False`. + :: + + $ cat skip.txt # doctest: +SKIP + 1 2 3 + 44 6 + 7 888 9 + + >>> np.genfromtxt("skip.txt", invalid_raise=False) # doctest: +SKIP + __main__:1: ConversionWarning: Some errors were detected ! + Line #2 (got 2 columns instead of 3) + array([[ 1., 2., 3.], + [ 7., 888., 9.]]) + + +* **The delimiter whitespace character is different from the whitespace that + indicates missing data**. For instance, if columns are delimited by ``\t``, + then missing data will be recognized if it consists of one + or more spaces. + :: + + $ cat tabs.txt # doctest: +SKIP + 1 2 3 + 44 6 + 7 888 9 + + # Showing the tabs (^I) and spaces + $ cat -T tabs.txt # doctest: +SKIP + 1^I2^I3 + 44^I ^I6 + 7^I888^I9 + + >>> np.genfromtxt("tabs.txt", delimiter="\t", missing_values=" +") # doctest: +SKIP + array([[ 1., 2., 3.], + [ 44., nan, 6.], + [ 7., 888., 9.]]) + +****************************************************************************** +Read a file in .npy or .npz format +****************************************************************************** + +Choices: + + - Use :func:`numpy.load`. It can read files generated by any of + :func:`numpy.save`, :func:`numpy.savez`, or :func:`numpy.savez_compressed`. + + - Use memory mapping. See `numpy.lib.format.open_memmap`. + +****************************************************************************** +Write to a file to be read back by NumPy +****************************************************************************** + +Binary +=============================================================================== + +Use +:func:`numpy.save`, or to store multiple arrays :func:`numpy.savez` +or :func:`numpy.savez_compressed`. + +For :ref:`security and portability `, set +``allow_pickle=False`` unless the dtype contains Python objects, which +requires pickling. + +Masked arrays :any:`can't currently be saved `, +nor can other arbitrary array subclasses. + +Human-readable +============================================================================== + +:func:`numpy.save` and :func:`numpy.savez` create binary files. To **write a +human-readable file**, use :func:`numpy.savetxt`. The array can only be 1- or +2-dimensional, and there's no ` savetxtz` for multiple files. + +Large arrays +============================================================================== + +See :ref:`how-to-io-large-arrays`. + +****************************************************************************** +Read an arbitrarily formatted binary file ("binary blob") +****************************************************************************** + +Use a :doc:`structured array `. + +**Example:** + +The ``.wav`` file header is a 44-byte block preceding ``data_size`` bytes of the +actual sound data:: + + chunk_id "RIFF" + chunk_size 4-byte unsigned little-endian integer + format "WAVE" + fmt_id "fmt " + fmt_size 4-byte unsigned little-endian integer + audio_fmt 2-byte unsigned little-endian integer + num_channels 2-byte unsigned little-endian integer + sample_rate 4-byte unsigned little-endian integer + byte_rate 4-byte unsigned little-endian integer + block_align 2-byte unsigned little-endian integer + bits_per_sample 2-byte unsigned little-endian integer + data_id "data" + data_size 4-byte unsigned little-endian integer + +The ``.wav`` file header as a NumPy structured dtype:: + + wav_header_dtype = np.dtype([ + ("chunk_id", (bytes, 4)), # flexible-sized scalar type, item size 4 + ("chunk_size", "`_ or `PyTables `_. +* **Zarr**: `here `_. +* **NetCDF**: :class:`scipy.io.netcdf_file`. + +For tradeoffs among memmap, Zarr, and HDF5, see +`pythonspeed.com `_. + +****************************************************************************** +Write files for reading by other (non-NumPy) tools +****************************************************************************** + +Formats for **exchanging data** with other tools include HDF5, Zarr, and +NetCDF (see :ref:`how-to-io-large-arrays`). + +****************************************************************************** +Write or read a JSON file +****************************************************************************** + +NumPy arrays are **not** directly +`JSON serializable `_. + + +.. _how-to-io-pickle-file: + +****************************************************************************** +Save/restore using a pickle file +****************************************************************************** + +Not recommended, due to lack of security and portability. + + * **security**: not secure against erroneous or maliciously constructed data + * **portability**: may not be loadable on different Python installations + +Use :func:`numpy.save` and :func:`numpy.load`. Set ``allow_pickle=False``, unless +the array dtype includes Python objects, in which case pickling is required. + +****************************************************************************** +Convert from a pandas DataFrame to a NumPy array +****************************************************************************** + +See :meth:`pandas.DataFrame.to_numpy`. + +****************************************************************************** + Save/restore using `~numpy.ndarray.tofile` and `~numpy.fromfile` +****************************************************************************** + +In general, prefer :func:`numpy.save` and :func:`numpy.load`. + +:func:`numpy.ndarray.tofile` and :func:`numpy.fromfile` lose information on +endianness and precision and so are unsuitable for anything but scratch +storage. + diff --git a/doc/source/user/howtos_index.rst b/doc/source/user/howtos_index.rst index 45e013e6fb6a..89a6f54e791c 100644 --- a/doc/source/user/howtos_index.rst +++ b/doc/source/user/howtos_index.rst @@ -12,4 +12,4 @@ the package, see the :ref:`API reference `. :maxdepth: 1 how-to-how-to - ionumpy + how-to-io diff --git a/doc/source/user/ionumpy.rst b/doc/source/user/ionumpy.rst deleted file mode 100644 index a31720322a77..000000000000 --- a/doc/source/user/ionumpy.rst +++ /dev/null @@ -1,20 +0,0 @@ -================================================ -How to read and write data using NumPy -================================================ - -.. currentmodule:: numpy - -.. testsetup:: - - import numpy as np - np.random.seed(1) - -**Objectives** - -- Writing NumPy arrays to files -- Reading NumPy arrays from files -- Dealing with encoding and dtype issues - -**Content** - -To be completed. From 9c799c4fd07c2b3d13c189fda5bee4d6b45e54bb Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Wed, 23 Sep 2020 09:06:09 -0400 Subject: [PATCH 086/409] DOC: Update pickle section in PR #17353 --- doc/source/user/how-to-io.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/doc/source/user/how-to-io.rst b/doc/source/user/how-to-io.rst index 49c0583c3b41..f8b4dee9f21f 100644 --- a/doc/source/user/how-to-io.rst +++ b/doc/source/user/how-to-io.rst @@ -296,13 +296,12 @@ NumPy arrays are **not** directly Save/restore using a pickle file ****************************************************************************** -Not recommended, due to lack of security and portability. +Avoid when possible; :doc:`pickles ` are not secure +against erroneous or maliciously constructed data. - * **security**: not secure against erroneous or maliciously constructed data - * **portability**: may not be loadable on different Python installations - -Use :func:`numpy.save` and :func:`numpy.load`. Set ``allow_pickle=False``, unless -the array dtype includes Python objects, in which case pickling is required. +Use :func:`numpy.save` and :func:`numpy.load`. Set ``allow_pickle=False``, +unless the array dtype includes Python objects, in which case pickling is +required. ****************************************************************************** Convert from a pandas DataFrame to a NumPy array From 9c5812b3eaa5a586bde4a19fa4845ee43f83ba8a Mon Sep 17 00:00:00 2001 From: Warren Weckesser Date: Wed, 23 Sep 2020 09:53:40 -0400 Subject: [PATCH 087/409] DOC: distutils: Remove an obsolete paragraph. --- doc/DISTUTILS.rst.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/doc/DISTUTILS.rst.txt b/doc/DISTUTILS.rst.txt index 01527374d182..f1f2704621a1 100644 --- a/doc/DISTUTILS.rst.txt +++ b/doc/DISTUTILS.rst.txt @@ -587,10 +587,6 @@ The header of a typical SciPy ``__init__.py`` is:: test = Tester().test bench = Tester().bench -Note that NumPy submodules still use a file named ``info.py`` in which the -module docstring and ``__all__`` dict are defined. These files will be removed -at some point. - Extra features in NumPy Distutils ''''''''''''''''''''''''''''''''' From 2fe46b4d25788a43499893c968e114bfa9be8a10 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Wed, 23 Sep 2020 11:10:39 -0400 Subject: [PATCH 088/409] DOC: Incorporate blob suggestions in PR #17353 --- doc/source/user/how-to-io.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/user/how-to-io.rst b/doc/source/user/how-to-io.rst index f8b4dee9f21f..64096bfc2a79 100644 --- a/doc/source/user/how-to-io.rst +++ b/doc/source/user/how-to-io.rst @@ -236,13 +236,13 @@ The ``.wav`` file header as a NumPy structured dtype:: ("block_align", " Date: Wed, 23 Sep 2020 18:41:32 +0200 Subject: [PATCH 089/409] TST: Fixed an `int`-related failure on 32-bit systems --- numpy/typing/tests/data/pass/ndarray_misc.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/numpy/typing/tests/data/pass/ndarray_misc.py b/numpy/typing/tests/data/pass/ndarray_misc.py index 1871aebba402..787370c0c6f2 100644 --- a/numpy/typing/tests/data/pass/ndarray_misc.py +++ b/numpy/typing/tests/data/pass/ndarray_misc.py @@ -45,8 +45,9 @@ class SubClass(np.ndarray): ... A.argsort() i8.choose([()]) -C.choose([[0, 1, 2], [3, 4, 5], [6, 7, 8]]) -C.choose([[0, 1, 2], [3, 4, 5], [6, 7, 8]], out=D) +_choices = np.array([[0, 1, 2], [3, 4, 5], [6, 7, 8]], dtype=np.int64) +C.choose(_choices) +C.choose(_choices, out=D) i8.clip(1) A.clip(1) From e67bbd2410927743db85c7f11b33db8129f865c0 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Wed, 23 Sep 2020 19:09:28 +0200 Subject: [PATCH 090/409] TST: Attempt #2 at fixing the `int`-related failure on 32-bit systems --- numpy/typing/tests/data/pass/ndarray_misc.py | 62 ++++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/numpy/typing/tests/data/pass/ndarray_misc.py b/numpy/typing/tests/data/pass/ndarray_misc.py index 787370c0c6f2..6c6f5d50b986 100644 --- a/numpy/typing/tests/data/pass/ndarray_misc.py +++ b/numpy/typing/tests/data/pass/ndarray_misc.py @@ -11,131 +11,131 @@ class SubClass(np.ndarray): ... -i8 = np.int64(1) -A = np.array([[1]], dtype=np.int64) -B0 = np.empty((), dtype=np.int64).view(SubClass) -B1 = np.empty((1,), dtype=np.int64).view(SubClass) -B2 = np.empty((1, 1), dtype=np.int64).view(SubClass) -C = np.array([0, 1, 2], dtype=np.int64) +i4 = np.int32(1) +A = np.array([[1]], dtype=np.int32) +B0 = np.empty((), dtype=np.int32).view(SubClass) +B1 = np.empty((1,), dtype=np.int32).view(SubClass) +B2 = np.empty((1, 1), dtype=np.int32).view(SubClass) +C = np.array([0, 1, 2], dtype=np.int32) D = np.empty(3).view(SubClass) -i8.all() +i4.all() A.all() A.all(axis=0) A.all(keepdims=True) A.all(out=B0) -i8.any() +i4.any() A.any() A.any(axis=0) A.any(keepdims=True) A.any(out=B0) -i8.argmax() +i4.argmax() A.argmax() A.argmax(axis=0) A.argmax(out=B0) -i8.argmin() +i4.argmin() A.argmin() A.argmin(axis=0) A.argmin(out=B0) -i8.argsort() +i4.argsort() A.argsort() -i8.choose([()]) -_choices = np.array([[0, 1, 2], [3, 4, 5], [6, 7, 8]], dtype=np.int64) +i4.choose([()]) +_choices = np.array([[0, 1, 2], [3, 4, 5], [6, 7, 8]], dtype=np.int32) C.choose(_choices) C.choose(_choices, out=D) -i8.clip(1) +i4.clip(1) A.clip(1) A.clip(None, 1) A.clip(1, out=B2) A.clip(None, 1, out=B2) -i8.compress([1]) +i4.compress([1]) A.compress([1]) A.compress([1], out=B1) -i8.conj() +i4.conj() A.conj() B0.conj() -i8.conjugate() +i4.conjugate() A.conjugate() B0.conjugate() -i8.cumprod() +i4.cumprod() A.cumprod() A.cumprod(out=B1) -i8.cumsum() +i4.cumsum() A.cumsum() A.cumsum(out=B1) -i8.max() +i4.max() A.max() A.max(axis=0) A.max(keepdims=True) A.max(out=B0) -i8.mean() +i4.mean() A.mean() A.mean(axis=0) A.mean(keepdims=True) A.mean(out=B0) -i8.min() +i4.min() A.min() A.min(axis=0) A.min(keepdims=True) A.min(out=B0) -i8.newbyteorder() +i4.newbyteorder() A.newbyteorder() B0.newbyteorder('|') -i8.prod() +i4.prod() A.prod() A.prod(axis=0) A.prod(keepdims=True) A.prod(out=B0) -i8.ptp() +i4.ptp() A.ptp() A.ptp(axis=0) A.ptp(keepdims=True) A.astype(int).ptp(out=B0) -i8.round() +i4.round() A.round() A.round(out=B2) -i8.repeat(1) +i4.repeat(1) A.repeat(1) B0.repeat(1) -i8.std() +i4.std() A.std() A.std(axis=0) A.std(keepdims=True) A.std(out=B0.astype(np.float64)) -i8.sum() +i4.sum() A.sum() A.sum(axis=0) A.sum(keepdims=True) A.sum(out=B0) -i8.take(0) +i4.take(0) A.take(0) A.take([0]) A.take(0, out=B0) A.take([0], out=B1) -i8.var() +i4.var() A.var() A.var(axis=0) A.var(keepdims=True) From 45aaf7f2dea425db3012aadd43376bd14a6486db Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 24 Sep 2020 08:51:56 -0500 Subject: [PATCH 091/409] TST: Add test for pickling using ufunc name as qualname --- numpy/core/src/umath/_umath_tests.c.src | 9 +++++++++ numpy/core/tests/test_ufunc.py | 13 +++++++++++++ 2 files changed, 22 insertions(+) diff --git a/numpy/core/src/umath/_umath_tests.c.src b/numpy/core/src/umath/_umath_tests.c.src index 660c296d6ddb..750fbeb92a7b 100644 --- a/numpy/core/src/umath/_umath_tests.c.src +++ b/numpy/core/src/umath/_umath_tests.c.src @@ -461,6 +461,15 @@ addUfuncs(PyObject *dictionary) { PyDict_SetItemString(dictionary, "cross1d", f); Py_DECREF(f); + f = PyUFunc_FromFuncAndDataAndSignature(NULL, NULL, + NULL, 0, 0, 0, PyUFunc_None, "_pickleable_module_global.ufunc", + "A dotted name for pickle testing, does nothing.", 0, NULL); + if (f == NULL) { + return -1; + } + PyDict_SetItemString(dictionary, "_pickleable_module_global_ufunc", f); + Py_DECREF(f); + return 0; } diff --git a/numpy/core/tests/test_ufunc.py b/numpy/core/tests/test_ufunc.py index 9eaa1a977715..0e9760853def 100644 --- a/numpy/core/tests/test_ufunc.py +++ b/numpy/core/tests/test_ufunc.py @@ -178,6 +178,10 @@ def __getattr__(self, attr): assert_array_equal(res_num.astype("O"), res_obj) +def _pickleable_module_global(): + pass + + class TestUfunc: def test_pickle(self): for proto in range(2, pickle.HIGHEST_PROTOCOL + 1): @@ -195,6 +199,15 @@ def test_pickle_withstring(self): b"(S'numpy.core.umath'\np1\nS'cos'\np2\ntp3\nRp4\n.") assert_(pickle.loads(astring) is np.cos) + def test_pickle_name_is_qualname(self): + # This tests that a simplification of our ufunc pickle code will + # lead to allowing qualnames as names. Future ufuncs should + # possible add a specific qualname, or a hook into pickling instead + # (dask+numba may benefit). + _pickleable_module_global.ufunc = umt._pickleable_module_global_ufunc + obj = pickle.loads(pickle.dumps(_pickleable_module_global.ufunc)) + assert obj is umt._pickleable_module_global_ufunc + def test_reduceat_shifting_sum(self): L = 6 x = np.arange(L) From bf71a62b52dedd8ceaa47a96c144a3584e1bd95b Mon Sep 17 00:00:00 2001 From: MelissaWM Date: Wed, 23 Sep 2020 23:09:46 -0300 Subject: [PATCH 092/409] BUG: Fixes module data docstrings. Fixes gh-15325 --- numpy/f2py/src/fortranobject.c | 6 ++++-- numpy/f2py/tests/src/module_data/mod.mod | Bin 0 -> 412 bytes .../src/module_data/module_data_docstring.f90 | 12 ++++++++++++ numpy/f2py/tests/test_block_docstring.py | 2 +- numpy/f2py/tests/test_module_doc.py | 16 ++++++++++++++++ 5 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 numpy/f2py/tests/src/module_data/mod.mod create mode 100644 numpy/f2py/tests/src/module_data/module_data_docstring.f90 create mode 100644 numpy/f2py/tests/test_module_doc.py diff --git a/numpy/f2py/src/fortranobject.c b/numpy/f2py/src/fortranobject.c index aa46c57d0b92..e420b827b0fb 100644 --- a/numpy/f2py/src/fortranobject.c +++ b/numpy/f2py/src/fortranobject.c @@ -213,6 +213,7 @@ format_def(char *buf, Py_ssize_t size, FortranDataDef def) return -1; } memcpy(p, notalloc, sizeof(notalloc)); + p += sizeof(notalloc); } return p - buf; @@ -255,7 +256,7 @@ fortran_doc(FortranDataDef def) } else { PyArray_Descr *d = PyArray_DescrFromType(def.type); - n = PyOS_snprintf(p, size, "'%c'-", d->type); + n = PyOS_snprintf(p, size, "%s : '%c'-", def.name, d->type); Py_DECREF(d); if (n < 0 || n >= size) { goto fail; @@ -264,7 +265,7 @@ fortran_doc(FortranDataDef def) size -= n; if (def.data == NULL) { - n = format_def(p, size, def) == -1; + n = format_def(p, size, def);// == -1; if (n < 0) { goto fail; } @@ -288,6 +289,7 @@ fortran_doc(FortranDataDef def) p += n; size -= n; } + } if (size <= 1) { goto fail; diff --git a/numpy/f2py/tests/src/module_data/mod.mod b/numpy/f2py/tests/src/module_data/mod.mod new file mode 100644 index 0000000000000000000000000000000000000000..8670a97e911c48ff2cae2cf83dd14f42f8a7004a GIT binary patch literal 412 zcmV;N0b~9jiwFP!000001GSW0Z-OushVT0;&RyYROTn+~vcl1jVKEi9c}1AuVlpLB zx9sPyl#lp1-Qv81mPp@ZV7twwwq`<~;4U^s!8gj|Hg zD3T3PHo`hCwbY_|)(GHGKn?`EpR)sWaU)yHeSiME3{GQoYlK~p(v`nqTyWzIMi1!MKNavb zzF2!@e0|MasAF;@XrC#v;wM)8!Wh`P+!(Z~bIjCM#Yth%sr+Hk)owB9ndBRfe&AZC G1pojXWXECv literal 0 HcmV?d00001 diff --git a/numpy/f2py/tests/src/module_data/module_data_docstring.f90 b/numpy/f2py/tests/src/module_data/module_data_docstring.f90 new file mode 100644 index 000000000000..4505e0cbc31e --- /dev/null +++ b/numpy/f2py/tests/src/module_data/module_data_docstring.f90 @@ -0,0 +1,12 @@ +module mod + integer :: i + integer :: x(4) + real, dimension(2,3) :: a + real, allocatable, dimension(:,:) :: b +contains + subroutine foo + integer :: k + k = 1 + a(1,2) = a(1,2)+3 + end subroutine foo +end module mod diff --git a/numpy/f2py/tests/test_block_docstring.py b/numpy/f2py/tests/test_block_docstring.py index e431f5ba6c97..7d725165b2fb 100644 --- a/numpy/f2py/tests/test_block_docstring.py +++ b/numpy/f2py/tests/test_block_docstring.py @@ -19,5 +19,5 @@ class TestBlockDocString(util.F2PyTest): @pytest.mark.xfail(IS_PYPY, reason="PyPy cannot modify tp_doc after PyType_Ready") def test_block_docstring(self): - expected = "'i'-array(2,3)\n" + expected = "bar : 'i'-array(2,3)\n" assert_equal(self.module.block.__doc__, expected) diff --git a/numpy/f2py/tests/test_module_doc.py b/numpy/f2py/tests/test_module_doc.py new file mode 100644 index 000000000000..f597929ad4dc --- /dev/null +++ b/numpy/f2py/tests/test_module_doc.py @@ -0,0 +1,16 @@ +import os +from . import util + +from numpy.testing import assert_equal + + +def _path(*a): + return os.path.join(*((os.path.dirname(__file__),) + a)) + + +class TestModuleDocString(util.F2PyTest): + sources = [_path('src', 'module_data', 'module_data_docstring.f90')] + + def test_module_docstring(self): + expected = "i : 'i'-scalar\nx : 'i'-array(4)\na : 'f'-array(2,3)\nb : 'f'-array(-1,-1), not allocated\x00\nfoo()\n\nWrapper for ``foo``.\n\n" + assert_equal(self.module.mod.__doc__, expected) From 16e8c26b86f29cb18470fccc57d95c942c30f642 Mon Sep 17 00:00:00 2001 From: MelissaWM Date: Thu, 24 Sep 2020 15:25:15 -0300 Subject: [PATCH 093/409] Fixing tests for pypy and win. --- doc/source/f2py/allocarr_session.dat | 9 ++++++--- doc/source/f2py/common_session.dat | 6 +++--- doc/source/f2py/moddata_session.dat | 14 +++++++++----- numpy/f2py/tests/test_module_doc.py | 10 ++++++++-- 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/doc/source/f2py/allocarr_session.dat b/doc/source/f2py/allocarr_session.dat index 754d9cb8b5a2..ba168c22aa12 100644 --- a/doc/source/f2py/allocarr_session.dat +++ b/doc/source/f2py/allocarr_session.dat @@ -1,8 +1,11 @@ >>> import allocarr >>> print(allocarr.mod.__doc__) -b - 'f'-array(-1,-1), not allocated -foo - Function signature: - foo() +b : 'f'-array(-1,-1), not allocated +foo() + +Wrapper for ``foo``. + + >>> allocarr.mod.foo() b is not allocated diff --git a/doc/source/f2py/common_session.dat b/doc/source/f2py/common_session.dat index 0a38bec27b8a..2595bfbd5b20 100644 --- a/doc/source/f2py/common_session.dat +++ b/doc/source/f2py/common_session.dat @@ -1,8 +1,8 @@ >>> import common >>> print(common.data.__doc__) -i - 'i'-scalar -x - 'i'-array(4) -a - 'f'-array(2,3) +i : 'i'-scalar +x : 'i'-array(4) +a : 'f'-array(2,3) >>> common.data.i = 5 >>> common.data.x[1] = 2 diff --git a/doc/source/f2py/moddata_session.dat b/doc/source/f2py/moddata_session.dat index e3c7580416f2..824bd86fc464 100644 --- a/doc/source/f2py/moddata_session.dat +++ b/doc/source/f2py/moddata_session.dat @@ -1,10 +1,14 @@ >>> import moddata >>> print(moddata.mod.__doc__) -i - 'i'-scalar -x - 'i'-array(4) -a - 'f'-array(2,3) -foo - Function signature: - foo() +i : 'i'-scalar +x : 'i'-array(4) +a : 'f'-array(2,3) +b : 'f'-array(-1,-1), not allocated +foo() + +Wrapper for ``foo``. + + >>> moddata.mod.i = 5 >>> moddata.mod.x[:2] = [1,2] diff --git a/numpy/f2py/tests/test_module_doc.py b/numpy/f2py/tests/test_module_doc.py index f597929ad4dc..9360b04a90a4 100644 --- a/numpy/f2py/tests/test_module_doc.py +++ b/numpy/f2py/tests/test_module_doc.py @@ -1,7 +1,9 @@ import os -from . import util +import sys +import pytest -from numpy.testing import assert_equal +from . import util +from numpy.testing import assert_equal, IS_PYPY def _path(*a): @@ -11,6 +13,10 @@ def _path(*a): class TestModuleDocString(util.F2PyTest): sources = [_path('src', 'module_data', 'module_data_docstring.f90')] + @pytest.mark.skipif(sys.platform=='win32', + reason='Fails with MinGW64 Gfortran (Issue #9673)') + @pytest.mark.xfail(IS_PYPY, + reason="PyPy cannot modify tp_doc after PyType_Ready") def test_module_docstring(self): expected = "i : 'i'-scalar\nx : 'i'-array(4)\na : 'f'-array(2,3)\nb : 'f'-array(-1,-1), not allocated\x00\nfoo()\n\nWrapper for ``foo``.\n\n" assert_equal(self.module.mod.__doc__, expected) From 2379f4c1a8f93b1f4e3e0cd40b587ca6e95d3f06 Mon Sep 17 00:00:00 2001 From: MelissaWM Date: Thu, 24 Sep 2020 19:43:24 -0300 Subject: [PATCH 094/409] Adding PR review comment. --- numpy/f2py/src/fortranobject.c | 1 + 1 file changed, 1 insertion(+) diff --git a/numpy/f2py/src/fortranobject.c b/numpy/f2py/src/fortranobject.c index e420b827b0fb..215473ed8641 100644 --- a/numpy/f2py/src/fortranobject.c +++ b/numpy/f2py/src/fortranobject.c @@ -214,6 +214,7 @@ format_def(char *buf, Py_ssize_t size, FortranDataDef def) } memcpy(p, notalloc, sizeof(notalloc)); p += sizeof(notalloc); + size -= sizeof(notalloc); } return p - buf; From 02e6b062027666b22c68d91e280b28f57ee9c2b5 Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Fri, 25 Sep 2020 14:21:31 +0200 Subject: [PATCH 095/409] DOC: random: Fix default_rng docstring (#17375) --- numpy/random/_generator.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/random/_generator.pyx b/numpy/random/_generator.pyx index e40dcefe34b7..bff023d7ffb6 100644 --- a/numpy/random/_generator.pyx +++ b/numpy/random/_generator.pyx @@ -4521,7 +4521,7 @@ def default_rng(seed=None): unpredictable entropy will be pulled from the OS. If an ``int`` or ``array_like[ints]`` is passed, then it will be passed to `SeedSequence` to derive the initial `BitGenerator` state. One may also - pass in a`SeedSequence` instance + pass in a `SeedSequence` instance. Additionally, when passed a `BitGenerator`, it will be wrapped by `Generator`. If passed a `Generator`, it will be returned unaltered. From dbe6d464dbde074d207804e3aed1b3cfaacf0eda Mon Sep 17 00:00:00 2001 From: MelissaWM Date: Sat, 26 Sep 2020 10:01:01 -0300 Subject: [PATCH 096/409] Removing extra comment. --- numpy/f2py/src/fortranobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/src/fortranobject.c b/numpy/f2py/src/fortranobject.c index 215473ed8641..3275f90ad2cb 100644 --- a/numpy/f2py/src/fortranobject.c +++ b/numpy/f2py/src/fortranobject.c @@ -266,7 +266,7 @@ fortran_doc(FortranDataDef def) size -= n; if (def.data == NULL) { - n = format_def(p, size, def);// == -1; + n = format_def(p, size, def); if (n < 0) { goto fail; } From 73e95ddd7879589fbc3ff98d962dd0099fa77c10 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Sat, 26 Sep 2020 09:01:12 -0400 Subject: [PATCH 097/409] DOC: Revise 'contiguous' in PR #16996 --- doc/source/glossary.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index d4db87723d3d..5f8de3988457 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -282,7 +282,7 @@ Glossary An array is contiguous if * it occupies an unbroken block of memory, and * array elements with higher indexes occupy higher addresses (that - is, the :term:`stride` is not negative). + is, no :term:`stride` is negative). copy From 3a94fff71006af722af305f03653876c1a7e32fd Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Sat, 26 Sep 2020 11:02:25 -0400 Subject: [PATCH 098/409] DOC: Bash examples in PR #17353 now Python And other suggestions by @eric-wieser. Also standardizes quotes as '"'. --- doc/source/user/how-to-io.rst | 56 +++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/doc/source/user/how-to-io.rst b/doc/source/user/how-to-io.rst index 64096bfc2a79..d962e1546ad9 100644 --- a/doc/source/user/how-to-io.rst +++ b/doc/source/user/how-to-io.rst @@ -83,17 +83,19 @@ that have missing values if * **Each field has a fixed width**: Use the width as the `delimiter` argument. :: - # File with width=4. The data does not have to be justified (for example, the - # 2 in row 1), the last column can be less than width (for example, the 6 in - # row 2), and no delimiting character is required (for instance 8888 and 9 in row 3) + # File with width=4. The data does not have to be justified (for example, + # the 2 in row 1), the last column can be less than width (for example, the 6 + # in row 2), and no delimiting character is required (for instance 8888 and 9 + # in row 3) - $cat fixedwidth.txt # doctest: +SKIP + >>> f = open("fixedwidth.txt").read() # doctest: +SKIP + >>> print(f) # doctest: +SKIP 1 2 3 44 6 7 88889 - # Showing spaces as '^' - $ tr ' ' '^' < fixedwidth.txt # doctest: +SKIP + # Showing spaces as ^ + >>> print(f.replace(" ","^")) # doctest: +SKIP 1^^^2^^^^^^3 44^^^^^^6 7^^^88889 @@ -107,12 +109,12 @@ that have missing values if `missing_values` argument. :: - $ cat nan.txt # doctest: +SKIP + >>> print(open("nan.txt").read()) # doctest: +SKIP 1 2 3 44 x 6 7 8888 9 - >>> np.genfromtxt("nan.txt", missing_values='x') # doctest: +SKIP + >>> np.genfromtxt("nan.txt", missing_values="x") # doctest: +SKIP array([[1.000e+00, 2.000e+00, 3.000e+00], [4.400e+01, nan, 6.000e+00], [7.000e+00, 8.888e+03, 9.000e+00]]) @@ -121,7 +123,7 @@ that have missing values if `invalid_raise=False`. :: - $ cat skip.txt # doctest: +SKIP + >>> print(open("skip.txt").read()) # doctest: +SKIP 1 2 3 44 6 7 888 9 @@ -139,21 +141,22 @@ that have missing values if or more spaces. :: - $ cat tabs.txt # doctest: +SKIP - 1 2 3 - 44 6 - 7 888 9 + >>> f = open("tabs.txt").read() # doctest: +SKIP + >>> print(f) # doctest: +SKIP + 1 2 3 + 44 6 + 7 888 9 - # Showing the tabs (^I) and spaces - $ cat -T tabs.txt # doctest: +SKIP - 1^I2^I3 - 44^I ^I6 - 7^I888^I9 + # Tabs vs. spaces + >>> print(f.replace("\t","^")) # doctest: +SKIP + 1^2^3 + 44^ ^6 + 7^888^9 - >>> np.genfromtxt("tabs.txt", delimiter="\t", missing_values=" +") # doctest: +SKIP - array([[ 1., 2., 3.], - [ 44., nan, 6.], - [ 7., 888., 9.]]) + >>> np.genfromtxt("tabs.txt", delimiter="\t", missing_values=" +") # doctest: +SKIP + array([[ 1., 2., 3.], + [ 44., nan, 6.], + [ 7., 888., 9.]]) ****************************************************************************** Read a file in .npy or .npz format @@ -226,7 +229,7 @@ The ``.wav`` file header as a NumPy structured dtype:: wav_header_dtype = np.dtype([ ("chunk_id", (bytes, 4)), # flexible-sized scalar type, item size 4 ("chunk_size", " Date: Sat, 26 Sep 2020 11:43:26 -0400 Subject: [PATCH 099/409] DOC: Italics sted monospace in PR #16996 --- doc/source/glossary.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 5f8de3988457..17071c8f175f 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -241,7 +241,7 @@ Glossary broadcast - ``broadcasting`` is NumPy's ability to process ndarrays of + *broadcasting* is NumPy's ability to process ndarrays of different sizes as if all were the same size. It permits an elegant do-what-I-mean behavior where, for instance, From 5d458ac949833246b6b4b3c3ac3117ec902c7611 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Sat, 26 Sep 2020 11:59:52 -0400 Subject: [PATCH 100/409] DOC: One more bash->python in PR #17353 --- doc/source/user/how-to-io.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/user/how-to-io.rst b/doc/source/user/how-to-io.rst index d962e1546ad9..1d4ded159422 100644 --- a/doc/source/user/how-to-io.rst +++ b/doc/source/user/how-to-io.rst @@ -36,7 +36,7 @@ With non-whitespace delimiters ------------------------------------------------------------------------------ :: - $ cat csv.txt # doctest: +SKIP + >>> print(open("csv.txt").read()) # doctest: +SKIP 1, 2, 3 4,, 6 7, 8, 9 From d267252186be8cfe1f4c7bc743d4b4523620bb42 Mon Sep 17 00:00:00 2001 From: MelissaWM Date: Sat, 26 Sep 2020 14:12:09 -0300 Subject: [PATCH 101/409] Added textwrap.dedent to test. --- numpy/f2py/tests/test_module_doc.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/numpy/f2py/tests/test_module_doc.py b/numpy/f2py/tests/test_module_doc.py index 9360b04a90a4..4b9555cee1fc 100644 --- a/numpy/f2py/tests/test_module_doc.py +++ b/numpy/f2py/tests/test_module_doc.py @@ -1,6 +1,7 @@ import os import sys import pytest +import textwrap from . import util from numpy.testing import assert_equal, IS_PYPY @@ -18,5 +19,12 @@ class TestModuleDocString(util.F2PyTest): @pytest.mark.xfail(IS_PYPY, reason="PyPy cannot modify tp_doc after PyType_Ready") def test_module_docstring(self): - expected = "i : 'i'-scalar\nx : 'i'-array(4)\na : 'f'-array(2,3)\nb : 'f'-array(-1,-1), not allocated\x00\nfoo()\n\nWrapper for ``foo``.\n\n" - assert_equal(self.module.mod.__doc__, expected) + assert_equal(self.module.mod.__doc__, + textwrap.dedent('''\ + i : 'i'-scalar + x : 'i'-array(4) + a : 'f'-array(2,3) + b : 'f'-array(-1,-1), not allocated\x00 + foo()\n + Wrapper for ``foo``.\n\n''') + ) From 4c94a5dbb1edb072af4776e2b193675de33cae06 Mon Sep 17 00:00:00 2001 From: Jack Date: Sun, 27 Sep 2020 00:30:49 -0700 Subject: [PATCH 102/409] Minor grammatical correction in quickstart doc. Single sentence grammatical correction. --- doc/source/user/quickstart.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/user/quickstart.rst b/doc/source/user/quickstart.rst index b1af8188675e..8e38234c598f 100644 --- a/doc/source/user/quickstart.rst +++ b/doc/source/user/quickstart.rst @@ -23,7 +23,7 @@ https://scipy.org/install.html for instructions. **Learner profile** This tutorial is intended as a quick overview of -algebra and arrays in NumPy and want to understand how n-dimensional +algebra and arrays in NumPy. It demonstrates how n-dimensional (:math:`n>=2`) arrays are represented and can be manipulated. In particular, if you don't know how to apply common functions to n-dimensional arrays (without using for-loops), or if you want to understand axis and shape properties for From 30a904198cd57251e46e6a60f3a6878cc4d0f531 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Sun, 27 Sep 2020 13:01:42 -0400 Subject: [PATCH 103/409] DOC: NumPy restyling for pydata theme --- doc/source/_static/numpy.css | 40 +++++++++++++++++++++++++++++++ doc/source/_templates/layout.html | 14 ++++------- 2 files changed, 44 insertions(+), 10 deletions(-) create mode 100644 doc/source/_static/numpy.css diff --git a/doc/source/_static/numpy.css b/doc/source/_static/numpy.css new file mode 100644 index 000000000000..22d08cc0dca6 --- /dev/null +++ b/doc/source/_static/numpy.css @@ -0,0 +1,40 @@ +@import url('https://fonts.googleapis.com/css2?family=Lato:ital,wght@0,400;0,700;0,900;1,400;1,700;1,900&family=Open+Sans:ital,wght@0,400;0,600;1,400;1,600&display=swap'); + +.navbar-brand img { + height: 75px; +} +.navbar-brand { + height: 75px; +} + +body { + font-family: 'Open Sans', sans-serif; + color:#4A4A4A; /* numpy.org body color */ +} + +pre, code { + font-size: 100%; + line-height: 155%; +} + +h1 { + font-style: "Lato", sans-serif; + color: #013243; /* warm black */ + font-weight: 700; + letter-spacing: -.04em; + text-align: right; + margin-top: 3rem; + margin-bottom: 4rem; + font-size: 3rem; +} + + +h2 { + color: #4d77cf; /* han blue */ + letter-spacing: -.03em; +} + +h3 { + color: #013243; /* warm black */ + letter-spacing: -.03em; +} diff --git a/doc/source/_templates/layout.html b/doc/source/_templates/layout.html index 0b0ba6271bdf..e2812fdd5ff3 100644 --- a/doc/source/_templates/layout.html +++ b/doc/source/_templates/layout.html @@ -1,16 +1,10 @@ {% extends "!layout.html" %} {%- block extrahead %} - +{{ super() }} + + + - -{{ super() }} {% endblock %} From 66685909f174c38ebfbf7f733f40bcf7d9e86d48 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Sun, 27 Sep 2020 13:01:41 -0500 Subject: [PATCH 104/409] NEP: Edit nep-0042 for more clarity (#17370) * DOC: Edit NEP-42 for clarity * DOC: Further edits of NEP 42 * DOC: Edit NEP-42 for clarity * DOC: Further edits of NEP 42 * WIP: Work out structure a bit and make intro longer * DOC: Round 3 of NEP 42 edits * Start with a new motivation/overview to be forced upon the rest * restructure along "top-level" categories and rename adjust_descriptors * Small fixups * Small clarity/correctness edits * Use `PyArrayMethod` instead of UFuncImpl I am starting to feel that calling this "array method" (even if it is in no way numpy array specific) is probably the best name. It is not ufunc specific, because ufuncs are dispatching + array method. Calling it ufuncimpl makes sense, but since the UFunc name which would also work perfectly is already taken, maybe a whole new name is better. * DOC: Fold in @seberg revisions * Fix typo * Make first example/overview inherit dtype instead of metaclass it could say `metaclass=DTypeMeta`, but its implicit. * DOC: Add "checklist" to NEP 42 Also anchor NEP 40 figure for reference in NEP 42. * Delete duplicated dtype-discovery section The section must have been duplicated in an merge conflict :/, I scanned through, and I think the version that was in the right place got most edits, although not 100% sure. * DOC: Reorganize scope, standardize heds * DOC: Retitle, revisions up to Casting Also anchor for parametric type discussion in NEP 40 * Some smaller changes/additions * DOC: Reorganize for better flow Also adds a NEP anchor at top. * DOC: Glossary, shuffle "Additional information" bullets * DOC: Small updates for sense and case * Fixup merge (some changes were not saved when finishing merge) * Answer bjnath's questions and motivate user impact It seems most answers bjnath already included himself, this adds a few tweaks. Added a note that tusers will basically have to implement the DType class as outlined is basically what the user will have to implement, so spell that out both at the top and in the C-API to clarify that. * NEP 41: Add subsection on dtype organization with parameters This might go a bit far, but I think the image goes very far in giving people the right idea of how to think of dtypes and DType instances. The question is if we want to even show the "alternative"... * Add bjnath as author * Add new header also to NEP 41 * DOC: Minor: delete horiz rules, tweak 2 headings * Rephrase NEP 41 bullt points in terms of NEP 42 Also changes those for NEP 43, to represent what is done, rather than what is solved (and thus fit better to the way its written for the NEP 42 part) * Try to rewerd paragraphs around setitem/getitem a bit some of these were very confusing, hopefully they are a bit better now. * Make a pass over the array-coercion part * Make careful pass up until array-coercion * Pass over the second half of the NEP 42 draft. * Use "given_descrs" which seems clearer and fix casting I always thought returning the casting safety is an awesome idea, but right now I am actually not so sure it helps much with anything. Its usually simpler, I guess. But if the user requested safe casting and we have an unsafe cast, for example for structured dtypes we will end up doing a lot of work for no reason. If we accept an error return without an error already set this is solved. My original reason was that setting a generic error in the resolve descriptor function would really be pretty annoying, so that returning the casting safety seemed like a good way to avoid worrying about most errors there. The "cast-is-view" output is important though. * Fix copy new intro note to NEP 40 (and fix it in NEP 41) Co-authored-by: Ben Nathanson --- .../_static/nep-0041-type-sketch-no-fonts.svg | 1110 ++++++++++ doc/neps/_static/nep-0041-type-sketch.svg | 523 +++++ doc/neps/nep-0040-legacy-datatype-impl.rst | 30 +- doc/neps/nep-0041-improved-dtype-support.rst | 106 +- doc/neps/nep-0042-new-dtypes.rst | 1928 +++++++++-------- 5 files changed, 2705 insertions(+), 992 deletions(-) create mode 100644 doc/neps/_static/nep-0041-type-sketch-no-fonts.svg create mode 100644 doc/neps/_static/nep-0041-type-sketch.svg diff --git a/doc/neps/_static/nep-0041-type-sketch-no-fonts.svg b/doc/neps/_static/nep-0041-type-sketch-no-fonts.svg new file mode 100644 index 000000000000..3250396c530a --- /dev/null +++ b/doc/neps/_static/nep-0041-type-sketch-no-fonts.svg @@ -0,0 +1,1110 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/neps/_static/nep-0041-type-sketch.svg b/doc/neps/_static/nep-0041-type-sketch.svg new file mode 100644 index 000000000000..9e597db9d9b2 --- /dev/null +++ b/doc/neps/_static/nep-0041-type-sketch.svg @@ -0,0 +1,523 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + Value Storage + Parameters andStorage options + Value Space andBehaviour + type + instance + + ABC + instance + + type + + DType + + + base dtype + element + + dtype + + element + + dtype + + Python type + + Python typewith ABC + NEP 41 Proposal + Alternative + + + diff --git a/doc/neps/nep-0040-legacy-datatype-impl.rst b/doc/neps/nep-0040-legacy-datatype-impl.rst index c247e3d62d10..39889109de40 100644 --- a/doc/neps/nep-0040-legacy-datatype-impl.rst +++ b/doc/neps/nep-0040-legacy-datatype-impl.rst @@ -13,15 +13,15 @@ NEP 40 — Legacy Datatype Implementation in NumPy .. note:: - This NEP is part of a series of NEPs encompassing first information - about the previous dtype implementation and issues with it in NEP 40 - (this document). - :ref:`NEP 41 ` then provides an overview and generic design choices - for the refactor. - Further NEPs 42 and 43 go into the technical details of the datatype - and universal function related internal and external API changes. - In some cases it may be necessary to consult the other NEPs for a full - picture of the desired changes and why these changes are necessary. + This NEP is first in a series: + + - NEP 40 (this document) explains the shortcomings of NumPy's dtype implementation. + + - :ref:`NEP 41 ` gives an overview of our proposed replacement. + + - :ref:`NEP 42 ` describes the new design's datatype-related APIs. + + - NEP 43 describes the new design's API for universal functions. @@ -44,6 +44,8 @@ of the current implementation of dtypes as well as a discussion. In many cases subsections will be split roughly to first describe the current implementation and then follow with an "Issues and Discussion" section. +.. _parametric-datatype-discussion: + Parametric Datatypes ^^^^^^^^^^^^^^^^^^^^ @@ -253,6 +255,8 @@ types such as ``np.inexact`` (see figure below). In fact, some control flow within NumPy currently uses ``issubclass(a.dtype.type, np.inexact)``. +.. _nep-0040_dtype-hierarchy: + .. figure:: _static/nep-0040_dtype-hierarchy.png **Figure:** Hierarchy of NumPy scalar types reproduced from the reference @@ -335,7 +339,7 @@ Each of these signatures is associated with a single inner-loop function defined in C, which does the actual calculation, and may be called multiple times. The main step in finding the correct inner-loop function is to call a -:c:type:`PyUFunc_TypeResolutionFunc` which retrieves the input dtypes from +:c:type:`PyUFunc_TypeResolutionFunc` which retrieves the input dtypes from the provided input arrays and will determine the full type signature (including output dtype) to be executed. @@ -366,7 +370,7 @@ It is currently only possible for user defined functions to be found/resolved if any of the inputs (or the outputs) has the user datatype, since it uses the `OO->O` signature. For example, given that a ufunc loop to implement ``fraction_divide(int, int) --> Fraction`` has been implemented, +-> Fraction`` has been implemented, the call ``fraction_divide(4, 5)`` (with no specific output dtype) will fail because the loop that includes the user datatype ``Fraction`` (as output) can only be found if any of @@ -572,7 +576,7 @@ Related Work ------------ * Julia types are an interesting blueprint for a type hierarchy, and define - abstract and concrete types [julia-types]_. + abstract and concrete types [julia-types]_. * In Julia promotion can occur based on abstract types. If a promoter is defined, it will cast the inputs and then Julia can then retry to find @@ -607,7 +611,7 @@ the following provides a subset for more recent ones: * https://hackmd.io/ok21UoAQQmOtSVk6keaJhw and https://hackmd.io/s/ryTFaOPHE (2019-04-30) Proposals for subclassing implementation approach. - + * Discussion about the calling convention of ufuncs and need for more powerful UFuncs: https://github.com/numpy/numpy/issues/12518 diff --git a/doc/neps/nep-0041-improved-dtype-support.rst b/doc/neps/nep-0041-improved-dtype-support.rst index 6dc4ea50c6fd..d7a08562d9c4 100644 --- a/doc/neps/nep-0041-improved-dtype-support.rst +++ b/doc/neps/nep-0041-improved-dtype-support.rst @@ -15,15 +15,15 @@ NEP 41 — First step towards a new Datatype System .. note:: - This NEP is part of a series of NEPs encompassing first information - about the previous dtype implementation and issues with it in - :ref:`NEP 40 `. - NEP 41 (this document) then provides an overview and generic design - choices for the refactor. - Further NEPs 42 and 43 go into the technical details of the datatype - and universal function related internal and external API changes. - In some cases it may be necessary to consult the other NEPs for a full - picture of the desired changes and why these changes are necessary. + This NEP is second in a series: + + - :ref:`NEP 40 ` explains the shortcomings of NumPy's dtype implementation. + + - NEP 41 (this document) gives an overview of our proposed replacement. + + - :ref:`NEP 42 ` describes the new design's datatype-related APIs. + + - NEP 43 describes the new design's API for universal functions. Abstract @@ -412,27 +412,28 @@ multiple development stages are required: * Phase II: Incrementally define or rework API - * Create a new and easily extensible API for defining new datatypes - and related functionality. (NEP 42) - - * Incrementally define all necessary functionality through the new API (NEP 42): - - * Defining operations such as ``np.common_type``. - * Allowing to define casting between datatypes. - * Add functionality necessary to create a numpy array from Python scalars - (i.e. ``np.array(...)``). - * … - - * Restructure how universal functions work (NEP 43), in order to: - - * make it possible to allow a `~numpy.ufunc` such as ``np.add`` to be - extended by user-defined datatypes such as Units. - - * allow efficient lookup for the correct implementation for user-defined - datatypes. - - * enable reuse of existing code. Units should be able to use the - normal math loops and add additional logic to determine output type. + * Incrementally define all necessary functionality through methods and + properties on the DType (NEP 42): + + * The properties of the class hierarchy and DType class itself, + including methods not covered by the following, most central, points. + * The functionality that will support dtype casting using ``arr.astype()`` + and casting related operations such as ``np.common_type``. + * The implementation of item access and storage, and the way shape and + dtype are determined when creating an array with ``np.array()`` + * Create a public C-API to define new DTypes. + + * Restructure how universal functions work (NEP 43), to allow extending + a `~numpy.ufunc` such as ``np.add`` for user-defined datatypes + such as Units: + + * Refactor how the low-level C functions are organized to make it + extensible and flexible enough for complicated DTypes such as Units. + * Implement registration and efficient lookup for these low-level C + functions as defined by the user. + * Define how promotion will be used to implement behaviour when casting + is required. For example ``np.float64(3) + np.int32(3)`` promotes the + ``int32`` to a ``float64``. * Phase III: Growth of NumPy and Scientific Python Ecosystem capabilities: @@ -583,7 +584,7 @@ special methods move from the dtype instances to methods on the new DType class. This is the typical design pattern used in Python. Organizing these methods and information in a more Pythonic way provides a solid foundation for refining and extending the API in the future. -The current API cannot be extended due to how it is exposed publically. +The current API cannot be extended due to how it is exposed publicly. This means for example that the methods currently stored in ``PyArray_ArrFuncs`` on each datatype (see :ref:`NEP 40 `) will be defined differently in the future and @@ -620,6 +621,49 @@ While DType and Scalar describe the same concept/type (e.g. an `int64`), it seems practical to split out the information and functionality necessary for numpy into the DType class. +The dtype instances provide parameters and storage options +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +From a computer science point of view a type defines the *value space* +(all possible values its instances can take) and their *behaviour*. +As proposed in this NEP, the DType class defines value space and behaviour. +The ``dtype`` instance can be seen as part of the value, so that the typical +Python ``instance`` corresponds to ``dtype + element`` (where *element* is the +data stored in the array). +An alternative view would be to define value space and behaviour on the +``dtype`` instances directly. +These two options are presented in the following figure and compared to +similar Python implementation patterns: + +.. image:: _static/nep-0041-type-sketch-no-fonts.svg + +The difference is in how parameters, such as string length or the datetime +units (``ms``, ``ns``, ...), and storage options, such as byte-order, are handled. +When implementing a Python (scalar) ``type`` parameters, for example the datetimes +unit, will be stored in the instance. +This is the design NEP 42 tries to mimic, however, the parameters are now part +of the dtype instance, meaning that part of the data stored in the instance +is shared by all array elements. +As mentioned previously, this means that the Python ``instance`` corresponds +to the ``dtype + element`` stored in a NumPy array. + +An more advanced approach in Python is to use a class factory and an abstract +base class (ABC). +This allows moving the parameter into the dynamically created ``type`` and +behaviour implementation may be specific to those parameters. +An alternative approach might use this model and implemented behaviour +directly on the ``dtype`` instance. + +We believe that the version as proposed here is easier to work with and understand. +Python class factories are not commonly used and NumPy does not use code +specialized for dtype parameters or byte-orders. +Making such specialization easier to implement such specialization does not +seem to be a priority. +One result of this choice is that some DTypes may only have a singleton instance +if they have no parameters or storage variation. +However, all of the NumPy dtypes require dynamically created instances due +to allowing metadata to be attached. + Scalars should not be instances of the datatypes (2) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/doc/neps/nep-0042-new-dtypes.rst b/doc/neps/nep-0042-new-dtypes.rst index b37555892b62..2d1e3a329e6c 100644 --- a/doc/neps/nep-0042-new-dtypes.rst +++ b/doc/neps/nep-0042-new-dtypes.rst @@ -1,9 +1,12 @@ -======================================== -NEP 42 — Implementation of New DataTypes -======================================== +.. _NEP42: -:title: Extensible Datatypes for NumPy +============================================================================== +NEP 42 — User-extensible dtypes +============================================================================== + +:title: User-extensible dtypes :Author: Sebastian Berg +:Author: Ben Nathanson :Author: Marten van Kerkwijk :Status: Draft :Type: Standard @@ -12,666 +15,439 @@ NEP 42 — Implementation of New DataTypes .. note:: - This NEP is part of a series of NEPs encompassing first information - about the previous dtype implementation and issues with it in - :ref:`NEP 40 `. - :ref:`NEP 41 ` then provides an overview and generic design - choices for the refactor. NEPs 42 (this document) - and 43 go into the technical details of the internal and external - API changes related to datatypes and universal functions, respectively. - In some cases it may be necessary to consult the other NEPs for a full - picture of the desired changes and why these changes are necessary. + This NEP is third in a series: + - :ref:`NEP 40 ` explains the shortcomings of NumPy's dtype implementation. -Abstract --------- - -NEP 40 and 41 detailed the need for the creation of a new datatype system within -NumPy to better serve downstream use-cases and improve the maintainability -and the extensibility of NumPy. -A main issue with the current dtype API is that datatypes are written as -a single Python class with special instances for each of the actual datatypes. -While this certainly has been a practical approach in implementing numerical -datatypes, it does not allow to naturally split up logic. For example, -functions such as ``can_cast`` have explicit logic for each datatype. -Because of this monolithic code structure user-defined datatypes do not have -the same capabilities as NumPy datatypes have. -The current structure also makes understanding and modifying datatypes harder. -The current datatypes are not well encapsulated, so modifications targeting -a single datatype inevitably touch code involving others. -As detailed in NEP 41, the desired general design is to create classes for -each of the NumPy-provided datatypes, meaning that ``np.dtype("float64")`` -returns an instance of a ``Float64`` class which is a subclass of ``np.dtype``. -``np.dtype[float64]`` will also be used to denote this class. -This will allow moving all logic into special methods on the ``np.dtype`` -subclasses. This ``DType`` class would then serve as the central -extension point for adding new dtypes to NumPy. - -This document proposes the new API for the datatypes itself. -A second proposal NEP 43 details proposed changes to the universal -functions. -Note that only the implementation of both NEPs will provide the desired full -functionality. + - :ref:`NEP 41 ` gives an overview of our proposed replacement. + - NEP 42 (this document) describes the new design's datatype-related APIs. -.. note:: + - NEP 43 describes the new design's API for universal functions. - At this time this NEP is in a preliminary state. Both internal and - external API may be adapted based on user input or implementation needs. - The general design principles and choices, while provisional, should not - be expected to change dramatically. +****************************************************************************** +Abstract +****************************************************************************** -Detailed Description --------------------- +NumPy's dtype architecture is monolithic, built around a single class that +handles each dtype as an instance. There's no principled way to expand it to +new dtypes, and the code is difficult to read and maintain. -NEP 41 layed out the creation of a class hierarchy for datatypes using the -new DType classes to provide all necessary implementations. -This NEP defines the specific choice of API necessary to define new DTypes. -Here, these are suggested as C-API slots; however, conceptually these -translate identically to Python methods. +As NEP 41 explains, we are proposing a new architecture that is modular and +open to user additions. dtypes will derive from a new ``DType`` class serving +as the extension point for new types. ``np.dtype("float64")`` will return an +instance of a ``Float64`` class, a subclass of root class ``np.dtype``. -Additionally, the NEP proposes to implement the notion of *abstract* DTypes. -Further, we detail – in part – how the proposed methods (C-API slots) -enable all necessary use cases. +This NEP is one of two that lay out the design and API of this new +architecture. This NEP addresses dtype implementation; NEP 43 addresses +universal functions. -Each section will begin with a short motivation of the issue or what -problem is addressed. This is followed by a description of the proposed -design choice, and then may list alternatives. +.. note:: + Details of the private and external APIs may change to reflect user + comments and implementation constraints. The underlying principles and + choices should not change significantly. -Nomenclature -"""""""""""" -As a brief note on nomenclature, it should be noted that ``dtype`` normally -denotes the dtype *instance*, which is the object attached to a numpy array. -On the other hand the ``DType`` class is the subclass of ``np.dtype``. -On the C-level we currently use the name ``descriptor`` or ``descr`` -interchangeably with *dtype instance*. ``descriptor`` or ``descr`` will be -used in proposed C-API names to differentiate dtype instances from DType -classes more clearly. -Note that the notion of dtype class is currently represented mainly as -the ``dtype.num`` and ``dtype.char``. -Please see the :ref:`dtype hierarchy figure ` for an -illustration of this distinction. +****************************************************************************** +Motivation and scope +****************************************************************************** -There are currently classes in NumPy for numeric types e.g. -``np.float64``; however, -these are not DTypes but the corresponding scalar classes -(see also NEP 40 and 41 for discussion on why these are largely unrelated to -the proposed changes). +Our goal is to allow user code to create fully featured dtypes for a broad +variety of uses, from physical units (such as meters) to domain-specific +representations of geometric objects. NEP 41 describes a number of these new +dtypes and their benefits. +Any design supporting dtypes must consider: -Proposed access to DType class -"""""""""""""""""""""""""""""" +- How shape and dtype are determined when an array is created +- How array elements are stored and accessed +- The rules for casting dtypes to other dtypes -**Motivation:** +In addition: -Currently we often call ``np.dtype`` to create the dtype instance -corresponding to a given scalar type (e.g. ``np.dtype(np.int64)``). -Adding the DType classes may require a way to access the classes conveniently. +- We want dtypes to comprise a class hierarchy open to new types and to + subhierarchies, as motivated in :ref:`NEP 41 `. -**Description:** +And to provide this, -To avoid duplication, but also to expose the classes conveniently to users -we propose the addition of:: +- We need to define a user API. - np.dtype[np.int64] +All these are the subjects of this NEP. -as a class getter. This can work both for user and NumPy DTypes, -although, in many cases a library may choose to provide a more direct -way to access the specific DType class. -This method may initially be limited to concrete DTypes. -The main reason for this choice is to provide a single -clear and future-proof way to find the DType class given the -Python (scalar) class. +- The class hierarchy, its relation to the Python scalar types, and its + important attributes are described in `DType class`_. -This should not be a common operation, so providing this class getter reduces -the pressure of adding the new DType classes into the namespace. +- The functionality that will support dtype casting is described in `Casting`_. -*Note: This is currently a possible extension and not yet decided.* +- The implementation of item access and storage, and the way shape and dtype + are determined when creating an array, are described in `Array coercion`_. +- The functionality for users to define their own DTypes is described in + `Public C-API`_. -Hierarchy of DTypes and Abstract DTypes -""""""""""""""""""""""""""""""""""""""" +The API here and in NEP 43 is entirely on the C side. A Python-side version +will be proposed in a future NEP. +A future Python API is expected to be similar, but provide a more convenient +API to reuse the functionality of existing DTypes. +It could also provide shorthands to create structured DTypes similar to python's +`dataclasses `_. -**Motivation:** -The creation of DType classes has already been decided in NEP 41. -Here we discuss the notion of **abstract** DTypes. -There are multiple reasons for this: -1. It allows the definition of a class hierarchy, in principle allowing checks like - ``isinstance(np.dtype("float64"), np.inexact)``. - **This hierarchy may be a prerequisite to implementing dispatching - for universal functions (NEP 43)** -2. Abstract DTypes can enable code such as - ``arr.astype(Complex)`` to express the desire to cast to a - complex data type of unspecified precision. -3. It anticipates the creation of families of DTypes by users. - For example allowing the creation of an abstract ``Unit`` class with a concrete - ``Float64Unit``. In which case ``Unit(np.float64, "m")`` could be - identical to ``Float64Unit("m")``. - -A very concrete example is the current Pandas ``Categorical`` DType, -which may benefit from abstraction to allow the differentiation of -a categorical of integer values and one of general object values. -The reason for this is that we may want to reject -``common_dtype(CategoricalInt64, String)``, but accept -``common_dtype(CategoricalObject, String)`` to be the ``object`` DType. -The current Pandas ``Categorical`` DType combines both and must remain -representable. The solution is thus to make ``Categorical`` abstract with -the two subclasses ``CategoricalInt64`` and ``CategoricalObject`` -distinguishing the two. - - -**Description:** - -The figure below shows the proposed datatype hierarchy. -It should be noted that abstract DTypes are distinct in two ways: - -1. They do not have instances. Instantiating an abstract DType has to return - a concrete subclass or raise an error (default, and possibly enforced - initially). -2. Unlike concrete DTypes, abstract DTypes can be superclasses, they may also - serve like Python's abstract base classes (ABC). - (It may be possible to simply use/inherit from Python ABCs.) - -These two rules are identical to the type choices made for example in the -`Julia language `_. -It allows for the creation of a datatype hierarchy, but avoids issues with -subclassing concrete DTypes directly. -For example, logic such as ``can_cast`` does not cleanly inherit from a -``Int64`` to a ``Datetime64`` even though the ``Datetime64`` could be seen -as an integer with only a unit attached (and thus implemented as a subclass). - -The main consequence for the DType implementer is that concrete DTypes can -never be subclasses of existing concrete DTypes. -End-users would not notice or need to know about this distinction. -However, subclassing may be a possible mechanism to extend the datatypes -in the future to allow specialized implementations for existing dtypes -such as a GPU float64 subclassing a NumPy float64. - -The combination of (initially) rejecting subclassing of concrete DTypes -while allowing it for abstract ones allows the transparent definition of -a class hierarchy, while avoiding potential issues with subclassing and -especially inheritance. - -As a technical implementation detail: the DType class will require C-side -storage of methods and additional information. -This requires the creation of a ``DTypeMeta`` class. -Each ``DType`` class is thus an instance of ``DTypeMeta`` with a well-defined -and extensible interface. -The end-user will not need to be aware of this. - -.. _hierarchy_figure: -.. figure:: _static/dtype_hierarchy.svg - :figclass: align-center - - -Methods/Slots defined for each DType -"""""""""""""""""""""""""""""""""""" - -NEP 41 detailed that all logic should be defined through special methods -on the DTypes. -This section will list a specific set of such methods (in the form of -Python methods). -The C-side equivalent slot signature will be summarized below after proposing -the general C-API for defining new Datatypes. -Note that while the slots are defined as special Python methods here, this is -for the readers convenience and *not* meant to imply the identical exposure -as a Python API. -This will need to be proposed in a separate, later, NEP. +****************************************************************************** +Backward compatibility +****************************************************************************** -Some of the methods may be similar or even reuse existing Python slots. -User-defined DType classes are discouraged from defining or using Python's -special slots without consulting the NumPy developers, in order to allow -defining them later. -For example ``dtype1 & dtype2`` could be a shorthand for -``np.common_dtype(dtype1, dtype2)``, and comparisons should be defined mainly -through casting logic. +The disruption is expected to be no greater than that of a typical NumPy +release. +- The main issues are noted in :ref:`NEP 41 ` and will mostly affect + heavy users of the NumPy C-API. -Additional Information -^^^^^^^^^^^^^^^^^^^^^^ +- Eventually we will want to deprecate the API currently used for creating + user-defined dtypes. -In addition to the more detailed methods below, the following general -information is currently provided and will be defined on the class: +- Small, rarely noticed inconsistencies are likely to change. Examples: -* ``cls.parametric`` (see also `NEP 40 `_): + - ``np.array(np.nan, dtype=np.int64)`` behaves differently from + ``np.array([np.nan], dtype=np.int64)`` with the latter raising an error. + This may require identical results (either both error or both succeed). + - ``np.array([array_like])`` sometimes behaves differently from + ``np.array([np.array(array_like)])`` + - array operations may or may not preserve dtype metadata - * Parametric will be a flag in the (private) C-API. However, the - Python API will instead use a ``ParametricDType`` class from - which to inherit. (This is similar to Python's type flags, which include - flags for some basic subclasses such as subclasses of ``float`` or ``tuple``) - * This flag is mainly to simplify DType creation and casting and - allow for performance tweaks. - * DTypes which are not parametric must define a canonical dtype instance - which should be a singleton. - * Parametric dtypes require some additional methods (below). +The new code must pass NumPy's regular test suite, giving some assurance that +the changes are compatible with existing code. -* ``self.canonical`` method (Alternative: new instance attribute) +****************************************************************************** +Usage and impact +****************************************************************************** - * Instead of byteorder, we may want a ``canonical`` flag (reusing the - ISNBO flag – "is native byte order" seems possible here). - This flag signals that the data are stored in the default/canonical way. - In practice this is always an NBO check, but generalization should be possible. - A potential use-case is a complex-conjugated instance of Complex which - stores ``real - imag`` instead of ``real + imag`` and is thus not - the canonical storage. +We believe the few structures in this section are sufficient to consolidate +NumPy's present functionality and also to support complex user-defined DTypes. -* ``ensure_canonical(self) -> dtype`` return a new dtype (or ``self``). - The returned dtype must have the ``canonical`` flag set. +The rest of the NEP fills in details and provides support for the claim. -* ``DType.type`` is the associated scalar type. ``dtype.type`` will be a - class attribute and the current ``dtype.type`` field will be considered - deprecated. This may be relaxed if a use-case arises. +Again, though Python is used for illustration, the implementation is a C API only; a +future NEP will tackle the Python API. -Additionally, existing methods (and C-side fields) will be provided. -However, the fields ``kind`` and ``char`` will be set to ``\0`` -(NULL character) on the C-side. -While discouraged, except for NumPy builtin types, ``kind`` both will return -the ``__qualname__`` of the object to ensure uniqueness for all DTypes. -(the replacement for ``kind`` will be to use ``isinstance`` checks). +After implementing this NEP, creating a DType will be possible by implementing +the following outlined DType base class, +that is further described in `DType class`_: -Another example of methods that should be moved to the DType class are the -various sorting functions, which shall be implemented by defining a method: +.. code-block:: python + :dedent: 0 -* ``dtype_get_sort_function(self, sortkind="stable") -> sortfunction`` + class DType(np.dtype): + type : type # Python scalar type + parametric : bool # (may be indicated by superclass) -which must return ``NotImplemented`` if the given ``sortkind`` is not known. -Similarly, any function implemented previously which cannot be removed will -be implemented as a special method. -Since these methods can be deprecated and new (renamed) replacements added, -the API is not defined here and it is acceptable if it changes over time. + @property + def canonical(self) -> bool: + raise NotImplementedError -For some of the current "methods" defined on the dtype, including sorting, -a long term solution may be to instead create generalized ufuncs to provide -the functionality. + def ensure_canonical(self : DType) -> DType: + raise NotImplementedError -**Alternatives:** +For casting, a large part of the functionality is provided by the "methods" stored +in ``_castingimpl`` -Some of these flags could be implemented by inheriting -for example from a ``ParametricDType`` class. However, on the C-side as -an implementation detail it seems simpler to provide a flag. -This does not preclude the possibility of creating a ``ParametricDType`` -to Python to represent the same thing. +.. code-block:: python + :dedent: 0 -**Example:** + @classmethod + def common_dtype(cls : DTypeMeta, other : DTypeMeta) -> DTypeMeta: + raise NotImplementedError -The ``datetime64`` DType is considered parametric, due to its unit, and -unlike a float64 has no canonical representation. The associated ``type`` -is the ``np.datetime64`` scalar. + def common_instance(self : DType, other : DType) -> DType: + raise NotImplementedError + # A mapping of "methods" each detailing how to cast to another DType + # (further specified at the end of the section) + _castingimpl = {} -**Issues and Details:** +For array-coercion, also part of casting: -A potential DType such as ``Categorical`` will not be required to have a clear type -associated with it. Instead, the ``type`` may be ``object`` and the -categorical's values are arbitrary objects. -Unlike with well-defined scalars, this ``type`` cannot -not be used for the dtype discovery necessary for coercion -(compare section `DType Discovery during Array Coercion`_). - - -Coercion to and from Python Objects -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. code-block:: python + :dedent: 0 -**Motivation:** + def __dtype_setitem__(self, item_pointer, value): + raise NotImplementedError -When storing a single value in an array or taking it out of the array, -it is necessary to coerce (convert) it to and from the low-level -representation inside the array. + def __dtype_getitem__(self, item_pointer, base_obj) -> object: + raise NotImplementedError -**Description:** + @classmethod + def __discover_descr_from_pyobject__(cls, obj : object) -> DType: + raise NotImplementedError -Coercing to and from Python scalars requires two to three methods: + # initially private: + @classmethod + def _known_scalar_type(cls, obj : object) -> bool: + raise NotImplementedError -1. ``__dtype_setitem__(self, item_pointer, value)`` -2. ``__dtype_getitem__(self, item_pointer, base_obj) -> object`` - The ``base_obj`` should be ignored normally, it is provided *only* for - memory management purposes, pointing to an object owning the data. - It exists only to allow support of structured datatypes with subarrays - within NumPy, which (currently) return views into the array. - The function returns an equivalent Python scalar (i.e. typically a NumPy - scalar). -3. ``__dtype_get_pyitem__(self, item_pointer, base_obj) -> object`` - (initially hidden for new-style user-defined datatypes, may be exposed - on user request). This corresponds to the ``arr.item()`` method which - is also used by ``arr.tolist()`` and returns e.g. Python floats instead of - NumPy floats. -(The above is meant for C-API. A Python-side API would have to use byte -buffers or similar to implement this, which may be useful for prototyping.) +Other elements of the casting implementation is the ``CastingImpl``: -These largely correspond to the current definitions. When a certain scalar -has a known (different) dtype, NumPy may in the future use casting instead -of ``__dtype_setitem__``. -A user datatype is (initially) expected to implement ``__dtype_setitem__`` -for its own ``DType.type`` and all basic Python scalars it wishes to support -(e.g. integers, floats, datetime). -In the future a function "``known_scalartype``" may be added to allow a user -dtype to signal which Python scalars it can store directly. +.. code-block:: python + :dedent: 0 + casting = Union["safe", "same_kind", "unsafe"] -**Implementation:** + class CastingImpl: + # Object describing and performing the cast + casting : casting -The pseudo-code implementation for setting a single item in an array -from an arbitrary Python object ``value`` is (note that some of the -functions are only defined below):: - - def PyArray_Pack(dtype, item_pointer, value): - DType = type(dtype) - if DType.type is type(value) or DType.known_scalartype(type(value)): - return dtype.__dtype_setitem__(item_pointer, value) - - # The dtype cannot handle the value, so try casting: - arr = np.array(value) - if arr.dtype is object or arr.ndim != 0: - # not a numpy or user scalar; try using the dtype after all: - return dtype.__dtype_setitem__(item_pointer, value) - - arr.astype(dtype) - item_pointer.write(arr[()]) - -where the call to ``np.array()`` represents the dtype discovery and is -not actually performed. + def resolve_descriptors(self, Tuple[DType] : input) -> (casting, Tuple[DType]): + raise NotImplementedError -**Example:** + # initially private: + def _get_loop(...) -> lowlevel_C_loop: + raise NotImplementedError -Current ``datetime64`` returns ``np.datetime64`` scalars and can be assigned -from ``np.datetime64``. -However, the datetime ``__dtype_setitem__`` also allows assignment from -date strings ("2016-05-01") or Python integers. -Additionally the datetime ``__dtype_get_pyitem__`` function actually returns -Python ``datetime.datetime`` object (most of the time). +which describes the casting from one DType to another. +In NEP 43 this ``CastingImpl`` object is used unchanged to support +universal functions. -**Alternatives:** +****************************************************************************** +Definitions +****************************************************************************** +.. glossary:: -This may be seen as simply a cast to and from the ``object`` dtype. -However, it seems slightly more complicated. This is because -in general a Python object could itself be a zero-dimensional array or -scalar with an associated DType. -Thus, representing it as a normal cast would either require that: + dtype + The dtype *instance*; this is the object attached to a numpy array. -* The implementor handles all Python classes, even those for which - ``np.array(scalar).astype(UserDType)`` already works because - ``np.array(scalar)`` returns, say, a datetime64 array. -* The cast is actually added between a typed-object to dtype. And even - in this case a generic fallback (for example ``float64`` can use - ``float(scalar)`` to do the cast) is also necessary. + DType + Any subclass of the base type ``np.dtype``. -It is certainly possible to describe the coercion to and from Python objects -using the general casting machinery. However, it seems special enough to -handle specifically. + coercion + Conversion of Python types to NumPy arrays and values stored in a NumPy + array. + cast + Conversion of an array to a different dtype. -**Further Issues and Discussion:** + promotion + Finding a dtype that can perform an operation on a mix of dtypes without + loss of information. -The setitem function currently duplicates some code, such as coercion -from a string. ``datetime64`` allows assignment from string, but the same -conversion also occurs for casts from the string dtype to ``datetime64``. -In the future, we may expose a way to signal whether a conversion is known, -and otherwise a normal cast is made so that the item is effectively set to ``np.array(scalar).astype(requested_dtype)``. + safe cast + A cast is safe if no information is lost when changing type. -There is a general issue about the handling of subclasses. We anticipate to not -automatically detect the dtype for ``np.array(float64_subclass)`` to be -float64. The user can still provide ``dtype=np.float64``. However, the above -"assign by casting" using ``np.array(scalar_subclass).astype(requested_dtype)`` -will fail. +On the C level we use ``descriptor`` or ``descr`` to mean +*dtype instance*. In the proposed C-API, these terms will distinguish +dtype instances from DType classes. .. note:: + Perhaps confusingly, NumPy already has a class hierarchy for numeric types, as + seen :ref:`in the figure ` of NEP 40, and the new + DType hierarchy will resemble it. But the existing hierarchy is for scalar + types, not DTypes, and its existence is largely irrelevant here, as NEP 40 and + 41 explain. - This means that ``np.complex256`` should not use ``__float__`` in its - ``__dtype_setitem__`` method in the future unless it is a known floating - point type. If the scalar is a subclass of a different high precision - floating point type (e.g. ``np.float128``) then this will lose precision. - - -DType Discovery during Array Coercion -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -An important step in the usage of NumPy arrays is the creation of the array -itself from collections of generic Python objects. - -**Motivation:** - -Although the distinction is not clear currently, there are two main needs:: - - np.array([1, 2, 3, 4.]) - -needs to guess the correct dtype based on the Python objects inside. -Such an array may include a mix of datatypes, as long as they can be clearly -promoted. -Currently not clearly distinct (but partially existing for strings) is the -use case of:: - - # np.dtype[np.str_] can also be spelled np.str_ or "S" (which works today) - np.array([object(), None], dtype=np.dtype[np.str_]) - -which forces each object to be interpreted as string. This is anticipated -to be useful for example for categorical datatypes:: - - np.array([1, 2, 1, 1, 2], dtype=Categorical) +.. _DType class: -to allow the discovery the of all unique values. -(For NumPy ``datetime64`` this is also currently used to allow string input.) - -There are three further issues to consider: - -1. It may be desirable that datatypes can be created which are associated - to normal Python scalars (such as ``datetime.datetime``), which do not - have a ``dtype`` attribute already. -2. In general, a datatype could represent a sequence, however, NumPy currently - assumes that sequences are always collections of elements (the sequence cannot be an - element itself). An example for this is would be a ``vector`` DType. -3. An array may itself contain arrays with a specific dtype (even - general Python objects). For example: - ``np.array([np.array(None, dtype=object)], dtype=np.String)`` - poses the issue of how to handle the included array. - -Some of these difficulties arise due to the fact that finding the correct shape -of the output array and finding the correct datatype are closely related. - -**Implementation:** - -There are two distinct cases given above: First, when the user has provided no -dtype information, and second when the user provided a DType class – -a notion that is currently represented e.g. by the parametric instance ``"S"`` -representing a string of any length. - -In the first case, it is necessary to establish a mapping from the Python type(s) -of the constituent elements to the DType class. -When the DType class is known, the correct dtype instance still needs to be found. -This shall be implemented by leveraging two pieces of information: - -1. ``DType.type``: The current type attribute to indicate which Python scalar - type is associated with the DType class (this is a *class* attribute that always - exists for any datatype and is not limited to array coercion). -2. The reverse lookup will remain hardcoded for the basic Python types initially. - Otherwise the ``type`` attribute will be used, and at least initially may - enforce deriving the scalar from a NumPy-provided scalar base class. - This method may be expanded later (see alternatives). -3. ``__discover_descr_from_pyobject__(cls, obj) -> dtype``: A classmethod that - returns the correct descriptor given the input object. - *Note that only parametric DTypes have to implement this*, most datatypes - can simply use a default (singleton) dtype instance which is found only - based on the ``type(obj)`` of the Python object. +****************************************************************************** +The DType class +****************************************************************************** -The Python type which is already associated with a DType through the -``DType.type`` attribute maps from the DType to the Python type. -A DType may choose to automatically discover from this Python type. -This will be achieved using a global a mapping (dictionary-like) of:: +This section reviews the structure underlying the proposed DType class, +including the type hierarchy and the use of abstract DTypes. - known_python_types[type] = DType +Class getter +============================================================================== -To anticipate the possibility of creating both a Python type (``pytype``) -and ``DType`` dynamically, and thus the potential desire to delete them again, -this mapping should generally be weak. -This requires that the ``pytype`` holds on to the ``DType`` explicitly. -Thus, in addition to building the global mapping, NumPy will store -the ``DType`` as ``pytype.__associated_array_dtype__`` in the Python type. -This does *not* define the mapping and should *not* be accessed directly. -In particular potential inheritance of the attribute does not mean that -NumPy will use the superclasses ``DType`` automatically. -A new ``DType`` must be created for the subclass. +To create a dtype instance from a scalar type users now call ``np.dtype`` (for +instance, ``np.dtype(np.int64)``). -.. note:: +To get the DType of a scalar type, we propose this getter syntax:: - Python integers do not have a clear/concrete NumPy type associated with - them right now. This is because during array coercion NumPy currently - finds the first type capable of representing their value in the list - of `long`, `unsigned long`, `int64`, `unsigned int64`, and `object` - (on many machines `long` is 64 bit). - - Instead they will need to be implemented using an - ``AbstractPyInt``. This DType class can then provide - ``__discover_descr_from_pyobject__`` and return the actual dtype which - is e.g. ``np.dtype("int64")``. - For dispatching/promotion in ufuncs, it will also be necessary - to dynamically create ``AbstractPyInt[value]`` classes (creation can be - cached), so that they can provide the current value based promotion - functionality provided by ``np.result_type(python_integer, array)`` [1]_. - -To allow for a DType to accept specific inputs as known scalars, we will -initially use a ``known_scalar_type`` method. -This allows discovery of a ``vector`` as a scalar (element) instead of -a sequence (for the command ``np.array(vector, dtype=VectorDType)``) -even when ``vector`` is itself a sequence or even an array subclass. -This will *not* be public API initially, but may be made public at a later -time. - -This will work similar to the following pseudo-code:: + np.dtype[np.int64] - def find_dtype(array_like): - common_dtype = None - for element in array_like: - # default to object dtype, if unknown - DType = known_python_types.get(type(element), np.dtype[object]) - dtype = DType.__discover_descr_from_pyobject__(element) +The notation works equally well with built-in and user-defined DTypes +and is inspired by and potentially useful for type hinting. - if common_dtype is None: - common_dtype = dtype - else: - common_dtype = np.promote_types(common_dtype, dtype) +This getter eliminates the need to create an explicit name for every +DType, crowding the ``np`` namespace; the getter itself signifies the type. -In practice, we have to find out whether an element is actually a sequence. -This means that instead of using the ``object`` dtype directly, we have to -check whether or not it is a sequence. +Since getter calls won't be needed often, this is unlikely to be burdensome. +Classes can also offer concise alternatives. -The full algorithm (without user provided dtype) thus looks more like:: +The initial implementation probably will return only concrete (not abstract) +DTypes. - def find_dtype_recursive(array_like, dtype=None): - """ - Recursively find the dtype for a nested sequences (arrays are not - supported here). - """ - DType = known_python_types.get(type(element), None) +*This item is still under review.* - if DType is None and is_array_like(array_like): - # Code for a sequence, an array_like may have a DType we - # can use directly: - for element in array_like: - dtype = find_dtype_recursive(element, dtype=dtype) - return dtype - elif DType is None: - DType = np.dtype[object] +Hierarchy and abstract classes +============================================================================== - # Same as above +We will use abstract classes as building blocks of our extensible DType class +hierarchy. -If the user provides ``DType``, then this DType will be tried first, and the -``dtype`` may need to be cast before the promotion is performed. +1. Abstract classes are inherited cleanly, in principle allowing checks like + ``isinstance(np.dtype("float64"), np.inexact)``. -**Limitations:** - -The above issue 3. is currently (sometimes) supported by NumPy so that -the values of an included array are inspected. -Support in those cases may be kept for compatibility, however, -it will not be exposed to user datatypes. -This means that if e.g. an array with a parametric string dtype is coerced above -(or cast) to an array of a fixed length string dtype (with unknown length), -this will result in an error. -Such a conversion will require passing the correct DType (fixed length of the -string) or providing a utility function to the user. - -The use of a global type map means that an error or warning has to be given -if two DTypes wish to map to the same Python type. In most cases user -DTypes should only be implemented for types defined within the same library to -avoid the potential for conflicts. -It will be the DType implementor's responsibility to be careful about this and use -the flag to disable registration when in doubt. - -**Alternatives:** - -The above proposes to add a global mapping, however, initially limiting it -to types deriving from a NumPy subclass (and a fixed set of Python types). -This could be relaxed in the future. -Alternatively, we could rely on the scalar belonging to the user dtype to -implement ``scalar.__associated_array_dtype__`` or similar. - -Initially, the exact implementation shall be *undefined*, if -scalars will have to derive from a NumPy scalar, they will also have -a ``.__associated_array_dtype__`` attribute. -At this time, a future update may to use this instead of a global mapping, -however, it makes NumPy a hard dependency for the scalar class. - -An initial alternative suggestion was to use a two-pass approach instead. -The first pass would only find the correct DType class, and the second pass -would then find correct dtype instance (the second pass is often not necessary). -The advantage of this is that the DType class information is vital for universal -functions to decide which loop to execute. -The first pass would provide the full information necessary for value-based -casting currently implemented for scalars, giving even the possibility of -expanding it to e.g. list inputs ``np.add(np.array([8], dtype="uint8"), [4])`` -giving a ``uint8`` result. -This is mainly related to the question to how the common dtype is found above. -It seems unlikely that this is useful, and similar to a global, could be -added later if deemed necessary. - -**Further Issues and Discussion:** - -While it is possible to create e.g. a DType such as Categorical, array, -or vector which can only be used if `dtype=DType` is provided, if this -is necessary these will not roundtrip correctly when converted back -and forth:: +2. Abstract classes allow a single piece of code to handle a multiplicity of + input types. Code written to accept Complex objects can work with numbers + of any precision; the precision of the results is determined by the + precision of the arguments. + +3. There is room for user-created families of DTypes. We can envision an + abstract ``Unit`` class for physical units, with a concrete subclass like + ``Float64Unit``. Calling ``Unit(np.float64, "m")`` (``m`` for meters) would + be equivalent to ``Float64Unit("m")``. + +4. The implementation of universal functions in NEP 43 may require + a class hierarchy. + +**Example:** A NumPy ``Categorical`` class would be a match for pandas +``Categorical`` objects, which can contain integers or general Python objects. +NumPy needs a DType that it can assign a Categorical to, but it also needs +DTypes like ``CategoricalInt64`` and ``CategoricalObject`` such that +``common_dtype(CategoricalInt64, String)`` raises an error, but +``common_dtype(CategoricalObject, String)`` returns an ``object`` DType. In +our scheme, ``Categorical`` is an abstract type with ``CategoricalInt64`` and +``CategoricalObject`` subclasses. + + +Rules for the class structure, illustrated :ref:`below `: + +1. Abstract DTypes cannot be instantiated. Instantiating an abstract DType + raises an error, or perhaps returns an instance of a concrete subclass. + Raising an error will be the default behavior and may be required initially. + +2. While abstract DTypes may be superclasses, they may also act like Python's + abstract base classes (ABC) allowing registration instead of subclassing. + It may be possible to simply use or inherit from Python ABCs. + +3. Concrete DTypes may not be subclassed. In the future this might be relaxed + to allow specialized implementations such as a GPU float64 subclassing a + NumPy float64. + +The +`Julia language `_ +has a similar prohibition against subclassing concrete types. +For example methods such as the later ``__common_instance__`` or +``__common_dtype__`` cannot work for a subclass unless they were designed +very carefully. +It helps avoid unintended vulnerabilities to implementation changes that +result from subclassing types that were not written to be subclassed. +We believe that the DType API should rather be extended to simplify wrapping +of existing functionality. + +The DType class requires C-side storage of methods and additional information, +to be implemented by a ``DTypeMeta`` class. Each ``DType`` class is an +instance of ``DTypeMeta`` with a well-defined and extensible interface; +end users ignore it. + +.. _nep42_hierarchy_figure: +.. figure:: _static/dtype_hierarchy.svg + :figclass: align-center - np.array(np.array(1, dtype=Categorical)[()]) -requires to pass the original ``dtype=Categorical`` or returns an array -with dtype ``object``. -While a general limitation, the round-tripping shall always be possible if -``dtype=old_dtype`` is provided. +Miscellaneous methods and attributes +============================================================================== + +This section collects definitions in the DType class that are not used in +casting and array coercion, which are described in detail below. + +* Existing dtype methods and C-side fields are preserved. + +* ``DType.type`` replaces ``dtype.type``. Unless a use case arises, + ``dtype.type`` will be deprecated. + This indicates a Python scalar type which represents the same values as + the DType. This is the same type as used in the proposed `Class getter`_ + and for `DType discovery during array coercion`_. + (This can may also be set for abstract DTypes, this is necessary + for array coercion.) + +* A new ``self.canonical`` property generalizes the notion of byte order to + indicate whether data has been stored in a default/canonical way. For + existing code, "canonical" will just signify native byte order, but it can + take on new meanings in new DTypes -- for instance, to distinguish a + complex-conjugated instance of Complex which stores ``real - imag`` instead + of ``real + imag`` and is thus not the canonical storage. The ISNBO ("is + native byte order") flag might be repurposed as the canonical flag. + +* Support is included for parametric DTypes. As explained in + :ref:`NEP 40 `, parametric types have a + value associated with them. A DType will be deemed parametric if it + inherits from ParametricDType. + + Strings are one example of a parametric type -- ``S8`` is different from + ``S4`` because ``S4`` cannot store a length 8 string such as ``"length 8"`` + while ``S8`` can. + Similarly, the ``datetime64`` DType is parametric, since its unit must be specified. + The associated ``type`` is the ``np.datetime64`` scalar. + +* DType methods may resemble or even reuse existing Python slots. Thus Python + special slots are off-limits for user-defined DTypes (for instance, defining + ``Unit("m") > Unit("cm")``), since we may want to develop a meaning for these + operators that is common to all DTypes. + +* Sorting functions are moved to the DType class. They may be implemented by + defining a method ``dtype_get_sort_function(self, sortkind="stable") -> + sortfunction`` that must return ``NotImplemented`` if the given ``sortkind`` + is not known. + +* Functions that cannot be removed are implemented as special methods. + Many of these were previously defined part of the :c:type:`PyArray_ArrFuncs` + slot of the dtype instance (``PyArray_Descr *``) and include functions + such as ``nonzero``, ``fill`` (used for ``np.arange``), and + ``fromstr`` (used to parse text files). + These old methods will be deprecated and replacements + following the new design principles added. + The API is not defined here. Since these methods can be deprecated and renamed + replacements added, it is acceptable if these new methods have to be modified. + +* Use of ``kind`` for non-built-in types is discouraged in favor of + ``isinstance`` checks. ``kind`` will return the ``__qualname__`` of the + object to ensure uniqueness for all DTypes. On the C side, ``kind`` and + ``char`` are set to ``\0`` (NULL character). + While ``kind`` will be discouraged, the current ``np.issubdtype`` + may remain the preferred method for this type of check. + +* A method ``ensure_canonical(self) -> dtype`` returns a new dtype (or + ``self``) with the ``canonical`` flag set. + +* Since NumPy's approach is to provide functionality through unfuncs, + functions like sorting that will be implemented in DTypes might eventually be + reimplemented as generalized ufuncs. + +.. _casting: + +****************************************************************************** +Casting +****************************************************************************** -**Example:** +We review here the operations related to casting arrays: -The current datetime DType requires a ``__discover_descr_from_pyobject__`` -which returns the correct unit for string inputs. This allows it to support -the current:: +- Finding the "common dtype," currently exposed by ``np.promote_types`` or + ``np.result_type`` - np.array(["2020-01-02", "2020-01-02 11:24"], dtype="M8") +- The result of calling ``np.can_cast`` -By inspecting the date strings. Together with the below common dtype -operation, this allows it to automatically find that the datetime64 unit -should be "minutes". +We show how casting arrays with ``arr.astype(new_dtype)`` will be implemented. +`Common DType` operations +============================================================================== -Common DType Operations -^^^^^^^^^^^^^^^^^^^^^^^ +Common-type operations are vital for array coercion when input types are +mixed. They determine the output dtype of ``np.concatenate()`` and are useful +in themselves. -NumPy currently provides functions like ``np.result_type`` and -``np.promote_types`` for determining common types. +NumPy provides ``np.result_type`` and +``np.promote_types``. These differ in that ``np.result_type`` can take arrays and scalars as input -and implements value based promotion [1]_. +and implements value-based promotion [1]_. -To distinguish between the promotion occurring during universal function application, -we will call it "common type" operation here. +To distinguish between the promotion occurring during universal function +application, we will call it "common type" operation here. **Motivation:** -Common type operations are vital for array coercion when different -input types are mixed. -They also provide the logic currently used to decide the output dtype of -``np.concatenate()`` and on their own are quite useful. Furthermore, common type operations may be used to find the correct dtype to use for functions with different inputs (including universal functions). @@ -684,32 +460,33 @@ This includes an interesting distinction: (Hypothetical example: ``float_arr + string_arr -> string``, but the output string length is not the same as ``np.concatenate(float_arr, string_arr)).dtype``.) -2. Array coercion and concatenation require the common dtype *instance*. -**Implementation:** -The implementation of the common dtype (instance) determination -has some overlap with casting. -Casting from a specific dtype (Float64) to a String needs to find -the correct string length (a step that is mainly necessary for parametric dtypes). +2. Array coercion and concatenation require the common dtype *instance*. + +**Implementation:** The implementation of the common dtype (instance) +determination has some overlap with casting. Casting from a specific dtype +(Float64) to a String needs to find the correct string length (a step that is +mainly necessary for parametric dtypes). We propose the following implementation: -1. ``__common_dtype__(cls, other : DTypeMeta) -> DTypeMeta`` answers what the common - DType class is given two DType class objects. - It may return ``NotImplemented`` to defer to ``other``. - (For abstract DTypes, subclasses get precedence, concrete types are always - leaves, so always get preference or are tried from left to right). +1. ``__common_dtype__(cls, other : DTypeMeta) -> DTypeMeta`` answers what the + common DType class is, given two DType class objects. It may return + ``NotImplemented`` to defer to ``other``. (For abstract DTypes, subclasses + get precedence, concrete types are never superclasses, so always get preference + or are tried from left to right). + 2. ``__common_instance__(self: SelfT, other : SelfT) -> SelfT`` is used when two instances of the same DType are given. - For builtin dtypes (that are not parametric), this - currently always returns ``self`` (but ensures native byte order). + For built-in dtypes (that are not parametric), this + currently always returns ``self`` (but ensures canonical representation). This is to preserve metadata. We can thus provide a default implementation for non-parametric user dtypes. These two cases do *not* cover the case where two different dtype instances -need to be promoted. For example `">float64"` and `"S8"`. -The solution is partially "outsourced" to the casting machinery by -splitting the operation up into three steps: +need to be promoted. For example `">float64"` and `"S8"`. The solution is +partially "outsourced" to the casting machinery by splitting the operation up +into three steps: 1. ``Float64.__common_dtype__(type(>float64), type(S8))`` returns `String` (or defers to ``String.__common_dtype__``). @@ -717,15 +494,13 @@ splitting the operation up into three steps: to `"S32"` (see below for how casting will be defined). 3. ``String.__common_instance__("S8", "S32")`` returns the final `"S32"`. -The main reason for this is to avoid the need to implement -identical functionality multiple times. -The design (together with casting) naturally separates the concerns of -different Datatypes. -In the above example, Float64 does not need to know about the cast. -While the casting machinery (``CastingImpl[Float64, String]``) -could include the third step, it is not required to do so and the string -can always be extended (e.g. with new encodings) without extending the -``CastingImpl[Float64, String]``. +The main reason for this is to avoid the need to implement identical +functionality multiple times. The design (together with casting) naturally +separates the concerns of different Datatypes. In the above example, Float64 +does not need to know about the cast. While the casting machinery +(``CastingImpl[Float64, String]``) could include the third step, it is not +required to do so and the string can always be extended (e.g. with new +encodings) without extending the ``CastingImpl[Float64, String]``. This means the implementation will work like this:: @@ -744,7 +519,7 @@ This means the implementation will work like this:: # Find what dtype1 is cast to when cast to the common DType # by using the CastingImpl as described below: castingimpl = get_castingimpl(type(dtype1), common) - safety, (_, dtype1) = castingimpl.adjust_descriptors((dtype1, None)) + safety, (_, dtype1) = castingimpl.resolve_descriptors((dtype1, None)) assert safety == "safe" # promotion should normally be a safe cast if type(dtype2) is not common: @@ -755,12 +530,10 @@ This means the implementation will work like this:: Some of these steps may be optimized for non-parametric DTypes. -**Note:** - -A currently implemented fallback for the ``__common_dtype__`` operation -is to use the "safe" casting logic. -Since ``int16`` can safely cast to ``int64``, it is clear that -``np.promote_types(int16, int64)`` should be ``int64``. +**Note:** A currently implemented fallback for the ``__common_dtype__`` +operation is to use the "safe" casting logic. Since ``int16`` can safely cast +to ``int64``, it is clear that ``np.promote_types(int16, int64)`` should be +``int64``. However, this cannot define all such operations, and will fail for example for:: @@ -771,39 +544,35 @@ in most cases a safe-cast implies that this will be the result of the ``__common_dtype__`` method. Note that some exceptions may apply. For example casting ``int32`` to -a (long enough) string is – at least at this time – considered "safe". +a (long enough) string is at least at this time considered "safe". However ``np.promote_types(int32, String)`` will *not* be defined. -**Alternatives:** - -The use of casting for common dtype (instance) determination neatly separates -the concerns and allows for a minimal set of duplicate functionality -being implemented. -In cases of mixed DType (classes), it also adds an additional step -to finding the common dtype. -The common dtype (of two instances) could thus be implemented explicitly to avoid -this indirection, potentially only as a fast-path. -The above suggestion assumes that this is, however, not a speed relevant path, -since in most cases, e.g. in array coercion, only a single Python type (and thus -dtype) is involved. -The proposed design hinges in the implementation of casting to be -separated into its own ufunc-like object as described below. +**Alternatives:** The use of casting for common dtype (instance) determination +neatly separates the concerns and allows for a minimal set of duplicate +functionality being implemented. In cases of mixed DType (classes), it also +adds an additional step to finding the common dtype. The common dtype (of two +instances) could thus be implemented explicitly to avoid this indirection, +potentially only as a fast-path. The above suggestion assumes that this is, +however, not a speed relevant path, since in most cases, e.g. in array +coercion, only a single Python type (and thus dtype) is involved. The proposed +design hinges in the implementation of casting to be separated into its own +ufunc-like object as described below. In principle common DType could be defined only based on "safe casting" rules, if we order all DTypes and find the first one both can cast to safely. However, the issue with this approach is that a newly added DType can change the behaviour of an existing program. For example, a new ``int24`` would be -the first valid common type for ``int16`` and ``uint16``, demoting the currently -defined behaviour of ``int32``. -This API extension could be allowed in the future, while adding it may be -more involved, the current proposal for defining casts is fully opaque in -this regard and thus extensible. - -**Example:** - -``object`` always chooses ``object`` as the common DType. For ``datetime64`` -type promotion is defined with no other datatype, but if someone were to -implement a new higher precision datetime, then:: +the first valid common type for ``int16`` and ``uint16``, demoting the +currently defined behavior of ``int32``. +Both, the need of a linear type hierarchy and the potential of changing +existing behaviour by adding a new DType, are a downside to using a generic +rule based on "safe casting". +However, a more generic common DType could be implemented in the future, since +``__common_dtype__`` can in principle use casting information internally. + +**Example:** ``object`` always chooses ``object`` as the common DType. For +``datetime64`` type promotion is defined with no other datatype, but if +someone were to implement a new higher precision datetime, then:: HighPrecisionDatetime.__common_dtype__(np.dtype[np.datetime64]) @@ -811,33 +580,31 @@ would return ``HighPrecisionDatetime``, and the below casting may need to decide how to handle the datetime unit. -Casting -^^^^^^^ +The cast operation +============================================================================== -Maybe the most complex and interesting operation which is provided -by DTypes is the ability to cast from one dtype to another. -The casting operation is much like a typical function (universal function) on -arrays converting one input to a new output. -There are mainly two distinctions: +Perhaps the most complex and interesting DType operation is casting. Casting +is much like a typical universal function on arrays, converting one input to a +new output. There are two key distinctions: -1. Casting always requires an explicit output datatype to be given. -2. The NumPy iterator API requires access to lower-level functions than - is currently necessary for universal functions. +1. Casting always requires an explicit output datatype. +2. The NumPy iterator API requires access to functions that are lower-level + than what universal functions currently need. -Casting from one dtype to another can be complex, and generally a casting -function may not implement all details of each input datatype (such as -non-native byte order or unaligned access). -Thus casting naturally is performed in up to three steps: +Casting can be complex, and may not implement all details of each input +datatype (such as non-native byte order or unaligned access). Thus casting +naturally is performed in up to three steps: -1. The input datatype is normalized and prepared for the actual cast. +1. The given datatype is normalized and prepared for the actual cast. 2. The cast is performed. 3. The cast result, which is in a normalized form, is cast to the requested form (non-native byte order). -although often only step 2. is required. +Often only step 2 is required. Further, NumPy provides different casting kinds or safety specifiers: +* "equivalent" * "safe" * "same_kind" * "unsafe" @@ -845,66 +612,63 @@ Further, NumPy provides different casting kinds or safety specifiers: and in some cases a cast may even be represented as a simple view. -**Motivation:** - -Similar to the common dtype/DType operation above, we again have two use cases: +**Motivation:** Similar to the common dtype/DType operation above, we again +have two use cases: 1. ``arr.astype(np.String)`` (current spelling ``arr.astype("S")``) -2. ``arr.astype(np.dtype("S8"))``. +2. ``arr.astype(np.dtype("S8"))`` -Where the first case is also noted in NEP 40 and 41 as a design goal, since +where the first case is also noted in NEP 40 and 41 as a design goal, since ``np.String`` could also be an abstract DType as mentioned above. The implementation of casting should also come with as little duplicate -implementation as necessary, i.e. to avoid unnecessary methods on the -DTypes. +implementation as necessary, i.e. to avoid unnecessary methods on the DTypes. Furthermore, it is desirable that casting is implemented similar to universal functions. Analogous to the above, the following also need to be defined: 1. ``np.can_cast(dtype, DType, "safe")`` (instance to class) -2. ``np.can_cast(dtype, other_dtype, "safe")`` (casting an instance to another instance) +2. ``np.can_cast(dtype, other_dtype, "safe")`` (casting an instance to another + instance) -overloading the meaning of ``dtype`` to mean either class or instance -(on the Python level). -The question of ``np.can_cast(DType, OtherDType, "safe")`` is also a -possibility and may be used internally. -However, it is initially not necessary to expose to Python. +overloading the meaning of ``dtype`` to mean either class or instance (on the +Python level). The question of ``np.can_cast(DType, OtherDType, "safe")`` is +also a possibility and may be used internally. However, it is initially not +necessary to expose to Python. -**Implementation:** - -During DType creation, DTypes will have the ability to pass a list of -``CastingImpl`` objects, which can define casting to and from the DType. -One of these ``CastingImpl`` objects is special because it should define -the cast within the same DType (from one instance to another). -A DType which does not define this, must have only a single implementation -and not be parametric. +**Implementation:** During DType creation, DTypes will have the ability to +pass a list of ``CastingImpl`` objects, which can define casting to and from +the DType. One of these ``CastingImpl`` objects is special because it should +define the cast within the same DType (from one instance to another). A DType +which does not define this, must have only a single implementation and not be +parametric. Each ``CastingImpl`` has a specific DType signature: -``CastingImpl[InputDtype, RequestedDtype]``. -And implements the following methods and attributes: +``CastingImpl[InputDtype, RequestedDtype]`` +and implements the following methods and attributes: -* ``adjust_descriptors(self, Tuple[DType] : input) -> casting, Tuple[DType]``. +* ``resolve_descriptors(self, Tuple[DType] : input) -> casting, Tuple[DType]``. Here ``casting`` signals the casting safeness (safe, unsafe, or same-kind) and the output dtype tuple is used for more multi-step casting (see below). * ``get_transferfunction(...) -> function handling cast`` (signature to be decided). This function returns a low-level implementation of a strided casting function ("transfer function"). -* ``cast_kind`` attribute with one of safe, unsafe, or same-kind. Used to +* ``casting`` attribute with one of equivalent, safe, unsafe, or same-kind. Used to quickly decide casting safety when this is relevant. -``adjust_descriptors`` provides information about whether or +``resolve_descriptors`` provides information about whether or not a cast is safe and is of importance mainly for parametric DTypes. ``get_transferfunction`` provides NumPy with a function capable of performing the actual cast. Initially the implementation of ``get_transferfunction`` -will be *private*, and users will only be able to provide contiguous loops +will be *private*, and users will only be able to provide strided loops with the signature. -**Performing the Cast:** +**Performing the cast** .. _cast_figure: + .. figure:: _static/casting_flow.svg :figclass: align-center @@ -916,141 +680,116 @@ an ``int24`` to an ``S8`` string (which can hold all 24bit integers). Due to this limited implementation, the full cast has to do multiple conversions. The full process is: -1. Call ``CastingImpl[Int24, String].adjust_descriptors((int24, "S20"))``. +1. Call ``CastingImpl[Int24, String].resolve_descriptors((int24, "S20"))``. This provides the information that ``CastingImpl[Int24, String]`` only - implements the cast of ``int24`` to ``"S8``. + implements the cast of ``int24`` to ``"S8"``. 2. Since ``"S8"`` does not match ``"S20"``, use ``CastingImpl[String, String].get_transferfunction()`` to find the transfer (casting) function to convert an ``"S8"`` into an ``"S20"`` 3. Fetch the transfer function to convert an ``int24`` to an ``"S8"`` using ``CastingImpl[Int24, String].get_transferfunction()`` 4. Perform the actual cast using the two transfer functions: - ``int24(42) -> S8("42") -> S20("42")``. + ``int24(42) -> S8("42") -> S20("42")``. -Note that in this example the ``adjust_descriptors`` function plays a less +Note that in this example the ``resolve_descriptors`` function plays a less central role. It becomes more important for ``np.can_cast``. -Further, ``adjust_descriptors`` allows the implementation for +Further, ``resolve_descriptors`` allows the implementation for ``np.array(42, dtype=int24).astype(String)`` to call -``CastingImpl[Int24, String].adjust_descriptors((int24, None))``. +``CastingImpl[Int24, String].resolve_descriptors((int24, None))``. In this case the result of ``(int24, "S8")`` defines the correct cast: ``np.array(42, dtype=int24),astype(String) == np.array("42", dtype="S8")``. -**Casting Safety:** +**Casting safety** -To answer the question of casting safety -``np.can_cast(int24, "S20", casting="safe")``, only the ``adjust_descriptors`` -function is required and called is in the same way as in -`the figure describing a cast `_. -In this case, the calls to ``adjust_descriptors``, will also provide the -information that ``int24 -> "S8"`` as well as ``"S8" -> "S20"`` are safe casts, -and thus also the ``int24 -> "S20"`` is a safe cast. +To answer the question of casting safety ``np.can_cast(int24, "S20", +casting="safe")``, only the ``resolve_descriptors`` function is required and +is called in the same way as in `the figure describing a cast `_. +In this case, the calls to ``resolve_descriptors``, will also provide the +information that ``int24 -> "S8"`` as well as ``"S8" -> "S20"`` are safe +casts, and thus also the ``int24 -> "S20"`` is a safe cast. -The casting safety can currently be "equivalent" when a cast is both safe -and can be performed using only a view. -The information that a cast is a simple "view" will instead be handled by -an additional flag. Thus the ``casting`` can have the 6 values in total: -safe, unsafe, same-kind as well as safe+view, unsafe+view, same-kind+view. -Where the current "equivalent" is the same as safe+view. +In some cases, no cast is necessary. For example, on most Linux systems +``np.dtype("long")`` and ``np.dtype("longlong")`` are different dtypes but are +both 64bit integers. +In this case, the cast can be performed using ``long_arr.view("longlong")``. +The information that a cast is a +"view" will be handled by an additional flag. Thus the ``casting`` +can have the 8 values in total: equivalent, safe, unsafe, same-kind as well as equivalent+view, safe+view, +unsafe+view, and same-kind+view. +NumPy currently defines ``dtype1 == dtype2`` to be True only if byte order matches. +This functionality can be replaced with the combination of "equivalent" casting +and the "view" flag. -(For more information on the ``adjust_descriptor`` signature see the -C-API section below.) +(For more information on the ``resolve_descriptors`` signature see the C-API +section below and NEP 43.) -**Casting between instances of the same DType:** +**Casting between instances of the same DType** -In general one of the casting implementations define by the DType implementor +In general one of the casting implementations defined by the DType implementor must be ``CastingImpl[DType, DType]`` (unless there is only a singleton -instance). -To keep the casting to as few steps as possible, this implementation must -be capable any conversions between all instances of this DType. +instance). To keep the casting to as few steps as possible, this +implementation must initially be capable of any conversions between all instances of this +DType. -**General Multi-Step Casting** +**General multistep casting** In general we could implement certain casts, such as ``int8`` to ``int24`` -even if the user only provides an ``int16 -> int24`` cast. -This proposal currently does not provide this functionality. However, -it could be extended in the future to either find such casts dynamically, -or at least allow ``adjust_descriptors`` to return arbitrary ``dtypes``. -If ``CastingImpl[Int8, Int24].adjust_descriptors((int8, int24))`` returns -``(int16, int24)``, the actual casting process could be extended to include -the ``int8 -> int16`` cast. Unlike the above example, which is limited -to at most three steps. - - -**Alternatives:** - -The choice of using only the DType classes in the first step of finding the -correct ``CastingImpl`` means that the default implementation of -``__common_dtype__`` has a reasonable definition of "safe casting" between +even if the user only provides an ``int16 -> int24`` cast. This proposal +currently does not provide this functionality. However, it could be extended +in the future to either find such casts dynamically, or at least allow +``resolve_descriptors`` to return arbitrary ``dtypes``. If ``CastingImpl[Int8, +Int24].resolve_descriptors((int8, int24))`` returns ``(int16, int24)``, the +actual casting process could be extended to include the ``int8 -> int16`` +cast. This adds an additional step to the casting process. + + +**Alternatives:** The choice of using only the DType classes in the first step +of finding the correct ``CastingImpl`` means that the default implementation +of ``__common_dtype__`` has a reasonable definition of "safe casting" between DTypes classes (although e.g. the concatenate operation using it may still fail when attempting to find the actual common instance or cast). -The split into multiple steps may seem to add complexity -rather than reduce it, however, it consolidates that we have the two distinct -signatures of ``np.can_cast(dtype, DTypeClass)`` and ``np.can_cast(dtype, other_dtype)``. +The split into multiple steps may seem to add complexity rather than reduce +it, however, it consolidates that we have the two distinct signatures of +``np.can_cast(dtype, DTypeClass)`` and ``np.can_cast(dtype, other_dtype)``. Further, the above API guarantees the separation of concerns for user DTypes. -The user ``Int24`` dtype does not have to handle all string lengths if it -does not wish to do so. Further, if an encoding was added to the ``String`` -DType, this does not affect the overall cast. -The ``adjust_descriptor`` function can keep returning the default encoding -and the ``CastingImpl[String, String]`` can take care of any necessary encoding -changes. +The user ``Int24`` dtype does not have to handle all string lengths if it does +not wish to do so. Further, if an encoding was added to the ``String`` DType, +this does not affect the overall cast. The ``resolve_descriptors`` function can +keep returning the default encoding and the ``CastingImpl[String, String]`` +can take care of any necessary encoding changes. The main alternative to the proposed design is to move most of the information -which is here pushed into the ``CastingImpl`` directly into methods -on the DTypes. This, however, will not allow the close similarity between casting -and universal functions. On the up side, it reduces the necessary indirection -as noted below. - -An initial proposal defined two methods ``__can_cast_to__(self, other)`` -to dynamically return ``CastingImpl``. -The advantage of this addition is that it removes the requirement to know all -possible casts at DType creation time (of one of the involved DTypes). -Such API could be added at a later time. It should be noted, however, -that it would be mainly useful for inheritance-like logic, which can be -problematic. As an example two different ``Float64WithUnit`` implementations -both could infer that they can unsafely cast between one another when in fact -some combinations should cast safely or preserve the Unit (both of which the -"base" ``Float64`` would discard). -In the proposed implementation this is not possible, since the two implementations -are not aware of each other. - - -**Notes:** - -The proposed ``CastingImpl`` is designed to be compatible with the -``UFuncImpl`` proposed in NEP 43. -While initially it will be a distinct object or C-struct, the aim is that -``CastingImpl`` can be a subclass or extension of ``UFuncImpl``. -Once this happens, this may naturally allow the use of a ``CastingImpl`` to -pass around a specialized casting function directly. - -In the future, we may consider adding a way to spell out that specific -casts are known to be *not* possible. - -In the above text ``CastingImpl`` is described as a Python object. In practice, -the current plan is to implement it as a C-side structure stored on the ``from`` -datatype. -A Python side API to get an equivalent ``CastingImpl`` object will be created, -but storing it (similar to the current implementation) on the ``from`` datatype -avoids the creation of cyclic reference counts. - -The way dispatching works for ``CastingImpl`` is planned to be limited initially -and fully opaque. -In the future, it may or may not be moved into a special UFunc, or behave -more like a universal function. - - -**Example:** - -The implementation for casting integers to datetime would currently generally -say that this cast is unsafe (it is always an unsafe cast). -Its ``adjust_descriptors`` functions may look like:: - - def adjust_descriptors(input): - from_dtype, to_dtype = input +which is here pushed into the ``CastingImpl`` directly into methods on the +DTypes. This, however, will not allow the close similarity between casting and +universal functions. On the up side, it reduces the necessary indirection as +noted below. + +An initial proposal defined two methods ``__can_cast_to__(self, other)`` to +dynamically return ``CastingImpl``. The advantage of this addition is that it +removes the requirement to define all possible casts at DType creation time (of +one of the involved DTypes). +Such API could be added at a later time. This is similar to Python which +provides ``__getattr__`` for additional control over attribute lookup. + +**Notes:** The proposed ``CastingImpl`` is designed to be identical to the +``PyArrayMethod`` proposed in NEP 43 as part of restructuring ufuncs to handle +new DTypes. + +The way dispatching works for ``CastingImpl`` is planned to be limited +initially and fully opaque. In the future, it may or may not be moved into a +special UFunc, or behave more like a universal function. + + +**Example:** The implementation for casting integers to datetime would generally +say that this cast is unsafe (because it is always an unsafe cast). +Its ``resolve_descriptors`` function may look like:: + + def resolve_descriptors(self, given_dtypes): + from_dtype, to_dtype = given_dtypes from_dtype = from_dtype.ensure_canonical() # ensure not byte-swapped if to_dtype is None: @@ -1065,26 +804,384 @@ Its ``adjust_descriptors`` functions may look like:: .. note:: - While NumPy currently defines some of these casts, with the possible - exception of the unit-less ``timedelta64`` it may be better to not - define these cast at all. In general we expect that user defined - DTypes will be using other methods such as ``unit.drop_unit(arr)`` - or ``arr * unit.seconds``. + While NumPy currently defines integer to datetime casts, with the possible + exception of the unit-less ``timedelta64`` it may be better to not define + these casts at all. In general we expect that user defined DTypes will be + using custom methods such as ``unit.drop_unit(arr)`` or ``arr * + unit.seconds``. + + +****************************************************************************** +Array coercion +****************************************************************************** + +The following sections discuss the two aspects related to creating an array from +arbitrary python objects. This requires a defined protocol to store data +inside the array. Further, it requires the ability to find the correct dtype +when a user does not provide the dtype explicitly. + +Coercion to and from Python objects +============================================================================== + +**Motivation:** When storing a single value in an array or taking it out, it +is necessary to coerce (convert) it to and from the low-level representation +inside the array. + +**Description:** Coercing to and from Python scalars requires two to three +methods: + +1. ``__dtype_setitem__(self, item_pointer, value)`` +2. ``__dtype_getitem__(self, item_pointer, base_obj) -> object``; + ``base_obj`` is for memory management and usually ignored; it points to + an object owning the data. Its only role is to support structured datatypes + with subarrays within NumPy, which currently return views into the array. + The function returns an equivalent Python scalar (i.e. typically a NumPy + scalar). +3. ``__dtype_get_pyitem__(self, item_pointer, base_obj) -> object`` (initially + hidden for new-style user-defined datatypes, may be exposed on user + request). This corresponds to the ``arr.item()`` method also used by + ``arr.tolist()`` and returns Python floats, for example, instead of NumPy + floats. + +(The above is meant for C-API. A Python-side API would have to use byte +buffers or similar to implement this, which may be useful for prototyping.) + +These largely correspond to the current definitions. When a certain scalar +has a known (different) dtype, NumPy may in the future use casting instead of +``__dtype_setitem__``. A user datatype is (initially) expected to implement +``__dtype_setitem__`` for its own ``DType.type`` and all basic Python scalars +it wishes to support (e.g. ``int`` and ``float``). In the future a +function "``known_scalartype``" may be made public to allow a user dtype to signal +which Python scalars it can store directly. + + +**Implementation:** The pseudocode implementation for setting a single item in +an array from an arbitrary Python object ``value`` is (note that some +functions are only defined below):: + + def PyArray_Pack(dtype, item_pointer, value): + DType = type(dtype) + if DType.type is type(value) or DType.known_scalartype(type(value)): + return dtype.__dtype_setitem__(item_pointer, value) + + # The dtype cannot handle the value, so try casting: + arr = np.array(value) + if arr.dtype is object or arr.ndim != 0: + # not a numpy or user scalar; try using the dtype after all: + return dtype.__dtype_setitem__(item_pointer, value) + + arr.astype(dtype) + item_pointer.write(arr[()]) + +where the call to ``np.array()`` represents the dtype discovery and is +not actually performed. + +**Example:** Current ``datetime64`` returns ``np.datetime64`` scalars and can +be assigned from ``np.datetime64``. However, the datetime +``__dtype_setitem__`` also allows assignment from date strings ("2016-05-01") +or Python integers. Additionally the datetime ``__dtype_get_pyitem__`` +function actually returns a Python ``datetime.datetime`` object (most of the +time). + + +**Alternatives:** This functionality could also be implemented as a cast to and +from the ``object`` dtype. +However, coercion is slightly more complex than typical casts. +One reason is that in general a Python object could itself be a +zero-dimensional array or scalar with an associated DType. +Such an object has a DType, and the correct cast to another DType is already +defined:: + + np.array(np.float32(4), dtype=object).astype(np.float64) + +is identical to:: + + np.array(4, dtype=np.float32).astype(np.float64) + +Implementing the first ``object`` to ``np.float64`` cast explicitly, +would require the user to take to duplicate or fall back to existing +casting functionality. + +It is certainly possible to describe the coercion to and from Python objects +using the general casting machinery, +but the ``object`` dtype is special and important enough to be handled by NumPy +using the presented methods. + +**Further Issues and Discussion:** The ``__dtype_setitem__`` function currently duplicates +some code, such as coercion from a string. ``datetime64`` allows assignment +from string, but the same conversion also occurs for casting from the string +dtype to ``datetime64``. In the future, we may expose the ``known_scalartype`` +function to allow the user to implement such duplication. +For example, NumPy would normally use ``np.array(np.string_("2019")).astype(datetime64)``, +but ``datetime64`` could choose to use its ``__dtype_setitem__`` instead, +e.g. for performance reasons. + +There is an issue about how subclasses of scalars should be handled. +We anticipate to stop automatically detecting the dtype for +``np.array(float64_subclass)`` to be float64. +The user can still provide ``dtype=np.float64``. +However, the above automatic casting using ``np.array(scalar_subclass).astype(requested_dtype)`` +will fail. +In many cases, this is not an issue, since the Python ``__float__`` protocol +can be used instead. But in some cases, this will mean that subclasses of +Python scalars will behave differently. + +.. note:: + + *Example:* ``np.complex256`` should not use ``__float__`` in its + ``__dtype_setitem__`` method in the future unless it is a known floating + point type. If the scalar is a subclass of a different high precision + floating point type (e.g. ``np.float128``) then this currently loses + precision without notifying the user. + In that case ``np.array(float128_subclass(3), dtype=np.complex256)`` + may fail unless the ``float128_subclass`` is first converted to the + ``np.float128`` base class. + + +DType discovery during array coercion +============================================================================== + +An important step in the use of NumPy arrays is creation of the array +from collections of generic Python objects. + +**Motivation:** Although the distinction is not clear currently, there are two main needs:: + + np.array([1, 2, 3, 4.]) + +needs to guess the correct dtype based on the Python objects inside. +Such an array may include a mix of datatypes, as long as they can be +promoted. +A second use case is when users provide the output DType class, but not the +specific DType instance:: + + np.array([object(), None], dtype=np.dtype[np.string_]) # (or `dtype="S"`) + +In this case the user indicates that ``object()`` and ``None`` should be +interpreted as strings. +The need to consider the user provided DType also arises for a future +``Categorical``:: + + np.array([1, 2, 1, 1, 2], dtype=Categorical) + +which must interpret the numbers as unique categorical values rather than +integers. + +There are three further issues to consider: + +1. It may be desirable to create datatypes associated + with normal Python scalars (such as ``datetime.datetime``) that do not + have a ``dtype`` attribute already. +2. In general, a datatype could represent a sequence, however, NumPy currently + assumes that sequences are always collections of elements + (the sequence cannot be an element itself). + An example would be a ``vector`` DType. +3. An array may itself contain arrays with a specific dtype (even + general Python objects). For example: + ``np.array([np.array(None, dtype=object)], dtype=np.String)`` + poses the issue of how to handle the included array. + +Some of these difficulties arise because finding the correct shape +of the output array and finding the correct datatype are closely related. + +**Implementation:** There are two distinct cases above: + +1. The user has provided no dtype information. +2. The user provided a DType class -- as represented, for example, by ``"S"`` + representing a string of any length. + +In the first case, it is necessary to establish a mapping from the Python type(s) +of the constituent elements to the DType class. +Once the DType class is known, the correct dtype instance needs to be found. +In the case of strings, this requires to find the string length. + +These two cases shall be implemented by leveraging two pieces of information: + +1. ``DType.type``: The current type attribute to indicate which Python scalar + type is associated with the DType class (this is a *class* attribute that always + exists for any datatype and is not limited to array coercion). +2. ``__discover_descr_from_pyobject__(cls, obj) -> dtype``: A classmethod that + returns the correct descriptor given the input object. + Note that only parametric DTypes have to implement this. + For non-parametric DTypes using the default instance will always be acceptable. + +The Python scalar type which is already associated with a DType through the +``DType.type`` attribute maps from the DType to the Python scalar type. +At registration time, a DType may choose to allow automatically discover for +this Python scalar type. +This requires a lookup in the opposite direction, which will be implemented +using global a mapping (dictionary-like) of:: + + known_python_types[type] = DType + +Correct garbage collection requires additional care. +If both the Python scalar type (``pytype``) and ``DType`` are created dynamically, +they will potentially be deleted again. +To allow this, it must be possible to make the above mapping weak. +This requires that the ``pytype`` holds a reference of ``DType`` explicitly. +Thus, in addition to building the global mapping, NumPy will store the ``DType`` as +``pytype.__associated_array_dtype__`` in the Python type. +This does *not* define the mapping and should *not* be accessed directly. +In particular potential inheritance of the attribute does not mean that NumPy will use the +superclasses ``DType`` automatically. A new ``DType`` must be created for the +subclass. + +.. note:: + + Python integers do not have a clear/concrete NumPy type associated right + now. This is because during array coercion NumPy currently finds the first + type capable of representing their value in the list of `long`, `unsigned + long`, `int64`, `unsigned int64`, and `object` (on many machines `long` is + 64 bit). + + Instead they will need to be implemented using an ``AbstractPyInt``. This + DType class can then provide ``__discover_descr_from_pyobject__`` and + return the actual dtype which is e.g. ``np.dtype("int64")``. For + dispatching/promotion in ufuncs, it will also be necessary to dynamically + create ``AbstractPyInt[value]`` classes (creation can be cached), so that + they can provide the current value based promotion functionality provided + by ``np.result_type(python_integer, array)`` [1]_. + +To allow for a DType to accept inputs as scalars that are not basic Python +types or instances of ``DType.type``, we use ``known_scalar_type`` method. +This can allow discovery of a ``vector`` as a scalar (element) instead of a sequence +(for the command ``np.array(vector, dtype=VectorDType)``) even when ``vector`` is itself a +sequence or even an array subclass. This will *not* be public API initially, +but may be made public at a later time. + +**Example:** The current datetime DType requires a +``__discover_descr_from_pyobject__`` which returns the correct unit for string +inputs. This allows it to support:: + + np.array(["2020-01-02", "2020-01-02 11:24"], dtype="M8") + +By inspecting the date strings. Together with the common dtype +operation, this allows it to automatically find that the datetime64 unit +should be "minutes". + + +**NumPy Internal Implementation:** The implementation to find the correct dtype +will work similar to the following pseudocode:: + + def find_dtype(array_like): + common_dtype = None + for element in array_like: + # default to object dtype, if unknown + DType = known_python_types.get(type(element), np.dtype[object]) + dtype = DType.__discover_descr_from_pyobject__(element) + + if common_dtype is None: + common_dtype = dtype + else: + common_dtype = np.promote_types(common_dtype, dtype) + +In practice, the input to ``np.array()`` is a mix of sequences and array-like +objects, so that deciding what is an element requires to check whether it +is a sequence. +The full algorithm (without user provided dtypes) thus looks more like:: + + def find_dtype_recursive(array_like, dtype=None): + """ + Recursively find the dtype for a nested sequences (arrays are not + supported here). + """ + DType = known_python_types.get(type(element), None) + + if DType is None and is_array_like(array_like): + # Code for a sequence, an array_like may have a DType we + # can use directly: + for element in array_like: + dtype = find_dtype_recursive(element, dtype=dtype) + return dtype + + elif DType is None: + DType = np.dtype[object] + + # dtype discovery and promotion as in `find_dtype` above + +If the user provides ``DType``, then this DType will be tried first, and the +``dtype`` may need to be cast before the promotion is performed. + +**Limitations:** The motivational point 3. of a nested array +``np.array([np.array(None, dtype=object)], dtype=np.String)`` is currently +(sometimes) supported by inspecting all elements of the nested array. +User DTypes will implicitly handle these correctly if the nested array +is of ``object`` dtype. +In some other cases NumPy will retain backward compatibility for existing +functionality only. +NumPy uses such functionality to allow code such as:: + + >>> np.array([np.array(["2020-05-05"], dtype="S")], dtype=np.datetime64) + array([['2020-05-05']], dtype='datetime64[D]') + +which discovers the datetime unit ``D`` (days). +This possibility will not be accessible to user DTypes without an +intermediate cast to ``object`` or a custom function. + +The use of a global type map means that an error or warning has to be given if +two DTypes wish to map to the same Python type. In most cases user DTypes +should only be implemented for types defined within the same library to avoid +the potential for conflicts. It will be the DType implementor's responsibility +to be careful about this and use avoid registration when in doubt. + +**Alternatives:** Instead of a global mapping, we could rely on the scalar +attribute ``scalar.__associated_array_dtype__``. +This only creates a difference in behaviour for subclasses and the exact +implementation can be undefined initially. +Scalars will be expected to derive from a NumPy scalar. +In principle NumPy could, for a time, still choose to rely on the attribute. + +An earlier proposal for the ``dtype`` discovery algorithm, +was to use a two-pass approach. +First finding only the correct ``DType`` class and only then discovering the parametric +``dtype`` instance. +This was rejected for unnecessary complexity. +The main advantage of this method is that it would have enabled value +based promotion in universal functions, allowing:: + + np.add(np.array([8], dtype="uint8"), [4]) + +to return a ``uint8`` result (instead of ``int16``), which currently happens for:: + + np.add(np.array([8], dtype="uint8"), 4) +(note the list ``[4]`` instead of scalar ``4``). +This is not a feature NumPy currently has or desires to support. -C-Side API -^^^^^^^^^^ +**Further Issues and Discussion:** It is possible to create a DType +such as Categorical, array, or vector which can only be used if ``dtype=DType`` +is provided. Such DTypes cannot roundtrip correctly. For example:: + + np.array(np.array(1, dtype=Categorical)[()]) + +will result in an integer array. To get the original ``Categorical`` array +``dtype=Categorical`` will need to be passed explicitly. +This is a general limitation, but round-tripping is always possible if +``dtype=original_arr.dtype`` is passed. + + +.. _c-api: + +****************************************************************************** +Public C-API +****************************************************************************** A Python side API shall not be defined here. This is a general side approach. DType creation -"""""""""""""" +============================================================================== + +To create a new DType the user will need to define all the methods and +attributes as presented above and outlined in the `Usage and impact`_ +section. +Some additional methods similar to those currently defined as part of +:c:type:`PyArray_ArrFuncs` will be necessary and part of the slots struct +below. -As already mentioned in NEP 41, the interface to define new DTypes in C -is modeled after the limited API in Python: the above-mentioned slots -and some additional necessary information will thus be passed within a slots -struct and identified by ``ssize_t`` integers:: +As already mentioned in NEP 41, the interface to define this DType class in C is +modeled after the `Python limited API `_: +the above-mentioned slots and some additional necessary information will +thus be passed within a slots struct and identified by ``ssize_t`` integers:: static struct PyArrayMethodDef slots[] = { {NPY_dt_method, method_implementation}, @@ -1093,10 +1190,8 @@ struct and identified by ``ssize_t`` integers:: } typedef struct{ - PyTypeObject *typeobj; /* type of python scalar */ - int is_parametric; /* Is the dtype parametric? */ - int is_abstract; /* Is the dtype abstract? */ - int flags /* flags (to be discussed) */ + PyTypeObject *typeobj; /* type of python scalar or NULL */ + int flags /* flags, including parametric and abstract */ /* NULL terminated CastingImpl; is copied and references are stolen */ CastingImpl *castingimpls[]; PyType_Slot *slots; @@ -1105,20 +1200,19 @@ struct and identified by ``ssize_t`` integers:: PyObject* PyArray_InitDTypeMetaFromSpec(PyArrayDTypeMeta_Spec *dtype_spec); -all of this information will be copied during instantiation. +All of this information will be copied. -**TODO:** The DType author should be able to at define new methods for -their DType, up to defining a full type object and in the future possibly even -extending the ``PyArrayDTypeMeta_Type`` struct. -We have to decide on how (and what) to make available to the user initially. -A proposed initial solution may be to simply allow inheriting from an existing -class. -Further this prevents overriding some slots, such as `==` which may not be -desirable. +**TODO:** The DType author should be able to define new methods for their +DType, up to defining a full type object and in the future possibly even +extending the ``PyArrayDTypeMeta_Type`` struct. We have to decide on how (and +what) to make available to the user initially. A possible initial solution may +be to only allow inheriting from an existing class: ``class MyDType(np.dtype, +MyBaseclass)``. If ``np.dtype`` is first in the method resolution order, this +also prevents overriding some slots, such as ``==`` which may not be desirable. -The proposed method slots are (prepended with ``NPY_dt_``), these are -detailed above and given here for summary: +The ``slots`` will be identified by names which are prefixed with ``NPY_dt_`` +and are: * ``is_canonical(self) -> {0, 1}`` * ``ensure_canonical(self) -> dtype`` @@ -1129,205 +1223,142 @@ detailed above and given here for summary: * ``common_dtype(cls, other) -> DType, NotImplemented, or NULL`` * ``common_instance(self, other) -> dtype or NULL`` -If not set, most slots are filled with slots which either error or defer automatically. +Where possible, a default implementation will be provided if the slot is +ommitted or set to ``NULL``. Non-parametric dtypes do not have to implement: * ``discover_descr_from_pyobject`` (uses ``default_descr`` instead) * ``common_instance`` (uses ``default_descr`` instead) -* ``ensure_canonical`` (uses ``default_descr`` instead) - -Which will be correct for most dtypes *which do not store metadata*. - -Other slots may be replaced by convenience versions, e.g. sorting methods -can be defined by providing: +* ``ensure_canonical`` (uses ``default_descr`` instead). -* ``compare(self, char *item_ptr1, char *item_ptr2, int *res) -> {-1, 0}`` - *TODO: We would like an error return, is this reasonable? (similar to old - python compare)* - -which uses generic sorting functionality. In general, we could add a -functions such as: +Sorting is expected to be implemented using: * ``get_sort_function(self, NPY_SORTKIND sort_kind) -> {out_sortfunction, NotImplemented, NULL}``. - If the sortkind is not understood it may be allowed to return ``NotImplemented``. -in the future. However, for example sorting is likely better solved by the -implementation of multiple generalized ufuncs which are called internally. +Although for convenience, it will be sufficient if the user implements only: + +* ``compare(self, char *item_ptr1, char *item_ptr2, int *res) -> {-1, 0, 1}`` -**Limitations:** -Using the above ``PyArrayDTypeMeta_Spec`` struct, the structure itself can -only be extended clumsily (e.g. by adding a version tag to the ``slots`` -to indicate a new, longer version of the struct). -We could also provide the struct using a function, which however will require -memory management but would allow ABI-compatible extension -(the struct is freed again when the DType is created). +**Limitations:** Using the above ``PyArrayDTypeMeta_Spec`` struct, the +structure itself can only be extended clumsily (e.g. by adding a version tag +to the ``slots`` to indicate a new, longer version of the struct). We could +also provide the struct using a function, which however will require memory +management but would allow ABI-compatible extension (the struct is freed again +when the DType is created). CastingImpl -""""""""""" +============================================================================== The external API for ``CastingImpl`` will be limited initially to defining: -* ``cast_kind`` attribute, which can be one of the supported casting kinds. +* ``casting`` attribute, which can be one of the supported casting kinds. This is the safest cast possible. For example casting between two NumPy strings is of course "safe" in general, but may be "same kind" in a specific instance if the second string is shorter. If neither type is parametric the - ``adjust_descriptors`` must use it. -* ``adjust_descriptors(dtypes_in[2], dtypes_out[2], casting_out) -> int {0, -1}`` - The out dtypes must be set correctly to dtypes which the strided loop - (transfer function) can handle. Initially the result must have be instances - of the same DType class as the ``CastingImpl`` is defined for. - The ``casting_out`` will be set to ``NPY_SAFE_CASTING``, ``NPY_UNSAFE_CASTING``, - or ``NPY_SAME_KIND_CASTING``. With a new, additional, flag ``NPY_CAST_IS_VIEW`` - which can be set to indicate that no cast is necessary, but a simple view - is sufficient to perform the cast. - The cast should return ``-1`` when a custom error message is set and - ``NPY_NO_CASTING`` to indicate that a generic casting error should be - set (this is in most cases preferable). -* ``strided_loop(char **args, npy_intp *dimensions, npy_intp *strides, dtypes[2]) -> int {0, nonzero}`` (must currently succeed) - -This is identical to the proposed API for ufuncs. By default the two dtypes -are passed in as the last argument. On error return (if no error is set) a -generic error will be given. -More optimized loops are in use internally, and will be made available to users -in the future (see notes) -The iterator API does not currently support casting errors: this is -a bug that needs to be fixed. Until it is fixed the loop should always -succeed (return 0). - -Although verbose, the API shall mimic the one for creating a new DType. -The ``PyArrayCastingImpl_Spec`` will include a field for ``dtypes`` and -identical to a ``PyArrayUFuncImpl_Spec``:: + ``resolve_descriptors`` must use it. + +* ``resolve_descriptors(self, given_descrs[2], loop_descrs[2]) -> int {casting, -1}``: + The ``loop_descrs`` must be set correctly to dtypes which the strided loop + (transfer function) can handle. Initially the result must have instances + of the same DType class as the ``CastingImpl`` is defined for. The + ``casting`` will be set to ``NPY_EQUIV_CASTING``, ``NPY_SAFE_CASTING``, + ``NPY_UNSAFE_CASTING``, or ``NPY_SAME_KIND_CASTING``. + A new, additional flag, ``NPY_CAST_IS_VIEW``, can be set to indicate that + no cast is necessary and a view is sufficient to perform the cast. + The return value shall be ``-1`` to indicate that the cast is not possible. + If no error is set, a generic error message will be given. If an error is + already set it will be chained and may provide additional information. + Note that ``self`` represents additional call information; details are given + in NEP 43. + +* ``strided_loop(char **args, npy_intp *dimensions, npy_intp *strides, + ...) -> int {0, -1}`` (signature will be fully defined in NEP 43) + +This is identical to the proposed API for ufuncs. The additional ``...`` +part of the signature will include information such as the two ``dtype``\s. +More optimized loops are in use internally, and +will be made available to users in the future (see notes). + +Although verbose, the API shall mimic the one for creating a new DType: + +.. code-block:: C typedef struct{ - int needs_api; /* whether the cast requires the API */ - PyArray_DTypeMeta *in_dtype; /* input DType class */ - PyArray_DTypeMeta *out_dtype; /* output DType class */ + int flags; /* e.g. whether the cast requires the API */ + int nin, nout; /* Number of Input and outputs (always 1) */ + NPY_CASTING casting; /* The default casting level */ + PyArray_DTypeMeta *dtypes; /* input and output DType class */ /* NULL terminated slots defining the methods */ PyType_Slot *slots; - } PyArrayUFuncImpl_Spec; - -The actual creation function ``PyArrayCastingImpl_FromSpec()`` will additionally -require a ``casting`` parameter to define the default (maximum) casting safety. -The internal representation of ufuncs and casting implementations may differ -initially if it makes implementation simpler, but should be kept opaque to -allow future merging. + } PyArrayMethod_Spec; -**TODO:** It may be possible to make this more close to the ufuncs or even -use a single FromSpec. This API shall only be finalized after/when NEP 43 -is finalized. +The focus differs between casting and general ufuncs. For example for casts +``nin == nout == 1`` is always correct, while for ufuncs ``casting`` is +expected to be usually `"safe"`. -**Notes:** - -We may initially allow users to define only a single loop. -However, internally NumPy optimizes far more, and this should be made -public incrementally, by either allowing to provide multiple versions, such -as: +**Notes:** We may initially allow users to define only a single loop. However, +internally NumPy optimizes far more, and this should be made public +incrementally, either by allowing multiple versions, such as: * contiguous inner loop * strided inner loop * scalar inner loop -or more likely through an additional ``get_inner_loop`` function which has -additional information, such as the fixed strides (similar to our internal API). +or more likely through exposure of the ``get_loop`` function which is passed +additional information, such as the fixed strides (similar to our internal +API). -The above example does not yet include the definition of setup/teardown -functionality, which may overlap with ``get_inner_loop``. -Since these are similar to the UFunc machinery, this should be defined in -detail in NEP 43 and then incorporated identically into casting. +The above example does not yet include potential setup and error handling +requirements. Since these are similar to the UFunc machinery, this will be +defined in detail in NEP 43 and then incorporated identically into casting. -Also the ``needs_api`` decision may actually be moved into a setup function, -and removed or mainly provided as a convenience flag. +The slots/methods used will be prefixed ``NPY_uf_`` for similarity to the +ufunc machinery. -The slots/methods used will be prefixed ``NPY_uf_`` for similarity to the ufunc -machinery. +**Alternatives:** Aside from name changes, and possible signature tweaks, +there seem to be few alternatives to the above structure. +The proposed API using ``*_FromSpec`` function is a good way to achieve a stable +and extensible API. The slots design is extensible and can be +changed without breaking binary compatibility. +Convenience functions can still be provided to allow creation with less code. -Alternatives -"""""""""""" - -Aside from name changes, and possible signature tweaks, there seem to -be few alternatives to the above structure. -Keeping the creation process close the Python limited API has some advantage. -Convenience functions could still be provided to allow creation with less -code. -The central point in the above design is that the enumerated slots design -is extensible and can be changed without breaking binary compatibility. -A downside is the possible need to pass in e.g. integer flags using a void -pointer inside this structure. - -A downside of this is that compilers cannot warn about function -pointer incompatibilities. There is currently no proposed solution to this. - - -Issues -^^^^^^ - -Any possible design decision will have issues. - -The above split into Python objects has the disadvantage that reference cycles -naturally occur. For example a potential ``CastingImpl`` object needs to -hold on to both ``DTypes``. Further, a scalar type may want to own a -strong reference to the corresponding ``DType`` while the ``DType`` *must* -hold a strong reference to the scalar. -We do not believe that these reference cycles are an issue. The may -require implementation of of cyclic reference counting at some point, but -cyclic reference resolution is very common in Python and dtypes (especially -classes) are only a small number of objects. - -In some cases, the new split will add additional indirections to the code, -since methods on the DType have to be looked up and called. -This should not have serious performance impact and seems necessary to -achieve the desired flexibility. - -From a user-perspective, a more serious downside is that handling certain -functionality in the ``DType`` rather than directly can mean that error -messages need to be raised from places where less context is available. -This may mean that error messages can be less specific. -This will be alleviated by exception chaining. Also decisions such as -returning the casting safety (even when it is impossible to cast) allow -most exceptions to be set at a point where more context is available -and ensures uniform errors messages. +One downside of this approach is that compilers cannot warn about function pointer +incompatibilities. +****************************************************************************** Implementation --------------- +****************************************************************************** -Internally a few implementation details have to be decided. These will be -fully opaque to the user and can be changed at a later time. +Steps for implementation are outlined in :ref:`NEP 41 `. This includes +internal restructuring for the new casting and array-coercion. +First, the NumPy will internally be rewritten using the above methods for +casting and array-coercion. -This includes: +After that, the new public API will be added incrementally. +We plan to expose it in a preliminary state initially to allow modification +after some experience can be gained. +In addition to the features presented in detail in this NEP, all functionality +currently implemented on the dtypes will be replaced systematically. -* How ``CastingImpl`` lookup, and thus the decision whether a cast is possible, - is defined. (This is speed relevant, although mainly during a transition - phase where UFuncs where NEP 43 is not yet implemented). - Thus, it is not very relevant to the NEP. It is only necessary to ensure fast - lookup during the transition phase for the current builtin Numerical types. -* How the mapping from a python scalar (e.g. ``3.``) to the DType is - implemented. - -The main steps for implementation are outlined in :ref:`NEP 41 `. -This includes the internal restructure for how casting and array-coercion -works. -After this the new public API will be added incrementally. -This includes replacements for certain slots which are occasionally -directly used on the dtype (e.g. ``dtype->f->setitem``). - - -Discussion ----------- +****************************************************************************** +Alternatives +****************************************************************************** -There is a large space of possible implementations with many discussions -in various places, as well as initial thoughts and design documents. -These are listed in the discussion of NEP 40 and not repeated here for -brevity. +The space of possible implementations is large, so there have been many +discussions, conceptions, and design documents. These are listed in NEP 40. +Since this NEP encompasses multiple individual decisions, alternatives +are discussed in the above individual sections. +****************************************************************************** References ----------- +****************************************************************************** .. [1] NumPy currently inspects the value to allow the operations:: @@ -1335,10 +1366,11 @@ References np.array([1.2], dtype=np.float32) + 1. to return a ``uint8`` or ``float32`` array respectively. This is - further described in the documentation of `numpy.result_type`. + further described in the documentation for :func:`numpy.result_type`. +****************************************************************************** Copyright ---------- +****************************************************************************** This document has been placed in the public domain. From e5a82c95742908899b0b4d7ee54617eb4c6b3972 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2020 07:16:02 +0000 Subject: [PATCH 105/409] MAINT: Bump hypothesis from 5.35.3 to 5.36.1 Bumps [hypothesis](https://github.com/HypothesisWorks/hypothesis) from 5.35.3 to 5.36.1. - [Release notes](https://github.com/HypothesisWorks/hypothesis/releases) - [Commits](https://github.com/HypothesisWorks/hypothesis/compare/hypothesis-python-5.35.3...hypothesis-python-5.36.1) Signed-off-by: dependabot-preview[bot] --- test_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index 036e9861fa4e..c86d46eb879a 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,7 +1,7 @@ cython==0.29.21 wheel setuptools<49.2.0 -hypothesis==5.35.3 +hypothesis==5.36.1 pytest==6.0.2 pytz==2020.1 pytest-cov==2.10.1 From 33e1dbee8d9e11a3e96efaae822ff6f3c44e3cef Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Mon, 28 Sep 2020 08:15:49 -0600 Subject: [PATCH 106/409] MAINT: Finish replacing PyInt_Check (#17364) * MAINT: Replace PyInt_Check in multiarray_tests.c.src * MAINT: Replace PyInt_Check in number.c * MAINT: Replace PyInt_Check in interators.c * MAINT: Replace PyInt_Check in refcount.c * MAINT: Replace PyInt_Check in hashdescr.c * MAINT: Replace PyInt_Check in scalarapi.c. * MAINT: Replace PyInt_Check in scalartypes.c.src. --- .../src/multiarray/_multiarray_tests.c.src | 27 ++++++--- numpy/core/src/multiarray/hashdescr.c | 4 +- numpy/core/src/multiarray/iterators.c | 2 +- numpy/core/src/multiarray/number.c | 13 +++- numpy/core/src/multiarray/refcount.c | 24 ++++---- numpy/core/src/multiarray/scalarapi.c | 60 +++++++++++-------- numpy/core/src/multiarray/scalartypes.c.src | 2 +- 7 files changed, 79 insertions(+), 53 deletions(-) diff --git a/numpy/core/src/multiarray/_multiarray_tests.c.src b/numpy/core/src/multiarray/_multiarray_tests.c.src index ea04c82bdf2e..0bf6958cde54 100644 --- a/numpy/core/src/multiarray/_multiarray_tests.c.src +++ b/numpy/core/src/multiarray/_multiarray_tests.c.src @@ -176,17 +176,20 @@ test_neighborhood_iterator(PyObject* NPY_UNUSED(self), PyObject* args) /* Compute boundaries for the neighborhood iterator */ for (i = 0; i < 2 * PyArray_NDIM(ax); ++i) { PyObject* bound; + bound = PySequence_GetItem(b, i); if (bound == NULL) { goto clean_itx; } - if (!PyInt_Check(bound)) { + /* PyLong_AsSsize checks for PyLong */ + bounds[i] = PyLong_AsSsize_t(bound); + if (error_converting(bounds[i])) { + PyErr_Clear(); PyErr_SetString(PyExc_ValueError, - "bound not long"); + "bound is invalid"); Py_DECREF(bound); goto clean_itx; } - bounds[i] = PyLong_AsSsize_t(bound); Py_DECREF(bound); } @@ -335,17 +338,20 @@ test_neighborhood_iterator_oob(PyObject* NPY_UNUSED(self), PyObject* args) /* Compute boundaries for the neighborhood iterator */ for (i = 0; i < 2 * PyArray_NDIM(ax); ++i) { PyObject* bound; + bound = PySequence_GetItem(b1, i); if (bound == NULL) { goto clean_itx; } - if (!PyInt_Check(bound)) { + /* PyLong_AsSsize checks for PyLong */ + bounds[i] = PyLong_AsSsize_t(bound); + if (error_converting(bounds[i])) { + PyErr_Clear(); PyErr_SetString(PyExc_ValueError, - "bound not long"); + "bound is invalid"); Py_DECREF(bound); goto clean_itx; } - bounds[i] = PyLong_AsSsize_t(bound); Py_DECREF(bound); } @@ -359,17 +365,20 @@ test_neighborhood_iterator_oob(PyObject* NPY_UNUSED(self), PyObject* args) for (i = 0; i < 2 * PyArray_NDIM(ax); ++i) { PyObject* bound; + bound = PySequence_GetItem(b2, i); if (bound == NULL) { goto clean_itx; } - if (!PyInt_Check(bound)) { + /* PyLong_AsSsize checks for PyLong */ + bounds[i] = PyLong_AsSsize_t(bound); + if (error_converting(bounds[i])) { + PyErr_Clear(); PyErr_SetString(PyExc_ValueError, - "bound not long"); + "bound is invalid"); Py_DECREF(bound); goto clean_itx; } - bounds[i] = PyLong_AsSsize_t(bound); Py_DECREF(bound); } diff --git a/numpy/core/src/multiarray/hashdescr.c b/numpy/core/src/multiarray/hashdescr.c index c596a7098a97..e9a99cc8fa8f 100644 --- a/numpy/core/src/multiarray/hashdescr.c +++ b/numpy/core/src/multiarray/hashdescr.c @@ -165,7 +165,7 @@ static int _array_descr_walk_fields(PyObject *names, PyObject* fields, PyObject* } foffset = PyTuple_GET_ITEM(value, 1); - if (!PyInt_Check(foffset)) { + if (!PyLong_Check(foffset)) { PyErr_SetString(PyExc_SystemError, "(Hash) Second item in compound dtype tuple not an int ???"); return -1; @@ -208,7 +208,7 @@ static int _array_descr_walk_subarray(PyArray_ArrayDescr* adescr, PyObject *l) PyList_Append(l, item); } } - else if (PyInt_Check(adescr->shape)) { + else if (PyLong_Check(adescr->shape)) { PyList_Append(l, adescr->shape); } else { diff --git a/numpy/core/src/multiarray/iterators.c b/numpy/core/src/multiarray/iterators.c index 31795b2d0faf..3ebd4c858974 100644 --- a/numpy/core/src/multiarray/iterators.c +++ b/numpy/core/src/multiarray/iterators.c @@ -597,7 +597,7 @@ iter_subscript(PyArrayIterObject *self, PyObject *ind) } /* Check for Integer or Slice */ - if (PyLong_Check(ind) || PyInt_Check(ind) || PySlice_Check(ind)) { + if (PyLong_Check(ind) || PySlice_Check(ind)) { start = parse_index_entry(ind, &step_size, &n_steps, self->size, 0, 1); if (start == -1) { diff --git a/numpy/core/src/multiarray/number.c b/numpy/core/src/multiarray/number.c index 87c3c9b0a708..a629dfe97fde 100644 --- a/numpy/core/src/multiarray/number.c +++ b/numpy/core/src/multiarray/number.c @@ -397,14 +397,21 @@ is_scalar_with_conversion(PyObject *o2, double* out_exponent) PyObject *temp; const int optimize_fpexps = 1; - if (PyInt_Check(o2)) { - *out_exponent = (double)PyLong_AsLong(o2); + if (PyLong_Check(o2)) { + long tmp = PyLong_AsLong(o2); + if (error_converting(tmp)) { + PyErr_Clear(); + return NPY_NOSCALAR; + } + *out_exponent = (double)tmp; return NPY_INTPOS_SCALAR; } + if (optimize_fpexps && PyFloat_Check(o2)) { *out_exponent = PyFloat_AsDouble(o2); return NPY_FLOAT_SCALAR; } + if (PyArray_Check(o2)) { if ((PyArray_NDIM((PyArrayObject *)o2) == 0) && ((PyArray_ISINTEGER((PyArrayObject *)o2) || @@ -442,7 +449,7 @@ is_scalar_with_conversion(PyObject *o2, double* out_exponent) else if (PyIndex_Check(o2)) { PyObject* value = PyNumber_Index(o2); Py_ssize_t val; - if (value==NULL) { + if (value == NULL) { if (PyErr_Occurred()) { PyErr_Clear(); } diff --git a/numpy/core/src/multiarray/refcount.c b/numpy/core/src/multiarray/refcount.c index 0f84449af012..41dd059b0ac1 100644 --- a/numpy/core/src/multiarray/refcount.c +++ b/numpy/core/src/multiarray/refcount.c @@ -292,20 +292,22 @@ static void _fillobject(char *optr, PyObject *obj, PyArray_Descr *dtype) { if (!PyDataType_FLAGCHK(dtype, NPY_ITEM_REFCOUNT)) { - if ((obj == Py_None) || (PyInt_Check(obj) && PyLong_AsLong(obj)==0)) { + PyObject *arr; + + if ((obj == Py_None) || + (PyLong_Check(obj) && PyLong_AsLong(obj) == 0)) { return; } - else { - PyObject *arr; - Py_INCREF(dtype); - arr = PyArray_NewFromDescr(&PyArray_Type, dtype, - 0, NULL, NULL, NULL, - 0, NULL); - if (arr!=NULL) { - dtype->f->setitem(obj, optr, arr); - } - Py_XDECREF(arr); + /* Clear possible long conversion error */ + PyErr_Clear(); + Py_INCREF(dtype); + arr = PyArray_NewFromDescr(&PyArray_Type, dtype, + 0, NULL, NULL, NULL, + 0, NULL); + if (arr!=NULL) { + dtype->f->setitem(obj, optr, arr); } + Py_XDECREF(arr); } if (dtype->type_num == NPY_OBJECT) { Py_XINCREF(obj); diff --git a/numpy/core/src/multiarray/scalarapi.c b/numpy/core/src/multiarray/scalarapi.c index b918786f22b6..f610ad4680e3 100644 --- a/numpy/core/src/multiarray/scalarapi.c +++ b/numpy/core/src/multiarray/scalarapi.c @@ -373,7 +373,8 @@ PyArray_FromScalar(PyObject *scalar, PyArray_Descr *outcode) NPY_NO_EXPORT PyObject * PyArray_ScalarFromObject(PyObject *object) { - PyObject *ret=NULL; + PyObject *ret = NULL; + if (PyArray_IsZeroDim(object)) { return PyArray_ToScalar(PyArray_DATA((PyArrayObject *)object), (PyArrayObject *)object); @@ -390,42 +391,49 @@ PyArray_ScalarFromObject(PyObject *object) PyArrayScalar_RETURN_FALSE; } } - else if (PyInt_Check(object)) { - ret = PyArrayScalar_New(Long); - if (ret == NULL) { - return NULL; + else if (PyLong_Check(object)) { + /* Check if fits in long */ + npy_long val_long = PyLong_AsLong(object); + if (!error_converting(val_long)) { + ret = PyArrayScalar_New(Long); + if (ret != NULL) { + PyArrayScalar_VAL(ret, Long) = val_long; + } + return ret; } - PyArrayScalar_VAL(ret, Long) = PyLong_AsLong(object); + PyErr_Clear(); + + /* Check if fits in long long */ + npy_longlong val_longlong = PyLong_AsLongLong(object); + if (!error_converting(val_longlong)) { + ret = PyArrayScalar_New(LongLong); + if (ret != NULL) { + PyArrayScalar_VAL(ret, LongLong) = val_longlong; + } + return ret; + } + PyErr_Clear(); + + return NULL; } else if (PyFloat_Check(object)) { ret = PyArrayScalar_New(Double); - if (ret == NULL) { - return NULL; + if (ret != NULL) { + PyArrayScalar_VAL(ret, Double) = PyFloat_AS_DOUBLE(object); } - PyArrayScalar_VAL(ret, Double) = PyFloat_AS_DOUBLE(object); + return ret; } else if (PyComplex_Check(object)) { ret = PyArrayScalar_New(CDouble); - if (ret == NULL) { - return NULL; + if (ret != NULL) { + PyArrayScalar_VAL(ret, CDouble).real = PyComplex_RealAsDouble(object); + PyArrayScalar_VAL(ret, CDouble).imag = PyComplex_ImagAsDouble(object); } - PyArrayScalar_VAL(ret, CDouble).real = PyComplex_RealAsDouble(object); - PyArrayScalar_VAL(ret, CDouble).imag = PyComplex_ImagAsDouble(object); + return ret; } - else if (PyLong_Check(object)) { - npy_longlong val; - val = PyLong_AsLongLong(object); - if (error_converting(val)) { - PyErr_Clear(); - return NULL; - } - ret = PyArrayScalar_New(LongLong); - if (ret == NULL) { - return NULL; - } - PyArrayScalar_VAL(ret, LongLong) = val; + else { + return NULL; } - return ret; } /*New reference */ diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src index 5a3f4922a66a..65672237dad4 100644 --- a/numpy/core/src/multiarray/scalartypes.c.src +++ b/numpy/core/src/multiarray/scalartypes.c.src @@ -2952,7 +2952,7 @@ void_arrtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds) * For a VOID scalar first see if obj is an integer or long * and create new memory of that size (filled with 0) for the scalar */ - if (PyLong_Check(obj) || PyInt_Check(obj) || + if (PyLong_Check(obj) || PyArray_IsScalar(obj, Integer) || (PyArray_Check(obj) && PyArray_NDIM((PyArrayObject *)obj)==0 && From 3d37ebccf696d8b186e11df99f3940441919f1c2 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Mon, 28 Sep 2020 08:39:02 -0600 Subject: [PATCH 107/409] MAINT: Remove old debug print statement. This looks like an old debug statement that should have been removed long ago. The C stdio.h header is nowhere imported and PyInt_Type no longer exists in Python 3. --- numpy/core/src/multiarray/multiarraymodule.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index f1d5ab694a80..ff2b796d2908 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -4409,12 +4409,6 @@ setup_scalartypes(PyObject *NPY_UNUSED(dict)) /* Timedelta is an integer with an associated unit */ SINGLE_INHERIT(Timedelta, SignedInteger); - /* - fprintf(stderr, - "tp_free = %p, PyObject_Del = %p, int_tp_free = %p, base.tp_free = %p\n", - PyIntArrType_Type.tp_free, PyObject_Del, PyInt_Type.tp_free, - PySignedIntegerArrType_Type.tp_free); - */ SINGLE_INHERIT(UByte, UnsignedInteger); SINGLE_INHERIT(UShort, UnsignedInteger); SINGLE_INHERIT(UInt, UnsignedInteger); From 5e2d8f47703c783e168bd845b65a92b9c8040a0f Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Mon, 28 Sep 2020 16:36:24 +0100 Subject: [PATCH 108/409] Apply suggestions from code review Co-authored-by: Sebastian Berg --- numpy/core/src/multiarray/datetime_busdaycal.c | 2 +- numpy/core/src/multiarray/datetime_strings.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/core/src/multiarray/datetime_busdaycal.c b/numpy/core/src/multiarray/datetime_busdaycal.c index cec7135ffea3..d48141d4cb7d 100644 --- a/numpy/core/src/multiarray/datetime_busdaycal.c +++ b/numpy/core/src/multiarray/datetime_busdaycal.c @@ -73,7 +73,7 @@ PyArray_WeekMaskConverter(PyObject *weekmask_in, npy_bool *weekmask) general_weekmask_string: /* a string like "SatSun" or "Mon Tue Wed" */ memset(weekmask, 0, 7); - for (int i = 0; i < len; i += 3) { + for (Py_ssize_t i = 0; i < len; i += 3) { while (isspace(str[i])) ++i; diff --git a/numpy/core/src/multiarray/datetime_strings.c b/numpy/core/src/multiarray/datetime_strings.c index 8665d329579f..360868568478 100644 --- a/numpy/core/src/multiarray/datetime_strings.c +++ b/numpy/core/src/multiarray/datetime_strings.c @@ -1399,7 +1399,7 @@ array_datetime_as_string(PyObject *NPY_UNUSED(self), PyObject *args, strobj = unit_in; } - Py_ssize_t len = 0; + Py_ssize_t len; char const *str = PyUnicode_AsUTF8AndSize(strobj, &len); if (str == NULL) { Py_DECREF(strobj); From 34d7d395d79f880dc9e156ff83af0cf5844867bf Mon Sep 17 00:00:00 2001 From: JMFT Date: Mon, 28 Sep 2020 18:43:07 +0100 Subject: [PATCH 109/409] DOC: Fix docstring for np.matmul (#17383) Update argument names to match ufunc signature in matmul docstring. --- numpy/core/code_generators/ufunc_docstrings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/core/code_generators/ufunc_docstrings.py b/numpy/core/code_generators/ufunc_docstrings.py index 82cd6fb273f2..1f8e4a6fb314 100644 --- a/numpy/core/code_generators/ufunc_docstrings.py +++ b/numpy/core/code_generators/ufunc_docstrings.py @@ -2646,8 +2646,8 @@ def add_newdoc(place, name, doc): Raises ------ ValueError - If the last dimension of `a` is not the same size as - the second-to-last dimension of `b`. + If the last dimension of `x1` is not the same size as + the second-to-last dimension of `x2`. If a scalar value is passed in. From 47c32b959ae71eb2c65c3b9b06796614105f5964 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Mon, 28 Sep 2020 15:55:23 -0400 Subject: [PATCH 110/409] DOC: Replace "About NumPy" with "Document conventions" Eliminates the unhelpful "About NumPy" title. All contents of the original appear more prominently elsewhere, except for the information about document conventions, which is now the title and subject. --- doc/source/_templates/indexcontent.html | 2 +- doc/source/about.rst | 62 ------------------------- doc/source/doc_conventions.rst | 23 +++++++++ doc/source/docs/howto_document.rst | 5 ++ doc/source/user/index.rst | 7 ++- 5 files changed, 34 insertions(+), 65 deletions(-) delete mode 100644 doc/source/about.rst create mode 100644 doc/source/doc_conventions.rst diff --git a/doc/source/_templates/indexcontent.html b/doc/source/_templates/indexcontent.html index 5929e755dee6..6633aa9bef90 100644 --- a/doc/source/_templates/indexcontent.html +++ b/doc/source/_templates/indexcontent.html @@ -56,7 +56,7 @@

{{ docstitle|e }}

Reporting bugs

Release Notes

-

About NumPy

+

Document conventions

License of NumPy

diff --git a/doc/source/about.rst b/doc/source/about.rst deleted file mode 100644 index 3e83833d178d..000000000000 --- a/doc/source/about.rst +++ /dev/null @@ -1,62 +0,0 @@ -About NumPy -=========== - -NumPy is the fundamental package -needed for scientific computing with Python. This package contains: - -- a powerful N-dimensional :ref:`array object ` -- sophisticated :ref:`(broadcasting) functions ` -- basic :ref:`linear algebra functions ` -- basic :ref:`Fourier transforms ` -- sophisticated :ref:`random number capabilities ` -- tools for integrating Fortran code -- tools for integrating C/C++ code - -Besides its obvious scientific uses, *NumPy* can also be used as an -efficient multi-dimensional container of generic data. Arbitrary -data types can be defined. This allows *NumPy* to seamlessly and -speedily integrate with a wide variety of databases. - -NumPy is a successor for two earlier scientific Python libraries: -Numeric and Numarray. - -NumPy community ---------------- - -NumPy is a distributed, volunteer, open-source project. *You* can help -us make it better; if you believe something should be improved either -in functionality or in documentation, don't hesitate to contact us --- or -even better, contact us and participate in fixing the problem. - -Our main means of communication are: - -- `scipy.org website `__ - -- `Mailing lists `__ - -- `NumPy Issues `__ (bug reports go here) - -- `Old NumPy Trac `__ (dead link) - -More information about the development of NumPy can be found at our `Developer Zone `__. - -The project management structure can be found at our :doc:`governance page ` - - -About this documentation -======================== - -Conventions ------------ - -Names of classes, objects, constants, etc. are given in **boldface** font. -Often they are also links to a more detailed documentation of the -referred object. - -This manual contains many examples of use, usually prefixed with the -Python prompt ``>>>`` (which is not a part of the example code). The -examples assume that you have first entered:: - ->>> import numpy as np - -before running the examples. diff --git a/doc/source/doc_conventions.rst b/doc/source/doc_conventions.rst new file mode 100644 index 000000000000..e2bc419d1691 --- /dev/null +++ b/doc/source/doc_conventions.rst @@ -0,0 +1,23 @@ +.. _documentation_conventions: + +############################################################################## +Documentation conventions +############################################################################## + +- Names that look like :func:`numpy.array` are links to detailed + documentation. + +- Examples often include the Python prompt ``>>>``. This is not part of the + code and will cause an error if typed or pasted into the Python + shell. It can be safely typed or pasted into the IPython shell; the ``>>>`` + is ignored. + +- Examples often use ``np`` as an alias for ``numpy``; that is, they assume + you've run:: + + >>> import numpy as np + +- If you're a code contributor writing a docstring, see :ref:`docstring_intro`. + +- If you're a writer contributing ordinary (non-docstring) documentation, see + :ref:`userdoc_guide`. diff --git a/doc/source/docs/howto_document.rst b/doc/source/docs/howto_document.rst index 9f9068ab32a7..5a36fa718db7 100644 --- a/doc/source/docs/howto_document.rst +++ b/doc/source/docs/howto_document.rst @@ -4,10 +4,15 @@ A Guide to NumPy/SciPy Documentation ==================================== +.. _userdoc_guide: + User documentation ******************* NumPy text documents should follow the `Google developer documentation style guide `_. + +.. _docstring_intro: + Docstrings ********** diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst index 3a79f0f2e9a5..11a019b489b8 100644 --- a/doc/source/user/index.rst +++ b/doc/source/user/index.rst @@ -26,7 +26,10 @@ classes contained in the package, see the :ref:`reference`. howtos_index -.. These are stuck here to avoid the "WARNING: document isn't included in any +.. Links to these files are placed directly in the top-level html + (doc/source/_templates/indexcontent.html, which appears for the URLs + numpy.org/devdocs and numpy.org/doc/XX) and are not in any toctree, so + we include them here to avoid a "WARNING: document isn't included in any toctree" message .. toctree:: @@ -39,5 +42,5 @@ classes contained in the package, see the :ref:`reference`. ../docs/index ../bugs ../release - ../about + ../doc_conventions ../license From d7b88ad4b695f7994f471d2f0b4e493d072d3cc1 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Mon, 28 Sep 2020 17:14:07 -0400 Subject: [PATCH 111/409] DOC: Update info on doc style rules --- doc/source/docs/howto_document.rst | 32 ++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/doc/source/docs/howto_document.rst b/doc/source/docs/howto_document.rst index 9f9068ab32a7..7cd97b954959 100644 --- a/doc/source/docs/howto_document.rst +++ b/doc/source/docs/howto_document.rst @@ -1,12 +1,36 @@ .. _howto-document: -A Guide to NumPy/SciPy Documentation -==================================== +A Guide to NumPy Documentation +============================== User documentation -******************* -NumPy text documents should follow the `Google developer documentation style guide `_. +****************** +- In general, we follow the + `Google developer documentation style guide `_. + +- NumPy style governs cases where: + + - Google has no guidance, or + - We prefer not to use the Google style + + Our current rules: + + - We pluralize *index* as *indices* rather than + `indexes `_, + following the precedent of :func:`numpy.indices`. + + - For consistency we also pluralize *matrix* as *matrices*. + +- Grammatical issues inadequately addressed by the NumPy or Google rules are + decided by the section on "Grammar and Usage" in the most recent edition of + the `Chicago Manual of Style + `_. + +- We welcome being + `alerted `_ to cases + we should add to the NumPy style rules. + Docstrings ********** From 127b262b0da014e624889532c3d4eb330ed2aa07 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Mon, 28 Sep 2020 18:12:14 -0500 Subject: [PATCH 112/409] BUG: Fix default void, datetime, and timedelta in array coercion When converting an empty sequence to an array, datetimes would use an incorrect itemsize (on master only). The legacy behaviour of void is that it uses 8 bytes, which is due to the fact that `[]` uses `float64` by default which has 8 bytes. --- numpy/core/src/multiarray/dtypemeta.c | 31 ++++++++++++++++++++++--- numpy/core/tests/test_array_coercion.py | 16 +++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/numpy/core/src/multiarray/dtypemeta.c b/numpy/core/src/multiarray/dtypemeta.c index 531f746d89a3..dbe5ba476887 100644 --- a/numpy/core/src/multiarray/dtypemeta.c +++ b/numpy/core/src/multiarray/dtypemeta.c @@ -204,8 +204,32 @@ nonparametric_default_descr(PyArray_DTypeMeta *cls) } +/* Ensure a copy of the singleton (just in case we do adapt it somewhere) */ static PyArray_Descr * -flexible_default_descr(PyArray_DTypeMeta *cls) +datetime_and_timedelta_default_descr(PyArray_DTypeMeta *cls) +{ + return PyArray_DescrNew(cls->singleton); +} + + +static PyArray_Descr * +void_default_descr(PyArray_DTypeMeta *cls) +{ + PyArray_Descr *res = PyArray_DescrNew(cls->singleton); + if (res == NULL) { + return NULL; + } + /* + * The legacy behaviour for `np.array([], dtype="V")` is to use "V8". + * This is because `[]` uses `float64` as dtype, and then that is used + * for the size of the requested void. + */ + res->elsize = 8; + return res; +} + +static PyArray_Descr * +string_and_unicode_default_descr(PyArray_DTypeMeta *cls) { PyArray_Descr *res = PyArray_DescrNewFromType(cls->type_num); if (res == NULL) { @@ -534,7 +558,7 @@ dtypemeta_wrap_legacy_descriptor(PyArray_Descr *descr) else if (PyTypeNum_ISDATETIME(descr->type_num)) { /* Datetimes are flexible, but were not considered previously */ dtype_class->parametric = NPY_TRUE; - dtype_class->default_descr = flexible_default_descr; + dtype_class->default_descr = datetime_and_timedelta_default_descr; dtype_class->discover_descr_from_pyobject = ( discover_datetime_and_timedelta_from_pyobject); dtype_class->common_dtype = datetime_common_dtype; @@ -545,13 +569,14 @@ dtypemeta_wrap_legacy_descriptor(PyArray_Descr *descr) } else if (PyTypeNum_ISFLEXIBLE(descr->type_num)) { dtype_class->parametric = NPY_TRUE; - dtype_class->default_descr = flexible_default_descr; if (descr->type_num == NPY_VOID) { + dtype_class->default_descr = void_default_descr; dtype_class->discover_descr_from_pyobject = ( void_discover_descr_from_pyobject); dtype_class->common_instance = void_common_instance; } else { + dtype_class->default_descr = string_and_unicode_default_descr; dtype_class->is_known_scalar_type = string_known_scalar_types; dtype_class->discover_descr_from_pyobject = ( string_discover_descr_from_pyobject); diff --git a/numpy/core/tests/test_array_coercion.py b/numpy/core/tests/test_array_coercion.py index a6c8cc8b2156..e0480c7bfcf5 100644 --- a/numpy/core/tests/test_array_coercion.py +++ b/numpy/core/tests/test_array_coercion.py @@ -324,6 +324,22 @@ def test_scalar_coercion_same_as_cast_and_assignment(self, cast_to): ass[()] = scalar assert_array_equal(ass, cast) + @pytest.mark.parametrize("dtype_char", np.typecodes["All"]) + def test_default_dtype_instance(self, dtype_char): + if dtype_char in "SU": + dtype = np.dtype(dtype_char + "1") + elif dtype_char == "V": + # Legacy behaviour was to use V8. The reason was float64 being the + # default dtype and that having 8 bytes. + dtype = np.dtype("V8") + else: + dtype = np.dtype(dtype_char) + + discovered_dtype, _ = _discover_array_parameters([], type(dtype)) + + assert discovered_dtype == dtype + assert discovered_dtype.itemsize == dtype.itemsize + class TestTimeScalars: @pytest.mark.parametrize("dtype", [np.int64, np.float32]) From 7a50de698a5cfcbd613fcb8660250fc87c24e4dc Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Mon, 28 Sep 2020 11:25:27 -0600 Subject: [PATCH 113/409] ENH: Add metastr_to_unicode function to datetime.c. This new function is intended to be used instead the existing append_metastr_to_string function. It returns the datetime metastr as a unicode string rather than appending it to an existing unicode string. Uses of append_metastr_to_string are replaced in datetime.c. --- numpy/core/src/multiarray/_datetime.h | 10 ++ numpy/core/src/multiarray/datetime.c | 158 ++++++++++++++++++-------- 2 files changed, 121 insertions(+), 47 deletions(-) diff --git a/numpy/core/src/multiarray/_datetime.h b/numpy/core/src/multiarray/_datetime.h index 4e7ade5edacc..be9bac48c811 100644 --- a/numpy/core/src/multiarray/_datetime.h +++ b/numpy/core/src/multiarray/_datetime.h @@ -199,6 +199,16 @@ NPY_NO_EXPORT int convert_pyobject_to_datetime_metadata(PyObject *obj, PyArray_DatetimeMetaData *out_meta); +/* + * Returns datetime metadata as a new reference a Unicode object. + * Returns NULL on error. + * + * If 'skip_brackets' is true, skips the '[]'. + * + */ +NPY_NO_EXPORT PyObject * +metastr_to_unicode(PyArray_DatetimeMetaData *meta, int skip_brackets); + /* * 'ret' is a PyUString containing the datetime string, and this * function appends the metadata string to it. diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index f2225809a904..99b6a24e9681 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -1434,18 +1434,20 @@ raise_if_datetime64_metadata_cast_error(char *object_type, return 0; } else { - PyObject *errmsg; - errmsg = PyUnicode_FromFormat("Cannot cast %s " - "from metadata ", object_type); - errmsg = append_metastr_to_string(src_meta, 0, errmsg); - PyUString_ConcatAndDel(&errmsg, - PyUnicode_FromString(" to ")); - errmsg = append_metastr_to_string(dst_meta, 0, errmsg); - PyUString_ConcatAndDel(&errmsg, - PyUnicode_FromFormat(" according to the rule %s", - npy_casting_to_string(casting))); - PyErr_SetObject(PyExc_TypeError, errmsg); - Py_DECREF(errmsg); + PyObject *src = metastr_to_unicode(src_meta, 0); + if (src == NULL) { + return -1; + } + PyObject *dst = metastr_to_unicode(dst_meta, 0); + if (dst == NULL) { + Py_DECREF(src); + return -1; + } + PyErr_Format(PyExc_TypeError, + "Cannot cast %s from metadata %S to %S according to the rule %s", + object_type, src, dst, npy_casting_to_string(casting)); + Py_DECREF(src); + Py_DECREF(dst); return -1; } } @@ -1466,18 +1468,20 @@ raise_if_timedelta64_metadata_cast_error(char *object_type, return 0; } else { - PyObject *errmsg; - errmsg = PyUnicode_FromFormat("Cannot cast %s " - "from metadata ", object_type); - errmsg = append_metastr_to_string(src_meta, 0, errmsg); - PyUString_ConcatAndDel(&errmsg, - PyUnicode_FromString(" to ")); - errmsg = append_metastr_to_string(dst_meta, 0, errmsg); - PyUString_ConcatAndDel(&errmsg, - PyUnicode_FromFormat(" according to the rule %s", - npy_casting_to_string(casting))); - PyErr_SetObject(PyExc_TypeError, errmsg); - Py_DECREF(errmsg); + PyObject *src = metastr_to_unicode(src_meta, 0); + if (src == NULL) { + return -1; + } + PyObject *dst = metastr_to_unicode(dst_meta, 0); + if (dst == NULL) { + Py_DECREF(src); + return -1; + } + PyErr_Format(PyExc_TypeError, + "Cannot cast %s from metadata %S to %S according to the rule %s", + object_type, src, dst, npy_casting_to_string(casting)); + Py_DECREF(src); + Py_DECREF(dst); return -1; } } @@ -1600,32 +1604,38 @@ compute_datetime_metadata_greatest_common_divisor( return 0; incompatible_units: { - PyObject *errmsg; - errmsg = PyUnicode_FromString("Cannot get " - "a common metadata divisor for " - "NumPy datetime metadata "); - errmsg = append_metastr_to_string(meta1, 0, errmsg); - PyUString_ConcatAndDel(&errmsg, - PyUnicode_FromString(" and ")); - errmsg = append_metastr_to_string(meta2, 0, errmsg); - PyUString_ConcatAndDel(&errmsg, - PyUnicode_FromString(" because they have " - "incompatible nonlinear base time units")); - PyErr_SetObject(PyExc_TypeError, errmsg); - Py_DECREF(errmsg); + PyObject *umeta1 = metastr_to_unicode(meta1, 0); + if (umeta1 == NULL) { + return -1; + } + PyObject *umeta2 = metastr_to_unicode(meta2, 0); + if (umeta2 == NULL) { + Py_DECREF(umeta1); + return -1; + } + PyErr_Format(PyExc_TypeError, + "Cannot get a common metadata divisor for Numpy datatime " + "metadata %S and %S because they have incompatible nonlinear " + "base time units.", umeta1, umeta2); + Py_DECREF(umeta1); + Py_DECREF(umeta2); return -1; } units_overflow: { - PyObject *errmsg; - errmsg = PyUnicode_FromString("Integer overflow " - "getting a common metadata divisor for " - "NumPy datetime metadata "); - errmsg = append_metastr_to_string(meta1, 0, errmsg); - PyUString_ConcatAndDel(&errmsg, - PyUnicode_FromString(" and ")); - errmsg = append_metastr_to_string(meta2, 0, errmsg); - PyErr_SetObject(PyExc_OverflowError, errmsg); - Py_DECREF(errmsg); + PyObject *umeta1 = metastr_to_unicode(meta1, 0); + if (umeta1 == NULL) { + return -1; + } + PyObject *umeta2 = metastr_to_unicode(meta2, 0); + if (umeta2 == NULL) { + Py_DECREF(umeta1); + return -1; + } + PyErr_Format(PyExc_OverflowError, + "Integer overflow getting a common metadata divisor for " + "NumPy datetime metadata %S and %S.", umeta1, umeta2); + Py_DECREF(umeta1); + Py_DECREF(umeta2); return -1; } } @@ -1949,6 +1959,60 @@ convert_pyobject_to_datetime_metadata(PyObject *obj, } } +/* + * Return the datetime metadata as a Unicode object. + * + * Returns new reference, NULL on error. + * + * If 'skip_brackets' is true, skips the '[]'. + * + */ +NPY_NO_EXPORT PyObject * +metastr_to_unicode(PyArray_DatetimeMetaData *meta, int skip_brackets) +{ + int num; + char const *basestr; + + if (meta->base == NPY_FR_GENERIC) { + /* Without brackets, give a string "generic" */ + if (skip_brackets) { + return PyUnicode_FromString("generic"); + } + /* But with brackets, append nothing */ + else { + return PyUnicode_FromString(""); + } + } + + num = meta->num; + if (meta->base >= 0 && meta->base < NPY_DATETIME_NUMUNITS) { + basestr = _datetime_strings[meta->base]; + } + else { + PyErr_SetString(PyExc_RuntimeError, + "NumPy datetime metadata is corrupted"); + return NULL; + } + + if (num == 1) { + if (skip_brackets) { + return PyUnicode_FromFormat("%s", basestr); + } + else { + return PyUnicode_FromFormat("[%s]", basestr); + } + } + else { + if (skip_brackets) { + return PyUnicode_FromFormat("%d%s", num, basestr); + } + else { + return PyUnicode_FromFormat("[%d%s]", num, basestr); + } + } +} + + /* * 'ret' is a PyUString containing the datetime string, and this * function appends the metadata string to it. From e77c8455bbf5590db0cae41707eba1b657d78d17 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Mon, 28 Sep 2020 16:16:51 -0600 Subject: [PATCH 114/409] MAINT: Replace append_metastr_to_string in scalartypes.c.src Use metastr_to_unicode instead. --- numpy/core/src/multiarray/scalartypes.c.src | 54 ++++++++++----------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src index 65672237dad4..74ee260afd2e 100644 --- a/numpy/core/src/multiarray/scalartypes.c.src +++ b/numpy/core/src/multiarray/scalartypes.c.src @@ -518,21 +518,15 @@ datetimetype_repr(PyObject *self) */ if ((scal->obmeta.num == 1 && scal->obmeta.base != NPY_FR_h) || scal->obmeta.base == NPY_FR_GENERIC) { - ret = PyUnicode_FromString("numpy.datetime64('"); - PyUString_ConcatAndDel(&ret, - PyUnicode_FromString(iso)); - PyUString_ConcatAndDel(&ret, - PyUnicode_FromString("')")); + ret = PyUnicode_FromFormat("numpy.datetime64('%s')", iso); } else { - ret = PyUnicode_FromString("numpy.datetime64('"); - PyUString_ConcatAndDel(&ret, - PyUnicode_FromString(iso)); - PyUString_ConcatAndDel(&ret, - PyUnicode_FromString("','")); - ret = append_metastr_to_string(&scal->obmeta, 1, ret); - PyUString_ConcatAndDel(&ret, - PyUnicode_FromString("')")); + PyObject *meta = metastr_to_unicode(&scal->obmeta, 1); + if (meta == NULL) { + return NULL; + } + ret = PyUnicode_FromFormat("numpy.datetime64('%s','%S')", iso, meta); + Py_DECREF(meta); } return ret; @@ -542,7 +536,7 @@ static PyObject * timedeltatype_repr(PyObject *self) { PyTimedeltaScalarObject *scal; - PyObject *ret; + PyObject *val, *ret; if (!PyArray_IsScalar(self, Timedelta)) { PyErr_SetString(PyExc_RuntimeError, @@ -554,32 +548,34 @@ timedeltatype_repr(PyObject *self) /* The value */ if (scal->obval == NPY_DATETIME_NAT) { - ret = PyUnicode_FromString("numpy.timedelta64('NaT'"); + val = PyUnicode_FromString("'NaT'"); } else { - /* - * Can't use "%lld" if HAVE_LONG_LONG is not defined - */ + /* Can't use "%lld" if HAVE_LONG_LONG is not defined */ #if defined(HAVE_LONG_LONG) - ret = PyUnicode_FromFormat("numpy.timedelta64(%lld", - (long long)scal->obval); + val = PyUnicode_FromFormat("%lld", (long long)scal->obval); #else - ret = PyUnicode_FromFormat("numpy.timedelta64(%ld", - (long)scal->obval); + val = PyUnicode_FromFormat("%ld", (long)scal->obval); #endif } + if (val == NULL) { + return NULL; + } + /* The metadata unit */ if (scal->obmeta.base == NPY_FR_GENERIC) { - PyUString_ConcatAndDel(&ret, - PyUnicode_FromString(")")); + ret = PyUnicode_FromFormat("numpy.timedelta64(%S)", val); } else { - PyUString_ConcatAndDel(&ret, - PyUnicode_FromString(",'")); - ret = append_metastr_to_string(&scal->obmeta, 1, ret); - PyUString_ConcatAndDel(&ret, - PyUnicode_FromString("')")); + PyObject *meta = metastr_to_unicode(&scal->obmeta, 1); + if (meta == NULL) { + Py_DECREF(val); + return NULL; + } + ret = PyUnicode_FromFormat("numpy.timedelta64(%S,'%S')", val, meta); + Py_DECREF(meta); } + Py_DECREF(val); return ret; } From a3fc7708f3808084bf7203f809998fa7448b30ef Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Mon, 28 Sep 2020 19:58:34 -0600 Subject: [PATCH 115/409] MAINT: Replace append_metastr_to_string in descriptor.c. Use metastr_to_unicode instead. --- numpy/core/src/multiarray/descriptor.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c index 6e378f626b6a..257ededaecac 100644 --- a/numpy/core/src/multiarray/descriptor.c +++ b/numpy/core/src/multiarray/descriptor.c @@ -1892,18 +1892,26 @@ arraydescr_protocol_typestr_get(PyArray_Descr *self) else { ret = PyUnicode_FromFormat("%c%c%d", endian, basic_, size); } + if (ret == NULL) { + return NULL; + } + if (PyDataType_ISDATETIME(self)) { PyArray_DatetimeMetaData *meta; - meta = get_datetime_metadata_from_dtype(self); if (meta == NULL) { Py_DECREF(ret); return NULL; } + PyObject *umeta = metastr_to_unicode(meta, 0); + if (umeta == NULL) { + Py_DECREF(ret); + return NULL; + } - ret = append_metastr_to_string(meta, 0, ret); + Py_SETREF(ret, PyUnicode_Concat(ret, umeta)); + Py_DECREF(umeta); } - return ret; } From 97a4bc1401d3cce1211ea986dff447c12c9a62fb Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Tue, 29 Sep 2020 07:27:40 -0600 Subject: [PATCH 116/409] MAINT: Revise comment in numpy.core._dtype.py Replace append_metastr_to_string by metastr_to_unicode. --- numpy/core/_dtype.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/_dtype.py b/numpy/core/_dtype.py index 50aeeb5bc921..4249071ffe98 100644 --- a/numpy/core/_dtype.py +++ b/numpy/core/_dtype.py @@ -176,7 +176,7 @@ def _byte_order_str(dtype): def _datetime_metadata_str(dtype): - # TODO: this duplicates the C append_metastr_to_string + # TODO: this duplicates the C metastr_to_unicode functionality unit, count = np.datetime_data(dtype) if unit == 'generic': return '' From 320cfb238c4bc160dad9c839eef8325618c89e23 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Tue, 29 Sep 2020 07:36:27 -0600 Subject: [PATCH 117/409] MAINT: Remove append_metastr_to_string function. It has been replaced by metastr_to_unicode. --- numpy/core/src/multiarray/_datetime.h | 12 ----- numpy/core/src/multiarray/datetime.c | 66 +-------------------------- 2 files changed, 1 insertion(+), 77 deletions(-) diff --git a/numpy/core/src/multiarray/_datetime.h b/numpy/core/src/multiarray/_datetime.h index be9bac48c811..421b03f93c5a 100644 --- a/numpy/core/src/multiarray/_datetime.h +++ b/numpy/core/src/multiarray/_datetime.h @@ -209,18 +209,6 @@ convert_pyobject_to_datetime_metadata(PyObject *obj, NPY_NO_EXPORT PyObject * metastr_to_unicode(PyArray_DatetimeMetaData *meta, int skip_brackets); -/* - * 'ret' is a PyUString containing the datetime string, and this - * function appends the metadata string to it. - * - * If 'skip_brackets' is true, skips the '[]'. - * - * This function steals the reference 'ret' - */ -NPY_NO_EXPORT PyObject * -append_metastr_to_string(PyArray_DatetimeMetaData *meta, - int skip_brackets, - PyObject *ret); /* * Tests for and converts a Python datetime.datetime or datetime.date diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index 99b6a24e9681..4afc45fb63e8 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -1978,7 +1978,7 @@ metastr_to_unicode(PyArray_DatetimeMetaData *meta, int skip_brackets) if (skip_brackets) { return PyUnicode_FromString("generic"); } - /* But with brackets, append nothing */ + /* But with brackets, return nothing */ else { return PyUnicode_FromString(""); } @@ -2013,70 +2013,6 @@ metastr_to_unicode(PyArray_DatetimeMetaData *meta, int skip_brackets) } -/* - * 'ret' is a PyUString containing the datetime string, and this - * function appends the metadata string to it. - * - * If 'skip_brackets' is true, skips the '[]'. - * - * This function steals the reference 'ret' - */ -NPY_NO_EXPORT PyObject * -append_metastr_to_string(PyArray_DatetimeMetaData *meta, - int skip_brackets, - PyObject *ret) -{ - PyObject *res; - int num; - char const *basestr; - - if (ret == NULL) { - return NULL; - } - - if (meta->base == NPY_FR_GENERIC) { - /* Without brackets, give a string "generic" */ - if (skip_brackets) { - PyUString_ConcatAndDel(&ret, PyUnicode_FromString("generic")); - return ret; - } - /* But with brackets, append nothing */ - else { - return ret; - } - } - - num = meta->num; - if (meta->base >= 0 && meta->base < NPY_DATETIME_NUMUNITS) { - basestr = _datetime_strings[meta->base]; - } - else { - PyErr_SetString(PyExc_RuntimeError, - "NumPy datetime metadata is corrupted"); - return NULL; - } - - if (num == 1) { - if (skip_brackets) { - res = PyUnicode_FromFormat("%s", basestr); - } - else { - res = PyUnicode_FromFormat("[%s]", basestr); - } - } - else { - if (skip_brackets) { - res = PyUnicode_FromFormat("%d%s", num, basestr); - } - else { - res = PyUnicode_FromFormat("[%d%s]", num, basestr); - } - } - - PyUString_ConcatAndDel(&ret, res); - return ret; -} - /* * Adjusts a datetimestruct based on a seconds offset. Assumes * the current values are valid. From 4c74f375936c5271d48ec101676de1b3ab0b743c Mon Sep 17 00:00:00 2001 From: leakec Date: Tue, 29 Sep 2020 12:43:46 -0500 Subject: [PATCH 118/409] BLD: Fixed ARGOUTVIEWM memory deallocation. Closes #17398 --- tools/swig/numpy.i | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tools/swig/numpy.i b/tools/swig/numpy.i index 6b69ce96e0ed..a2e7a335fdb0 100644 --- a/tools/swig/numpy.i +++ b/tools/swig/numpy.i @@ -2492,9 +2492,9 @@ if (!array) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$2), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$2), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -2562,9 +2562,9 @@ if (!array) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$3), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$3), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -2632,9 +2632,9 @@ if (!array || !require_fortran(array)) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$3), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$3), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -2706,9 +2706,9 @@ if (!array) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$4), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$4), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -2780,9 +2780,9 @@ if (!array || !require_fortran(array)) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$4), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$4), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -2856,9 +2856,9 @@ if (!array) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$5), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$5), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -2932,9 +2932,9 @@ if (!array || !require_fortran(array)) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$5), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$5), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -3008,9 +3008,9 @@ if (!array) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$5), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$5), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -3084,9 +3084,9 @@ if (!array || !require_fortran(array)) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$5), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$5), free); %#endif %#if NPY_API_VERSION < 0x00000007 From ab81a55dba9294aac6926d2921b147c791488ae3 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Tue, 29 Sep 2020 15:08:09 -0700 Subject: [PATCH 119/409] DOC: rm incorrect alias from recarray user article. --- doc/source/user/basics.rec.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/user/basics.rec.rst b/doc/source/user/basics.rec.rst index f579b0d85141..7380ef6359dc 100644 --- a/doc/source/user/basics.rec.rst +++ b/doc/source/user/basics.rec.rst @@ -576,7 +576,7 @@ Record Arrays As an optional convenience numpy provides an ndarray subclass, :class:`numpy.recarray`, and associated helper functions in the -:mod:`numpy.lib.recfunctions` submodule (aliased as ``numpy.rec``), that allows +:mod:`numpy.lib.recfunctions` submodule, that allows access to fields of structured arrays by attribute instead of only by index. Record arrays also use a special datatype, :class:`numpy.record`, that allows field access by attribute on the structured scalars obtained from the array. From d1d725b10a568da98ade43e81deeef3e41789ab8 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Tue, 29 Sep 2020 20:06:08 -0700 Subject: [PATCH 120/409] DOC: Add arraysetops to autosummary. * Generate stub for module, fixes 7 broken links * Modify arraysetops module docstring w/ rst formatting * Remove listing of functions from module docstring. --- doc/source/reference/routines.set.rst | 5 +++++ numpy/lib/arraysetops.py | 21 +++++---------------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/doc/source/reference/routines.set.rst b/doc/source/reference/routines.set.rst index b12d3d5f5e79..149c33a8b610 100644 --- a/doc/source/reference/routines.set.rst +++ b/doc/source/reference/routines.set.rst @@ -3,6 +3,11 @@ Set routines .. currentmodule:: numpy +.. autosummary:: + :toctree: generated/ + + lib.arraysetops + Making proper sets ------------------ .. autosummary:: diff --git a/numpy/lib/arraysetops.py b/numpy/lib/arraysetops.py index 6a2ad004cbf0..9464692e0b2b 100644 --- a/numpy/lib/arraysetops.py +++ b/numpy/lib/arraysetops.py @@ -1,28 +1,17 @@ """ Set operations for arrays based on sorting. -:Contains: - unique, - isin, - ediff1d, - intersect1d, - setxor1d, - in1d, - union1d, - setdiff1d - -:Notes: +Notes +----- For floating point arrays, inaccurate results may appear due to usual round-off and floating point comparison issues. Speed could be gained in some operations by an implementation of -sort(), that can provide directly the permutation vectors, avoiding -thus calls to argsort(). +`numpy.sort`, that can provide directly the permutation vectors, thus avoiding +calls to `numpy.argsort`. -To do: Optionally return indices analogously to unique for all functions. - -:Author: Robert Cimrman +Original author: Robert Cimrman """ import functools From 3599af53d7fbf3976020b8443424ed5ffb955693 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Tue, 29 Sep 2020 20:34:42 -0700 Subject: [PATCH 121/409] Incorporate feedback from review. --- doc/source/user/basics.rec.rst | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/doc/source/user/basics.rec.rst b/doc/source/user/basics.rec.rst index 7380ef6359dc..bb4ed89e999a 100644 --- a/doc/source/user/basics.rec.rst +++ b/doc/source/user/basics.rec.rst @@ -575,11 +575,14 @@ Record Arrays ============= As an optional convenience numpy provides an ndarray subclass, -:class:`numpy.recarray`, and associated helper functions in the -:mod:`numpy.lib.recfunctions` submodule, that allows -access to fields of structured arrays by attribute instead of only by index. -Record arrays also use a special datatype, :class:`numpy.record`, that allows +:class:`numpy.recarray` that allows access to fields of structured arrays by +attribute instead of only by index. +Record arrays use a special datatype, :class:`numpy.record`, that allows field access by attribute on the structured scalars obtained from the array. +The :mod:`numpy.rec` module provides functions for creating recarrays from +various objects. +Additional helper functions for creating and manipulating structured arrays +can be found in :mod:`numpy.lib.recfunctions`. The simplest way to create a record array is with ``numpy.rec.array``:: From e13747d8dc27c359f11485b4d6e764d7f8231562 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Tue, 29 Sep 2020 15:31:41 -0600 Subject: [PATCH 122/409] MAINT: Replace PyUString_ConcatAndDel in nditer_constr.c. --- numpy/core/src/multiarray/nditer_constr.c | 234 ++++++++++------------ 1 file changed, 108 insertions(+), 126 deletions(-) diff --git a/numpy/core/src/multiarray/nditer_constr.c b/numpy/core/src/multiarray/nditer_constr.c index 4bc6d2ca1c5c..b379a28ac3ae 100644 --- a/numpy/core/src/multiarray/nditer_constr.c +++ b/numpy/core/src/multiarray/nditer_constr.c @@ -1750,73 +1750,70 @@ npyiter_fill_axisdata(NpyIter *iter, npy_uint32 flags, npyiter_opitflags *op_itf return 1; broadcast_error: { - PyObject *errmsg, *tmp; npy_intp remdims[NPY_MAXDIMS]; - char *tmpstr; if (op_axes == NULL) { - errmsg = PyUnicode_FromString("operands could not be broadcast " - "together with shapes "); - if (errmsg == NULL) { + PyObject *shape1 = PyUnicode_FromString(""); + if (shape1 == NULL) { return 0; } for (iop = 0; iop < nop; ++iop) { if (op[iop] != NULL) { - tmp = convert_shape_to_string(PyArray_NDIM(op[iop]), - PyArray_DIMS(op[iop]), - " "); + int ndims = PyArray_NDIM(op[iop]); + npy_intp *dims = PyArray_DIMS(op[iop]); + PyObject *tmp = convert_shape_to_string(ndims, dims, " "); if (tmp == NULL) { - Py_DECREF(errmsg); + Py_DECREF(shape1); return 0; } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { + Py_SETREF(shape1, PyUnicode_Concat(shape1, tmp)); + Py_DECREF(tmp); + if (shape1 == NULL) { return 0; } } } - if (itershape != NULL) { - tmp = PyUnicode_FromString("and requested shape "); - if (tmp == NULL) { - Py_DECREF(errmsg); - return 0; - } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { - return 0; - } - - tmp = convert_shape_to_string(ndim, itershape, ""); - if (tmp == NULL) { - Py_DECREF(errmsg); - return 0; - } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { + if (itershape == NULL) { + PyErr_Format(PyExc_ValueError, + "operands could not be broadcast together with " + "shapes %S", shape1); + Py_DECREF(shape1); + return 0; + } + else { + PyObject *shape2 = convert_shape_to_string(ndim, itershape, ""); + if (shape2 == NULL) { + Py_DECREF(shape1); return 0; } - + PyErr_Format(PyExc_ValueError, + "operands could not be broadcast together with " + "shapes %S and requested shape %S", shape1, shape2); + Py_DECREF(shape1); + Py_DECREF(shape2); + return 0; } - PyErr_SetObject(PyExc_ValueError, errmsg); - Py_DECREF(errmsg); } else { - errmsg = PyUnicode_FromString("operands could not be broadcast " - "together with remapped shapes " - "[original->remapped]: "); + PyObject *shape1 = PyUnicode_FromString(""); + if (shape1 == NULL) { + return 0; + } for (iop = 0; iop < nop; ++iop) { if (op[iop] != NULL) { int *axes = op_axes[iop]; + int ndims = PyArray_NDIM(op[iop]); + npy_intp *dims = PyArray_DIMS(op[iop]); + char *tmpstr = (axes == NULL) ? " " : "->"; - tmpstr = (axes == NULL) ? " " : "->"; - tmp = convert_shape_to_string(PyArray_NDIM(op[iop]), - PyArray_DIMS(op[iop]), - tmpstr); + PyObject *tmp = convert_shape_to_string(ndims, dims, tmpstr); if (tmp == NULL) { + Py_DECREF(shape1); return 0; } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { + Py_SETREF(shape1, PyUnicode_Concat(shape1, tmp)); + Py_DECREF(tmp); + if (shape1 == NULL) { return 0; } @@ -1831,80 +1828,83 @@ broadcast_error: { remdims[idim] = -1; } } - tmp = convert_shape_to_string(ndim, remdims, " "); + PyObject *tmp = convert_shape_to_string(ndim, remdims, " "); if (tmp == NULL) { + Py_DECREF(shape1); return 0; } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { + Py_SETREF(shape1, PyUnicode_Concat(shape1, tmp)); + Py_DECREF(tmp); + if (shape1 == NULL) { return 0; } } } } - if (itershape != NULL) { - tmp = PyUnicode_FromString("and requested shape "); - if (tmp == NULL) { - Py_DECREF(errmsg); - return 0; - } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { - return 0; - } - - tmp = convert_shape_to_string(ndim, itershape, ""); - if (tmp == NULL) { - Py_DECREF(errmsg); - return 0; - } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { + if (itershape == NULL) { + PyErr_Format(PyExc_ValueError, + "operands could not be broadcast together with " + "remapped shapes [original->remapped]: %S", shape1); + Py_DECREF(shape1); + return 0; + } + else { + PyObject *shape2 = convert_shape_to_string(ndim, itershape, ""); + if (shape2 == NULL) { + Py_DECREF(shape1); return 0; } - + PyErr_Format(PyExc_ValueError, + "operands could not be broadcast together with " + "remapped shapes [original->remapped]: %S and " + "requested shape %S", shape1, shape2); + Py_DECREF(shape1); + Py_DECREF(shape2); + return 0; } - PyErr_SetObject(PyExc_ValueError, errmsg); - Py_DECREF(errmsg); } - - return 0; } operand_different_than_broadcast: { - npy_intp remdims[NPY_MAXDIMS]; - PyObject *errmsg, *tmp; - - /* Start of error message */ - if (op_flags[iop] & NPY_ITER_READONLY) { - errmsg = PyUnicode_FromString("non-broadcastable operand " - "with shape "); - } - else { - errmsg = PyUnicode_FromString("non-broadcastable output " - "operand with shape "); - } - if (errmsg == NULL) { + /* operand shape */ + int ndims = PyArray_NDIM(op[iop]); + npy_intp *dims = PyArray_DIMS(op[iop]); + PyObject *shape1 = convert_shape_to_string(ndims, dims, ""); + if (shape1 == NULL) { return 0; } - /* Operand shape */ - tmp = convert_shape_to_string(PyArray_NDIM(op[iop]), - PyArray_DIMS(op[iop]), ""); - if (tmp == NULL) { + /* Broadcast shape */ + PyObject *shape2 = convert_shape_to_string(ndim, broadcast_shape, ""); + if (shape2 == NULL) { + Py_DECREF(shape1); return 0; } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { + + if (op_axes == NULL || op_axes[iop] == NULL) { + /* operand shape not remapped */ + + if (op_flags[iop] & NPY_ITER_READONLY) { + PyErr_Format(PyExc_ValueError, + "non-broadcastable operand with shape %S doesn't " + "match the broadcast shape %S", shape1, shape2); + } + else { + PyErr_Format(PyExc_ValueError, + "non-broadcastable output operand with shape %S doesn't " + "match the broadcast shape %S", shape1, shape2); + } + Py_DECREF(shape1); + Py_DECREF(shape2); return 0; } - /* Remapped operand shape */ - if (op_axes != NULL && op_axes[iop] != NULL) { - int *axes = op_axes[iop]; + else { + /* operand shape remapped */ + npy_intp remdims[NPY_MAXDIMS]; + int *axes = op_axes[iop]; for (idim = 0; idim < ndim; ++idim) { - npy_intp i = axes[ndim-idim-1]; - + npy_intp i = axes[ndim - idim - 1]; if (i >= 0 && i < PyArray_NDIM(op[iop])) { remdims[idim] = PyArray_DIM(op[iop], i); } @@ -1913,48 +1913,30 @@ operand_different_than_broadcast: { } } - tmp = PyUnicode_FromString(" [remapped to "); - if (tmp == NULL) { - return 0; - } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { + PyObject *shape3 = convert_shape_to_string(ndim, remdims, ""); + if (shape3 == NULL) { + Py_DECREF(shape1); + Py_DECREF(shape2); return 0; } - tmp = convert_shape_to_string(ndim, remdims, "]"); - if (tmp == NULL) { - return 0; + if (op_flags[iop] & NPY_ITER_READONLY) { + PyErr_Format(PyExc_ValueError, + "non-broadcastable operand with shape %S " + "[remapped to %S] doesn't match the broadcast shape %S", + shape1, shape3, shape2); } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { - return 0; + else { + PyErr_Format(PyExc_ValueError, + "non-broadcastable output operand with shape %S " + "[remapped to %S] doesn't match the broadcast shape %S", + shape1, shape3, shape2); } - } - - tmp = PyUnicode_FromString(" doesn't match the broadcast shape "); - if (tmp == NULL) { - return 0; - } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { + Py_DECREF(shape1); + Py_DECREF(shape2); + Py_DECREF(shape3); return 0; } - - /* Broadcast shape */ - tmp = convert_shape_to_string(ndim, broadcast_shape, ""); - if (tmp == NULL) { - return 0; - } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { - return 0; - } - - PyErr_SetObject(PyExc_ValueError, errmsg); - Py_DECREF(errmsg); - - return 0; } } From 6d97fdda4875a723366d58db6c5e5febfc67b5da Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Tue, 29 Sep 2020 12:35:46 -0600 Subject: [PATCH 123/409] MAINT: Replace PyUString_ConcatAndDel in mapping.c. --- numpy/core/src/multiarray/mapping.c | 72 +++++++++++------------------ 1 file changed, 27 insertions(+), 45 deletions(-) diff --git a/numpy/core/src/multiarray/mapping.c b/numpy/core/src/multiarray/mapping.c index 0998a6b495c8..cb5c3823dccf 100644 --- a/numpy/core/src/multiarray/mapping.c +++ b/numpy/core/src/multiarray/mapping.c @@ -1418,10 +1418,7 @@ _get_field_view(PyArrayObject *arr, PyObject *ind, PyArrayObject **view) return 0; } else if (tup == NULL){ - PyObject *errmsg = PyUnicode_FromString("no field of name "); - PyUString_Concat(&errmsg, ind); - PyErr_SetObject(PyExc_ValueError, errmsg); - Py_DECREF(errmsg); + PyErr_Format(PyExc_ValueError, "no field of name %S", ind); return 0; } if (_unpack_field(tup, &fieldtype, &offset) < 0) { @@ -2345,7 +2342,6 @@ mapiter_fill_info(PyArrayMapIterObject *mit, npy_index_info *indices, int consec_status = -1; int axis, broadcast_axis; npy_intp dimension; - PyObject *errmsg, *tmp; for (i = 0; i < mit->nd_fancy; i++) { mit->dimensions[i] = 1; @@ -2433,35 +2429,38 @@ mapiter_fill_info(PyArrayMapIterObject *mit, npy_index_info *indices, return 0; - broadcast_error: +broadcast_error: ; // Declarations cannot follow labels, add empty statement. /* * Attempt to set a meaningful exception. Could also find out * if a boolean index was converted. */ - errmsg = PyUnicode_FromString("shape mismatch: indexing arrays could not " - "be broadcast together with shapes "); + PyObject *errmsg = PyUnicode_FromString(""); if (errmsg == NULL) { return -1; } - for (i = 0; i < index_num; i++) { if (!(indices[i].type & HAS_FANCY)) { continue; } - tmp = convert_shape_to_string( - PyArray_NDIM((PyArrayObject *)indices[i].object), - PyArray_SHAPE((PyArrayObject *)indices[i].object), - " "); + + int ndim = PyArray_NDIM((PyArrayObject *)indices[i].object); + npy_intp *shape = PyArray_SHAPE((PyArrayObject *)indices[i].object); + PyObject *tmp = convert_shape_to_string(ndim, shape, " "); if (tmp == NULL) { + Py_DECREF(errmsg); return -1; } - PyUString_ConcatAndDel(&errmsg, tmp); + + Py_SETREF(errmsg, PyUnicode_Concat(errmsg, tmp)); + Py_DECREF(tmp); if (errmsg == NULL) { return -1; } } - PyErr_SetObject(PyExc_IndexError, errmsg); + PyErr_Format(PyExc_IndexError, + "shape mismatch: indexing arrays could not " + "be broadcast together with shapes %S", errmsg); Py_DECREF(errmsg); return -1; } @@ -2653,7 +2652,6 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, npy_uint32 extra_op_flags, PyArrayObject *extra_op, PyArray_Descr *extra_op_dtype) { - PyObject *errmsg, *tmp; /* For shape reporting on error */ PyArrayObject *original_extra_op = extra_op; @@ -3183,45 +3181,29 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, goto finish; broadcast_error: - errmsg = PyUnicode_FromString("shape mismatch: value array " - "of shape "); - if (errmsg == NULL) { - goto finish; - } - /* Report the shape of the original array if it exists */ if (original_extra_op == NULL) { original_extra_op = extra_op; } - tmp = convert_shape_to_string(PyArray_NDIM(original_extra_op), - PyArray_DIMS(original_extra_op), " "); - if (tmp == NULL) { - goto finish; - } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { + int extra_ndim = PyArray_NDIM(original_extra_op); + npy_intp *extra_dims = PyArray_DIMS(original_extra_op); + PyObject *shape1 = convert_shape_to_string(extra_ndim, extra_dims, " "); + if (shape1 == NULL) { goto finish; } - tmp = PyUnicode_FromString("could not be broadcast to indexing " - "result of shape "); - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { + PyObject *shape2 = convert_shape_to_string(mit->nd, mit->dimensions, ""); + if (shape2 == NULL) + Py_DECREF(shape1); goto finish; - } - tmp = convert_shape_to_string(mit->nd, mit->dimensions, ""); - if (tmp == NULL) { - goto finish; - } - PyUString_ConcatAndDel(&errmsg, tmp); - if (errmsg == NULL) { - goto finish; - } + PyErr_Format(PyExc_ValueError, + "shape mismatch: value array of shape %S could not be broadcast " + "to indexing result of shape %S", shape1, shape2); - PyErr_SetObject(PyExc_ValueError, errmsg); - Py_DECREF(errmsg); + Py_DECREF(shape1); + Py_DECREF(shape2); finish: Py_XDECREF(extra_op); @@ -3320,7 +3302,7 @@ PyArray_MapIterArrayCopyIfOverlap(PyArrayObject * a, PyObject * index, Py_XDECREF(a_copy); Py_XDECREF(subspace); Py_XDECREF((PyObject *)mit); - for (i=0; i < index_num; i++) { + for (i = 0; i < index_num; i++) { Py_XDECREF(indices[i].object); } return NULL; From c909c478f4a1974a573843d912c2027d034770d0 Mon Sep 17 00:00:00 2001 From: Jun Kudo Date: Thu, 1 Oct 2020 03:50:03 +0900 Subject: [PATCH 124/409] ENH: Replace the module-level `__getattr__` with explicit type annotations #17367 --- numpy/__init__.pyi | 320 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 318 insertions(+), 2 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index d4eda6b3150f..7fdd1a5ece74 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -154,8 +154,324 @@ __all__ = [ "var", ] -# TODO: remove when the full numpy namespace is defined -def __getattr__(name: str) -> Any: ... +DataSource: Any +False_: Any +MachAr: Any +ScalarType: Any +True_: Any +UFUNC_PYVALS_NAME: Any +abs: Any +angle: Any +append: Any +apply_along_axis: Any +apply_over_axes: Any +arange: Any +array2string: Any +array_repr: Any +array_split: Any +array_str: Any +asanyarray: Any +asarray: Any +asarray_chkfinite: Any +ascontiguousarray: Any +asfarray: Any +asfortranarray: Any +asmatrix: Any +asscalar: Any +atleast_1d: Any +atleast_2d: Any +atleast_3d: Any +average: Any +bartlett: Any +bincount: Any +bitwise_not: Any +blackman: Any +block: Any +bmat: Any +bool8: Any +broadcast: Any +broadcast_arrays: Any +broadcast_to: Any +busday_count: Any +busday_offset: Any +busdaycalendar: Any +byte: Any +byte_bounds: Any +bytes0: Any +c_: Any +can_cast: Any +cast: Any +cdouble: Any +cfloat: Any +char: Any +chararray: Any +clongdouble: Any +clongfloat: Any +column_stack: Any +common_type: Any +compare_chararrays: Any +compat: Any +complex256: Any +complex_: Any +concatenate: Any +conj: Any +copy: Any +copyto: Any +corrcoef: Any +cov: Any +csingle: Any +ctypeslib: Any +cumproduct: Any +datetime_as_string: Any +datetime_data: Any +delete: Any +deprecate: Any +deprecate_with_doc: Any +diag: Any +diag_indices: Any +diag_indices_from: Any +diagflat: Any +diff: Any +digitize: Any +disp: Any +divide: Any +dot: Any +double: Any +dsplit: Any +dstack: Any +ediff1d: Any +einsum: Any +einsum_path: Any +emath: Any +errstate: Any +expand_dims: Any +extract: Any +eye: Any +fft: Any +fill_diagonal: Any +finfo: Any +fix: Any +flip: Any +fliplr: Any +flipud: Any +float128: Any +float_: Any +format_float_positional: Any +format_float_scientific: Any +format_parser: Any +frombuffer: Any +fromfile: Any +fromiter: Any +frompyfunc: Any +fromregex: Any +fromstring: Any +genfromtxt: Any +geomspace: Any +get_include: Any +get_printoptions: Any +getbufsize: Any +geterr: Any +geterrcall: Any +geterrobj: Any +gradient: Any +half: Any +hamming: Any +hanning: Any +histogram: Any +histogram2d: Any +histogram_bin_edges: Any +histogramdd: Any +hsplit: Any +hstack: Any +i0: Any +iinfo: Any +imag: Any +in1d: Any +index_exp: Any +info: Any +inner: Any +insert: Any +int0: Any +int_: Any +intc: Any +interp: Any +intersect1d: Any +intp: Any +is_busday: Any +iscomplex: Any +iscomplexobj: Any +isin: Any +isneginf: Any +isposinf: Any +isreal: Any +isrealobj: Any +iterable: Any +ix_: Any +kaiser: Any +kron: Any +lexsort: Any +lib: Any +linalg: Any +linspace: Any +load: Any +loads: Any +loadtxt: Any +logspace: Any +longcomplex: Any +longdouble: Any +longfloat: Any +longlong: Any +lookfor: Any +ma: Any +mafromtxt: Any +mask_indices: Any +mat: Any +math: Any +matrix: Any +matrixlib: Any +max: Any +may_share_memory: Any +median: Any +memmap: Any +meshgrid: Any +mgrid: Any +min: Any +min_scalar_type: Any +mintypecode: Any +mod: Any +msort: Any +nan_to_num: Any +nanargmax: Any +nanargmin: Any +nancumprod: Any +nancumsum: Any +nanmax: Any +nanmean: Any +nanmedian: Any +nanmin: Any +nanpercentile: Any +nanprod: Any +nanquantile: Any +nanstd: Any +nansum: Any +nanvar: Any +nbytes: Any +ndenumerate: Any +ndfromtxt: Any +ndindex: Any +nditer: Any +nested_iters: Any +newaxis: Any +numarray: Any +object0: Any +ogrid: Any +packbits: Any +pad: Any +percentile: Any +piecewise: Any +place: Any +poly: Any +poly1d: Any +polyadd: Any +polyder: Any +polydiv: Any +polyfit: Any +polyint: Any +polymul: Any +polynomial: Any +polysub: Any +polyval: Any +printoptions: Any +product: Any +promote_types: Any +put_along_axis: Any +putmask: Any +quantile: Any +r_: Any +random: Any +ravel_multi_index: Any +real: Any +real_if_close: Any +rec: Any +recarray: Any +recfromcsv: Any +recfromtxt: Any +record: Any +require: Any +result_type: Any +roots: Any +rot90: Any +round: Any +round_: Any +row_stack: Any +s_: Any +save: Any +savetxt: Any +savez: Any +savez_compressed: Any +sctypeDict: Any +sctypeNA: Any +sctypes: Any +select: Any +set_printoptions: Any +set_string_function: Any +setbufsize: Any +setdiff1d: Any +seterr: Any +seterrcall: Any +seterrobj: Any +setxor1d: Any +shares_memory: Any +short: Any +show_config: Any +sinc: Any +single: Any +singlecomplex: Any +sort_complex: Any +source: Any +split: Any +stack: Any +str0: Any +string_: Any +sys: Any +take_along_axis: Any +testing: Any +tile: Any +trapz: Any +tri: Any +tril: Any +tril_indices: Any +tril_indices_from: Any +trim_zeros: Any +triu: Any +triu_indices: Any +triu_indices_from: Any +typeDict: Any +typeNA: Any +typecodes: Any +typename: Any +ubyte: Any +uint: Any +uint0: Any +uintc: Any +uintp: Any +ulonglong: Any +unicode_: Any +union1d: Any +unique: Any +unpackbits: Any +unravel_index: Any +unwrap: Any +ushort: Any +vander: Any +vdot: Any +vectorize: Any +version: Any +void0: Any +vsplit: Any +vstack: Any +where: Any +who: Any _NdArraySubClass = TypeVar("_NdArraySubClass", bound=ndarray) _ByteOrder = Literal["S", "<", ">", "=", "|", "L", "B", "N", "I"] From 8528a10d0e0b436f2238039e9c46012395a2958e Mon Sep 17 00:00:00 2001 From: Jun Kudo Date: Thu, 1 Oct 2020 04:44:22 +0900 Subject: [PATCH 125/409] ENH: remove unnecessary annotations in __init__.pyi. --- numpy/__init__.pyi | 3 --- 1 file changed, 3 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 7fdd1a5ece74..6a1c5b72b58e 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -160,7 +160,6 @@ MachAr: Any ScalarType: Any True_: Any UFUNC_PYVALS_NAME: Any -abs: Any angle: Any append: Any apply_along_axis: Any @@ -410,7 +409,6 @@ savetxt: Any savez: Any savez_compressed: Any sctypeDict: Any -sctypeNA: Any sctypes: Any select: Any set_printoptions: Any @@ -447,7 +445,6 @@ triu: Any triu_indices: Any triu_indices_from: Any typeDict: Any -typeNA: Any typecodes: Any typename: Any ubyte: Any From 0996962236506342d16730580e126efe2dfbf98d Mon Sep 17 00:00:00 2001 From: Jun Kudo Date: Thu, 1 Oct 2020 05:02:13 +0900 Subject: [PATCH 126/409] TST: Add test to verify that getting a non-existent objects indeed results in an error. --- numpy/typing/tests/data/fail/modules.py | 315 ++++++++++++++++++++++++ 1 file changed, 315 insertions(+) diff --git a/numpy/typing/tests/data/fail/modules.py b/numpy/typing/tests/data/fail/modules.py index e7ffe89207f0..5b3d3ffb3075 100644 --- a/numpy/typing/tests/data/fail/modules.py +++ b/numpy/typing/tests/data/fail/modules.py @@ -1,3 +1,318 @@ import numpy as np np.testing.bob # E: Module has no attribute +np.DataSource # E: Module has no attribute +np.False_ # E: Module has no attribute +np.MachAr # E: Module has no attribute +np.ScalarType # E: Module has no attribute +np.True_ # E: Module has no attribute +np.UFUNC_PYVALS_NAME # E: Module has no attribute +np.angle # E: Module has no attribute +np.append # E: Module has no attribute +np.apply_along_axis # E: Module has no attribute +np.apply_over_axes # E: Module has no attribute +np.arange # E: Module has no attribute +np.array2string # E: Module has no attribute +np.array_repr # E: Module has no attribute +np.array_split # E: Module has no attribute +np.array_str # E: Module has no attribute +np.asanyarray # E: Module has no attribute +np.asarray # E: Module has no attribute +np.asarray_chkfinite # E: Module has no attribute +np.ascontiguousarray # E: Module has no attribute +np.asfarray # E: Module has no attribute +np.asfortranarray # E: Module has no attribute +np.asmatrix # E: Module has no attribute +np.asscalar # E: Module has no attribute +np.atleast_1d # E: Module has no attribute +np.atleast_2d # E: Module has no attribute +np.atleast_3d # E: Module has no attribute +np.average # E: Module has no attribute +np.bartlett # E: Module has no attribute +np.bincount # E: Module has no attribute +np.bitwise_not # E: Module has no attribute +np.blackman # E: Module has no attribute +np.block # E: Module has no attribute +np.bmat # E: Module has no attribute +np.bool8 # E: Module has no attribute +np.broadcast # E: Module has no attribute +np.broadcast_arrays # E: Module has no attribute +np.broadcast_to # E: Module has no attribute +np.busday_count # E: Module has no attribute +np.busday_offset # E: Module has no attribute +np.busdaycalendar # E: Module has no attribute +np.byte # E: Module has no attribute +np.byte_bounds # E: Module has no attribute +np.bytes0 # E: Module has no attribute +np.c_ # E: Module has no attribute +np.can_cast # E: Module has no attribute +np.cast # E: Module has no attribute +np.cdouble # E: Module has no attribute +np.cfloat # E: Module has no attribute +np.char # E: Module has no attribute +np.chararray # E: Module has no attribute +np.clongdouble # E: Module has no attribute +np.clongfloat # E: Module has no attribute +np.column_stack # E: Module has no attribute +np.common_type # E: Module has no attribute +np.compare_chararrays # E: Module has no attribute +np.compat # E: Module has no attribute +np.complex256 # E: Module has no attribute +np.complex_ # E: Module has no attribute +np.concatenate # E: Module has no attribute +np.conj # E: Module has no attribute +np.copy # E: Module has no attribute +np.copyto # E: Module has no attribute +np.corrcoef # E: Module has no attribute +np.cov # E: Module has no attribute +np.csingle # E: Module has no attribute +np.ctypeslib # E: Module has no attribute +np.cumproduct # E: Module has no attribute +np.datetime_as_string # E: Module has no attribute +np.datetime_data # E: Module has no attribute +np.delete # E: Module has no attribute +np.deprecate # E: Module has no attribute +np.deprecate_with_doc # E: Module has no attribute +np.diag # E: Module has no attribute +np.diag_indices # E: Module has no attribute +np.diag_indices_from # E: Module has no attribute +np.diagflat # E: Module has no attribute +np.diff # E: Module has no attribute +np.digitize # E: Module has no attribute +np.disp # E: Module has no attribute +np.divide # E: Module has no attribute +np.dot # E: Module has no attribute +np.double # E: Module has no attribute +np.dsplit # E: Module has no attribute +np.dstack # E: Module has no attribute +np.ediff1d # E: Module has no attribute +np.einsum # E: Module has no attribute +np.einsum_path # E: Module has no attribute +np.emath # E: Module has no attribute +np.errstate # E: Module has no attribute +np.expand_dims # E: Module has no attribute +np.extract # E: Module has no attribute +np.eye # E: Module has no attribute +np.fft # E: Module has no attribute +np.fill_diagonal # E: Module has no attribute +np.finfo # E: Module has no attribute +np.fix # E: Module has no attribute +np.flip # E: Module has no attribute +np.fliplr # E: Module has no attribute +np.flipud # E: Module has no attribute +np.float128 # E: Module has no attribute +np.float_ # E: Module has no attribute +np.format_float_positional # E: Module has no attribute +np.format_float_scientific # E: Module has no attribute +np.format_parser # E: Module has no attribute +np.frombuffer # E: Module has no attribute +np.fromfile # E: Module has no attribute +np.fromiter # E: Module has no attribute +np.frompyfunc # E: Module has no attribute +np.fromregex # E: Module has no attribute +np.fromstring # E: Module has no attribute +np.genfromtxt # E: Module has no attribute +np.geomspace # E: Module has no attribute +np.get_include # E: Module has no attribute +np.get_printoptions # E: Module has no attribute +np.getbufsize # E: Module has no attribute +np.geterr # E: Module has no attribute +np.geterrcall # E: Module has no attribute +np.geterrobj # E: Module has no attribute +np.gradient # E: Module has no attribute +np.half # E: Module has no attribute +np.hamming # E: Module has no attribute +np.hanning # E: Module has no attribute +np.histogram # E: Module has no attribute +np.histogram2d # E: Module has no attribute +np.histogram_bin_edges # E: Module has no attribute +np.histogramdd # E: Module has no attribute +np.hsplit # E: Module has no attribute +np.hstack # E: Module has no attribute +np.i0 # E: Module has no attribute +np.iinfo # E: Module has no attribute +np.imag # E: Module has no attribute +np.in1d # E: Module has no attribute +np.index_exp # E: Module has no attribute +np.info # E: Module has no attribute +np.inner # E: Module has no attribute +np.insert # E: Module has no attribute +np.int0 # E: Module has no attribute +np.int_ # E: Module has no attribute +np.intc # E: Module has no attribute +np.interp # E: Module has no attribute +np.intersect1d # E: Module has no attribute +np.intp # E: Module has no attribute +np.is_busday # E: Module has no attribute +np.iscomplex # E: Module has no attribute +np.iscomplexobj # E: Module has no attribute +np.isin # E: Module has no attribute +np.isneginf # E: Module has no attribute +np.isposinf # E: Module has no attribute +np.isreal # E: Module has no attribute +np.isrealobj # E: Module has no attribute +np.iterable # E: Module has no attribute +np.ix_ # E: Module has no attribute +np.kaiser # E: Module has no attribute +np.kron # E: Module has no attribute +np.lexsort # E: Module has no attribute +np.lib # E: Module has no attribute +np.linalg # E: Module has no attribute +np.linspace # E: Module has no attribute +np.load # E: Module has no attribute +np.loads # E: Module has no attribute +np.loadtxt # E: Module has no attribute +np.logspace # E: Module has no attribute +np.longcomplex # E: Module has no attribute +np.longdouble # E: Module has no attribute +np.longfloat # E: Module has no attribute +np.longlong # E: Module has no attribute +np.lookfor # E: Module has no attribute +np.ma # E: Module has no attribute +np.mafromtxt # E: Module has no attribute +np.mask_indices # E: Module has no attribute +np.mat # E: Module has no attribute +np.math # E: Module has no attribute +np.matrix # E: Module has no attribute +np.matrixlib # E: Module has no attribute +np.max # E: Module has no attribute +np.may_share_memory # E: Module has no attribute +np.median # E: Module has no attribute +np.memmap # E: Module has no attribute +np.meshgrid # E: Module has no attribute +np.mgrid # E: Module has no attribute +np.min # E: Module has no attribute +np.min_scalar_type # E: Module has no attribute +np.mintypecode # E: Module has no attribute +np.mod # E: Module has no attribute +np.msort # E: Module has no attribute +np.nan_to_num # E: Module has no attribute +np.nanargmax # E: Module has no attribute +np.nanargmin # E: Module has no attribute +np.nancumprod # E: Module has no attribute +np.nancumsum # E: Module has no attribute +np.nanmax # E: Module has no attribute +np.nanmean # E: Module has no attribute +np.nanmedian # E: Module has no attribute +np.nanmin # E: Module has no attribute +np.nanpercentile # E: Module has no attribute +np.nanprod # E: Module has no attribute +np.nanquantile # E: Module has no attribute +np.nanstd # E: Module has no attribute +np.nansum # E: Module has no attribute +np.nanvar # E: Module has no attribute +np.nbytes # E: Module has no attribute +np.ndenumerate # E: Module has no attribute +np.ndfromtxt # E: Module has no attribute +np.ndindex # E: Module has no attribute +np.nditer # E: Module has no attribute +np.nested_iters # E: Module has no attribute +np.newaxis # E: Module has no attribute +np.numarray # E: Module has no attribute +np.object0 # E: Module has no attribute +np.ogrid # E: Module has no attribute +np.packbits # E: Module has no attribute +np.pad # E: Module has no attribute +np.percentile # E: Module has no attribute +np.piecewise # E: Module has no attribute +np.place # E: Module has no attribute +np.poly # E: Module has no attribute +np.poly1d # E: Module has no attribute +np.polyadd # E: Module has no attribute +np.polyder # E: Module has no attribute +np.polydiv # E: Module has no attribute +np.polyfit # E: Module has no attribute +np.polyint # E: Module has no attribute +np.polymul # E: Module has no attribute +np.polynomial # E: Module has no attribute +np.polysub # E: Module has no attribute +np.polyval # E: Module has no attribute +np.printoptions # E: Module has no attribute +np.product # E: Module has no attribute +np.promote_types # E: Module has no attribute +np.put_along_axis # E: Module has no attribute +np.putmask # E: Module has no attribute +np.quantile # E: Module has no attribute +np.r_ # E: Module has no attribute +np.random # E: Module has no attribute +np.ravel_multi_index # E: Module has no attribute +np.real # E: Module has no attribute +np.real_if_close # E: Module has no attribute +np.rec # E: Module has no attribute +np.recarray # E: Module has no attribute +np.recfromcsv # E: Module has no attribute +np.recfromtxt # E: Module has no attribute +np.record # E: Module has no attribute +np.require # E: Module has no attribute +np.result_type # E: Module has no attribute +np.roots # E: Module has no attribute +np.rot90 # E: Module has no attribute +np.round # E: Module has no attribute +np.round_ # E: Module has no attribute +np.row_stack # E: Module has no attribute +np.s_ # E: Module has no attribute +np.save # E: Module has no attribute +np.savetxt # E: Module has no attribute +np.savez # E: Module has no attribute +np.savez_compressed # E: Module has no attribute +np.sctypeDict # E: Module has no attribute +np.sctypes # E: Module has no attribute +np.select # E: Module has no attribute +np.set_printoptions # E: Module has no attribute +np.set_string_function # E: Module has no attribute +np.setbufsize # E: Module has no attribute +np.setdiff1d # E: Module has no attribute +np.seterr # E: Module has no attribute +np.seterrcall # E: Module has no attribute +np.seterrobj # E: Module has no attribute +np.setxor1d # E: Module has no attribute +np.shares_memory # E: Module has no attribute +np.short # E: Module has no attribute +np.show_config # E: Module has no attribute +np.sinc # E: Module has no attribute +np.single # E: Module has no attribute +np.singlecomplex # E: Module has no attribute +np.sort_complex # E: Module has no attribute +np.source # E: Module has no attribute +np.split # E: Module has no attribute +np.stack # E: Module has no attribute +np.str0 # E: Module has no attribute +np.string_ # E: Module has no attribute +np.sys # E: Module has no attribute +np.take_along_axis # E: Module has no attribute +np.testing # E: Module has no attribute +np.tile # E: Module has no attribute +np.trapz # E: Module has no attribute +np.tri # E: Module has no attribute +np.tril # E: Module has no attribute +np.tril_indices # E: Module has no attribute +np.tril_indices_from # E: Module has no attribute +np.trim_zeros # E: Module has no attribute +np.triu # E: Module has no attribute +np.triu_indices # E: Module has no attribute +np.triu_indices_from # E: Module has no attribute +np.typeDict # E: Module has no attribute +np.typecodes # E: Module has no attribute +np.typename # E: Module has no attribute +np.ubyte # E: Module has no attribute +np.uint # E: Module has no attribute +np.uint0 # E: Module has no attribute +np.uintc # E: Module has no attribute +np.uintp # E: Module has no attribute +np.ulonglong # E: Module has no attribute +np.unicode_ # E: Module has no attribute +np.union1d # E: Module has no attribute +np.unique # E: Module has no attribute +np.unpackbits # E: Module has no attribute +np.unravel_index # E: Module has no attribute +np.unwrap # E: Module has no attribute +np.ushort # E: Module has no attribute +np.vander # E: Module has no attribute +np.vdot # E: Module has no attribute +np.vectorize # E: Module has no attribute +np.version # E: Module has no attribute +np.void0 # E: Module has no attribute +np.vsplit # E: Module has no attribute +np.vstack # E: Module has no attribute +np.where # E: Module has no attribute +np.who # E: Module has no attribute \ No newline at end of file From d120110a0e0dff81a54a982f82574f1e072c345f Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Wed, 30 Sep 2020 21:35:26 +0100 Subject: [PATCH 127/409] DOC: in PR template, set expectations for PR review timeline --- .github/PULL_REQUEST_TEMPLATE.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 528580a8e6df..3634292f8ce7 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -9,3 +9,10 @@ http://www.numpy.org/devdocs/dev/development_workflow.html#writing-the-commit-me + + From 618456137a265d0dd84b85dd33a588bc69babc99 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Wed, 30 Sep 2020 15:18:59 -0600 Subject: [PATCH 128/409] MAINT: Cleanup PyUString_ConcatAndDel in scalartypes.c.src. --- numpy/core/src/multiarray/scalartypes.c.src | 38 ++++++++++----------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src index 74ee260afd2e..1a50927a8cd4 100644 --- a/numpy/core/src/multiarray/scalartypes.c.src +++ b/numpy/core/src/multiarray/scalartypes.c.src @@ -660,14 +660,12 @@ timedeltatype_str(PyObject *self) * Can't use "%lld" if HAVE_LONG_LONG is not defined */ #if defined(HAVE_LONG_LONG) - ret = PyUnicode_FromFormat("%lld ", - (long long)(scal->obval * scal->obmeta.num)); + ret = PyUnicode_FromFormat("%lld %s", + (long long)(scal->obval * scal->obmeta.num), basestr); #else - ret = PyUnicode_FromFormat("%ld ", - (long)(scal->obval * scal->obmeta.num)); + ret = PyUnicode_FromFormat("%ld %s", + (long)(scal->obval * scal->obmeta.num), basestr); #endif - PyUString_ConcatAndDel(&ret, - PyUnicode_FromString(basestr)); } return ret; @@ -886,7 +884,7 @@ static PyObject * static PyObject * c@name@type_@kind@(PyObject *self) { - PyObject *rstr, *istr, *ret; + PyObject *rstr, *istr; npy_c@name@ val = PyArrayScalar_VAL(self, C@Name@); TrimMode trim = TrimMode_DptZeros; @@ -899,16 +897,13 @@ c@name@type_@kind@(PyObject *self) if (istr == NULL) { return NULL; } - - PyUString_ConcatAndDel(&istr, PyUnicode_FromString("j")); - return istr; + PyObject *ret = PyUnicode_FromFormat("%Sj", istr); + Py_DECREF(istr); + return ret; } if (npy_isfinite(val.real)) { rstr = @name@type_@kind@_either(val.real, trim, trim, 0); - if (rstr == NULL) { - return NULL; - } } else if (npy_isnan(val.real)) { rstr = PyUnicode_FromString("nan"); @@ -919,12 +914,12 @@ c@name@type_@kind@(PyObject *self) else { rstr = PyUnicode_FromString("-inf"); } + if (rstr == NULL) { + return NULL; + } if (npy_isfinite(val.imag)) { istr = @name@type_@kind@_either(val.imag, trim, trim, 1); - if (istr == NULL) { - return NULL; - } } else if (npy_isnan(val.imag)) { istr = PyUnicode_FromString("+nan"); @@ -935,11 +930,14 @@ c@name@type_@kind@(PyObject *self) else { istr = PyUnicode_FromString("-inf"); } + if (istr == NULL) { + Py_DECREF(rstr); + return NULL; + } - ret = PyUnicode_FromString("("); - PyUString_ConcatAndDel(&ret, rstr); - PyUString_ConcatAndDel(&ret, istr); - PyUString_ConcatAndDel(&ret, PyUnicode_FromString("j)")); + PyObject *ret = PyUnicode_FromFormat("(%S%Sj)", rstr, istr); + Py_DECREF(rstr); + Py_DECREF(istr); return ret; } From fc6b5916dc261cc8deb953a0ca85828ddfff36cc Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Wed, 30 Sep 2020 16:00:11 -0600 Subject: [PATCH 129/409] MAINT: Replace PyUString_ConcatAndDel in shape.c. --- numpy/core/src/multiarray/shape.c | 36 +++++++++++++------------------ 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/numpy/core/src/multiarray/shape.c b/numpy/core/src/multiarray/shape.c index 1a38fe956f4d..9dda899131d9 100644 --- a/numpy/core/src/multiarray/shape.c +++ b/numpy/core/src/multiarray/shape.c @@ -458,14 +458,12 @@ _attempt_nocopy_reshape(PyArrayObject *self, int newnd, const npy_intp *newdims, static void raise_reshape_size_mismatch(PyArray_Dims *newshape, PyArrayObject *arr) { - PyObject *msg = PyUnicode_FromFormat("cannot reshape array of size %zd " - "into shape ", PyArray_SIZE(arr)); PyObject *tmp = convert_shape_to_string(newshape->len, newshape->ptr, ""); - - PyUString_ConcatAndDel(&msg, tmp); - if (msg != NULL) { - PyErr_SetObject(PyExc_ValueError, msg); - Py_DECREF(msg); + if (tmp != NULL) { + PyErr_Format(PyExc_ValueError, + "cannot reshape array of size %zd into shape %S", + PyArray_SIZE(arr), tmp); + Py_DECREF(tmp); } } @@ -984,29 +982,25 @@ NPY_NO_EXPORT PyObject * build_shape_string(npy_intp n, npy_intp const *vals) { npy_intp i; - PyObject *ret, *tmp; /* * Negative dimension indicates "newaxis", which can * be discarded for printing if it's a leading dimension. * Find the first non-"newaxis" dimension. */ - i = 0; - while (i < n && vals[i] < 0) { - ++i; - } + for (i = 0; i < n && vals[i] < 0; ++i); if (i == n) { return PyUnicode_FromFormat("()"); } - else { - ret = PyUnicode_FromFormat("(%" NPY_INTP_FMT, vals[i++]); - if (ret == NULL) { - return NULL; - } - } + PyObject *ret = PyUnicode_FromFormat("%" NPY_INTP_FMT, vals[i++]); + if (ret == NULL) { + return NULL; + } for (; i < n; ++i) { + PyObject *tmp; + if (vals[i] < 0) { tmp = PyUnicode_FromString(",newaxis"); } @@ -1018,14 +1012,14 @@ build_shape_string(npy_intp n, npy_intp const *vals) return NULL; } - PyUString_ConcatAndDel(&ret, tmp); + Py_SETREF(ret, PyUnicode_Concat(ret, tmp)); + Py_DECREF(tmp); if (ret == NULL) { return NULL; } } - tmp = PyUnicode_FromFormat(")"); - PyUString_ConcatAndDel(&ret, tmp); + Py_SETREF(ret, PyUnicode_FromFormat("(%S)", ret)); return ret; } From b8b9b56a8112abd56b33a4dcdbfa9ed651c50e0d Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Wed, 30 Sep 2020 16:56:08 -0600 Subject: [PATCH 130/409] MAINT: Replace PyUString_ConcatAndDel in array_assign.c --- numpy/core/src/common/array_assign.c | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/numpy/core/src/common/array_assign.c b/numpy/core/src/common/array_assign.c index 67abcae24268..c55f6bdb4624 100644 --- a/numpy/core/src/common/array_assign.c +++ b/numpy/core/src/common/array_assign.c @@ -64,19 +64,22 @@ broadcast_strides(int ndim, npy_intp const *shape, return 0; broadcast_error: { - PyObject *errmsg; - - errmsg = PyUnicode_FromFormat("could not broadcast %s from shape ", - strides_name); - PyUString_ConcatAndDel(&errmsg, - build_shape_string(strides_ndim, strides_shape)); - PyUString_ConcatAndDel(&errmsg, - PyUnicode_FromString(" into shape ")); - PyUString_ConcatAndDel(&errmsg, - build_shape_string(ndim, shape)); - PyErr_SetObject(PyExc_ValueError, errmsg); - Py_DECREF(errmsg); + PyObject *shape1 = convert_shape_to_string(strides_ndim, + strides_shape, ""); + if (shape1 == NULL) { + return -1; + } + PyObject *shape2 = convert_shape_to_string(ndim, shape, ""); + if (shape2 == NULL) { + Py_DECREF(shape1); + return -1; + } + PyErr_Format(PyExc_ValueError, + "could not broadcast %s from shape %S into shape %S", + strides_name, shape1, shape2); + Py_DECREF(shape1); + Py_DECREF(shape2); return -1; } } From 278ed3eb540529c5cb5dc30c317c9e4558c05c64 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Wed, 30 Sep 2020 17:36:20 -0600 Subject: [PATCH 131/409] MAINT: Replace PyUString_ConcatAndDel in common.c. --- numpy/core/src/multiarray/common.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/numpy/core/src/multiarray/common.c b/numpy/core/src/multiarray/common.c index 6af71f351ab0..841ed799db54 100644 --- a/numpy/core/src/multiarray/common.c +++ b/numpy/core/src/multiarray/common.c @@ -233,7 +233,6 @@ NPY_NO_EXPORT PyObject * convert_shape_to_string(npy_intp n, npy_intp const *vals, char *ending) { npy_intp i; - PyObject *ret, *tmp; /* * Negative dimension indicates "newaxis", which can @@ -245,14 +244,14 @@ convert_shape_to_string(npy_intp n, npy_intp const *vals, char *ending) if (i == n) { return PyUnicode_FromFormat("()%s", ending); } - else { - ret = PyUnicode_FromFormat("(%" NPY_INTP_FMT, vals[i++]); - if (ret == NULL) { - return NULL; - } - } + PyObject *ret = PyUnicode_FromFormat("%" NPY_INTP_FMT, vals[i++]); + if (ret == NULL) { + return NULL; + } for (; i < n; ++i) { + PyObject *tmp; + if (vals[i] < 0) { tmp = PyUnicode_FromString(",newaxis"); } @@ -264,19 +263,19 @@ convert_shape_to_string(npy_intp n, npy_intp const *vals, char *ending) return NULL; } - PyUString_ConcatAndDel(&ret, tmp); + Py_SETREF(ret, PyUnicode_Concat(ret, tmp)); + Py_DECREF(tmp); if (ret == NULL) { return NULL; } } if (i == 1) { - tmp = PyUnicode_FromFormat(",)%s", ending); + Py_SETREF(ret, PyUnicode_FromFormat("(%S,)%s", ret, ending)); } else { - tmp = PyUnicode_FromFormat(")%s", ending); + Py_SETREF(ret, PyUnicode_FromFormat("(%S)%s", ret, ending)); } - PyUString_ConcatAndDel(&ret, tmp); return ret; } From d79cbd8c1c818d9135e1d3b69526d7f04f259922 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Wed, 30 Sep 2020 18:03:41 -0600 Subject: [PATCH 132/409] MAINT: Replace PyUString_ConcatAndDel in descriptor.c. --- numpy/core/src/multiarray/descriptor.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c index 257ededaecac..24a3507bc408 100644 --- a/numpy/core/src/multiarray/descriptor.c +++ b/numpy/core/src/multiarray/descriptor.c @@ -2898,14 +2898,13 @@ arraydescr_setstate(PyArray_Descr *self, PyObject *args) } if (PyDataType_ISDATETIME(self) && (metadata != NULL)) { - PyObject *old_metadata, *errmsg; + PyObject *old_metadata; PyArray_DatetimeMetaData temp_dt_data; if ((! PyTuple_Check(metadata)) || (PyTuple_Size(metadata) != 2)) { - errmsg = PyUnicode_FromString("Invalid datetime dtype (metadata, c_metadata): "); - PyUString_ConcatAndDel(&errmsg, PyObject_Repr(metadata)); - PyErr_SetObject(PyExc_ValueError, errmsg); - Py_DECREF(errmsg); + PyErr_Format(PyExc_ValueError, + "Invalid datetime dtype (metadata, c_metadata): %R", + metadata); return NULL; } From 2dd8693721f9781d4837ed14461d122bdd5c7874 Mon Sep 17 00:00:00 2001 From: Jun Kudo Date: Thu, 1 Oct 2020 11:24:21 +0900 Subject: [PATCH 133/409] Revert "TST: Add test to verify that getting a non-existent objects indeed results in an error." This reverts commit 0996962236506342d16730580e126efe2dfbf98d. --- numpy/typing/tests/data/fail/modules.py | 315 ------------------------ 1 file changed, 315 deletions(-) diff --git a/numpy/typing/tests/data/fail/modules.py b/numpy/typing/tests/data/fail/modules.py index 5b3d3ffb3075..e7ffe89207f0 100644 --- a/numpy/typing/tests/data/fail/modules.py +++ b/numpy/typing/tests/data/fail/modules.py @@ -1,318 +1,3 @@ import numpy as np np.testing.bob # E: Module has no attribute -np.DataSource # E: Module has no attribute -np.False_ # E: Module has no attribute -np.MachAr # E: Module has no attribute -np.ScalarType # E: Module has no attribute -np.True_ # E: Module has no attribute -np.UFUNC_PYVALS_NAME # E: Module has no attribute -np.angle # E: Module has no attribute -np.append # E: Module has no attribute -np.apply_along_axis # E: Module has no attribute -np.apply_over_axes # E: Module has no attribute -np.arange # E: Module has no attribute -np.array2string # E: Module has no attribute -np.array_repr # E: Module has no attribute -np.array_split # E: Module has no attribute -np.array_str # E: Module has no attribute -np.asanyarray # E: Module has no attribute -np.asarray # E: Module has no attribute -np.asarray_chkfinite # E: Module has no attribute -np.ascontiguousarray # E: Module has no attribute -np.asfarray # E: Module has no attribute -np.asfortranarray # E: Module has no attribute -np.asmatrix # E: Module has no attribute -np.asscalar # E: Module has no attribute -np.atleast_1d # E: Module has no attribute -np.atleast_2d # E: Module has no attribute -np.atleast_3d # E: Module has no attribute -np.average # E: Module has no attribute -np.bartlett # E: Module has no attribute -np.bincount # E: Module has no attribute -np.bitwise_not # E: Module has no attribute -np.blackman # E: Module has no attribute -np.block # E: Module has no attribute -np.bmat # E: Module has no attribute -np.bool8 # E: Module has no attribute -np.broadcast # E: Module has no attribute -np.broadcast_arrays # E: Module has no attribute -np.broadcast_to # E: Module has no attribute -np.busday_count # E: Module has no attribute -np.busday_offset # E: Module has no attribute -np.busdaycalendar # E: Module has no attribute -np.byte # E: Module has no attribute -np.byte_bounds # E: Module has no attribute -np.bytes0 # E: Module has no attribute -np.c_ # E: Module has no attribute -np.can_cast # E: Module has no attribute -np.cast # E: Module has no attribute -np.cdouble # E: Module has no attribute -np.cfloat # E: Module has no attribute -np.char # E: Module has no attribute -np.chararray # E: Module has no attribute -np.clongdouble # E: Module has no attribute -np.clongfloat # E: Module has no attribute -np.column_stack # E: Module has no attribute -np.common_type # E: Module has no attribute -np.compare_chararrays # E: Module has no attribute -np.compat # E: Module has no attribute -np.complex256 # E: Module has no attribute -np.complex_ # E: Module has no attribute -np.concatenate # E: Module has no attribute -np.conj # E: Module has no attribute -np.copy # E: Module has no attribute -np.copyto # E: Module has no attribute -np.corrcoef # E: Module has no attribute -np.cov # E: Module has no attribute -np.csingle # E: Module has no attribute -np.ctypeslib # E: Module has no attribute -np.cumproduct # E: Module has no attribute -np.datetime_as_string # E: Module has no attribute -np.datetime_data # E: Module has no attribute -np.delete # E: Module has no attribute -np.deprecate # E: Module has no attribute -np.deprecate_with_doc # E: Module has no attribute -np.diag # E: Module has no attribute -np.diag_indices # E: Module has no attribute -np.diag_indices_from # E: Module has no attribute -np.diagflat # E: Module has no attribute -np.diff # E: Module has no attribute -np.digitize # E: Module has no attribute -np.disp # E: Module has no attribute -np.divide # E: Module has no attribute -np.dot # E: Module has no attribute -np.double # E: Module has no attribute -np.dsplit # E: Module has no attribute -np.dstack # E: Module has no attribute -np.ediff1d # E: Module has no attribute -np.einsum # E: Module has no attribute -np.einsum_path # E: Module has no attribute -np.emath # E: Module has no attribute -np.errstate # E: Module has no attribute -np.expand_dims # E: Module has no attribute -np.extract # E: Module has no attribute -np.eye # E: Module has no attribute -np.fft # E: Module has no attribute -np.fill_diagonal # E: Module has no attribute -np.finfo # E: Module has no attribute -np.fix # E: Module has no attribute -np.flip # E: Module has no attribute -np.fliplr # E: Module has no attribute -np.flipud # E: Module has no attribute -np.float128 # E: Module has no attribute -np.float_ # E: Module has no attribute -np.format_float_positional # E: Module has no attribute -np.format_float_scientific # E: Module has no attribute -np.format_parser # E: Module has no attribute -np.frombuffer # E: Module has no attribute -np.fromfile # E: Module has no attribute -np.fromiter # E: Module has no attribute -np.frompyfunc # E: Module has no attribute -np.fromregex # E: Module has no attribute -np.fromstring # E: Module has no attribute -np.genfromtxt # E: Module has no attribute -np.geomspace # E: Module has no attribute -np.get_include # E: Module has no attribute -np.get_printoptions # E: Module has no attribute -np.getbufsize # E: Module has no attribute -np.geterr # E: Module has no attribute -np.geterrcall # E: Module has no attribute -np.geterrobj # E: Module has no attribute -np.gradient # E: Module has no attribute -np.half # E: Module has no attribute -np.hamming # E: Module has no attribute -np.hanning # E: Module has no attribute -np.histogram # E: Module has no attribute -np.histogram2d # E: Module has no attribute -np.histogram_bin_edges # E: Module has no attribute -np.histogramdd # E: Module has no attribute -np.hsplit # E: Module has no attribute -np.hstack # E: Module has no attribute -np.i0 # E: Module has no attribute -np.iinfo # E: Module has no attribute -np.imag # E: Module has no attribute -np.in1d # E: Module has no attribute -np.index_exp # E: Module has no attribute -np.info # E: Module has no attribute -np.inner # E: Module has no attribute -np.insert # E: Module has no attribute -np.int0 # E: Module has no attribute -np.int_ # E: Module has no attribute -np.intc # E: Module has no attribute -np.interp # E: Module has no attribute -np.intersect1d # E: Module has no attribute -np.intp # E: Module has no attribute -np.is_busday # E: Module has no attribute -np.iscomplex # E: Module has no attribute -np.iscomplexobj # E: Module has no attribute -np.isin # E: Module has no attribute -np.isneginf # E: Module has no attribute -np.isposinf # E: Module has no attribute -np.isreal # E: Module has no attribute -np.isrealobj # E: Module has no attribute -np.iterable # E: Module has no attribute -np.ix_ # E: Module has no attribute -np.kaiser # E: Module has no attribute -np.kron # E: Module has no attribute -np.lexsort # E: Module has no attribute -np.lib # E: Module has no attribute -np.linalg # E: Module has no attribute -np.linspace # E: Module has no attribute -np.load # E: Module has no attribute -np.loads # E: Module has no attribute -np.loadtxt # E: Module has no attribute -np.logspace # E: Module has no attribute -np.longcomplex # E: Module has no attribute -np.longdouble # E: Module has no attribute -np.longfloat # E: Module has no attribute -np.longlong # E: Module has no attribute -np.lookfor # E: Module has no attribute -np.ma # E: Module has no attribute -np.mafromtxt # E: Module has no attribute -np.mask_indices # E: Module has no attribute -np.mat # E: Module has no attribute -np.math # E: Module has no attribute -np.matrix # E: Module has no attribute -np.matrixlib # E: Module has no attribute -np.max # E: Module has no attribute -np.may_share_memory # E: Module has no attribute -np.median # E: Module has no attribute -np.memmap # E: Module has no attribute -np.meshgrid # E: Module has no attribute -np.mgrid # E: Module has no attribute -np.min # E: Module has no attribute -np.min_scalar_type # E: Module has no attribute -np.mintypecode # E: Module has no attribute -np.mod # E: Module has no attribute -np.msort # E: Module has no attribute -np.nan_to_num # E: Module has no attribute -np.nanargmax # E: Module has no attribute -np.nanargmin # E: Module has no attribute -np.nancumprod # E: Module has no attribute -np.nancumsum # E: Module has no attribute -np.nanmax # E: Module has no attribute -np.nanmean # E: Module has no attribute -np.nanmedian # E: Module has no attribute -np.nanmin # E: Module has no attribute -np.nanpercentile # E: Module has no attribute -np.nanprod # E: Module has no attribute -np.nanquantile # E: Module has no attribute -np.nanstd # E: Module has no attribute -np.nansum # E: Module has no attribute -np.nanvar # E: Module has no attribute -np.nbytes # E: Module has no attribute -np.ndenumerate # E: Module has no attribute -np.ndfromtxt # E: Module has no attribute -np.ndindex # E: Module has no attribute -np.nditer # E: Module has no attribute -np.nested_iters # E: Module has no attribute -np.newaxis # E: Module has no attribute -np.numarray # E: Module has no attribute -np.object0 # E: Module has no attribute -np.ogrid # E: Module has no attribute -np.packbits # E: Module has no attribute -np.pad # E: Module has no attribute -np.percentile # E: Module has no attribute -np.piecewise # E: Module has no attribute -np.place # E: Module has no attribute -np.poly # E: Module has no attribute -np.poly1d # E: Module has no attribute -np.polyadd # E: Module has no attribute -np.polyder # E: Module has no attribute -np.polydiv # E: Module has no attribute -np.polyfit # E: Module has no attribute -np.polyint # E: Module has no attribute -np.polymul # E: Module has no attribute -np.polynomial # E: Module has no attribute -np.polysub # E: Module has no attribute -np.polyval # E: Module has no attribute -np.printoptions # E: Module has no attribute -np.product # E: Module has no attribute -np.promote_types # E: Module has no attribute -np.put_along_axis # E: Module has no attribute -np.putmask # E: Module has no attribute -np.quantile # E: Module has no attribute -np.r_ # E: Module has no attribute -np.random # E: Module has no attribute -np.ravel_multi_index # E: Module has no attribute -np.real # E: Module has no attribute -np.real_if_close # E: Module has no attribute -np.rec # E: Module has no attribute -np.recarray # E: Module has no attribute -np.recfromcsv # E: Module has no attribute -np.recfromtxt # E: Module has no attribute -np.record # E: Module has no attribute -np.require # E: Module has no attribute -np.result_type # E: Module has no attribute -np.roots # E: Module has no attribute -np.rot90 # E: Module has no attribute -np.round # E: Module has no attribute -np.round_ # E: Module has no attribute -np.row_stack # E: Module has no attribute -np.s_ # E: Module has no attribute -np.save # E: Module has no attribute -np.savetxt # E: Module has no attribute -np.savez # E: Module has no attribute -np.savez_compressed # E: Module has no attribute -np.sctypeDict # E: Module has no attribute -np.sctypes # E: Module has no attribute -np.select # E: Module has no attribute -np.set_printoptions # E: Module has no attribute -np.set_string_function # E: Module has no attribute -np.setbufsize # E: Module has no attribute -np.setdiff1d # E: Module has no attribute -np.seterr # E: Module has no attribute -np.seterrcall # E: Module has no attribute -np.seterrobj # E: Module has no attribute -np.setxor1d # E: Module has no attribute -np.shares_memory # E: Module has no attribute -np.short # E: Module has no attribute -np.show_config # E: Module has no attribute -np.sinc # E: Module has no attribute -np.single # E: Module has no attribute -np.singlecomplex # E: Module has no attribute -np.sort_complex # E: Module has no attribute -np.source # E: Module has no attribute -np.split # E: Module has no attribute -np.stack # E: Module has no attribute -np.str0 # E: Module has no attribute -np.string_ # E: Module has no attribute -np.sys # E: Module has no attribute -np.take_along_axis # E: Module has no attribute -np.testing # E: Module has no attribute -np.tile # E: Module has no attribute -np.trapz # E: Module has no attribute -np.tri # E: Module has no attribute -np.tril # E: Module has no attribute -np.tril_indices # E: Module has no attribute -np.tril_indices_from # E: Module has no attribute -np.trim_zeros # E: Module has no attribute -np.triu # E: Module has no attribute -np.triu_indices # E: Module has no attribute -np.triu_indices_from # E: Module has no attribute -np.typeDict # E: Module has no attribute -np.typecodes # E: Module has no attribute -np.typename # E: Module has no attribute -np.ubyte # E: Module has no attribute -np.uint # E: Module has no attribute -np.uint0 # E: Module has no attribute -np.uintc # E: Module has no attribute -np.uintp # E: Module has no attribute -np.ulonglong # E: Module has no attribute -np.unicode_ # E: Module has no attribute -np.union1d # E: Module has no attribute -np.unique # E: Module has no attribute -np.unpackbits # E: Module has no attribute -np.unravel_index # E: Module has no attribute -np.unwrap # E: Module has no attribute -np.ushort # E: Module has no attribute -np.vander # E: Module has no attribute -np.vdot # E: Module has no attribute -np.vectorize # E: Module has no attribute -np.version # E: Module has no attribute -np.void0 # E: Module has no attribute -np.vsplit # E: Module has no attribute -np.vstack # E: Module has no attribute -np.where # E: Module has no attribute -np.who # E: Module has no attribute \ No newline at end of file From 72172ce9fd0fc11378cb65d7ed033407ac722ca7 Mon Sep 17 00:00:00 2001 From: Jun Kudo Date: Thu, 1 Oct 2020 14:43:40 +0900 Subject: [PATCH 134/409] TST: add test to verify that getting a non-existent objects of top-level indeed results in an error --- numpy/typing/tests/data/fail/modules.py | 1 + 1 file changed, 1 insertion(+) diff --git a/numpy/typing/tests/data/fail/modules.py b/numpy/typing/tests/data/fail/modules.py index e7ffe89207f0..be031e6e15e5 100644 --- a/numpy/typing/tests/data/fail/modules.py +++ b/numpy/typing/tests/data/fail/modules.py @@ -1,3 +1,4 @@ import numpy as np np.testing.bob # E: Module has no attribute +np.bob # E: Module has no attribute \ No newline at end of file From 5d8d296e9b7694c1254792dac564be14d47a46a9 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Thu, 1 Oct 2020 10:56:53 +0200 Subject: [PATCH 135/409] TST: Mark the typing tests as slow --- numpy/typing/tests/test_typing.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/numpy/typing/tests/test_typing.py b/numpy/typing/tests/test_typing.py index beb53ddecefa..cba1dc1be7eb 100644 --- a/numpy/typing/tests/test_typing.py +++ b/numpy/typing/tests/test_typing.py @@ -36,6 +36,7 @@ def get_test_cases(directory): ) +@pytest.mark.slow @pytest.mark.skipif(NO_MYPY, reason="Mypy is not installed") @pytest.mark.parametrize("path", get_test_cases(PASS_DIR)) def test_success(path): @@ -50,6 +51,7 @@ def test_success(path): assert re.match(r"Success: no issues found in \d+ source files?", stdout.strip()) +@pytest.mark.slow @pytest.mark.skipif(NO_MYPY, reason="Mypy is not installed") @pytest.mark.parametrize("path", get_test_cases(FAIL_DIR)) def test_fail(path): @@ -99,6 +101,7 @@ def test_fail(path): pytest.fail(f"Error {repr(errors[lineno])} not found") +@pytest.mark.slow @pytest.mark.skipif(NO_MYPY, reason="Mypy is not installed") @pytest.mark.parametrize("path", get_test_cases(REVEAL_DIR)) def test_reveal(path): @@ -130,6 +133,7 @@ def test_reveal(path): assert marker in error_line +@pytest.mark.slow @pytest.mark.skipif(NO_MYPY, reason="Mypy is not installed") @pytest.mark.parametrize("path", get_test_cases(PASS_DIR)) def test_code_runs(path): From d03a906ed6d220fb9a359c51bf8413dfb3012a49 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Sun, 30 Aug 2020 00:45:55 +0200 Subject: [PATCH 136/409] ENH,WIP: Added type hints to `np.core._asarray` --- numpy/core/_asarray.pyi | 70 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 numpy/core/_asarray.pyi diff --git a/numpy/core/_asarray.pyi b/numpy/core/_asarray.pyi new file mode 100644 index 000000000000..bcfc5dbd13d4 --- /dev/null +++ b/numpy/core/_asarray.pyi @@ -0,0 +1,70 @@ +import sys +from typing import TypeVar, Optional, Union, Iterable, Tuple, overload + +from numpy import ndarray +from numpy.typing import ArrayLike, DtypeLike + +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + +_ArrayType = TypeVar("_ArrayType", bound=ndarray) + +def asarray( + a: object, + dtype: DtypeLike = ..., + order: Optional[str] = ..., + *, + like: ArrayLike = ... +) -> ndarray: ... +@overload +def asanyarray( + a: _ArrayType, + dtype: None = ..., + order: Optional[str] = ..., + *, + like: ArrayLike = ... +) -> _ArrayType: ... +@overload +def asanyarray( + a: object, + dtype: DtypeLike = ..., + order: Optional[str] = ..., + *, + like: ArrayLike = ... +) -> ndarray: ... +def ascontiguousarray( + a: object, dtype: DtypeLike = ..., *, like: ArrayLike = ... +) -> ndarray: ... +def asfortranarray( + a: object, dtype: DtypeLike = ..., *, like: ArrayLike = ... +) -> ndarray: ... + +_Requirements = Literal["F", "C", "A", "W", "O"] +_E = Literal["E"] + +@overload +def require( + a: object, + dtype: DtypeLike = ..., + requirements: Union[_E, Iterable[Union[_E, _Requirements]]] = ..., + *, + like: ArrayLike = ... +) -> ndarray: ... +@overload +def require( + a: _ArrayType, + dtype: None = ..., + requirements: Union[None, _Requirements, Iterable[_Requirements]] = ..., + *, + like: ArrayLike = ... +) -> _ArrayType: ... +@overload +def require( + a: object, + dtype: DtypeLike = ..., + requirements: Union[None, _Requirements, Iterable[_Requirements]] = ..., + *, + like: ArrayLike = ... +) -> ndarray: ... From abe1429ee329571b640bf2b59b9be361e4f84c86 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Sun, 30 Aug 2020 00:49:55 +0200 Subject: [PATCH 137/409] ENH: Use literals where possible --- numpy/core/_asarray.pyi | 8 ++++---- numpy/typing/tests/data/pass/literal.py | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/numpy/core/_asarray.pyi b/numpy/core/_asarray.pyi index bcfc5dbd13d4..57081b76018f 100644 --- a/numpy/core/_asarray.pyi +++ b/numpy/core/_asarray.pyi @@ -1,7 +1,7 @@ import sys from typing import TypeVar, Optional, Union, Iterable, Tuple, overload -from numpy import ndarray +from numpy import ndarray, _OrderKACF from numpy.typing import ArrayLike, DtypeLike if sys.version_info >= (3, 8): @@ -14,7 +14,7 @@ _ArrayType = TypeVar("_ArrayType", bound=ndarray) def asarray( a: object, dtype: DtypeLike = ..., - order: Optional[str] = ..., + order: _OrderKACF = ..., *, like: ArrayLike = ... ) -> ndarray: ... @@ -22,7 +22,7 @@ def asarray( def asanyarray( a: _ArrayType, dtype: None = ..., - order: Optional[str] = ..., + order: _OrderKACF = ..., *, like: ArrayLike = ... ) -> _ArrayType: ... @@ -30,7 +30,7 @@ def asanyarray( def asanyarray( a: object, dtype: DtypeLike = ..., - order: Optional[str] = ..., + order: _OrderKACF = ..., *, like: ArrayLike = ... ) -> ndarray: ... diff --git a/numpy/typing/tests/data/pass/literal.py b/numpy/typing/tests/data/pass/literal.py index 321ce3c2bc0c..8eaeb6afb2ad 100644 --- a/numpy/typing/tests/data/pass/literal.py +++ b/numpy/typing/tests/data/pass/literal.py @@ -31,6 +31,8 @@ (KACF, partial(np.add, 1, 1)), # i.e. np.ufunc.__call__ (ACF, partial(np.reshape, AR, 1)), (KACF, partial(np.ravel, AR)), + (KACF, partial(np.asarray, 1)), + (KACF, partial(np.asanyarray, 1)), ] for order_set, func in order_list: From 37acf5f73c05bfbd115eeefc6201c2212b240933 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Wed, 16 Sep 2020 11:43:38 +0200 Subject: [PATCH 138/409] TST: Added tests for `np.core._asarray` --- .../tests/data/fail/array_constructors.py | 7 ++++ .../tests/data/pass/array_constructors.py | 38 +++++++++++++++++++ .../tests/data/reveal/array_constructors.py | 38 +++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 numpy/typing/tests/data/fail/array_constructors.py create mode 100644 numpy/typing/tests/data/pass/array_constructors.py create mode 100644 numpy/typing/tests/data/reveal/array_constructors.py diff --git a/numpy/typing/tests/data/fail/array_constructors.py b/numpy/typing/tests/data/fail/array_constructors.py new file mode 100644 index 000000000000..04e869863328 --- /dev/null +++ b/numpy/typing/tests/data/fail/array_constructors.py @@ -0,0 +1,7 @@ +import numpy as np + +a: np.ndarray + +np.require(a, requirements=1) # E: No overload variant +np.require(a, requirements="TEST") # E: incompatible type + diff --git a/numpy/typing/tests/data/pass/array_constructors.py b/numpy/typing/tests/data/pass/array_constructors.py new file mode 100644 index 000000000000..d5f7c35ca74a --- /dev/null +++ b/numpy/typing/tests/data/pass/array_constructors.py @@ -0,0 +1,38 @@ +from typing import List +import numpy as np + +class SubClass(np.ndarray): ... + +A = np.array([1]) +B = A.view(SubClass).copy() +C = [1] + +np.asarray(A) +np.asarray(B) +np.asarray(C) + +np.asanyarray(A) +np.asanyarray(B) +np.asanyarray(B, dtype=int) +np.asanyarray(C) + +np.ascontiguousarray(A) +np.ascontiguousarray(B) +np.ascontiguousarray(C) + +np.asfortranarray(A) +np.asfortranarray(B) +np.asfortranarray(C) + +np.require(A) +np.require(B) +np.require(B, dtype=int) +np.require(B, requirements=None) +np.require(B, requirements="E") +np.require(B, requirements=["ENSUREARRAY"]) +np.require(B, requirements={"F", "E"}) +np.require(B, requirements=["C", "OWNDATA"]) +np.require(B, requirements="W") +np.require(B, requirements="A") +np.require(C) + diff --git a/numpy/typing/tests/data/reveal/array_constructors.py b/numpy/typing/tests/data/reveal/array_constructors.py new file mode 100644 index 000000000000..ffbcc869c40a --- /dev/null +++ b/numpy/typing/tests/data/reveal/array_constructors.py @@ -0,0 +1,38 @@ +from typing import List +import numpy as np + +class SubClass(np.ndarray): ... + +A: np.ndarray +B: SubClass +C: List[int] + +reveal_type(np.asarray(A)) # E: ndarray +reveal_type(np.asarray(B)) # E: ndarray +reveal_type(np.asarray(C)) # E: ndarray + +reveal_type(np.asanyarray(A)) # E: ndarray +reveal_type(np.asanyarray(B)) # E: SubClass +reveal_type(np.asanyarray(B, dtype=int)) # E: ndarray +reveal_type(np.asanyarray(C)) # E: ndarray + +reveal_type(np.ascontiguousarray(A)) # E: ndarray +reveal_type(np.ascontiguousarray(B)) # E: ndarray +reveal_type(np.ascontiguousarray(C)) # E: ndarray + +reveal_type(np.asfortranarray(A)) # E: ndarray +reveal_type(np.asfortranarray(B)) # E: ndarray +reveal_type(np.asfortranarray(C)) # E: ndarray + +reveal_type(np.require(A)) # E: ndarray +reveal_type(np.require(B)) # E: SubClass +reveal_type(np.require(B, requirements=None)) # E: SubClass +reveal_type(np.require(B, dtype=int)) # E: ndarray +reveal_type(np.require(B, requirements="E")) # E: ndarray +reveal_type(np.require(B, requirements=["ENSUREARRAY"])) # E: ndarray +reveal_type(np.require(B, requirements={"F", "E"})) # E: ndarray +reveal_type(np.require(B, requirements=["C", "OWNDATA"])) # E: SubClass +reveal_type(np.require(B, requirements="W")) # E: SubClass +reveal_type(np.require(B, requirements="A")) # E: SubClass +reveal_type(np.require(C)) # E: ndarray + From e142934543713507bab9418d53290d8896455b79 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Wed, 16 Sep 2020 11:44:43 +0200 Subject: [PATCH 139/409] TST: Consolidate all array construction tests --- .../tests/data/fail/array_constructors.py | 19 +++++++++++++ numpy/typing/tests/data/fail/linspace.py | 13 --------- numpy/typing/tests/data/fail/simple.py | 12 -------- .../tests/data/pass/array_constructors.py | 28 +++++++++++++++++++ numpy/typing/tests/data/pass/linspace.py | 22 --------------- numpy/typing/tests/data/pass/simple.py | 9 ------ .../tests/data/reveal/array_constructors.py | 4 +++ numpy/typing/tests/data/reveal/linspace.py | 6 ---- 8 files changed, 51 insertions(+), 62 deletions(-) delete mode 100644 numpy/typing/tests/data/fail/linspace.py delete mode 100644 numpy/typing/tests/data/fail/simple.py delete mode 100644 numpy/typing/tests/data/pass/linspace.py delete mode 100644 numpy/typing/tests/data/reveal/linspace.py diff --git a/numpy/typing/tests/data/fail/array_constructors.py b/numpy/typing/tests/data/fail/array_constructors.py index 04e869863328..5218572a61c2 100644 --- a/numpy/typing/tests/data/fail/array_constructors.py +++ b/numpy/typing/tests/data/fail/array_constructors.py @@ -5,3 +5,22 @@ np.require(a, requirements=1) # E: No overload variant np.require(a, requirements="TEST") # E: incompatible type +np.zeros("test") # E: incompatible type +np.zeros() # E: Too few arguments + +np.ones("test") # E: incompatible type +np.ones() # E: Too few arguments + +np.array(0, float, True) # E: Too many positional + +np.linspace(None, 'bob') # E: No overload variant +np.linspace(0, 2, num=10.0) # E: No overload variant +np.linspace(0, 2, endpoint='True') # E: No overload variant +np.linspace(0, 2, retstep=b'False') # E: No overload variant +np.linspace(0, 2, dtype=0) # E: No overload variant +np.linspace(0, 2, axis=None) # E: No overload variant + +np.logspace(None, 'bob') # E: Argument 1 +np.logspace(0, 2, base=None) # E: Argument "base" + +np.geomspace(None, 'bob') # E: Argument 1 diff --git a/numpy/typing/tests/data/fail/linspace.py b/numpy/typing/tests/data/fail/linspace.py deleted file mode 100644 index a9769c5d6fb1..000000000000 --- a/numpy/typing/tests/data/fail/linspace.py +++ /dev/null @@ -1,13 +0,0 @@ -import numpy as np - -np.linspace(None, 'bob') # E: No overload variant -np.linspace(0, 2, num=10.0) # E: No overload variant -np.linspace(0, 2, endpoint='True') # E: No overload variant -np.linspace(0, 2, retstep=b'False') # E: No overload variant -np.linspace(0, 2, dtype=0) # E: No overload variant -np.linspace(0, 2, axis=None) # E: No overload variant - -np.logspace(None, 'bob') # E: Argument 1 -np.logspace(0, 2, base=None) # E: Argument "base" - -np.geomspace(None, 'bob') # E: Argument 1 diff --git a/numpy/typing/tests/data/fail/simple.py b/numpy/typing/tests/data/fail/simple.py deleted file mode 100644 index 57c08fb7db7e..000000000000 --- a/numpy/typing/tests/data/fail/simple.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Simple expression that should fail with mypy.""" - -import numpy as np - -# Array creation routines checks -np.zeros("test") # E: incompatible type -np.zeros() # E: Too few arguments - -np.ones("test") # E: incompatible type -np.ones() # E: Too few arguments - -np.array(0, float, True) # E: Too many positional diff --git a/numpy/typing/tests/data/pass/array_constructors.py b/numpy/typing/tests/data/pass/array_constructors.py index d5f7c35ca74a..bf29e52b91b9 100644 --- a/numpy/typing/tests/data/pass/array_constructors.py +++ b/numpy/typing/tests/data/pass/array_constructors.py @@ -1,12 +1,24 @@ from typing import List import numpy as np +class Index: + def __index__(self) -> int: + return 0 + class SubClass(np.ndarray): ... A = np.array([1]) B = A.view(SubClass).copy() C = [1] +np.array(1, dtype=float) +np.array(1, copy=False) +np.array(1, order='F') +np.array(1, order=None) +np.array(1, subok=True) +np.array(1, ndmin=3) +np.array(1, str, copy=True, order='C', subok=False, ndmin=2) + np.asarray(A) np.asarray(B) np.asarray(C) @@ -36,3 +48,19 @@ class SubClass(np.ndarray): ... np.require(B, requirements="A") np.require(C) +np.linspace(0, 2) +np.linspace(0.5, [0, 1, 2]) +np.linspace([0, 1, 2], 3) +np.linspace(0j, 2) +np.linspace(0, 2, num=10) +np.linspace(0, 2, endpoint=True) +np.linspace(0, 2, retstep=True) +np.linspace(0j, 2j, retstep=True) +np.linspace(0, 2, dtype=bool) +np.linspace([0, 1], [2, 3], axis=Index()) + +np.logspace(0, 2, base=2) +np.logspace(0, 2, base=2) +np.logspace(0, 2, base=[1j, 2j], num=2) + +np.geomspace(1, 2) diff --git a/numpy/typing/tests/data/pass/linspace.py b/numpy/typing/tests/data/pass/linspace.py deleted file mode 100644 index 8c6d0d56b93b..000000000000 --- a/numpy/typing/tests/data/pass/linspace.py +++ /dev/null @@ -1,22 +0,0 @@ -import numpy as np - -class Index: - def __index__(self) -> int: - return 0 - -np.linspace(0, 2) -np.linspace(0.5, [0, 1, 2]) -np.linspace([0, 1, 2], 3) -np.linspace(0j, 2) -np.linspace(0, 2, num=10) -np.linspace(0, 2, endpoint=True) -np.linspace(0, 2, retstep=True) -np.linspace(0j, 2j, retstep=True) -np.linspace(0, 2, dtype=bool) -np.linspace([0, 1], [2, 3], axis=Index()) - -np.logspace(0, 2, base=2) -np.logspace(0, 2, base=2) -np.logspace(0, 2, base=[1j, 2j], num=2) - -np.geomspace(1, 2) diff --git a/numpy/typing/tests/data/pass/simple.py b/numpy/typing/tests/data/pass/simple.py index 52705055702c..4d397bd29f29 100644 --- a/numpy/typing/tests/data/pass/simple.py +++ b/numpy/typing/tests/data/pass/simple.py @@ -17,15 +17,6 @@ def ndarray_func(x): array == 1 array.dtype == float -# Array creation routines checks -np.array(1, dtype=float) -np.array(1, copy=False) -np.array(1, order='F') -np.array(1, order=None) -np.array(1, subok=True) -np.array(1, ndmin=3) -np.array(1, str, copy=True, order='C', subok=False, ndmin=2) - ndarray_func(np.zeros([1, 2])) ndarray_func(np.ones([1, 2])) ndarray_func(np.empty([1, 2])) diff --git a/numpy/typing/tests/data/reveal/array_constructors.py b/numpy/typing/tests/data/reveal/array_constructors.py index ffbcc869c40a..ba8a8eda1e94 100644 --- a/numpy/typing/tests/data/reveal/array_constructors.py +++ b/numpy/typing/tests/data/reveal/array_constructors.py @@ -36,3 +36,7 @@ class SubClass(np.ndarray): ... reveal_type(np.require(B, requirements="A")) # E: SubClass reveal_type(np.require(C)) # E: ndarray +reveal_type(np.linspace(0, 10)) # E: numpy.ndarray +reveal_type(np.linspace(0, 10, retstep=True)) # E: Tuple[numpy.ndarray, numpy.inexact] +reveal_type(np.logspace(0, 10)) # E: numpy.ndarray +reveal_type(np.geomspace(1, 10)) # E: numpy.ndarray diff --git a/numpy/typing/tests/data/reveal/linspace.py b/numpy/typing/tests/data/reveal/linspace.py deleted file mode 100644 index cfbbdf390264..000000000000 --- a/numpy/typing/tests/data/reveal/linspace.py +++ /dev/null @@ -1,6 +0,0 @@ -import numpy as np - -reveal_type(np.linspace(0, 10)) # E: numpy.ndarray -reveal_type(np.linspace(0, 10, retstep=True)) # E: Tuple[numpy.ndarray, numpy.inexact] -reveal_type(np.logspace(0, 10)) # E: numpy.ndarray -reveal_type(np.geomspace(1, 10)) # E: numpy.ndarray From da2a85480b7920b059f1cf0b82c2fe98ea798bee Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Wed, 16 Sep 2020 11:45:10 +0200 Subject: [PATCH 140/409] ENH: Update the `np.__all__` --- numpy/__init__.pyi | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 6a1c5b72b58e..c9f45472d18e 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -107,6 +107,14 @@ from numpy.core.fromnumeric import ( var, ) +from numpy.core._asarray import ( + asarray, + asanyarray, + ascontiguousarray, + asfortranarray, + require, +) + # Add an object to `__all__` if their stubs are defined in an external file; # their stubs will not be recognized otherwise. # NOTE: This is redundant for objects defined within this file. @@ -152,6 +160,11 @@ __all__ = [ "mean", "std", "var", + "asarray", + "asanyarray", + "ascontiguousarray", + "asfortranarray", + "require", ] DataSource: Any From 97c221685fd4a8773bfe24e974a59cca6babf8c5 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Wed, 16 Sep 2020 11:45:36 +0200 Subject: [PATCH 141/409] MAINT: Minor fixes; removed unused imports --- numpy/core/_asarray.pyi | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/numpy/core/_asarray.pyi b/numpy/core/_asarray.pyi index 57081b76018f..5eb5dac36b59 100644 --- a/numpy/core/_asarray.pyi +++ b/numpy/core/_asarray.pyi @@ -1,5 +1,5 @@ import sys -from typing import TypeVar, Optional, Union, Iterable, Tuple, overload +from typing import TypeVar, Union, Iterable, overload from numpy import ndarray, _OrderKACF from numpy.typing import ArrayLike, DtypeLike @@ -41,17 +41,15 @@ def asfortranarray( a: object, dtype: DtypeLike = ..., *, like: ArrayLike = ... ) -> ndarray: ... -_Requirements = Literal["F", "C", "A", "W", "O"] -_E = Literal["E"] +_Requirements = Literal[ + "C", "C_CONTIGUOUS", "CONTIGUOUS", + "F", "F_CONTIGUOUS", "FORTRAN", + "A", "ALIGNED", + "W", "WRITEABLE", + "O", "OWNDATA" +] +_E = Literal["E", "ENSUREARRAY"] -@overload -def require( - a: object, - dtype: DtypeLike = ..., - requirements: Union[_E, Iterable[Union[_E, _Requirements]]] = ..., - *, - like: ArrayLike = ... -) -> ndarray: ... @overload def require( a: _ArrayType, @@ -61,6 +59,14 @@ def require( like: ArrayLike = ... ) -> _ArrayType: ... @overload +def require( + a: object, + dtype: DtypeLike = ..., + requirements: Union[_E, Iterable[Union[_E, _Requirements]]] = ..., + *, + like: ArrayLike = ... +) -> ndarray: ... +@overload def require( a: object, dtype: DtypeLike = ..., From 7794d059f3531965b05cc11052376d3930d1c518 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Wed, 16 Sep 2020 12:59:50 +0200 Subject: [PATCH 142/409] ENH: Add an alias for `Union[_Requirements, _E]` Addresses https://github.com/numpy/numpy/pull/17326#discussion_r489322146 --- numpy/core/_asarray.pyi | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/numpy/core/_asarray.pyi b/numpy/core/_asarray.pyi index 5eb5dac36b59..e074d69d2c23 100644 --- a/numpy/core/_asarray.pyi +++ b/numpy/core/_asarray.pyi @@ -49,6 +49,7 @@ _Requirements = Literal[ "O", "OWNDATA" ] _E = Literal["E", "ENSUREARRAY"] +_RequirementsWithE = Union[_Requirements, _E] @overload def require( @@ -62,7 +63,7 @@ def require( def require( a: object, dtype: DtypeLike = ..., - requirements: Union[_E, Iterable[Union[_E, _Requirements]]] = ..., + requirements: Union[_E, Iterable[_RequirementsWithE]] = ..., *, like: ArrayLike = ... ) -> ndarray: ... From 2af59625c802ba72f9557a6667112f34cc10fbc3 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Wed, 23 Sep 2020 11:41:55 +0200 Subject: [PATCH 143/409] MAINT: Replaced `__all__` with `import x as x` Objects defined in an external file will be picked up if they are either specified in `__all__` or imported as `import x as x`. The latter notation is more compact and is less prone to merge conflicts. --- numpy/__init__.pyi | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index c9f45472d18e..9baa4f4bd307 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -108,11 +108,11 @@ from numpy.core.fromnumeric import ( ) from numpy.core._asarray import ( - asarray, - asanyarray, - ascontiguousarray, - asfortranarray, - require, + asarray as asarray, + asanyarray as asanyarray, + ascontiguousarray as ascontiguousarray, + asfortranarray as asfortranarray, + require as require, ) # Add an object to `__all__` if their stubs are defined in an external file; @@ -160,11 +160,6 @@ __all__ = [ "mean", "std", "var", - "asarray", - "asanyarray", - "ascontiguousarray", - "asfortranarray", - "require", ] DataSource: Any From be77442c1facae452d3742415161c4f8203f4a86 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Thu, 1 Oct 2020 13:57:35 +0200 Subject: [PATCH 144/409] MAINT: Removed placeholder annotations --- numpy/__init__.pyi | 5 ----- 1 file changed, 5 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 9baa4f4bd307..30d15ed12338 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -177,12 +177,8 @@ array2string: Any array_repr: Any array_split: Any array_str: Any -asanyarray: Any -asarray: Any asarray_chkfinite: Any -ascontiguousarray: Any asfarray: Any -asfortranarray: Any asmatrix: Any asscalar: Any atleast_1d: Any @@ -404,7 +400,6 @@ recarray: Any recfromcsv: Any recfromtxt: Any record: Any -require: Any result_type: Any roots: Any rot90: Any From 247245ebec0513ba1450900024db2691698f20f4 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Wed, 30 Sep 2020 18:32:03 -0600 Subject: [PATCH 145/409] MAINT: Replace PyUString_ConcatAndDel in ufunc_object.c. --- numpy/core/src/umath/ufunc_object.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index f693eb5c246f..8660ee413ee9 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -5977,6 +5977,7 @@ _typecharfromnum(int num) { return ret; } + static PyObject * ufunc_get_doc(PyUFuncObject *ufunc) { @@ -5997,18 +5998,18 @@ ufunc_get_doc(PyUFuncObject *ufunc) * introspection on name and nin + nout to automate the first part * of it the doc string shouldn't need the calling convention */ - doc = PyObject_CallFunctionObjArgs( - _sig_formatter, (PyObject *)ufunc, NULL); + doc = PyObject_CallFunctionObjArgs(_sig_formatter, + (PyObject *)ufunc, NULL); if (doc == NULL) { return NULL; } if (ufunc->doc != NULL) { - PyUString_ConcatAndDel(&doc, - PyUnicode_FromFormat("\n\n%s", ufunc->doc)); + Py_SETREF(doc, PyUnicode_FromFormat("%S\n\n%s", doc, ufunc->doc)); } return doc; } + static PyObject * ufunc_get_nin(PyUFuncObject *ufunc) { From e6030ff2101e12abd01bd93b11a949821e26c248 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Wed, 30 Sep 2020 17:35:05 -0600 Subject: [PATCH 146/409] MAINT: Remove the build_shape_string function. The build_shape_string function is effectively a duplicate of the convert_shape_to_string function in common.c. All uses have now been replaced, so remove it. --- numpy/core/src/multiarray/shape.c | 45 ------------------------------- numpy/core/src/multiarray/shape.h | 7 ----- 2 files changed, 52 deletions(-) diff --git a/numpy/core/src/multiarray/shape.c b/numpy/core/src/multiarray/shape.c index 9dda899131d9..02c349759528 100644 --- a/numpy/core/src/multiarray/shape.c +++ b/numpy/core/src/multiarray/shape.c @@ -977,51 +977,6 @@ PyArray_Flatten(PyArrayObject *a, NPY_ORDER order) return (PyObject *)ret; } -/* See shape.h for parameters documentation */ -NPY_NO_EXPORT PyObject * -build_shape_string(npy_intp n, npy_intp const *vals) -{ - npy_intp i; - - /* - * Negative dimension indicates "newaxis", which can - * be discarded for printing if it's a leading dimension. - * Find the first non-"newaxis" dimension. - */ - for (i = 0; i < n && vals[i] < 0; ++i); - - if (i == n) { - return PyUnicode_FromFormat("()"); - } - - PyObject *ret = PyUnicode_FromFormat("%" NPY_INTP_FMT, vals[i++]); - if (ret == NULL) { - return NULL; - } - for (; i < n; ++i) { - PyObject *tmp; - - if (vals[i] < 0) { - tmp = PyUnicode_FromString(",newaxis"); - } - else { - tmp = PyUnicode_FromFormat(",%" NPY_INTP_FMT, vals[i]); - } - if (tmp == NULL) { - Py_DECREF(ret); - return NULL; - } - - Py_SETREF(ret, PyUnicode_Concat(ret, tmp)); - Py_DECREF(tmp); - if (ret == NULL) { - return NULL; - } - } - - Py_SETREF(ret, PyUnicode_FromFormat("(%S)", ret)); - return ret; -} /*NUMPY_API * diff --git a/numpy/core/src/multiarray/shape.h b/numpy/core/src/multiarray/shape.h index d252925569ba..875b5430f2e8 100644 --- a/numpy/core/src/multiarray/shape.h +++ b/numpy/core/src/multiarray/shape.h @@ -1,13 +1,6 @@ #ifndef _NPY_ARRAY_SHAPE_H_ #define _NPY_ARRAY_SHAPE_H_ -/* - * Builds a string representation of the shape given in 'vals'. - * A negative value in 'vals' gets interpreted as newaxis. - */ -NPY_NO_EXPORT PyObject * -build_shape_string(npy_intp n, npy_intp const *vals); - /* * Creates a sorted stride perm matching the KEEPORDER behavior * of the NpyIter object. Because this operates based on multiple From a2e76ff3dc1e19203b2529d939f489ca04ab98a0 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 30 Sep 2020 13:55:54 -0500 Subject: [PATCH 147/409] API: Special case how numpy scalars are coerced to signed integer This removes one of the larger changes to array-coercion, which meant that NumPy scalars were always coerced like a 0-D array would be (i.e. using normal casting). When the assignment is explicitly an integer, now `scalar.__int__()` will be used instead (as was the case previously). Since previously this was handled differently, a *single* scalar is still converted using casting: np.array(np.float64(np.nan), dtype=np.int64) succeeds, but any other thing fails, such as: np.array([np.float64(np.nan)], dtype=np.int64) arr1d_int64[()] = np.float64(np.nan) np.array(np.array(np.nan), dtype=np.int64) This does not affect Python scalars, that always raise, because they always are converted using `scalar.__int__()`. Unsigned integers always supported casting from their signed equivalent, so the difference is much less visible for them and this chooses to always use the casting behaviour. The main reason for this change is to help pands: https://github.com/pandas-dev/pandas/issues/35481 --- .../upcoming_changes/16200.compatibility.rst | 22 +++++++-- numpy/core/src/multiarray/ctors.c | 28 ++++++++++- numpy/core/src/multiarray/dtypemeta.c | 17 +++++++ numpy/core/tests/test_array_coercion.py | 47 +++++++++++++++++-- 4 files changed, 104 insertions(+), 10 deletions(-) diff --git a/doc/release/upcoming_changes/16200.compatibility.rst b/doc/release/upcoming_changes/16200.compatibility.rst index d0fd51265f6e..2bbdd883ed48 100644 --- a/doc/release/upcoming_changes/16200.compatibility.rst +++ b/doc/release/upcoming_changes/16200.compatibility.rst @@ -8,14 +8,26 @@ error:: np.array([np.float64(np.nan)], dtype=np.int64) -will succeed at this time (this may change) and return an undefined result -(usually the smallest possible integer). This also affects assignments:: +will succeed and return an undefined result (usually the smallest possible +integer). This also affects assignments:: arr[0] = np.float64(np.nan) -Note, this already happened for ``np.array(np.float64(np.nan), dtype=np.int64)`` -and that the behaviour is unchanged for ``np.nan`` itself which is a Python -float. +At this time, NumPy retains the behaviour for:: + + np.array(np.float64(np.nan), dtype=np.int64) + +The above changes do not affect Python scalars: + + np.array([float("NaN")], dtype=np.int64) + +remains unaffected (``np.nan`` is a Python ``float``, not a NumPy one). +Unlike signed integers, unsigned integers do not retain this special case, +since they always behaved more like casting. +The following code stops raising an error:: + + np.array([np.float64(np.nan)], dtype=np.uint64) + To avoid backward compatibility issues, at this time assignment from ``datetime64`` scalar to strings of too short length remains supported. This means that ``np.asarray(np.datetime64("2020-10-10"), dtype="S5")`` diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 956dfd3bbf9e..f543d02d0041 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1460,6 +1460,31 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, ((PyVoidScalarObject *)op)->flags, NULL, op); } + else if (cache == 0 && newtype != NULL && + PyDataType_ISSIGNED(newtype) && PyArray_IsScalar(op, Generic)) { + assert(ndim == 0); + /* + * This is an (possible) inconsistency where: + * + * np.array(np.float64(np.nan), dtype=np.int64) + * + * behaves differently from: + * + * np.array([np.float64(np.nan)], dtype=np.int64) + * arr1d_int64[0] = np.float64(np.nan) + * np.array(np.array(np.nan), dtype=np.int64) + * + * by not raising an error instead of using typical casting. + * The error is desirable, but to always error seems like a + * larger change to be considered at some other time and it is + * undesirable that 0-D arrays behave differently from scalars. + * This retains the behaviour, largely due to issues in pandas + * which relied on a try/except (although hopefully that will + * have a better solution at some point): + * https://github.com/pandas-dev/pandas/issues/35481 + */ + return PyArray_FromScalar(op, dtype); + } /* There was no array (or array-like) passed in directly. */ if ((flags & NPY_ARRAY_WRITEBACKIFCOPY) || @@ -1480,7 +1505,8 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, if (cache == NULL) { /* This is a single item. Set it directly. */ assert(ndim == 0); - if (PyArray_Pack(PyArray_DESCR(ret), PyArray_DATA(ret), op) < 0) { + + if (PyArray_Pack(PyArray_DESCR(ret), PyArray_BYTES(ret), op) < 0) { Py_DECREF(ret); return NULL; } diff --git a/numpy/core/src/multiarray/dtypemeta.c b/numpy/core/src/multiarray/dtypemeta.c index dbe5ba476887..109f4a225d64 100644 --- a/numpy/core/src/multiarray/dtypemeta.c +++ b/numpy/core/src/multiarray/dtypemeta.c @@ -304,6 +304,18 @@ python_builtins_are_known_scalar_types( } +static int +signed_integers_is_known_scalar_types( + PyArray_DTypeMeta *cls, PyTypeObject *pytype) +{ + if (python_builtins_are_known_scalar_types(cls, pytype)) { + return 1; + } + /* Convert our scalars (raise on too large unsigned and NaN, etc.) */ + return PyType_IsSubtype(pytype, &PyGenericArrType_Type); +} + + static int datetime_known_scalar_types( PyArray_DTypeMeta *cls, PyTypeObject *pytype) @@ -549,6 +561,11 @@ dtypemeta_wrap_legacy_descriptor(PyArray_Descr *descr) dtype_class->common_dtype = default_builtin_common_dtype; dtype_class->common_instance = NULL; + if (PyTypeNum_ISSIGNED(dtype_class->type_num)) { + /* Convert our scalars (raise on too large unsigned and NaN, etc.) */ + dtype_class->is_known_scalar_type = signed_integers_is_known_scalar_types; + } + if (PyTypeNum_ISUSERDEF(descr->type_num)) { dtype_class->common_dtype = legacy_userdtype_common_dtype_function; } diff --git a/numpy/core/tests/test_array_coercion.py b/numpy/core/tests/test_array_coercion.py index e0480c7bfcf5..ce66589cacab 100644 --- a/numpy/core/tests/test_array_coercion.py +++ b/numpy/core/tests/test_array_coercion.py @@ -309,6 +309,13 @@ def test_scalar_coercion_same_as_cast_and_assignment(self, cast_to): # coercion should also raise (error type may change) with pytest.raises(Exception): np.array(scalar, dtype=dtype) + + if (isinstance(scalar, rational) and + np.issubdtype(dtype, np.signedinteger)): + return + + with pytest.raises(Exception): + np.array([scalar], dtype=dtype) # assignment should also raise res = np.zeros((), dtype=dtype) with pytest.raises(Exception): @@ -340,6 +347,30 @@ def test_default_dtype_instance(self, dtype_char): assert discovered_dtype == dtype assert discovered_dtype.itemsize == dtype.itemsize + @pytest.mark.parametrize("dtype", np.typecodes["Integer"]) + def test_scalar_to_int_coerce_does_not_cast(self, dtype): + """ + Signed integers are currently different in that they do not cast other + NumPy scalar, but instead use scalar.__int__(). The harcoded + exception to this rule is `np.array(scalar, dtype=integer)`. + """ + dtype = np.dtype(dtype) + invalid_int = np.ulonglong(-1) + + float_nan = np.float64(np.nan) + + for scalar in [float_nan, invalid_int]: + # This is a special case using casting logic and thus not failing: + coerced = np.array(scalar, dtype=dtype) + cast = np.array(scalar).astype(dtype) + assert_array_equal(coerced, cast) + + # However these fail: + with pytest.raises((ValueError, OverflowError)): + np.array([scalar], dtype=dtype) + with pytest.raises((ValueError, OverflowError)): + cast[()] = scalar + class TestTimeScalars: @pytest.mark.parametrize("dtype", [np.int64, np.float32]) @@ -349,13 +380,21 @@ class TestTimeScalars: param(np.datetime64("NaT", "generic"), id="datetime64[generic](NaT)"), param(np.datetime64(1, "D"), id="datetime64[D]")],) def test_coercion_basic(self, dtype, scalar): + # Note the `[scalar]` is there because np.array(scalar) uses stricter + # `scalar.__int__()` rules for backward compatibility right now. arr = np.array(scalar, dtype=dtype) cast = np.array(scalar).astype(dtype) - ass = np.ones((), dtype=dtype) - ass[()] = scalar # raises, as would np.array([scalar], dtype=dtype) - assert_array_equal(arr, cast) - assert_array_equal(cast, cast) + + ass = np.ones((), dtype=dtype) + if issubclass(dtype, np.integer): + with pytest.raises(TypeError): + # raises, as would np.array([scalar], dtype=dtype), this is + # conversion from times, but behaviour of integers. + ass[()] = scalar + else: + ass[()] = scalar + assert_array_equal(ass, cast) @pytest.mark.parametrize("dtype", [np.int64, np.float32]) @pytest.mark.parametrize("scalar", From 57adb4bb6a8eb46fed597d4a781e47bcb86ebe11 Mon Sep 17 00:00:00 2001 From: Bas van Beek <43369155+BvB93@users.noreply.github.com> Date: Thu, 1 Oct 2020 18:23:53 +0200 Subject: [PATCH 148/409] DOC: Fix a parameter type in the `putmask` docs (#17412) --- numpy/core/multiarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/multiarray.py b/numpy/core/multiarray.py index 540d1ea9b5f6..6b335f1a64bb 100644 --- a/numpy/core/multiarray.py +++ b/numpy/core/multiarray.py @@ -1100,7 +1100,7 @@ def putmask(a, mask, values): Parameters ---------- - a : array_like + a : ndarray Target array. mask : array_like Boolean mask array. It has to be the same shape as `a`. From e31ae7ff70c72dec2cf7ba3bd817982ad1b68713 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 1 Oct 2020 15:39:11 -0500 Subject: [PATCH 149/409] DEP: Deprecate coercion to subarray dtypes When coercing to subarray dtypes, e.g. using `np.array(obj, dtype)`, but also `arr.astype(dtype)`, the behaviour was only well defined with tuple inputs, but not with array-like inputs. In particular, `arr.astype(dtype)` had arguably surprising behaviour of not converting by element, but rather attempting (and often failing) to broadcast `arr` to the result array with added dimensions. This deprecates all of these cases, the main issue would be for users relying on stranger inputs with broadcasted tuples contained in sequences: ``` np.array([((0, 1), (1, 2)), ((2,),)], dtype='(2,2)f4') ``` In most cases, where the tuples have the correct output shape, the new base dtype can be directly used since the discovered shape should match. However, there is no work-around for the above case. Closes gh-17173 --- .../upcoming_changes/17419.deprecation.rst | 24 +++ numpy/core/src/multiarray/ctors.c | 202 ++++++++++++++++++ numpy/core/src/multiarray/methods.c | 14 ++ numpy/core/tests/test_deprecations.py | 41 ++++ 4 files changed, 281 insertions(+) create mode 100644 doc/release/upcoming_changes/17419.deprecation.rst diff --git a/doc/release/upcoming_changes/17419.deprecation.rst b/doc/release/upcoming_changes/17419.deprecation.rst new file mode 100644 index 000000000000..fcab3a8adef9 --- /dev/null +++ b/doc/release/upcoming_changes/17419.deprecation.rst @@ -0,0 +1,24 @@ +Arrays cannot be using subarray dtypes +-------------------------------------- +Array creation and casting using ``np.array(obj, dtype)`` +and ``arr.astype(dtype)`` will not support ``dtype`` +to be a subarray dtype such as ``np.dtype("(2)i,")``. + +For such a ``dtype`` the following behaviour occurs currently:: + + res = np.array(obj, dtype) + + res.dtype is not dtype + res.dtype is dtype.base + res.shape[-dtype.ndim:] == dtype.shape + +The shape of the dtype is included into the array. +This leads to inconsistencies when ``obj`` is: + +* a scalar, such as ``np.array(1, dtype="(2)i")`` +* an array, such as ``np.array(np.array([1]), dtype="(2)i")`` + +In most cases the work-around is to pass the output dtype directly +and possibly check ``res.shape[-dtype.ndim:] == dtype.shape``. +If this is insufficient, please open an issue on the NumPy issue +tracker. diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 956dfd3bbf9e..014de27a5d71 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1367,6 +1367,160 @@ PyArray_GetArrayParamsFromObject(PyObject *NPY_UNUSED(op), } +/* + * This function is a legacy implementation to retain subarray dtype + * behaviour in array coercion. The behaviour here makes sense if tuples + * of matching dimensionality are being coerced. Due to the difficulty + * that the result is ill-defined for lists of array-likes, this is deprecated. + * + * WARNING: Do not use this function, it exists purely to support a deprecated + * code path. + */ +static int +setArrayFromSequence(PyArrayObject *a, PyObject *s, + int dim, PyArrayObject * dst) +{ + Py_ssize_t i, slen; + int res = -1; + + /* first recursion, view equal destination */ + if (dst == NULL) + dst = a; + + /* + * This code is to ensure that the sequence access below will + * return a lower-dimensional sequence. + */ + + /* INCREF on entry DECREF on exit */ + Py_INCREF(s); + + PyObject *seq = NULL; + + if (PyArray_Check(s)) { + if (!(PyArray_CheckExact(s))) { + /* + * make sure a base-class array is used so that the dimensionality + * reduction assumption is correct. + */ + /* This will DECREF(s) if replaced */ + s = PyArray_EnsureArray(s); + if (s == NULL) { + goto fail; + } + } + + /* dst points to correct array subsection */ + if (PyArray_CopyInto(dst, (PyArrayObject *)s) < 0) { + goto fail; + } + + Py_DECREF(s); + return 0; + } + + if (dim > PyArray_NDIM(a)) { + PyErr_Format(PyExc_ValueError, + "setArrayFromSequence: sequence/array dimensions mismatch."); + goto fail; + } + + /* Try __array__ before using s as a sequence */ + PyObject *tmp = _array_from_array_like(s, NULL, 0, NULL); + if (tmp == NULL) { + goto fail; + } + else if (tmp == Py_NotImplemented) { + Py_DECREF(tmp); + } + else { + int r = PyArray_CopyInto(dst, (PyArrayObject *)tmp); + Py_DECREF(tmp); + if (r < 0) { + goto fail; + } + Py_DECREF(s); + return 0; + } + + seq = PySequence_Fast(s, "Could not convert object to sequence"); + if (seq == NULL) { + goto fail; + } + slen = PySequence_Fast_GET_SIZE(seq); + + /* + * Either the dimensions match, or the sequence has length 1 and can + * be broadcast to the destination. + */ + if (slen != PyArray_DIMS(a)[dim] && slen != 1) { + PyErr_Format(PyExc_ValueError, + "cannot copy sequence with size %zd to array axis " + "with dimension %" NPY_INTP_FMT, slen, PyArray_DIMS(a)[dim]); + goto fail; + } + + /* Broadcast the one element from the sequence to all the outputs */ + if (slen == 1) { + PyObject *o = PySequence_Fast_GET_ITEM(seq, 0); + npy_intp alen = PyArray_DIM(a, dim); + + for (i = 0; i < alen; i++) { + if ((PyArray_NDIM(a) - dim) > 1) { + PyArrayObject * tmp = + (PyArrayObject *)array_item_asarray(dst, i); + if (tmp == NULL) { + goto fail; + } + + res = setArrayFromSequence(a, o, dim+1, tmp); + Py_DECREF(tmp); + } + else { + char * b = (PyArray_BYTES(dst) + i * PyArray_STRIDES(dst)[0]); + res = PyArray_SETITEM(dst, b, o); + } + if (res < 0) { + goto fail; + } + } + } + /* Copy element by element */ + else { + for (i = 0; i < slen; i++) { + PyObject * o = PySequence_Fast_GET_ITEM(seq, i); + if ((PyArray_NDIM(a) - dim) > 1) { + PyArrayObject * tmp = + (PyArrayObject *)array_item_asarray(dst, i); + if (tmp == NULL) { + goto fail; + } + + res = setArrayFromSequence(a, o, dim+1, tmp); + Py_DECREF(tmp); + } + else { + char * b = (PyArray_BYTES(dst) + i * PyArray_STRIDES(dst)[0]); + res = PyArray_SETITEM(dst, b, o); + } + if (res < 0) { + goto fail; + } + } + } + + Py_DECREF(seq); + Py_DECREF(s); + return 0; + + fail: + Py_XDECREF(seq); + Py_DECREF(s); + return res; +} + + + /*NUMPY_API * Does not check for NPY_ARRAY_ENSURECOPY and NPY_ARRAY_NOTSWAPPED in flags * Steals a reference to newtype --- which can be NULL @@ -1407,6 +1561,54 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, if (ndim < 0) { return NULL; } + + if (NPY_UNLIKELY(fixed_descriptor != NULL && PyDataType_HASSUBARRAY(dtype))) { + /* + * When a subarray dtype was passed in, its dimensions are absorbed + * into the array dimension (causing a dimension mismatch). + * We can't reasonably handle this because of inconsistencies in + * how it was handled (depending on nested list vs. embed array-likes). + * So we give a deprecation warning and fall back to legacy code. + */ + ret = (PyArrayObject *)PyArray_NewFromDescr( + &PyArray_Type, dtype, ndim, dims, NULL, NULL, + flags&NPY_ARRAY_F_CONTIGUOUS, NULL); + if (ret == NULL) { + npy_free_coercion_cache(cache); + return NULL; + } + assert(PyArray_NDIM(ret) != ndim); + + if (cache == NULL) { + /* This is a single item. Sets only first subarray element. */ + assert(ndim == 0); + if (PyArray_Pack(PyArray_DESCR(ret), PyArray_DATA(ret), op) < 0) { + Py_DECREF(ret); + return NULL; + } + } + else { + npy_free_coercion_cache(cache); + if (setArrayFromSequence(ret, op, 0, NULL) < 0) { + Py_DECREF(ret); + return NULL; + } + } + /* NumPy 1.20, 2020-10-01 */ + if (DEPRECATE( + "using a dtype with a subarray field is deprecated. " + "This can lead to inconsistent behaviour due to the resulting " + "dtype being different from the input dtype. " + "You may try to use `dtype=dtype.base`, which should give the " + "same result for most inputs, but does not guarantee the " + "output dimensions to match the subarray ones. " + "(Deprecated NumPy 1.20)")) { + Py_DECREF(ret); + return NULL; + } + return (PyObject *)ret; + } + if (dtype == NULL) { dtype = PyArray_DescrFromType(NPY_DEFAULT_TYPE); } diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index f7cb2185b5b0..e4421b41b7df 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -844,6 +844,20 @@ array_astype(PyArrayObject *self, PyObject *args, PyObject *kwds) if (ret == NULL) { return NULL; } + /* NumPy 1.20, 2020-10-01 */ + if ((PyArray_NDIM(self) != PyArray_NDIM(ret)) && DEPRECATE( + "using a dtype with a subarray field is deprecated. " + "This can lead to inconsistent behaviour due to the resulting " + "dtype being different from the input dtype. " + "You may try to use `dtype=dtype.base`, which should give the " + "same result for most inputs, but does not guarantee the " + "output dimensions to match the subarray ones. " + "For `arr.astype()` the old, surprising, behaviour can be " + "retained using `res = np.empty(arr.shape, dtype)` followed" + "by `res[...] = arr`. (Deprecated NumPy 1.20)")) { + Py_DECREF(ret); + return NULL; + } if (PyArray_CopyInto(ret, self) < 0) { Py_DECREF(ret); diff --git a/numpy/core/tests/test_deprecations.py b/numpy/core/tests/test_deprecations.py index 17391e80cdc4..91fd95d4d8f2 100644 --- a/numpy/core/tests/test_deprecations.py +++ b/numpy/core/tests/test_deprecations.py @@ -81,6 +81,8 @@ def assert_deprecated(self, function, num=1, ignore_others=False, kwargs : dict Keyword arguments for `function` """ + __tracebackhide__ = True # Hide traceback for py.test + # reset the log self.log[:] = [] @@ -728,3 +730,42 @@ def test_not_deprecated(self): np.concatenate(([0.], [1.]), out=np.empty(2, dtype=np.int64), casting="same_kind") + +class TestDeprecateSubarrayDTypeDuringArrayCoercion(_DeprecationTestCase): + message = "using a dtype with a subarray field is deprecated" + + @pytest.mark.parametrize(["obj", "dtype"], + [([((0, 1), (1, 2)), ((2,),)], '(2,2)f4'), + (["1", "2"], "(2)i,")]) + def test_deprecated_sequence(self, obj, dtype): + dtype = np.dtype(dtype) + self.assert_deprecated(lambda: np.array(obj, dtype=dtype)) + with pytest.warns(DeprecationWarning): + res = np.array(obj, dtype=dtype) + + # Using `arr.astype(subarray_dtype)` is also deprecated, because + # it uses broadcasting instead of casting each element. + self.assert_deprecated(lambda: res.astype(dtype)) + expected = np.empty(len(obj), dtype=dtype) + for i in range(len(expected)): + expected[i] = obj[i] + + assert_array_equal(res, expected) + + def test_deprecated_array(self): + # Arrays are more complex, since they "broadcast" on success: + arr = np.array([1, 2]) + self.assert_deprecated(lambda: np.array(arr, dtype="(2)i,")) + with pytest.warns(DeprecationWarning): + res = np.array(arr, dtype="(2)i,") + + assert_array_equal(res, [[1, 2], [1, 2]]) + + def test_not_deprecated(self): + # These error paths are not deprecated, the tests should be retained + # when the deprecation is finalized. + arr = np.arange(5 * 2).reshape(5, 2) + with pytest.raises(ValueError): + arr.astype("(2,2)f") + with pytest.raises(ValueError): + np.array(arr, dtype="(2,2)f") From 0e4a4d4026e9fba72c5e063c59f243a755d8979a Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 1 Oct 2020 23:30:30 -0500 Subject: [PATCH 150/409] BUG: Fix memory leak in array-coercion error paths Also uses PyMem_*, since those seem the more approriate functions. --- numpy/core/src/multiarray/array_coercion.c | 4 ++-- numpy/core/src/multiarray/ctors.c | 3 +++ numpy/core/tests/test_array_coercion.py | 15 +++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/numpy/core/src/multiarray/array_coercion.c b/numpy/core/src/multiarray/array_coercion.c index aae8d5141412..64a06d58b12a 100644 --- a/numpy/core/src/multiarray/array_coercion.c +++ b/numpy/core/src/multiarray/array_coercion.c @@ -538,7 +538,7 @@ npy_new_coercion_cache( cache = _coercion_cache_cache[_coercion_cache_num]; } else { - cache = PyObject_MALLOC(sizeof(coercion_cache_obj)); + cache = PyMem_Malloc(sizeof(coercion_cache_obj)); } if (cache == NULL) { PyErr_NoMemory(); @@ -570,7 +570,7 @@ npy_unlink_coercion_cache(coercion_cache_obj *current) _coercion_cache_num++; } else { - PyObject_FREE(current); + PyMem_Free(current); } return next; } diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 956dfd3bbf9e..55c0a31f0c8a 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -610,6 +610,7 @@ PyArray_AssignFromCache(PyArrayObject *self, coercion_cache_obj *cache) { PyErr_SetString(PyExc_RuntimeError, "Inconsistent object during array creation? " "Content of sequences changed (cache not consumed)."); + npy_free_coercion_cache(cache); return -1; } return 0; @@ -1467,6 +1468,7 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, PyErr_SetString(PyExc_TypeError, "WRITEBACKIFCOPY used for non-array input."); Py_DECREF(dtype); + npy_free_coercion_cache(cache); return NULL; } @@ -1475,6 +1477,7 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, &PyArray_Type, dtype, ndim, dims, NULL, NULL, flags&NPY_ARRAY_F_CONTIGUOUS, NULL); if (ret == NULL) { + npy_free_coercion_cache(cache); return NULL; } if (cache == NULL) { diff --git a/numpy/core/tests/test_array_coercion.py b/numpy/core/tests/test_array_coercion.py index e0480c7bfcf5..79954b998fb3 100644 --- a/numpy/core/tests/test_array_coercion.py +++ b/numpy/core/tests/test_array_coercion.py @@ -635,3 +635,18 @@ def __array__(self): assert arr[()] is ArrayLike arr = np.array([ArrayLike]) assert arr[0] is ArrayLike + + @pytest.mark.skipif( + np.dtype(np.intp).itemsize < 8, reason="Needs 64bit platform") + def test_too_large_array_error_paths(self): + """Test the error paths, including for memory leaks""" + arr = np.array(0, dtype="uint8") + # Guarantees that a contiguous copy won't work: + arr = np.broadcast_to(arr, 2**62) + + for i in range(5): + # repeat, to ensure caching cannot have an effect: + with pytest.raises(MemoryError): + np.array(arr) + with pytest.raises(MemoryError): + np.array([arr]) From c71edcec18d2fdbe391822e7658db43d783e8db0 Mon Sep 17 00:00:00 2001 From: Jovial Joe Jayarson Date: Fri, 2 Oct 2020 11:02:54 +0530 Subject: [PATCH 151/409] fix: chains nested try-except-raise --- numpy/ma/core.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/numpy/ma/core.py b/numpy/ma/core.py index b5371f51a6bc..4e320576bdb3 100644 --- a/numpy/ma/core.py +++ b/numpy/ma/core.py @@ -443,9 +443,9 @@ def _check_fill_value(fill_value, ndtype): if isinstance(fill_value, (ndarray, np.void)): try: fill_value = np.array(fill_value, copy=False, dtype=ndtype) - except ValueError: + except ValueError as e: err_msg = "Unable to transform %s to dtype %s" - raise ValueError(err_msg % (fill_value, ndtype)) + raise ValueError(err_msg % (fill_value, ndtype)) from e else: fill_value = np.asarray(fill_value, dtype=object) fill_value = np.array(_recursive_set_fill_value(fill_value, ndtype), @@ -460,12 +460,12 @@ def _check_fill_value(fill_value, ndtype): # Also in case of converting string arrays. try: fill_value = np.array(fill_value, copy=False, dtype=ndtype) - except (OverflowError, ValueError): + except (OverflowError, ValueError) as e: # Raise TypeError instead of OverflowError or ValueError. # OverflowError is seldom used, and the real problem here is # that the passed fill_value is not compatible with the ndtype. err_msg = "Cannot convert fill_value %s to dtype %s" - raise TypeError(err_msg % (fill_value, ndtype)) + raise TypeError(err_msg % (fill_value, ndtype)) from e return np.array(fill_value) From 01e48bf1222794454a8fad7c44e6d2ac19c7a9da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrique=20Mat=C3=ADas=20S=C3=A1nchez=20=28Quique=29?= Date: Fri, 2 Oct 2020 09:31:55 +0200 Subject: [PATCH 152/409] DOC: Remove bogus reference to _a_ This will make this docstring coherent with the one for ndarray.argmax(). --- numpy/core/_add_newdocs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py index 879b3645d74d..b8cf12c60c1c 100644 --- a/numpy/core/_add_newdocs.py +++ b/numpy/core/_add_newdocs.py @@ -2618,7 +2618,7 @@ """ a.argmin(axis=None, out=None) - Return indices of the minimum values along the given axis of `a`. + Return indices of the minimum values along the given axis. Refer to `numpy.argmin` for detailed documentation. From cd20e7460ec84bb3b1b18b881ce09a851e492a08 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Fri, 2 Oct 2020 11:39:53 +0100 Subject: [PATCH 153/409] DOC: Fix formatting issues in description of .c.src files --- doc/DISTUTILS.rst.txt | 48 +++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/doc/DISTUTILS.rst.txt b/doc/DISTUTILS.rst.txt index 01527374d182..81ad8496397b 100644 --- a/doc/DISTUTILS.rst.txt +++ b/doc/DISTUTILS.rst.txt @@ -394,37 +394,37 @@ and ``/**end repeat**/`` lines, which may also be nested using consecutively numbered delimiting lines such as ``/**begin repeat1`` and ``/**end repeat1**/``: -1. "/\**begin repeat "on a line by itself marks the beginning of -a segment that should be repeated. +1. ``/**begin repeat ``on a line by itself marks the beginning of + a segment that should be repeated. 2. Named variable expansions are defined using ``#name=item1, item2, item3, -..., itemN#`` and placed on successive lines. These variables are -replaced in each repeat block with corresponding word. All named -variables in the same repeat block must define the same number of -words. + ..., itemN#`` and placed on successive lines. These variables are + replaced in each repeat block with corresponding word. All named + variables in the same repeat block must define the same number of + words. 3. In specifying the repeat rule for a named variable, ``item*N`` is short- -hand for ``item, item, ..., item`` repeated N times. In addition, -parenthesis in combination with \*N can be used for grouping several -items that should be repeated. Thus, #name=(item1, item2)*4# is -equivalent to #name=item1, item2, item1, item2, item1, item2, item1, -item2# + hand for ``item, item, ..., item`` repeated N times. In addition, + parenthesis in combination with ``*N`` can be used for grouping several + items that should be repeated. Thus, ``#name=(item1, item2)*4#`` is + equivalent to ``#name=item1, item2, item1, item2, item1, item2, item1, + item2#``. -4. "\*/ "on a line by itself marks the end of the variable expansion -naming. The next line is the first line that will be repeated using -the named rules. +4. ``*/`` on a line by itself marks the end of the variable expansion + naming. The next line is the first line that will be repeated using + the named rules. 5. Inside the block to be repeated, the variables that should be expanded -are specified as ``@name@`` + are specified as ``@name@``. -6. "/\**end repeat**/ "on a line by itself marks the previous line -as the last line of the block to be repeated. +6. ``/**end repeat**/`` on a line by itself marks the previous line + as the last line of the block to be repeated. 7. A loop in the NumPy C source code may have a ``@TYPE@`` variable, targeted -for string substitution, which is preprocessed to a number of otherwise -identical loops with several strings such as INT, LONG, UINT, ULONG. The -``@TYPE@`` style syntax thus reduces code duplication and maintenance burden by -mimicking languages that have generic type support. + for string substitution, which is preprocessed to a number of otherwise + identical loops with several strings such as ``INT``, ``LONG``, ``UINT``, + ``ULONG``. The ``@TYPE@`` style syntax thus reduces code duplication and + maintenance burden by mimicking languages that have generic type support. The above rules may be clearer in the following template source example: @@ -464,13 +464,13 @@ The above rules may be clearer in the following template source example: /**end repeat**/ -The preprocessing of generically typed C source files (whether in NumPy +The preprocessing of generically-typed C source files (whether in NumPy proper or in any third party package using NumPy Distutils) is performed by `conv_template.py`_. -The type specific C files generated (extension: .c) +The type-specific C files generated (extension: ``.c``) by these modules during the build process are ready to be compiled. This form of generic typing is also supported for C header files (preprocessed -to produce .h files). +to produce ``.h`` files). .. _conv_template.py: https://github.com/numpy/numpy/blob/master/numpy/distutils/conv_template.py From d846cdabf5e0f76d53cb941bf075c21ecb98d573 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Fri, 2 Oct 2020 12:24:36 +0100 Subject: [PATCH 154/409] Update doc/DISTUTILS.rst.txt --- doc/DISTUTILS.rst.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/DISTUTILS.rst.txt b/doc/DISTUTILS.rst.txt index 81ad8496397b..dbc6bf8b8ddd 100644 --- a/doc/DISTUTILS.rst.txt +++ b/doc/DISTUTILS.rst.txt @@ -394,7 +394,7 @@ and ``/**end repeat**/`` lines, which may also be nested using consecutively numbered delimiting lines such as ``/**begin repeat1`` and ``/**end repeat1**/``: -1. ``/**begin repeat ``on a line by itself marks the beginning of +1. ``/**begin repeat`` on a line by itself marks the beginning of a segment that should be repeated. 2. Named variable expansions are defined using ``#name=item1, item2, item3, From f691175d8fd836d1f65f95db4774370cc1ec3a88 Mon Sep 17 00:00:00 2001 From: danbeibei <57632244+danbeibei@users.noreply.github.com> Date: Fri, 2 Oct 2020 15:57:18 +0200 Subject: [PATCH 155/409] Update numpy/distutils/fcompiler/nv.py Co-authored-by: Eric Wieser --- numpy/distutils/fcompiler/nv.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/numpy/distutils/fcompiler/nv.py b/numpy/distutils/fcompiler/nv.py index 4cf30f5ef100..c5f203ee8ddb 100644 --- a/numpy/distutils/fcompiler/nv.py +++ b/numpy/distutils/fcompiler/nv.py @@ -1,15 +1,18 @@ -# https://developer.nvidia.com/hpc-sdk import sys from numpy.distutils.fcompiler import FCompiler compilers = ['NVHPCFCompiler'] -""" -Since august 2020 the NVIDIA HPC SDK includes the compilers formely known as The Portland Group compilers. -https://www.pgroup.com/index.htm -""" class NVHPCFCompiler(FCompiler): + """ NVIDIA High Performance Computing (HPC) SDK Fortran Compiler + + https://developer.nvidia.com/hpc-sdk + + Since august 2020 the NVIDIA HPC SDK includes the compilers formerly known as The Portland Group compilers, + https://www.pgroup.com/index.htm. + See also `numpy.distutils.fcompiler.pg`. + """ compiler_type = 'nv' description = 'NVIDIA HPC SDK' From 47a918f644da4ed3947ee2f797c790027041f29b Mon Sep 17 00:00:00 2001 From: Bas van Beek <43369155+BvB93@users.noreply.github.com> Date: Fri, 2 Oct 2020 23:25:14 +0200 Subject: [PATCH 156/409] ENH: Annotate the arithmetic operations of `ndarray` and `generic` (#17273) * ENH: Added annotations for arithmetic-based magic methods * TST: Added arithmetic tests * TST: Moved a number of tests to `arithmetic.py` * ENH: Ensure that objects annotated as `number` support arithmetic operations * MAINT: Arithmetic operations on 0d arrays return scalars * MAINT: Clarify the type of generics returned by `ufunc.__call__` * TST: Added more arithmetic tests * MAINT: Use `_CharLike` when both `str` and `bytes` are accepted * MAINT: Change the `timedelta64` baseclass to `generic` * MAINT: Add aliases for common scalar unions * MAINT: Update the defition of `_NumberLike` * MAINT: Replace `_NumberLike` with `_ComplexLike` in the `complexfloating` annotations * MAINT: Move the callback protocols to their own module * MAINT: Make `typing._callback` available at runtime * DOC: Provide further clarification about callback protocols * MAINT: Replace `_callback` with `_callable` Addresses https://github.com/numpy/numpy/pull/17273#discussion_r485821346 The use of `__call__`-defining protocols is not limited to callbacks. The module name name & docstring now reflects this. * MAINT: Removed `__add__` from `str_` and `bytes_` Most `np.bytes_` / `np.str_` methods return their builtin `bytes` / `str` counterpart. This includes addition. * MAINT: Fix the return type of boolean division Addresses https://github.com/numpy/numpy/pull/17273#discussion_r486271220 Dividing a `np.bool_` by an integer (or vice versa) always returns `float64` * MAINT: Renamed all `_Arithmetic` protocols to `_Op Addresses https://github.com/numpy/numpy/pull/17273#discussion_r486272745 * TST: Add tests for boolean division * ENH: Make `np.number` generic w.r.t. its precision * ENH,WIP: Add a mypy plugin for casting `np.number` instances to appropiate subclasses * Revert "ENH,WIP: Add a mypy plugin for casting `np.number` instances to appropiate subclasses" This reverts commit c526fb619d20902bfd77709c8983c7a7d5477c95. * Revert "ENH: Make `np.number` generic w.r.t. its precision" This reverts commit dbf20183cf7ff71e379cd1a165d07e1a1d643135. * MAINT: Narow the definition of `_ComplexLike` Addresses https://github.com/numpy/numpy/pull/17273#discussion_r490440238 * MAINT: Refined the return type of `unint + int` ops `unsignedinteger + signedinteger` generally returns a `signedinteger` subclass. The exception to this is `uint64 + signedinteger`, which returns `float64`. Addresses https://github.com/numpy/numpy/pull/17273#discussion_r490442023 * MAINT: Use `_IntLike` and `_FloatLike` in the definition of `_ComplexLike` --- numpy/__init__.pyi | 181 ++++++++++--- numpy/typing/_callable.py | 136 ++++++++++ numpy/typing/tests/data/fail/arithmetic.py | 19 ++ numpy/typing/tests/data/fail/scalars.py | 16 -- numpy/typing/tests/data/pass/arithmetic.py | 257 +++++++++++++++++++ numpy/typing/tests/data/pass/scalars.py | 13 - numpy/typing/tests/data/pass/ufuncs.py | 5 +- numpy/typing/tests/data/reveal/arithmetic.py | 256 ++++++++++++++++++ numpy/typing/tests/data/reveal/scalars.py | 17 -- 9 files changed, 811 insertions(+), 89 deletions(-) create mode 100644 numpy/typing/_callable.py create mode 100644 numpy/typing/tests/data/fail/arithmetic.py create mode 100644 numpy/typing/tests/data/pass/arithmetic.py create mode 100644 numpy/typing/tests/data/reveal/arithmetic.py diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 30d15ed12338..e712801eba7e 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -5,6 +5,18 @@ from abc import abstractmethod from numpy.core._internal import _ctypes from numpy.typing import ArrayLike, DtypeLike, _Shape, _ShapeLike +from numpy.typing._callable import ( + _BoolOp, + _BoolSub, + _BoolTrueDiv, + _TD64Div, + _IntTrueDiv, + _UnsignedIntOp, + _SignedIntOp, + _FloatOp, + _ComplexOp, + _NumberOp, +) from typing import ( Any, @@ -646,23 +658,10 @@ class _ArrayOrScalarCommon( def __ne__(self, other): ... def __gt__(self, other): ... def __ge__(self, other): ... - def __add__(self, other): ... - def __radd__(self, other): ... - def __sub__(self, other): ... - def __rsub__(self, other): ... - def __mul__(self, other): ... - def __rmul__(self, other): ... - def __truediv__(self, other): ... - def __rtruediv__(self, other): ... - def __floordiv__(self, other): ... - def __rfloordiv__(self, other): ... def __mod__(self, other): ... def __rmod__(self, other): ... def __divmod__(self, other): ... def __rdivmod__(self, other): ... - # NumPy's __pow__ doesn't handle a third argument - def __pow__(self, other): ... - def __rpow__(self, other): ... def __lshift__(self, other): ... def __rlshift__(self, other): ... def __rshift__(self, other): ... @@ -834,14 +833,26 @@ class ndarray(_ArrayOrScalarCommon, Iterable, Sized, Container): def __matmul__(self, other): ... def __imatmul__(self, other): ... def __rmatmul__(self, other): ... + def __add__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __radd__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __sub__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __rsub__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __mul__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __rmul__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __floordiv__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __rfloordiv__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __pow__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __rpow__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __truediv__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __rtruediv__(self, other: ArrayLike) -> Union[ndarray, generic]: ... # `np.generic` does not support inplace operations - def __iadd__(self, other): ... - def __isub__(self, other): ... - def __imul__(self, other): ... - def __itruediv__(self, other): ... - def __ifloordiv__(self, other): ... + def __iadd__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... + def __isub__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... + def __imul__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... + def __itruediv__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... + def __ifloordiv__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... + def __ipow__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... def __imod__(self, other): ... - def __ipow__(self, other): ... def __ilshift__(self, other): ... def __irshift__(self, other): ... def __iand__(self, other): ... @@ -857,6 +868,11 @@ class ndarray(_ArrayOrScalarCommon, Iterable, Sized, Container): # See https://github.com/numpy/numpy-stubs/pull/80 for more details. _CharLike = Union[str, bytes] +_BoolLike = Union[bool, bool_] +_IntLike = Union[int, integer] +_FloatLike = Union[_IntLike, float, floating] +_ComplexLike = Union[_FloatLike, complex, complexfloating] +_NumberLike = Union[int, float, complex, number, bool_] class generic(_ArrayOrScalarCommon): @abstractmethod @@ -869,6 +885,19 @@ class number(generic): # type: ignore def real(self: _ArraySelf) -> _ArraySelf: ... @property def imag(self: _ArraySelf) -> _ArraySelf: ... + # Ensure that objects annotated as `number` support arithmetic operations + __add__: _NumberOp + __radd__: _NumberOp + __sub__: _NumberOp + __rsub__: _NumberOp + __mul__: _NumberOp + __rmul__: _NumberOp + __floordiv__: _NumberOp + __rfloordiv__: _NumberOp + __pow__: _NumberOp + __rpow__: _NumberOp + __truediv__: _NumberOp + __rtruediv__: _NumberOp class bool_(generic): def __init__(self, __value: object = ...) -> None: ... @@ -876,6 +905,18 @@ class bool_(generic): def real(self: _ArraySelf) -> _ArraySelf: ... @property def imag(self: _ArraySelf) -> _ArraySelf: ... + __add__: _BoolOp[bool_] + __radd__: _BoolOp[bool_] + __sub__: _BoolSub + __rsub__: _BoolSub + __mul__: _BoolOp[bool_] + __rmul__: _BoolOp[bool_] + __floordiv__: _BoolOp[int8] + __rfloordiv__: _BoolOp[int8] + __pow__: _BoolOp[int8] + __rpow__: _BoolOp[int8] + __truediv__: _BoolTrueDiv + __rtruediv__: _BoolTrueDiv class object_(generic): def __init__(self, __value: object = ...) -> None: ... @@ -892,10 +933,18 @@ class datetime64(generic): __format: Union[_CharLike, Tuple[_CharLike, _IntLike]] = ..., ) -> None: ... @overload - def __init__(self, __value: int, __format: Union[_CharLike, Tuple[_CharLike, _IntLike]]) -> None: ... - def __add__(self, other: Union[timedelta64, int]) -> datetime64: ... - def __sub__(self, other: Union[timedelta64, datetime64, int]) -> timedelta64: ... - def __rsub__(self, other: Union[datetime64, int]) -> timedelta64: ... + def __init__( + self, + __value: int, + __format: Union[_CharLike, Tuple[_CharLike, _IntLike]] + ) -> None: ... + def __add__(self, other: Union[timedelta64, _IntLike, _BoolLike]) -> datetime64: ... + def __radd__(self, other: Union[timedelta64, _IntLike, _BoolLike]) -> datetime64: ... + @overload + def __sub__(self, other: datetime64) -> timedelta64: ... + @overload + def __sub__(self, other: Union[timedelta64, _IntLike, _BoolLike]) -> datetime64: ... + def __rsub__(self, other: datetime64) -> timedelta64: ... # Support for `__index__` was added in python 3.8 (bpo-20092) if sys.version_info >= (3, 8): @@ -911,8 +960,20 @@ class integer(number): # type: ignore # NOTE: `__index__` is technically defined in the bottom-most # sub-classes (`int64`, `uint32`, etc) def __index__(self) -> int: ... - -class signedinteger(integer): ... # type: ignore + __truediv__: _IntTrueDiv + __rtruediv__: _IntTrueDiv + +class signedinteger(integer): # type: ignore + __add__: _SignedIntOp + __radd__: _SignedIntOp + __sub__: _SignedIntOp + __rsub__: _SignedIntOp + __mul__: _SignedIntOp + __rmul__: _SignedIntOp + __floordiv__: _SignedIntOp + __rfloordiv__: _SignedIntOp + __pow__: _SignedIntOp + __rpow__: _SignedIntOp class int8(signedinteger): def __init__(self, __value: _IntValue = ...) -> None: ... @@ -926,24 +987,36 @@ class int32(signedinteger): class int64(signedinteger): def __init__(self, __value: _IntValue = ...) -> None: ... -class timedelta64(signedinteger): +class timedelta64(generic): def __init__( self, __value: Union[None, int, _CharLike, dt.timedelta, timedelta64] = ..., __format: Union[_CharLike, Tuple[_CharLike, _IntLike]] = ..., ) -> None: ... - @overload - def __add__(self, other: Union[timedelta64, int]) -> timedelta64: ... - @overload - def __add__(self, other: datetime64) -> datetime64: ... - def __sub__(self, other: Union[timedelta64, int]) -> timedelta64: ... - @overload - def __truediv__(self, other: timedelta64) -> float: ... - @overload - def __truediv__(self, other: float) -> timedelta64: ... + def __add__(self, other: Union[timedelta64, _IntLike, _BoolLike]) -> timedelta64: ... + def __radd__(self, other: Union[timedelta64, _IntLike, _BoolLike]) -> timedelta64: ... + def __sub__(self, other: Union[timedelta64, _IntLike, _BoolLike]) -> timedelta64: ... + def __rsub__(self, other: Union[timedelta64, _IntLike, _BoolLike]) -> timedelta64: ... + def __mul__(self, other: Union[_FloatLike, _BoolLike]) -> timedelta64: ... + def __rmul__(self, other: Union[_FloatLike, _BoolLike]) -> timedelta64: ... + __truediv__: _TD64Div[float64] + __floordiv__: _TD64Div[signedinteger] + def __rtruediv__(self, other: timedelta64) -> float64: ... + def __rfloordiv__(self, other: timedelta64) -> signedinteger: ... def __mod__(self, other: timedelta64) -> timedelta64: ... -class unsignedinteger(integer): ... # type: ignore +class unsignedinteger(integer): # type: ignore + # NOTE: `uint64 + signedinteger -> float64` + __add__: _UnsignedIntOp + __radd__: _UnsignedIntOp + __sub__: _UnsignedIntOp + __rsub__: _UnsignedIntOp + __mul__: _UnsignedIntOp + __rmul__: _UnsignedIntOp + __floordiv__: _UnsignedIntOp + __rfloordiv__: _UnsignedIntOp + __pow__: _UnsignedIntOp + __rpow__: _UnsignedIntOp class uint8(unsignedinteger): def __init__(self, __value: _IntValue = ...) -> None: ... @@ -958,7 +1031,20 @@ class uint64(unsignedinteger): def __init__(self, __value: _IntValue = ...) -> None: ... class inexact(number): ... # type: ignore -class floating(inexact): ... # type: ignore + +class floating(inexact): # type: ignore + __add__: _FloatOp + __radd__: _FloatOp + __sub__: _FloatOp + __rsub__: _FloatOp + __mul__: _FloatOp + __rmul__: _FloatOp + __truediv__: _FloatOp + __rtruediv__: _FloatOp + __floordiv__: _FloatOp + __rfloordiv__: _FloatOp + __pow__: _FloatOp + __rpow__: _FloatOp _FloatType = TypeVar('_FloatType', bound=floating) @@ -977,6 +1063,18 @@ class complexfloating(inexact, Generic[_FloatType]): # type: ignore @property def imag(self) -> _FloatType: ... # type: ignore[override] def __abs__(self) -> _FloatType: ... # type: ignore[override] + __add__: _ComplexOp + __radd__: _ComplexOp + __sub__: _ComplexOp + __rsub__: _ComplexOp + __mul__: _ComplexOp + __rmul__: _ComplexOp + __truediv__: _ComplexOp + __rtruediv__: _ComplexOp + __floordiv__: _ComplexOp + __rfloordiv__: _ComplexOp + __pow__: _ComplexOp + __rpow__: _ComplexOp class complex64(complexfloating[float32]): def __init__(self, __value: _ComplexValue = ...) -> None: ... @@ -987,7 +1085,7 @@ class complex128(complexfloating[float64], complex): class flexible(generic): ... # type: ignore class void(flexible): - def __init__(self, __value: Union[int, integer, bool_, bytes]): ... + def __init__(self, __value: Union[_IntLike, _BoolLike, bytes]): ... @property def real(self: _ArraySelf) -> _ArraySelf: ... @property @@ -995,6 +1093,9 @@ class void(flexible): class character(flexible): ... # type: ignore +# NOTE: Most `np.bytes_` / `np.str_` methods return their +# builtin `bytes` / `str` counterpart + class bytes_(character, bytes): @overload def __init__(self, __value: object = ...) -> None: ... @@ -1396,7 +1497,3 @@ def sctype2char(sctype: object) -> str: ... def find_common_type( array_types: Sequence[DtypeLike], scalar_types: Sequence[DtypeLike] ) -> dtype: ... - -_NumberLike = Union[int, float, complex, number, bool_] -_IntLike = Union[int, integer] -_BoolLike = Union[bool, bool_] diff --git a/numpy/typing/_callable.py b/numpy/typing/_callable.py new file mode 100644 index 000000000000..5e14b708f1a6 --- /dev/null +++ b/numpy/typing/_callable.py @@ -0,0 +1,136 @@ +""" +A module with various ``typing.Protocol`` subclasses that implement +the ``__call__`` magic method. + +See the `Mypy documentation`_ on protocols for more details. + +.. _`Mypy documentation`: https://mypy.readthedocs.io/en/stable/protocols.html#callback-protocols + +""" + +import sys +from typing import Union, TypeVar, overload, Any + +from numpy import ( + _BoolLike, + _IntLike, + _FloatLike, + _ComplexLike, + _NumberLike, + generic, + bool_, + timedelta64, + number, + integer, + unsignedinteger, + signedinteger, + int32, + int64, + floating, + float32, + float64, + complexfloating, + complex128, +) + +if sys.version_info >= (3, 8): + from typing import Protocol + HAVE_PROTOCOL = True +else: + try: + from typing_extensions import Protocol + except ImportError: + HAVE_PROTOCOL = False + else: + HAVE_PROTOCOL = True + +if HAVE_PROTOCOL: + _NumberType = TypeVar("_NumberType", bound=number) + _NumberType_co = TypeVar("_NumberType_co", covariant=True, bound=number) + _GenericType_co = TypeVar("_GenericType_co", covariant=True, bound=generic) + + class _BoolOp(Protocol[_GenericType_co]): + @overload + def __call__(self, __other: _BoolLike) -> _GenericType_co: ... + @overload # platform dependent + def __call__(self, __other: int) -> Union[int32, int64]: ... + @overload + def __call__(self, __other: float) -> float64: ... + @overload + def __call__(self, __other: complex) -> complex128: ... + @overload + def __call__(self, __other: _NumberType) -> _NumberType: ... + + class _BoolSub(Protocol): + # Note that `__other: bool_` is absent here + @overload # platform dependent + def __call__(self, __other: int) -> Union[int32, int64]: ... + @overload + def __call__(self, __other: float) -> float64: ... + @overload + def __call__(self, __other: complex) -> complex128: ... + @overload + def __call__(self, __other: _NumberType) -> _NumberType: ... + + class _BoolTrueDiv(Protocol): + @overload + def __call__(self, __other: Union[float, _IntLike, _BoolLike]) -> float64: ... + @overload + def __call__(self, __other: complex) -> complex128: ... + @overload + def __call__(self, __other: _NumberType) -> _NumberType: ... + + class _TD64Div(Protocol[_NumberType_co]): + @overload + def __call__(self, __other: timedelta64) -> _NumberType_co: ... + @overload + def __call__(self, __other: _FloatLike) -> timedelta64: ... + + class _IntTrueDiv(Protocol): + @overload + def __call__(self, __other: Union[_IntLike, float]) -> floating: ... + @overload + def __call__(self, __other: complex) -> complexfloating[floating]: ... + + class _UnsignedIntOp(Protocol): + # NOTE: `uint64 + signedinteger -> float64` + @overload + def __call__(self, __other: Union[bool, unsignedinteger]) -> unsignedinteger: ... + @overload + def __call__(self, __other: Union[int, signedinteger]) -> Union[signedinteger, float64]: ... + @overload + def __call__(self, __other: float) -> floating: ... + @overload + def __call__(self, __other: complex) -> complexfloating[floating]: ... + + class _SignedIntOp(Protocol): + @overload + def __call__(self, __other: Union[int, signedinteger]) -> signedinteger: ... + @overload + def __call__(self, __other: float) -> floating: ... + @overload + def __call__(self, __other: complex) -> complexfloating[floating]: ... + + class _FloatOp(Protocol): + @overload + def __call__(self, __other: _FloatLike) -> floating: ... + @overload + def __call__(self, __other: complex) -> complexfloating[floating]: ... + + class _ComplexOp(Protocol): + def __call__(self, __other: _ComplexLike) -> complexfloating[floating]: ... + + class _NumberOp(Protocol): + def __call__(self, __other: _NumberLike) -> number: ... + +else: + _BoolOp = Any + _BoolSub = Any + _BoolTrueDiv = Any + _TD64Div = Any + _IntTrueDiv = Any + _UnsignedIntOp = Any + _SignedIntOp = Any + _FloatOp = Any + _ComplexOp = Any + _NumberOp = Any diff --git a/numpy/typing/tests/data/fail/arithmetic.py b/numpy/typing/tests/data/fail/arithmetic.py new file mode 100644 index 000000000000..169e104f9a22 --- /dev/null +++ b/numpy/typing/tests/data/fail/arithmetic.py @@ -0,0 +1,19 @@ +import numpy as np + +b_ = np.bool_() +dt = np.datetime64(0, "D") +td = np.timedelta64(0, "D") + +b_ - b_ # E: No overload variant + +dt + dt # E: Unsupported operand types +td - dt # E: Unsupported operand types +td % 1 # E: Unsupported operand types +td / dt # E: No overload + +# NOTE: The 1 tests below currently don't work due to the broad +# (i.e. untyped) signature of `.__mod__()`. +# TODO: Revisit this once annotations are added to the +# `_ArrayOrScalarCommon` magic methods. + +# td % dt # E: Unsupported operand types diff --git a/numpy/typing/tests/data/fail/scalars.py b/numpy/typing/tests/data/fail/scalars.py index 47c031163636..13bb45483353 100644 --- a/numpy/typing/tests/data/fail/scalars.py +++ b/numpy/typing/tests/data/fail/scalars.py @@ -28,22 +28,6 @@ np.datetime64(0) # E: non-matching overload -dt_64 = np.datetime64(0, "D") -td_64 = np.timedelta64(1, "h") - -dt_64 + dt_64 # E: Unsupported operand types -td_64 - dt_64 # E: Unsupported operand types -td_64 % 1 # E: Unsupported operand types - -# NOTE: The 2 tests below currently don't work due to the broad -# (i.e. untyped) signature of `generic.__truediv__()` and `.__mod__()`. -# TODO: Revisit this once annotations are added to the -# `_ArrayOrScalarCommon` magic methods. - -# td_64 / dt_64 # E: No overload -# td_64 % dt_64 # E: Unsupported operand types - - class A: def __float__(self): return 1.0 diff --git a/numpy/typing/tests/data/pass/arithmetic.py b/numpy/typing/tests/data/pass/arithmetic.py new file mode 100644 index 000000000000..f26eab879255 --- /dev/null +++ b/numpy/typing/tests/data/pass/arithmetic.py @@ -0,0 +1,257 @@ +import numpy as np + +c16 = np.complex128(1) +f8 = np.float64(1) +i8 = np.int64(1) +u8 = np.uint64(1) + +c8 = np.complex64(1) +f4 = np.float32(1) +i4 = np.int32(1) +u4 = np.uint32(1) + +dt = np.datetime64(1, "D") +td = np.timedelta64(1, "D") + +b_ = np.bool_(1) + +b = bool(1) +c = complex(1) +f = float(1) +i = int(1) + +AR = np.ones(1, dtype=np.float64) +AR.setflags(write=False) + +# Time structures + +dt + td +dt + i +dt + i4 +dt + i8 +dt - dt +dt - i +dt - i4 +dt - i8 + +td + td +td + i +td + i4 +td + i8 +td - td +td - i +td - i4 +td - i8 +td / f +td / f4 +td / f8 +td / td +td // td +td % td + + +# boolean + +b_ / b +b_ / b_ +b_ / i +b_ / i8 +b_ / i4 +b_ / u8 +b_ / u4 +b_ / f +b_ / f8 +b_ / f4 +b_ / c +b_ / c16 +b_ / c8 + +b / b_ +b_ / b_ +i / b_ +i8 / b_ +i4 / b_ +u8 / b_ +u4 / b_ +f / b_ +f8 / b_ +f4 / b_ +c / b_ +c16 / b_ +c8 / b_ + +# Complex + +c16 + c16 +c16 + f8 +c16 + i8 +c16 + c8 +c16 + f4 +c16 + i4 +c16 + b_ +c16 + b +c16 + c +c16 + f +c16 + i +c16 + AR + +c16 + c16 +f8 + c16 +i8 + c16 +c8 + c16 +f4 + c16 +i4 + c16 +b_ + c16 +b + c16 +c + c16 +f + c16 +i + c16 +AR + c16 + +c8 + c16 +c8 + f8 +c8 + i8 +c8 + c8 +c8 + f4 +c8 + i4 +c8 + b_ +c8 + b +c8 + c +c8 + f +c8 + i +c8 + AR + +c16 + c8 +f8 + c8 +i8 + c8 +c8 + c8 +f4 + c8 +i4 + c8 +b_ + c8 +b + c8 +c + c8 +f + c8 +i + c8 +AR + c8 + +# Float + +f8 + f8 +f8 + i8 +f8 + f4 +f8 + i4 +f8 + b_ +f8 + b +f8 + c +f8 + f +f8 + i +f8 + AR + +f8 + f8 +i8 + f8 +f4 + f8 +i4 + f8 +b_ + f8 +b + f8 +c + f8 +f + f8 +i + f8 +AR + f8 + +f4 + f8 +f4 + i8 +f4 + f4 +f4 + i4 +f4 + b_ +f4 + b +f4 + c +f4 + f +f4 + i +f4 + AR + +f8 + f4 +i8 + f4 +f4 + f4 +i4 + f4 +b_ + f4 +b + f4 +c + f4 +f + f4 +i + f4 +AR + f4 + +# Int + +i8 + i8 +i8 + u8 +i8 + i4 +i8 + u4 +i8 + b_ +i8 + b +i8 + c +i8 + f +i8 + i +i8 + AR + +u8 + u8 +u8 + i4 +u8 + u4 +u8 + b_ +u8 + b +u8 + c +u8 + f +u8 + i +u8 + AR + +i8 + i8 +u8 + i8 +i4 + i8 +u4 + i8 +b_ + i8 +b + i8 +c + i8 +f + i8 +i + i8 +AR + i8 + +u8 + u8 +i4 + u8 +u4 + u8 +b_ + u8 +b + u8 +c + u8 +f + u8 +i + u8 +AR + u8 + +i4 + i8 +i4 + i4 +i4 + i +i4 + b_ +i4 + b +i4 + AR + +u4 + i8 +u4 + i4 +u4 + u8 +u4 + u4 +u4 + i +u4 + b_ +u4 + b +u4 + AR + +i8 + i4 +i4 + i4 +i + i4 +b_ + i4 +b + i4 +AR + i4 + +i8 + u4 +i4 + u4 +u8 + u4 +u4 + u4 +b_ + u4 +b + u4 +i + u4 +AR + u4 diff --git a/numpy/typing/tests/data/pass/scalars.py b/numpy/typing/tests/data/pass/scalars.py index c02e1ed36f28..49ddb8ed91f5 100644 --- a/numpy/typing/tests/data/pass/scalars.py +++ b/numpy/typing/tests/data/pass/scalars.py @@ -108,19 +108,6 @@ def __float__(self) -> float: np.timedelta64(None) np.timedelta64(None, "D") -dt_64 = np.datetime64(0, "D") -td_64 = np.timedelta64(1, "h") - -dt_64 + td_64 -dt_64 - dt_64 -dt_64 - td_64 - -td_64 + td_64 -td_64 - td_64 -td_64 / 1.0 -td_64 / td_64 -td_64 % td_64 - np.void(1) np.void(np.int64(1)) np.void(True) diff --git a/numpy/typing/tests/data/pass/ufuncs.py b/numpy/typing/tests/data/pass/ufuncs.py index 82172952a61d..ad4d483d4a70 100644 --- a/numpy/typing/tests/data/pass/ufuncs.py +++ b/numpy/typing/tests/data/pass/ufuncs.py @@ -6,7 +6,10 @@ np.matmul(np.ones((2, 2, 2)), np.ones((2, 2, 2)), axes=[(0, 1), (0, 1), (0, 1)]) np.sin(1, signature="D") np.sin(1, extobj=[16, 1, lambda: None]) -np.sin(1) + np.sin(1) +# NOTE: `np.generic` subclasses are not guaranteed to support addition; +# re-enable this we can infer the exact return type of `np.sin(...)`. +# +# np.sin(1) + np.sin(1) np.sin.types[0] np.sin.__name__ diff --git a/numpy/typing/tests/data/reveal/arithmetic.py b/numpy/typing/tests/data/reveal/arithmetic.py new file mode 100644 index 000000000000..b8c457aafb53 --- /dev/null +++ b/numpy/typing/tests/data/reveal/arithmetic.py @@ -0,0 +1,256 @@ +import numpy as np + +c16 = np.complex128() +f8 = np.float64() +i8 = np.int64() +u8 = np.uint64() + +c8 = np.complex64() +f4 = np.float32() +i4 = np.int32() +u4 = np.uint32() + +dt = np.datetime64(0, "D") +td = np.timedelta64(0, "D") + +b_ = np.bool_() + +b = bool() +c = complex() +f = float() +i = int() + +AR = np.array([0], dtype=np.float64) +AR.setflags(write=False) + +# Time structures + +reveal_type(dt + td) # E: numpy.datetime64 +reveal_type(dt + i) # E: numpy.datetime64 +reveal_type(dt + i4) # E: numpy.datetime64 +reveal_type(dt + i8) # E: numpy.datetime64 +reveal_type(dt - dt) # E: numpy.timedelta64 +reveal_type(dt - i) # E: numpy.datetime64 +reveal_type(dt - i4) # E: numpy.datetime64 +reveal_type(dt - i8) # E: numpy.datetime64 + +reveal_type(td + td) # E: numpy.timedelta64 +reveal_type(td + i) # E: numpy.timedelta64 +reveal_type(td + i4) # E: numpy.timedelta64 +reveal_type(td + i8) # E: numpy.timedelta64 +reveal_type(td - td) # E: numpy.timedelta64 +reveal_type(td - i) # E: numpy.timedelta64 +reveal_type(td - i4) # E: numpy.timedelta64 +reveal_type(td - i8) # E: numpy.timedelta64 +reveal_type(td / f) # E: numpy.timedelta64 +reveal_type(td / f4) # E: numpy.timedelta64 +reveal_type(td / f8) # E: numpy.timedelta64 +reveal_type(td / td) # E: float64 +reveal_type(td // td) # E: signedinteger +reveal_type(td % td) # E: numpy.timedelta64 + +# boolean + +reveal_type(b_ / b) # E: float64 +reveal_type(b_ / b_) # E: float64 +reveal_type(b_ / i) # E: float64 +reveal_type(b_ / i8) # E: float64 +reveal_type(b_ / i4) # E: float64 +reveal_type(b_ / u8) # E: float64 +reveal_type(b_ / u4) # E: float64 +reveal_type(b_ / f) # E: float64 +reveal_type(b_ / f8) # E: float64 +reveal_type(b_ / f4) # E: float32 +reveal_type(b_ / c) # E: complex128 +reveal_type(b_ / c16) # E: complex128 +reveal_type(b_ / c8) # E: complex64 + +reveal_type(b / b_) # E: float64 +reveal_type(b_ / b_) # E: float64 +reveal_type(i / b_) # E: float64 +reveal_type(i8 / b_) # E: float64 +reveal_type(i4 / b_) # E: float64 +reveal_type(u8 / b_) # E: float64 +reveal_type(u4 / b_) # E: float64 +reveal_type(f / b_) # E: float64 +reveal_type(f8 / b_) # E: float64 +reveal_type(f4 / b_) # E: float32 +reveal_type(c / b_) # E: complex128 +reveal_type(c16 / b_) # E: complex128 +reveal_type(c8 / b_) # E: complex64 + +# Complex + +reveal_type(c16 + c16) # E: complexfloating +reveal_type(c16 + f8) # E: complexfloating +reveal_type(c16 + i8) # E: complexfloating +reveal_type(c16 + c8) # E: complexfloating +reveal_type(c16 + f4) # E: complexfloating +reveal_type(c16 + i4) # E: complexfloating +reveal_type(c16 + b_) # E: complex128 +reveal_type(c16 + b) # E: complexfloating +reveal_type(c16 + c) # E: complexfloating +reveal_type(c16 + f) # E: complexfloating +reveal_type(c16 + i) # E: complexfloating +reveal_type(c16 + AR) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(c16 + c16) # E: complexfloating +reveal_type(f8 + c16) # E: complexfloating +reveal_type(i8 + c16) # E: complexfloating +reveal_type(c8 + c16) # E: complexfloating +reveal_type(f4 + c16) # E: complexfloating +reveal_type(i4 + c16) # E: complexfloating +reveal_type(b_ + c16) # E: complex128 +reveal_type(b + c16) # E: complexfloating +reveal_type(c + c16) # E: complexfloating +reveal_type(f + c16) # E: complexfloating +reveal_type(i + c16) # E: complexfloating +reveal_type(AR + c16) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(c8 + c16) # E: complexfloating +reveal_type(c8 + f8) # E: complexfloating +reveal_type(c8 + i8) # E: complexfloating +reveal_type(c8 + c8) # E: complexfloating +reveal_type(c8 + f4) # E: complexfloating +reveal_type(c8 + i4) # E: complexfloating +reveal_type(c8 + b_) # E: complex64 +reveal_type(c8 + b) # E: complexfloating +reveal_type(c8 + c) # E: complexfloating +reveal_type(c8 + f) # E: complexfloating +reveal_type(c8 + i) # E: complexfloating +reveal_type(c8 + AR) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(c16 + c8) # E: complexfloating +reveal_type(f8 + c8) # E: complexfloating +reveal_type(i8 + c8) # E: complexfloating +reveal_type(c8 + c8) # E: complexfloating +reveal_type(f4 + c8) # E: complexfloating +reveal_type(i4 + c8) # E: complexfloating +reveal_type(b_ + c8) # E: complex64 +reveal_type(b + c8) # E: complexfloating +reveal_type(c + c8) # E: complexfloating +reveal_type(f + c8) # E: complexfloating +reveal_type(i + c8) # E: complexfloating +reveal_type(AR + c8) # E: Union[numpy.ndarray, numpy.generic] + +# Float + +reveal_type(f8 + f8) # E: floating +reveal_type(f8 + i8) # E: floating +reveal_type(f8 + f4) # E: floating +reveal_type(f8 + i4) # E: floating +reveal_type(f8 + b_) # E: float64 +reveal_type(f8 + b) # E: floating +reveal_type(f8 + c) # E: complexfloating +reveal_type(f8 + f) # E: floating +reveal_type(f8 + i) # E: floating +reveal_type(f8 + AR) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(f8 + f8) # E: floating +reveal_type(i8 + f8) # E: floating +reveal_type(f4 + f8) # E: floating +reveal_type(i4 + f8) # E: floating +reveal_type(b_ + f8) # E: float64 +reveal_type(b + f8) # E: floating +reveal_type(c + f8) # E: complexfloating +reveal_type(f + f8) # E: floating +reveal_type(i + f8) # E: floating +reveal_type(AR + f8) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(f4 + f8) # E: floating +reveal_type(f4 + i8) # E: floating +reveal_type(f4 + f4) # E: floating +reveal_type(f4 + i4) # E: floating +reveal_type(f4 + b_) # E: float32 +reveal_type(f4 + b) # E: floating +reveal_type(f4 + c) # E: complexfloating +reveal_type(f4 + f) # E: floating +reveal_type(f4 + i) # E: floating +reveal_type(f4 + AR) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(f8 + f4) # E: floating +reveal_type(i8 + f4) # E: floating +reveal_type(f4 + f4) # E: floating +reveal_type(i4 + f4) # E: floating +reveal_type(b_ + f4) # E: float32 +reveal_type(b + f4) # E: floating +reveal_type(c + f4) # E: complexfloating +reveal_type(f + f4) # E: floating +reveal_type(i + f4) # E: floating +reveal_type(AR + f4) # E: Union[numpy.ndarray, numpy.generic] + +# Int + +reveal_type(i8 + i8) # E: signedinteger +reveal_type(i8 + u8) # E: Union[numpy.signedinteger, numpy.float64] +reveal_type(i8 + i4) # E: signedinteger +reveal_type(i8 + u4) # E: Union[numpy.signedinteger, numpy.float64] +reveal_type(i8 + b_) # E: int64 +reveal_type(i8 + b) # E: signedinteger +reveal_type(i8 + c) # E: complexfloating +reveal_type(i8 + f) # E: floating +reveal_type(i8 + i) # E: signedinteger +reveal_type(i8 + AR) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(u8 + u8) # E: unsignedinteger +reveal_type(u8 + i4) # E: Union[numpy.signedinteger, numpy.float64] +reveal_type(u8 + u4) # E: unsignedinteger +reveal_type(u8 + b_) # E: uint64 +reveal_type(u8 + b) # E: unsignedinteger +reveal_type(u8 + c) # E: complexfloating +reveal_type(u8 + f) # E: floating +reveal_type(u8 + i) # E: Union[numpy.signedinteger, numpy.float64] +reveal_type(u8 + AR) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(i8 + i8) # E: signedinteger +reveal_type(u8 + i8) # E: Union[numpy.signedinteger, numpy.float64] +reveal_type(i4 + i8) # E: signedinteger +reveal_type(u4 + i8) # E: Union[numpy.signedinteger, numpy.float64] +reveal_type(b_ + i8) # E: int64 +reveal_type(b + i8) # E: signedinteger +reveal_type(c + i8) # E: complexfloating +reveal_type(f + i8) # E: floating +reveal_type(i + i8) # E: signedinteger +reveal_type(AR + i8) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(u8 + u8) # E: unsignedinteger +reveal_type(i4 + u8) # E: Union[numpy.signedinteger, numpy.float64] +reveal_type(u4 + u8) # E: unsignedinteger +reveal_type(b_ + u8) # E: uint64 +reveal_type(b + u8) # E: unsignedinteger +reveal_type(c + u8) # E: complexfloating +reveal_type(f + u8) # E: floating +reveal_type(i + u8) # E: Union[numpy.signedinteger, numpy.float64] +reveal_type(AR + u8) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(i4 + i8) # E: signedinteger +reveal_type(i4 + i4) # E: signedinteger +reveal_type(i4 + i) # E: signedinteger +reveal_type(i4 + b_) # E: int32 +reveal_type(i4 + b) # E: signedinteger +reveal_type(i4 + AR) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(u4 + i8) # E: Union[numpy.signedinteger, numpy.float64] +reveal_type(u4 + i4) # E: Union[numpy.signedinteger, numpy.float64] +reveal_type(u4 + u8) # E: unsignedinteger +reveal_type(u4 + u4) # E: unsignedinteger +reveal_type(u4 + i) # E: Union[numpy.signedinteger, numpy.float64] +reveal_type(u4 + b_) # E: uint32 +reveal_type(u4 + b) # E: unsignedinteger +reveal_type(u4 + AR) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(i8 + i4) # E: signedinteger +reveal_type(i4 + i4) # E: signedinteger +reveal_type(i + i4) # E: signedinteger +reveal_type(b_ + i4) # E: int32 +reveal_type(b + i4) # E: signedinteger +reveal_type(AR + i4) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(i8 + u4) # E: Union[numpy.signedinteger, numpy.float64] +reveal_type(i4 + u4) # E: Union[numpy.signedinteger, numpy.float64] +reveal_type(u8 + u4) # E: unsignedinteger +reveal_type(u4 + u4) # E: unsignedinteger +reveal_type(b_ + u4) # E: uint32 +reveal_type(b + u4) # E: unsignedinteger +reveal_type(i + u4) # E: Union[numpy.signedinteger, numpy.float64] +reveal_type(AR + u4) # E: Union[numpy.ndarray, numpy.generic] diff --git a/numpy/typing/tests/data/reveal/scalars.py b/numpy/typing/tests/data/reveal/scalars.py index 882fe96128d6..ec3713b0fc42 100644 --- a/numpy/typing/tests/data/reveal/scalars.py +++ b/numpy/typing/tests/data/reveal/scalars.py @@ -12,22 +12,5 @@ reveal_type(x.shape) # E: tuple[builtins.int] reveal_type(x.strides) # E: tuple[builtins.int] -# Time structures -dt = np.datetime64(0, "D") -td = np.timedelta64(0, "D") - -reveal_type(dt + td) # E: numpy.datetime64 -reveal_type(dt + 1) # E: numpy.datetime64 -reveal_type(dt - dt) # E: numpy.timedelta64 -reveal_type(dt - 1) # E: numpy.timedelta64 - -reveal_type(td + td) # E: numpy.timedelta64 -reveal_type(td + 1) # E: numpy.timedelta64 -reveal_type(td - td) # E: numpy.timedelta64 -reveal_type(td - 1) # E: numpy.timedelta64 -reveal_type(td / 1.0) # E: numpy.timedelta64 -reveal_type(td / td) # E: float -reveal_type(td % td) # E: numpy.timedelta64 - reveal_type(np.complex64().real) # E: numpy.float32 reveal_type(np.complex128().imag) # E: numpy.float64 From e4312341c1ad9c102383a1d8a86a56490a3019c8 Mon Sep 17 00:00:00 2001 From: Safouane Chergui <31566323+chsafouane@users.noreply.github.com> Date: Fri, 2 Oct 2020 23:41:16 +0200 Subject: [PATCH 157/409] nep-0029 typo correction to provides -> to provide --- doc/neps/nep-0029-deprecation_policy.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/neps/nep-0029-deprecation_policy.rst b/doc/neps/nep-0029-deprecation_policy.rst index 4674d24ec663..957674ee6e20 100644 --- a/doc/neps/nep-0029-deprecation_policy.rst +++ b/doc/neps/nep-0029-deprecation_policy.rst @@ -77,7 +77,7 @@ release in November 2020 should support Python 3.7 and newer. The current Python release cadence is 18 months so a 42 month window ensures that there will always be at least two minor versions of Python in the window. The window is extended 6 months beyond the anticipated two-release -interval for Python to provides resilience against small fluctuations / +interval for Python to provide resilience against small fluctuations / delays in its release schedule. Because Python minor version support is based only on historical From f632c948ff954892acda4d7849b8aa0e632c0a25 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 2 Oct 2020 19:45:27 -0500 Subject: [PATCH 158/409] BUG: Fix memoryleaks related to NEP 37 function overrides This adds a few missing DECREFs/XDECREFs, all of these are already covered by the tests and can be found running a leak-checker. --- numpy/core/src/multiarray/arrayfunction_override.c | 4 ++++ numpy/core/src/multiarray/multiarraymodule.c | 2 ++ 2 files changed, 6 insertions(+) diff --git a/numpy/core/src/multiarray/arrayfunction_override.c b/numpy/core/src/multiarray/arrayfunction_override.c index 613fe6b3f09f..8e3bde78f714 100644 --- a/numpy/core/src/multiarray/arrayfunction_override.c +++ b/numpy/core/src/multiarray/arrayfunction_override.c @@ -388,15 +388,18 @@ array_implement_c_array_function_creation( PyObject *numpy_module = PyImport_Import(npy_ma_str_numpy); if (numpy_module == NULL) { + Py_DECREF(relevant_args); return NULL; } PyObject *public_api = PyObject_GetAttrString(numpy_module, function_name); Py_DECREF(numpy_module); if (public_api == NULL) { + Py_DECREF(relevant_args); return NULL; } if (!PyCallable_Check(public_api)) { + Py_DECREF(relevant_args); Py_DECREF(public_api); return PyErr_Format(PyExc_RuntimeError, "numpy.%s is not callable.", @@ -406,6 +409,7 @@ array_implement_c_array_function_creation( PyObject* result = array_implement_array_function_internal( public_api, relevant_args, args, kwargs); + Py_DECREF(relevant_args); Py_DECREF(public_api); return result; } diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index ff2b796d2908..1aad70dc65bb 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -2296,6 +2296,7 @@ array_fromiter(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds) array_function_result = array_implement_c_array_function_creation( "fromiter", args, keywds); if (array_function_result != Py_NotImplemented) { + Py_DECREF(descr); return array_function_result; } @@ -2942,6 +2943,7 @@ array_arange(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kws) { array_function_result = array_implement_c_array_function_creation( "arange", args, kws); if (array_function_result != Py_NotImplemented) { + Py_XDECREF(typecode); return array_function_result; } From 24d368a809827cd2cd787a470a65b4a7cd721d10 Mon Sep 17 00:00:00 2001 From: Takanori H Date: Sat, 3 Oct 2020 14:39:31 +0900 Subject: [PATCH 159/409] DOC: Fix the links for ``Ellipsis`` (#17431) --- doc/source/reference/arrays.indexing.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/source/reference/arrays.indexing.rst b/doc/source/reference/arrays.indexing.rst index 180a79dae18c..b2a9f1d219f3 100644 --- a/doc/source/reference/arrays.indexing.rst +++ b/doc/source/reference/arrays.indexing.rst @@ -34,7 +34,7 @@ Basic Slicing and Indexing Basic slicing extends Python's basic concept of slicing to N dimensions. Basic slicing occurs when *obj* is a :class:`slice` object (constructed by ``start:stop:step`` notation inside of brackets), an -integer, or a tuple of slice objects and integers. :const:`Ellipsis` +integer, or a tuple of slice objects and integers. :py:data:`Ellipsis` and :const:`newaxis` objects can be interspersed with these as well. @@ -43,7 +43,7 @@ well. In order to remain backward compatible with a common usage in Numeric, basic slicing is also initiated if the selection object is any non-ndarray and non-tuple sequence (such as a :class:`list`) containing - :class:`slice` objects, the :const:`Ellipsis` object, or the :const:`newaxis` + :class:`slice` objects, the :py:data:`Ellipsis` object, or the :const:`newaxis` object, but not for integer arrays or other embedded sequences. .. index:: @@ -129,7 +129,7 @@ concepts to remember include: [5], [6]]]) -- :const:`Ellipsis` expands to the number of ``:`` objects needed for the +- :py:data:`Ellipsis` expands to the number of ``:`` objects needed for the selection tuple to index all dimensions. In most cases, this means that length of the expanded selection tuple is ``x.ndim``. There may only be a single ellipsis present. @@ -333,7 +333,7 @@ the subspace defined by the basic indexing (excluding integers) and the subspace from the advanced indexing part. Two cases of index combination need to be distinguished: -* The advanced indexes are separated by a slice, :const:`Ellipsis` or :const:`newaxis`. +* The advanced indexes are separated by a slice, :py:data:`Ellipsis` or :const:`newaxis`. For example ``x[arr1, :, arr2]``. * The advanced indexes are all next to each other. For example ``x[..., arr1, arr2, :]`` but *not* ``x[arr1, :, 1]`` From 35c67f6d33aa82d1ee8bd4fbdc5b925ef560de2a Mon Sep 17 00:00:00 2001 From: Alex Rogozhnikov Date: Sat, 3 Oct 2020 02:28:51 -0700 Subject: [PATCH 160/409] DOC: add references to einops and opt_einsum Following discussion in mailing list, this PR adds mentions to third-party libs einops and opt_einsum to einsum documentation --- numpy/core/einsumfunc.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/numpy/core/einsumfunc.py b/numpy/core/einsumfunc.py index f65f4015c928..e0942becaaa5 100644 --- a/numpy/core/einsumfunc.py +++ b/numpy/core/einsumfunc.py @@ -1062,6 +1062,17 @@ def einsum(*operands, out=None, optimize=False, **kwargs): -------- einsum_path, dot, inner, outer, tensordot, linalg.multi_dot + einops: + similar verbose interface is provided by + `einops `_ package to cover + additional operations: transpose, reshape/flatten, repeat/tile, + squeeze/unsqueeze and reductions. + + opt_einsum: + `opt_einsum `_ + optimizes contraction order for einsum-like expressions + in backend-agnostic manner. + Notes ----- .. versionadded:: 1.6.0 From 30aca02691baf56dd9b5d81dcefebdb3e575678e Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Sat, 3 Oct 2020 08:16:10 -0600 Subject: [PATCH 161/409] MAINT : Disable 32 bit PyPy CI testing on Windows. The test is consistently failing at the moment, we can enable it again when PyPy gets fixed. --- azure-pipelines.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index da57649b884b..9382ac83c129 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -217,11 +217,11 @@ stages: BITS: 64 NPY_USE_BLAS_ILP64: '1' OPENBLAS_SUFFIX: '64_' - PyPy36-32bit: - PYTHON_VERSION: 'PyPy3.6' - PYTHON_ARCH: 'x32' - TEST_MODE: fast - BITS: 32 + #PyPy36-32bit: + #PYTHON_VERSION: 'PyPy3.6' + #PYTHON_ARCH: 'x32' + #TEST_MODE: fast + #BITS: 32 steps: - template: azure-steps-windows.yml - job: Linux_PyPy3 From 92a3c76976a7a791933160544bb73cd37c28da6a Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Sat, 3 Oct 2020 12:16:08 -0400 Subject: [PATCH 162/409] DOC: Security warning for issues template Tag for "Bug report" now warns not to file security vulnerabilities. Fixes gh-17434. --- .github/ISSUE_TEMPLATE/bug-report.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index d2df08689da5..78ffc1063eaf 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -1,6 +1,6 @@ --- -name: "Bug Report" -about: Submit a bug report to help us improve NumPy +name: "Bug report" +about: Report a bug. Not for security vulnerabilities -- see below. --- From d55243ea48b06d82584ec0ead0de5dc72b70be8f Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Sat, 3 Oct 2020 12:26:22 -0400 Subject: [PATCH 163/409] DOC: Fix "Feature request" spelling in issue templates "Request" had an upper-case R, inconsistent with Google style and other template items. --- .github/ISSUE_TEMPLATE/feature-request.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md index 0be94f92893d..00c6f59c5faf 100644 --- a/.github/ISSUE_TEMPLATE/feature-request.md +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -1,5 +1,5 @@ --- -name: "Feature Request" +name: "Feature request" about: Check instructions for submitting your idea on the mailing list first. --- From 3ebcf252efb8189e5d2b114685280fd5fff007ef Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Sat, 3 Oct 2020 15:17:31 -0400 Subject: [PATCH 164/409] DOC: Revise credit line per CC Adds "CC BY 4.0" credit to the Pauli Vertanen example. --- doc/source/user/how-to-io.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/user/how-to-io.rst b/doc/source/user/how-to-io.rst index 1d4ded159422..ca9fc41f07f8 100644 --- a/doc/source/user/how-to-io.rst +++ b/doc/source/user/how-to-io.rst @@ -250,7 +250,8 @@ The ``.wav`` file header as a NumPy structured dtype:: This ``.wav`` example is for illustration; to read a ``.wav`` file in real life, use Python's built-in module :mod:`wave`. -Credit: Pauli Virtanen, :ref:`advanced_numpy`. +(Adapted from Pauli Virtanen, :ref:`advanced_numpy`, licensed +under `CC BY 4.0 `_.) .. _how-to-io-large-arrays: From fe01a9b1cca286f67e01807786b991dd22b1b7f7 Mon Sep 17 00:00:00 2001 From: Abhishek Singh <34937403+devhpes@users.noreply.github.com> Date: Sun, 4 Oct 2020 01:42:11 +0530 Subject: [PATCH 165/409] fix exception chaining in format.py --- numpy/lib/format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/lib/format.py b/numpy/lib/format.py index afbd3784a9ad..5d951e262570 100644 --- a/numpy/lib/format.py +++ b/numpy/lib/format.py @@ -746,7 +746,7 @@ def read_array(fp, allow_pickle=False, pickle_kwargs=None): # Friendlier error message raise UnicodeError("Unpickling a python object failed: %r\n" "You may need to pass the encoding= option " - "to numpy.load" % (err,)) + "to numpy.load" % (err,)) from err else: if isfileobj(fp): # We can use the fast fromfile() function. From 93b5da05bcaa39d76b07c334690f85d5d54f54df Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Sat, 3 Oct 2020 17:03:18 -0400 Subject: [PATCH 166/409] DOC: Cleaner PR template Make the instructions easy to see and hard to delete unread. Moves some text to new sections in dev/development_workflow.rst. The dashes above and below the heading, which in monospace seem too long, fit it well in Chromium browsers and reasonably in Firefox. --- .github/PULL_REQUEST_TEMPLATE.md | 40 ++++++++++++++----------- doc/source/dev/development_workflow.rst | 35 ++++++++++++++++------ 2 files changed, 48 insertions(+), 27 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 3634292f8ce7..075d2519b2ce 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,18 +1,22 @@ - - - - - - - + diff --git a/doc/source/dev/development_workflow.rst b/doc/source/dev/development_workflow.rst index d5a49a9f9512..53f3eef9136b 100644 --- a/doc/source/dev/development_workflow.rst +++ b/doc/source/dev/development_workflow.rst @@ -188,6 +188,16 @@ Standard acronyms to start the commit message with are:: REL: related to releasing numpy +.. _workflow_mailing_list: + +Get the mailing list's opinion +======================================================= + +If you plan a new feature or API change, it's wisest to first email the +NumPy `mailing list `_ +asking for comment. If you haven't heard back in a week, it's +OK to ping the list again. + .. _asking-for-merging: Asking for your changes to be merged with the main repo @@ -197,15 +207,22 @@ When you feel your work is finished, you can create a pull request (PR). Github has a nice help page that outlines the process for `filing pull requests`_. If your changes involve modifications to the API or addition/modification of a -function, you should - -- send an email to the `NumPy mailing list`_ with a link to your PR along with - a description of and a motivation for your changes. This may generate - changes and feedback. It might be prudent to start with this step if your - change may be controversial. -- add a release note to the ``doc/release/upcoming_changes/`` directory, - following the instructions and format in the - ``doc/release/upcoming_changes/README.rst`` file. +function, add a release note to the ``doc/release/upcoming_changes/`` +directory, following the instructions and format in the +``doc/release/upcoming_changes/README.rst`` file. + + +Getting your PR reviewed +======================== + +We review pull requests as soon as we can, typically within a week. If you get +no review comments within two weeks, feel free to ask for feedback by +adding a comment on your PR (this will notify maintainers). + +If your PR is large or complicated, asking for input on the numpy-discussion +mailing list may also be useful. + + .. _rebasing-on-master: From 1edd9693c2879e41d1f787a0731b49e286985c22 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Sat, 3 Oct 2020 17:11:32 -0400 Subject: [PATCH 167/409] DOC: Fixes for PR 17440 Colon missing in heading. Added sphinx anchor for PR-timeline (html anchor was OK). --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- doc/source/dev/development_workflow.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 075d2519b2ce..1f8d35aabe49 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -9,7 +9,7 @@ http://www.numpy.org/devdocs/dev/development_workflow.html#writing-the-commit-message -* IF IT'S A NEW FEATURE OR API CHANGE, TEST THE WATERS +* IF IT'S A NEW FEATURE OR API CHANGE, TEST THE WATERS: http://www.numpy.org/devdocs/dev/development_workflow.html#get-the-mailing-list-s-opinion diff --git a/doc/source/dev/development_workflow.rst b/doc/source/dev/development_workflow.rst index 53f3eef9136b..1665cfddb9b7 100644 --- a/doc/source/dev/development_workflow.rst +++ b/doc/source/dev/development_workflow.rst @@ -212,6 +212,8 @@ directory, following the instructions and format in the ``doc/release/upcoming_changes/README.rst`` file. +.. _workflow_PR_timeline: + Getting your PR reviewed ======================== From 49b10b0e32473441d84810fff236dbb0a5d11768 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Sat, 3 Oct 2020 17:16:17 -0400 Subject: [PATCH 168/409] DOC: Fix indents in PE #17440 --- .github/PULL_REQUEST_TEMPLATE.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 1f8d35aabe49..dee33ee5fd13 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -10,13 +10,13 @@ * IF IT'S A NEW FEATURE OR API CHANGE, TEST THE WATERS: - http://www.numpy.org/devdocs/dev/development_workflow.html#get-the-mailing-list-s-opinion + http://www.numpy.org/devdocs/dev/development_workflow.html#get-the-mailing-list-s-opinion * HIT ALL THE GUIDELINES: - https://numpy.org/devdocs/dev/index.html#guidelines + https://numpy.org/devdocs/dev/index.html#guidelines * WHAT TO DO IF WE HAVEN'T GOTTEN BACK TO YOU: - http://www.numpy.org/devdocs/dev/development_workflow.html#getting-your-pr-reviewed + http://www.numpy.org/devdocs/dev/development_workflow.html#getting-your-pr-reviewed --> From f3a6b33144e62ce77aa767b73d930dec420a3511 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 4 Oct 2020 12:09:52 +0300 Subject: [PATCH 169/409] ENH: Warn on unsupported Python 3.9+ --- setup.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1f5212676310..b4c06c402186 100755 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ import sys import subprocess import textwrap -import sysconfig +import warnings if sys.version_info[:2] < (3, 6): @@ -59,6 +59,13 @@ ISRELEASED = False VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) +if sys.version_info >= (3, 9): + warnings.warn( + f"NumPy {VERSION} does not support Python " + f"{sys.version_info.major}.{sys.version_info.minor}.", + RuntimeWarning, + ) + # Return the git revision as a string def git_version(): From 2ebb45374d845ad6d4843e977c0ca03e53674d25 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 4 Oct 2020 12:41:47 +0300 Subject: [PATCH 170/409] MAINT: Fix Flake8 --- setup.py | 52 +++++++++++++++++++++++++--------------------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/setup.py b/setup.py index b4c06c402186..a4e498af09a3 100755 --- a/setup.py +++ b/setup.py @@ -95,6 +95,7 @@ def _minimal_ext_cmd(cmd): return GIT_REVISION + # BEFORE importing setuptools, remove MANIFEST. Otherwise it may not be # properly updated when the contents of directories change (true for distutils, # not sure about setuptools). @@ -157,7 +158,7 @@ def write_version_py(filename='numpy/version.py'): a.close() -def configuration(parent_package='',top_path=None): +def configuration(parent_package='', top_path=None): from numpy.distutils.misc_util import Configuration config = Configuration(None, parent_package, top_path) @@ -170,7 +171,7 @@ def configuration(parent_package='',top_path=None): config.add_data_files(('numpy', 'LICENSE.txt')) config.add_data_files(('numpy', 'numpy/*.pxd')) - config.get_version('numpy/version.py') # sets config.version + config.get_version('numpy/version.py') # sets config.version return config @@ -182,13 +183,12 @@ def check_submodules(): if not os.path.exists('.git'): return with open('.gitmodules') as f: - for l in f: - if 'path' in l: - p = l.split('=')[-1].strip() + for line in f: + if 'path' in line: + p = line.split('=')[-1].strip() if not os.path.exists(p): raise ValueError('Submodule {} missing'.format(p)) - proc = subprocess.Popen(['git', 'submodule', 'status'], stdout=subprocess.PIPE) status, _ = proc.communicate() @@ -280,9 +280,9 @@ def generate_cython(): print("Cythonizing sources") for d in ('random',): p = subprocess.call([sys.executable, - os.path.join(cwd, 'tools', 'cythonize.py'), - 'numpy/{0}'.format(d)], - cwd=cwd) + os.path.join(cwd, 'tools', 'cythonize.py'), + 'numpy/{0}'.format(d)], + cwd=cwd) if p != 0: raise RuntimeError("Running cythonize failed!") @@ -353,7 +353,6 @@ def parse_setuppy_commands(): """)) return False - # The following commands aren't supported. They can only be executed when # the user explicitly adds a --force command-line argument. bad_commands = dict( @@ -391,8 +390,8 @@ def parse_setuppy_commands(): ) bad_commands['nosetests'] = bad_commands['test'] for command in ('upload_docs', 'easy_install', 'bdist', 'bdist_dumb', - 'register', 'check', 'install_data', 'install_headers', - 'install_lib', 'install_scripts', ): + 'register', 'check', 'install_data', 'install_headers', + 'install_lib', 'install_scripts', ): bad_commands[command] = "`setup.py %s` is not supported" % command for command in bad_commands.keys(): @@ -412,7 +411,8 @@ def parse_setuppy_commands(): # If we got here, we didn't detect what setup.py command was given import warnings warnings.warn("Unrecognized setuptools command, proceeding with " - "generating Cython sources and expanding templates", stacklevel=2) + "generating Cython sources and expanding templates", + stacklevel=2) return True @@ -447,25 +447,24 @@ def setup_package(): 'f2py%s.%s = numpy.f2py.f2py2e:main' % sys.version_info[:2], ] - cmdclass={"sdist": sdist_checked, - } + cmdclass = {"sdist": sdist_checked, } metadata = dict( - name = 'numpy', - maintainer = "NumPy Developers", - maintainer_email = "numpy-discussion@python.org", - description = DOCLINES[0], - long_description = "\n".join(DOCLINES[2:]), - url = "https://www.numpy.org", - author = "Travis E. Oliphant et al.", - download_url = "https://pypi.python.org/pypi/numpy", + name='numpy', + maintainer="NumPy Developers", + maintainer_email="numpy-discussion@python.org", + description=DOCLINES[0], + long_description="\n".join(DOCLINES[2:]), + url="https://www.numpy.org", + author="Travis E. Oliphant et al.", + download_url="https://pypi.python.org/pypi/numpy", project_urls={ "Bug Tracker": "https://github.com/numpy/numpy/issues", "Documentation": get_docs_url(), "Source Code": "https://github.com/numpy/numpy", }, - license = 'BSD', + license='BSD', classifiers=[_f for _f in CLASSIFIERS.split('\n') if _f], - platforms = ["Windows", "Linux", "Solaris", "Mac OS-X", "Unix"], + platforms=["Windows", "Linux", "Solaris", "Mac OS-X", "Unix"], test_suite='pytest', cmdclass=cmdclass, python_requires='>=3.6', @@ -486,8 +485,7 @@ def setup_package(): # patches distutils, even though we don't use it import setuptools # noqa: F401 from numpy.distutils.core import setup - cwd = os.path.abspath(os.path.dirname(__file__)) - if not 'sdist' in sys.argv: + if 'sdist' not in sys.argv: # Generate Cython sources, unless we're generating an sdist generate_cython() From c549dbb47d8b4e2d9ee6572ddaa78e494ec41238 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 4 Oct 2020 14:51:01 +0300 Subject: [PATCH 171/409] Add comment to remind to keep consistent with classifiers Co-authored-by: Eric Wieser --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index a4e498af09a3..e4fe91faec32 100755 --- a/setup.py +++ b/setup.py @@ -59,6 +59,7 @@ ISRELEASED = False VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) +# keep this consistent with the `Programming Language :: Python :: ...` classifiers above if sys.version_info >= (3, 9): warnings.warn( f"NumPy {VERSION} does not support Python " From a30c410b1743e38ae32501f199a643629708cbd7 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 4 Oct 2020 14:52:21 +0300 Subject: [PATCH 172/409] Clarify warning, it might work before official support Co-authored-by: Eric Wieser --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e4fe91faec32..ffa3107afd00 100755 --- a/setup.py +++ b/setup.py @@ -62,7 +62,7 @@ # keep this consistent with the `Programming Language :: Python :: ...` classifiers above if sys.version_info >= (3, 9): warnings.warn( - f"NumPy {VERSION} does not support Python " + f"NumPy {VERSION} may not yet support Python " f"{sys.version_info.major}.{sys.version_info.minor}.", RuntimeWarning, ) From cac2df03a9f5caac03482b95f4db4c145ac2aa8a Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 4 Oct 2020 14:53:20 +0300 Subject: [PATCH 173/409] Already compatible with 3.10 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ffa3107afd00..09c66a4eef65 100755 --- a/setup.py +++ b/setup.py @@ -60,7 +60,7 @@ VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) # keep this consistent with the `Programming Language :: Python :: ...` classifiers above -if sys.version_info >= (3, 9): +if sys.version_info >= (3, 10): warnings.warn( f"NumPy {VERSION} may not yet support Python " f"{sys.version_info.major}.{sys.version_info.minor}.", From 7178a1dc24484248a86525f22a8d80bf0d4510f3 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 4 Oct 2020 14:59:45 +0300 Subject: [PATCH 174/409] Update comment Co-authored-by: Eric Wieser --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 09c66a4eef65..fccc2277863d 100755 --- a/setup.py +++ b/setup.py @@ -59,7 +59,7 @@ ISRELEASED = False VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) -# keep this consistent with the `Programming Language :: Python :: ...` classifiers above +# The first version not in the `Programming Language :: Python :: ...` classifiers above if sys.version_info >= (3, 10): warnings.warn( f"NumPy {VERSION} may not yet support Python " From 79a8e16ed2de533dc91e474c912cfaef8e1a1e92 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 4 Oct 2020 14:59:53 +0300 Subject: [PATCH 175/409] Add 3.10 classifier --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index fccc2277863d..e99313502afc 100755 --- a/setup.py +++ b/setup.py @@ -43,6 +43,7 @@ Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 +Programming Language :: Python :: 3.9 Programming Language :: Python :: 3 :: Only Programming Language :: Python :: Implementation :: CPython Topic :: Software Development From e592c27da4e3747184aa29acc799d06bcc77519b Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Sun, 4 Oct 2020 15:10:42 +0200 Subject: [PATCH 176/409] ENH: Add `Typing :: Typed` to the PyPi classifier --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 1f5212676310..cf0858e5f9ab 100755 --- a/setup.py +++ b/setup.py @@ -47,6 +47,7 @@ Programming Language :: Python :: Implementation :: CPython Topic :: Software Development Topic :: Scientific/Engineering +Typing :: Typed Operating System :: Microsoft :: Windows Operating System :: POSIX Operating System :: Unix From d84ff867373b60927d0dc3ff511a22828a9ac8dc Mon Sep 17 00:00:00 2001 From: takanori-pskq Date: Sun, 4 Oct 2020 21:39:30 +0900 Subject: [PATCH 177/409] DOC: Fix the references for macros --- doc/source/reference/arrays.interface.rst | 8 ++++---- doc/source/reference/c-api/array.rst | 8 +++++++- doc/source/reference/c-api/config.rst | 6 ++++++ 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/doc/source/reference/arrays.interface.rst b/doc/source/reference/arrays.interface.rst index 73e4aef0c9f8..6d12cf999790 100644 --- a/doc/source/reference/arrays.interface.rst +++ b/doc/source/reference/arrays.interface.rst @@ -231,10 +231,10 @@ as:: The flags member may consist of 5 bits showing how the data should be interpreted and one bit showing how the Interface should be -interpreted. The data-bits are :const:`CONTIGUOUS` (0x1), -:const:`FORTRAN` (0x2), :const:`ALIGNED` (0x100), :const:`NOTSWAPPED` -(0x200), and :const:`WRITEABLE` (0x400). A final flag -:const:`ARR_HAS_DESCR` (0x800) indicates whether or not this structure +interpreted. The data-bits are :c:macro:`NPY_ARRAY_C_CONTIGUOUS` (0x1), +:c:macro:`NPY_ARRAY_F_CONTIGUOUS` (0x2), :c:macro:`NPY_ARRAY_ALIGNED` (0x100), +:c:macro:`NPY_ARRAY_NOTSWAPPED` (0x200), and :c:macro:`NPY_ARRAY_WRITEABLE` (0x400). A final flag +:c:macro:`NPY_ARR_HAS_DESCR` (0x800) indicates whether or not this structure has the arrdescr field. The field should not be accessed unless this flag is present. diff --git a/doc/source/reference/c-api/array.rst b/doc/source/reference/c-api/array.rst index cfe4d2d51fa5..9fe45d2def92 100644 --- a/doc/source/reference/c-api/array.rst +++ b/doc/source/reference/c-api/array.rst @@ -3259,6 +3259,8 @@ Memory management :c:data:`NPY_USE_PYMEM` is 0, if :c:data:`NPY_USE_PYMEM` is 1, then the Python memory allocator is used. + .. c:macro:: NPY_USE_PYMEM + .. c:function:: int PyArray_ResolveWritebackIfCopy(PyArrayObject* obj) If ``obj.flags`` has :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` or (deprecated) @@ -3289,9 +3291,13 @@ be accomplished using two groups of macros. Typically, if one macro in a group is used in a code block, all of them must be used in the same code block. Currently, :c:data:`NPY_ALLOW_THREADS` is defined to the python-defined :c:data:`WITH_THREADS` constant unless the environment -variable :c:data:`NPY_NOSMP` is set in which case +variable ``NPY_NOSMP`` is set in which case :c:data:`NPY_ALLOW_THREADS` is defined to be 0. +.. c:macro:: NPY_ALLOW_THREADS + +.. c:macro:: WITH_THREADS + Group 1 """"""" diff --git a/doc/source/reference/c-api/config.rst b/doc/source/reference/c-api/config.rst index c3e2c98af078..5fdbc0ff1ece 100644 --- a/doc/source/reference/c-api/config.rst +++ b/doc/source/reference/c-api/config.rst @@ -102,6 +102,12 @@ Platform information One of :c:data:`NPY_CPU_BIG`, :c:data:`NPY_CPU_LITTLE`, or :c:data:`NPY_CPU_UNKNOWN_ENDIAN`. + .. c:macro:: NPY_CPU_BIG + + .. c:macro:: NPY_CPU_LITTLE + + .. c:macro:: NPY_CPU_UNKNOWN_ENDIAN + Compiler directives ------------------- From 7fb0b02df1ca5b388f81f88b22f478d65ce1b74e Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Sun, 4 Oct 2020 13:13:05 -0400 Subject: [PATCH 178/409] DOC: Remove CoC pages from Sphinx code_of_conduct.rst and report_handling_manual.rst duplicate pages now on Hugo. --- doc/source/dev/conduct/code_of_conduct.rst | 163 ------------- .../dev/conduct/report_handling_manual.rst | 220 ------------------ doc/source/dev/index.rst | 2 - 3 files changed, 385 deletions(-) delete mode 100644 doc/source/dev/conduct/code_of_conduct.rst delete mode 100644 doc/source/dev/conduct/report_handling_manual.rst diff --git a/doc/source/dev/conduct/code_of_conduct.rst b/doc/source/dev/conduct/code_of_conduct.rst deleted file mode 100644 index f2f0a536dc56..000000000000 --- a/doc/source/dev/conduct/code_of_conduct.rst +++ /dev/null @@ -1,163 +0,0 @@ -NumPy Code of Conduct -===================== - - -Introduction ------------- - -This code of conduct applies to all spaces managed by the NumPy project, -including all public and private mailing lists, issue trackers, wikis, blogs, -Twitter, and any other communication channel used by our community. The NumPy -project does not organise in-person events, however events related to our -community should have a code of conduct similar in spirit to this one. - -This code of conduct should be honored by everyone who participates in -the NumPy community formally or informally, or claims any affiliation with the -project, in any project-related activities and especially when representing the -project, in any role. - -This code is not exhaustive or complete. It serves to distill our common -understanding of a collaborative, shared environment and goals. Please try to -follow this code in spirit as much as in letter, to create a friendly and -productive environment that enriches the surrounding community. - - -Specific Guidelines -------------------- - -We strive to: - -1. Be open. We invite anyone to participate in our community. We prefer to use - public methods of communication for project-related messages, unless - discussing something sensitive. This applies to messages for help or - project-related support, too; not only is a public support request much more - likely to result in an answer to a question, it also ensures that any - inadvertent mistakes in answering are more easily detected and corrected. - -2. Be empathetic, welcoming, friendly, and patient. We work together to resolve - conflict, and assume good intentions. We may all experience some frustration - from time to time, but we do not allow frustration to turn into a personal - attack. A community where people feel uncomfortable or threatened is not a - productive one. - -3. Be collaborative. Our work will be used by other people, and in turn we will - depend on the work of others. When we make something for the benefit of the - project, we are willing to explain to others how it works, so that they can - build on the work to make it even better. Any decision we make will affect - users and colleagues, and we take those consequences seriously when making - decisions. - -4. Be inquisitive. Nobody knows everything! Asking questions early avoids many - problems later, so we encourage questions, although we may direct them to - the appropriate forum. We will try hard to be responsive and helpful. - -5. Be careful in the words that we choose. We are careful and respectful in - our communication and we take responsibility for our own speech. Be kind to - others. Do not insult or put down other participants. We will not accept - harassment or other exclusionary behaviour, such as: - - - Violent threats or language directed against another person. - - Sexist, racist, or otherwise discriminatory jokes and language. - - Posting sexually explicit or violent material. - - Posting (or threatening to post) other people's personally identifying information ("doxing"). - - Sharing private content, such as emails sent privately or non-publicly, - or unlogged forums such as IRC channel history, without the sender's consent. - - Personal insults, especially those using racist or sexist terms. - - Unwelcome sexual attention. - - Excessive profanity. Please avoid swearwords; people differ greatly in their sensitivity to swearing. - - Repeated harassment of others. In general, if someone asks you to stop, then stop. - - Advocating for, or encouraging, any of the above behaviour. - - -Diversity Statement -------------------- - -The NumPy project welcomes and encourages participation by everyone. We are -committed to being a community that everyone enjoys being part of. Although -we may not always be able to accommodate each individual's preferences, we try -our best to treat everyone kindly. - -No matter how you identify yourself or how others perceive you: we welcome you. -Though no list can hope to be comprehensive, we explicitly honour diversity in: -age, culture, ethnicity, genotype, gender identity or expression, language, -national origin, neurotype, phenotype, political beliefs, profession, race, -religion, sexual orientation, socioeconomic status, subculture and technical -ability, to the extent that these do not conflict with this code of conduct. - - -Though we welcome people fluent in all languages, NumPy development is -conducted in English. - -Standards for behaviour in the NumPy community are detailed in the Code of -Conduct above. Participants in our community should uphold these standards -in all their interactions and help others to do so as well (see next section). - - -Reporting Guidelines --------------------- - -We know that it is painfully common for internet communication to start at or -devolve into obvious and flagrant abuse. We also recognize that sometimes -people may have a bad day, or be unaware of some of the guidelines in this Code -of Conduct. Please keep this in mind when deciding on how to respond to a -breach of this Code. - -For clearly intentional breaches, report those to the Code of Conduct committee -(see below). For possibly unintentional breaches, you may reply to the person -and point out this code of conduct (either in public or in private, whatever is -most appropriate). If you would prefer not to do that, please feel free to -report to the Code of Conduct Committee directly, or ask the Committee for -advice, in confidence. - -You can report issues to the NumPy Code of Conduct committee, at -numpy-conduct@googlegroups.com. Currently, the committee consists of: - -- Stefan van der Walt -- Melissa Weber Mendonça -- Anirudh Subramanian - -If your report involves any members of the committee, or if they feel they have -a conflict of interest in handling it, then they will recuse themselves from -considering your report. Alternatively, if for any reason you feel -uncomfortable making a report to the committee, then you can also contact: - -- Senior `NumFOCUS staff `__: conduct@numfocus.org - - -Incident reporting resolution & Code of Conduct enforcement ------------------------------------------------------------ - -*This section summarizes the most important points, more details can be found -in* :ref:`CoC_reporting_manual`. - -We will investigate and respond to all complaints. The NumPy Code of Conduct -Committee and the NumPy Steering Committee (if involved) will protect the -identity of the reporter, and treat the content of complaints as confidential -(unless the reporter agrees otherwise). - -In case of severe and obvious breaches, e.g. personal threat or violent, sexist -or racist language, we will immediately disconnect the originator from NumPy -communication channels; please see the manual for details. - -In cases not involving clear severe and obvious breaches of this code of -conduct, the process for acting on any received code of conduct violation -report will be: - -1. acknowledge report is received -2. reasonable discussion/feedback -3. mediation (if feedback didn't help, and only if both reporter and reportee agree to this) -4. enforcement via transparent decision (see :ref:`CoC_resolutions`) by the - Code of Conduct Committee - -The committee will respond to any report as soon as possible, and at most -within 72 hours. - - -Endnotes --------- - -We are thankful to the groups behind the following documents, from which we -drew content and inspiration: - -- `The SciPy Code of Conduct `_ - diff --git a/doc/source/dev/conduct/report_handling_manual.rst b/doc/source/dev/conduct/report_handling_manual.rst deleted file mode 100644 index d39b615bb8c1..000000000000 --- a/doc/source/dev/conduct/report_handling_manual.rst +++ /dev/null @@ -1,220 +0,0 @@ -:orphan: - -.. _CoC_reporting_manual: - -NumPy Code of Conduct - How to follow up on a report ----------------------------------------------------- - -This is the manual followed by NumPy's Code of Conduct Committee. It's used -when we respond to an issue to make sure we're consistent and fair. - -Enforcing the Code of Conduct impacts our community today and for the future. -It's an action that we do not take lightly. When reviewing enforcement -measures, the Code of Conduct Committee will keep the following values and -guidelines in mind: - -* Act in a personal manner rather than impersonal. The Committee can engage - the parties to understand the situation, while respecting the privacy and any - necessary confidentiality of reporters. However, sometimes it is necessary - to communicate with one or more individuals directly: the Committee's goal is - to improve the health of our community rather than only produce a formal - decision. - -* Emphasize empathy for individuals rather than judging behavior, avoiding - binary labels of "good" and "bad/evil". Overt, clear-cut aggression and - harassment exists and we will be address that firmly. But many scenarios - that can prove challenging to resolve are those where normal disagreements - devolve into unhelpful or harmful behavior from multiple parties. - Understanding the full context and finding a path that re-engages all is - hard, but ultimately the most productive for our community. - -* We understand that email is a difficult medium and can be isolating. - Receiving criticism over email, without personal contact, can be - particularly painful. This makes it especially important to keep an - atmosphere of open-minded respect of the views of others. It also means - that we must be transparent in our actions, and that we will do everything - in our power to make sure that all our members are treated fairly and with - sympathy. - -* Discrimination can be subtle and it can be unconscious. It can show itself - as unfairness and hostility in otherwise ordinary interactions. We know - that this does occur, and we will take care to look out for it. We would - very much like to hear from you if you feel you have been treated unfairly, - and we will use these procedures to make sure that your complaint is heard - and addressed. - -* Help increase engagement in good discussion practice: try to identify where - discussion may have broken down and provide actionable information, pointers - and resources that can lead to positive change on these points. - -* Be mindful of the needs of new members: provide them with explicit support - and consideration, with the aim of increasing participation from - underrepresented groups in particular. - -* Individuals come from different cultural backgrounds and native languages. - Try to identify any honest misunderstandings caused by a non-native speaker - and help them understand the issue and what they can change to avoid causing - offence. Complex discussion in a foreign language can be very intimidating, - and we want to grow our diversity also across nationalities and cultures. - -*Mediation*: voluntary, informal mediation is a tool at our disposal. In -contexts such as when two or more parties have all escalated to the point of -inappropriate behavior (something sadly common in human conflict), it may be -useful to facilitate a mediation process. This is only an example: the -Committee can consider mediation in any case, mindful that the process is meant -to be strictly voluntary and no party can be pressured to participate. If the -Committee suggests mediation, it should: - -* Find a candidate who can serve as a mediator. -* Obtain the agreement of the reporter(s). The reporter(s) have complete - freedom to decline the mediation idea, or to propose an alternate mediator. -* Obtain the agreement of the reported person(s). -* Settle on the mediator: while parties can propose a different mediator than - the suggested candidate, only if common agreement is reached on all terms can - the process move forward. -* Establish a timeline for mediation to complete, ideally within two weeks. - -The mediator will engage with all the parties and seek a resolution that is -satisfactory to all. Upon completion, the mediator will provide a report -(vetted by all parties to the process) to the Committee, with recommendations -on further steps. The Committee will then evaluate these results (whether -satisfactory resolution was achieved or not) and decide on any additional -action deemed necessary. - - -How the committee will respond to reports -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -When the committee (or a committee member) receives a report, they will first -determine whether the report is about a clear and severe breach (as defined -below). If so, immediate action needs to be taken in addition to the regular -report handling process. - -Clear and severe breach actions -+++++++++++++++++++++++++++++++ - -We know that it is painfully common for internet communication to start at or -devolve into obvious and flagrant abuse. We will deal quickly with clear and -severe breaches like personal threats, violent, sexist or racist language. - -When a member of the Code of Conduct committee becomes aware of a clear and -severe breach, they will do the following: - -* Immediately disconnect the originator from all NumPy communication channels. -* Reply to the reporter that their report has been received and that the - originator has been disconnected. -* In every case, the moderator should make a reasonable effort to contact the - originator, and tell them specifically how their language or actions - qualify as a "clear and severe breach". The moderator should also say - that, if the originator believes this is unfair or they want to be - reconnected to NumPy, they have the right to ask for a review, as below, by - the Code of Conduct Committee. - The moderator should copy this explanation to the Code of Conduct Committee. -* The Code of Conduct Committee will formally review and sign off on all cases - where this mechanism has been applied to make sure it is not being used to - control ordinary heated disagreement. - -Report handling -+++++++++++++++ - -When a report is sent to the committee they will immediately reply to the -reporter to confirm receipt. This reply must be sent within 72 hours, and the -group should strive to respond much quicker than that. - -If a report doesn't contain enough information, the committee will obtain all -relevant data before acting. The committee is empowered to act on the Steering -Council’s behalf in contacting any individuals involved to get a more complete -account of events. - -The committee will then review the incident and determine, to the best of their -ability: - -* What happened. -* Whether this event constitutes a Code of Conduct violation. -* Who are the responsible party(ies). -* Whether this is an ongoing situation, and there is a threat to anyone's - physical safety. - -This information will be collected in writing, and whenever possible the -group's deliberations will be recorded and retained (i.e. chat transcripts, -email discussions, recorded conference calls, summaries of voice conversations, -etc). - -It is important to retain an archive of all activities of this committee to -ensure consistency in behavior and provide institutional memory for the -project. To assist in this, the default channel of discussion for this -committee will be a private mailing list accessible to current and future -members of the committee as well as members of the Steering Council upon -justified request. If the Committee finds the need to use off-list -communications (e.g. phone calls for early/rapid response), it should in all -cases summarize these back to the list so there's a good record of the process. - -The Code of Conduct Committee should aim to have a resolution agreed upon within -two weeks. In the event that a resolution can't be determined in that time, the -committee will respond to the reporter(s) with an update and projected timeline -for resolution. - - -.. _CoC_resolutions: - -Resolutions -~~~~~~~~~~~ - -The committee must agree on a resolution by consensus. If the group cannot reach -consensus and deadlocks for over a week, the group will turn the matter over to -the Steering Council for resolution. - - -Possible responses may include: - -* Taking no further action - - - if we determine no violations have occurred. - - if the matter has been resolved publicly while the committee was considering responses. - -* Coordinating voluntary mediation: if all involved parties agree, the - Committee may facilitate a mediation process as detailed above. -* Remind publicly, and point out that some behavior/actions/language have been - judged inappropriate and why in the current context, or can but hurtful to - some people, requesting the community to self-adjust. -* A private reprimand from the committee to the individual(s) involved. In this - case, the group chair will deliver that reprimand to the individual(s) over - email, cc'ing the group. -* A public reprimand. In this case, the committee chair will deliver that - reprimand in the same venue that the violation occurred, within the limits of - practicality. E.g., the original mailing list for an email violation, but - for a chat room discussion where the person/context may be gone, they can be - reached by other means. The group may choose to publish this message - elsewhere for documentation purposes. -* A request for a public or private apology, assuming the reporter agrees to - this idea: they may at their discretion refuse further contact with the - violator. The chair will deliver this request. The committee may, if it - chooses, attach "strings" to this request: for example, the group may ask a - violator to apologize in order to retain one’s membership on a mailing list. -* A "mutually agreed upon hiatus" where the committee asks the individual to - temporarily refrain from community participation. If the individual chooses - not to take a temporary break voluntarily, the committee may issue a - "mandatory cooling off period". -* A permanent or temporary ban from some or all NumPy spaces (mailing lists, - gitter.im, etc.). The group will maintain records of all such bans so that - they may be reviewed in the future or otherwise maintained. - -Once a resolution is agreed upon, but before it is enacted, the committee will -contact the original reporter and any other affected parties and explain the -proposed resolution. The committee will ask if this resolution is acceptable, -and must note feedback for the record. - -Finally, the committee will make a report to the NumPy Steering Council (as -well as the NumPy core team in the event of an ongoing resolution, such as a -ban). - -The committee will never publicly discuss the issue; all public statements will -be made by the chair of the Code of Conduct Committee or the NumPy Steering -Council. - - -Conflicts of Interest -~~~~~~~~~~~~~~~~~~~~~ - -In the event of any conflict of interest, a committee member must immediately -notify the other members, and recuse themselves if necessary. diff --git a/doc/source/dev/index.rst b/doc/source/dev/index.rst index c4f35b68ff6b..020df0b2bd29 100644 --- a/doc/source/dev/index.rst +++ b/doc/source/dev/index.rst @@ -9,7 +9,6 @@ Contributing to NumPy .. toctree:: :hidden: - conduct/code_of_conduct Git Basics development_environment development_workflow @@ -293,7 +292,6 @@ The rest of the story .. toctree:: :maxdepth: 2 - conduct/code_of_conduct Git Basics development_environment development_workflow From 5ee77370b92f24b06a221774c6b1c4292e6b09ef Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Sun, 4 Oct 2020 19:15:08 +0100 Subject: [PATCH 179/409] DOC: Improve [source] links for C extension types Previously, these would all link to `numpy/core/__init__.py`. Now the scalar type and `ndarray` link to the files where the `PyTypeObject` is defined. In future, we should do this for all extension types, probably automatically. --- doc/source/conf.py | 43 +++++++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index fe7ea096740d..5bb8870b4f8c 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -310,6 +310,17 @@ def setup(app): else: print("NOTE: linkcode extension not found -- no links to source generated") + +def _get_c_source_file(obj): + if issubclass(obj, numpy.generic): + return r"core/src/multiarray/scalartypes.c.src" + elif obj is numpy.ndarray: + return r"core/src/multiarray/arrayobject.c" + else: + # todo: come up with a better way to generate these + return None + + def linkcode_resolve(domain, info): """ Determine the URL corresponding to Python object @@ -340,25 +351,33 @@ def linkcode_resolve(domain, info): else: obj = unwrap(obj) - try: - fn = inspect.getsourcefile(obj) - except Exception: - fn = None - if not fn: - return None + fn = None + lineno = None - try: - source, lineno = inspect.getsourcelines(obj) - except Exception: - lineno = None + # Make a poor effort at linking C extension types + if isinstance(obj, type) and obj.__module__ == 'numpy': + fn = _get_c_source_file(obj) + + if fn is None: + try: + fn = inspect.getsourcefile(obj) + except Exception: + fn = None + if not fn: + return None + + try: + source, lineno = inspect.getsourcelines(obj) + except Exception: + lineno = None + + fn = relpath(fn, start=dirname(numpy.__file__)) if lineno: linespec = "#L%d-L%d" % (lineno, lineno + len(source) - 1) else: linespec = "" - fn = relpath(fn, start=dirname(numpy.__file__)) - if 'dev' in numpy.__version__: return "https://github.com/numpy/numpy/blob/master/numpy/%s%s" % ( fn, linespec) From 2877fb04ecd9816edadac9d42555e06aaa807f2d Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Sun, 4 Oct 2020 21:01:54 +0100 Subject: [PATCH 180/409] DOC: Change the value of type.__name__ on the scalar types for sphinx builds By default, the `.__name__` of the numeric `np.generic` subclasses is their bitlength name, such as `np.int64`. This is convenient when working interactively, because it lets users see the size of their array easily; but in docs it is confusing, as the sizes of the integers in the doc build may not match their size on the platform of the user reading them. Without this change, `..autoclass:: numpy.short` would just display "alias of uint16", which is backwards. Rather than changing the names globally, or adding a build flag to change the names, this uses `ctypes` to modify the scalar names at startup. This resembles the approach taken by the `forbiddenfruit` module for patching builtin slots, although that would be overkill here. The timing of when we perform this patching is important - we can't do it until after `numpy.core._umath_multiarray` has been loaded, but we need to do it before `numpy.core._add_newdocs` generates the name-based docstrings. Similarly, we can't just disable `numpy.core._add_newdocs` until later, as it populates docstrings in `ndarray` on which `numpy.ma.core` does further processing. To resolve this, we split out the scalar docstrings in `numpy.core._add_newdocs` into a new module `numpy.core._add_newdocs_scalars` that _is_ safe to disable until later. --- doc/source/conf.py | 56 +++++++++ numpy/core/__init__.py | 1 + numpy/core/_add_newdocs.py | 182 --------------------------- numpy/core/_add_newdocs_scalars.py | 195 +++++++++++++++++++++++++++++ 4 files changed, 252 insertions(+), 182 deletions(-) create mode 100644 numpy/core/_add_newdocs_scalars.py diff --git a/doc/source/conf.py b/doc/source/conf.py index 5bb8870b4f8c..98a2565dbbb1 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -6,6 +6,62 @@ # Minimum version, enforced by sphinx needs_sphinx = '2.2.0' + +# This is a nasty hack to use platform-agnostic names for types in the +# documentation. + +# must be kept alive to hold the patched names +_name_cache = {} + +def replace_scalar_type_names(): + """ Rename numpy types to use the canonical names to make sphinx behave """ + import ctypes + + Py_ssize_t = ctypes.c_int64 if ctypes.sizeof(ctypes.c_void_p) == 8 else ctypes.c_int32 + + class PyObject(ctypes.Structure): + pass + + class PyTypeObject(ctypes.Structure): + pass + + PyObject._fields_ = [ + ('ob_refcnt', Py_ssize_t), + ('ob_type', ctypes.POINTER(PyTypeObject)), + ] + + + PyTypeObject._fields_ = [ + # varhead + ('ob_base', PyObject), + ('ob_size', Py_ssize_t), + # declaration + ('tp_name', ctypes.c_char_p), + ] + + # prevent numpy attaching docstrings to the scalar types + assert 'numpy.core._add_newdocs_scalars' not in sys.modules + sys.modules['numpy.core._add_newdocs_scalars'] = object() + + import numpy + + # change the __name__ of the scalar types + for name in [ + 'byte', 'short', 'intc', 'int_', 'longlong', + 'ubyte', 'ushort', 'uintc', 'uint', 'ulonglong', + 'half', 'single', 'double', 'longdouble', + 'half', 'csingle', 'cdouble', 'clongdouble', + ]: + typ = getattr(numpy, name) + c_typ = PyTypeObject.from_address(id(typ)) + c_typ.tp_name = _name_cache[typ] = b"numpy." + name.encode('utf8') + + # now generate the docstrings as usual + del sys.modules['numpy.core._add_newdocs_scalars'] + import numpy.core._add_newdocs_scalars + +replace_scalar_type_names() + # ----------------------------------------------------------------------------- # General configuration # ----------------------------------------------------------------------------- diff --git a/numpy/core/__init__.py b/numpy/core/__init__.py index a0769cc89112..e8d3a381b602 100644 --- a/numpy/core/__init__.py +++ b/numpy/core/__init__.py @@ -96,6 +96,7 @@ # do this after everything else, to minimize the chance of this misleadingly # appearing in an import-time traceback from . import _add_newdocs +from . import _add_newdocs_scalars # add these for module-freeze analysis (like PyInstaller) from . import _dtype_ctypes from . import _internal diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py index b8cf12c60c1c..aa858761da78 100644 --- a/numpy/core/_add_newdocs.py +++ b/numpy/core/_add_newdocs.py @@ -9,8 +9,6 @@ """ -from numpy.core import numerictypes as _numerictypes -from numpy.core import dtype from numpy.core.function_base import add_newdoc from numpy.core.overrides import array_function_like_doc @@ -6283,183 +6281,3 @@ def refer_to_array_attribute(attr, method=True): Abstract base class of all character string scalar types. """) - - -############################################################################## -# -# Documentation for concrete scalar classes -# -############################################################################## - -def numeric_type_aliases(aliases): - def type_aliases_gen(): - for alias, doc in aliases: - try: - alias_type = getattr(_numerictypes, alias) - except AttributeError: - # The set of aliases that actually exist varies between platforms - pass - else: - yield (alias_type, alias, doc) - return list(type_aliases_gen()) - - -possible_aliases = numeric_type_aliases([ - ('int8', '8-bit signed integer (-128 to 127)'), - ('int16', '16-bit signed integer (-32768 to 32767)'), - ('int32', '32-bit signed integer (-2147483648 to 2147483647)'), - ('int64', '64-bit signed integer (-9223372036854775808 to 9223372036854775807)'), - ('intp', 'Signed integer large enough to fit pointer, compatible with C ``intptr_t``'), - ('uint8', '8-bit unsigned integer (0 to 255)'), - ('uint16', '16-bit unsigned integer (0 to 65535)'), - ('uint32', '32-bit unsigned integer (0 to 4294967295)'), - ('uint64', '64-bit unsigned integer (0 to 18446744073709551615)'), - ('uintp', 'Unsigned integer large enough to fit pointer, compatible with C ``uintptr_t``'), - ('float16', '16-bit-precision floating-point number type: sign bit, 5 bits exponent, 10 bits mantissa'), - ('float32', '32-bit-precision floating-point number type: sign bit, 8 bits exponent, 23 bits mantissa'), - ('float64', '64-bit precision floating-point number type: sign bit, 11 bits exponent, 52 bits mantissa'), - ('float96', '96-bit extended-precision floating-point number type'), - ('float128', '128-bit extended-precision floating-point number type'), - ('complex64', 'Complex number type composed of 2 32-bit-precision floating-point numbers'), - ('complex128', 'Complex number type composed of 2 64-bit-precision floating-point numbers'), - ('complex192', 'Complex number type composed of 2 96-bit extended-precision floating-point numbers'), - ('complex256', 'Complex number type composed of 2 128-bit extended-precision floating-point numbers'), - ]) - - -def add_newdoc_for_scalar_type(obj, fixed_aliases, doc): - o = getattr(_numerictypes, obj) - - character_code = dtype(o).char - canonical_name_doc = "" if obj == o.__name__ else "Canonical name: ``np.{}``.\n ".format(obj) - alias_doc = ''.join("Alias: ``np.{}``.\n ".format(alias) for alias in fixed_aliases) - alias_doc += ''.join("Alias *on this platform*: ``np.{}``: {}.\n ".format(alias, doc) - for (alias_type, alias, doc) in possible_aliases if alias_type is o) - - docstring = """ - {doc} - Character code: ``'{character_code}'``. - {canonical_name_doc}{alias_doc} - """.format(doc=doc.strip(), character_code=character_code, - canonical_name_doc=canonical_name_doc, alias_doc=alias_doc) - - add_newdoc('numpy.core.numerictypes', obj, docstring) - - -add_newdoc_for_scalar_type('bool_', ['bool8'], - """ - Boolean type (True or False), stored as a byte. - """) - -add_newdoc_for_scalar_type('byte', [], - """ - Signed integer type, compatible with C ``char``. - """) - -add_newdoc_for_scalar_type('short', [], - """ - Signed integer type, compatible with C ``short``. - """) - -add_newdoc_for_scalar_type('intc', [], - """ - Signed integer type, compatible with C ``int``. - """) - -add_newdoc_for_scalar_type('int_', [], - """ - Signed integer type, compatible with Python `int` anc C ``long``. - """) - -add_newdoc_for_scalar_type('longlong', [], - """ - Signed integer type, compatible with C ``long long``. - """) - -add_newdoc_for_scalar_type('ubyte', [], - """ - Unsigned integer type, compatible with C ``unsigned char``. - """) - -add_newdoc_for_scalar_type('ushort', [], - """ - Unsigned integer type, compatible with C ``unsigned short``. - """) - -add_newdoc_for_scalar_type('uintc', [], - """ - Unsigned integer type, compatible with C ``unsigned int``. - """) - -add_newdoc_for_scalar_type('uint', [], - """ - Unsigned integer type, compatible with C ``unsigned long``. - """) - -add_newdoc_for_scalar_type('ulonglong', [], - """ - Signed integer type, compatible with C ``unsigned long long``. - """) - -add_newdoc_for_scalar_type('half', [], - """ - Half-precision floating-point number type. - """) - -add_newdoc_for_scalar_type('single', [], - """ - Single-precision floating-point number type, compatible with C ``float``. - """) - -add_newdoc_for_scalar_type('double', ['float_'], - """ - Double-precision floating-point number type, compatible with Python `float` - and C ``double``. - """) - -add_newdoc_for_scalar_type('longdouble', ['longfloat'], - """ - Extended-precision floating-point number type, compatible with C - ``long double`` but not necessarily with IEEE 754 quadruple-precision. - """) - -add_newdoc_for_scalar_type('csingle', ['singlecomplex'], - """ - Complex number type composed of two single-precision floating-point - numbers. - """) - -add_newdoc_for_scalar_type('cdouble', ['cfloat', 'complex_'], - """ - Complex number type composed of two double-precision floating-point - numbers, compatible with Python `complex`. - """) - -add_newdoc_for_scalar_type('clongdouble', ['clongfloat', 'longcomplex'], - """ - Complex number type composed of two extended-precision floating-point - numbers. - """) - -add_newdoc_for_scalar_type('object_', [], - """ - Any Python object. - """) - -# TODO: work out how to put this on the base class, np.floating -for float_name in ('half', 'single', 'double', 'longdouble'): - add_newdoc('numpy.core.numerictypes', float_name, ('as_integer_ratio', - """ - {ftype}.as_integer_ratio() -> (int, int) - - Return a pair of integers, whose ratio is exactly equal to the original - floating point number, and with a positive denominator. - Raise OverflowError on infinities and a ValueError on NaNs. - - >>> np.{ftype}(10.0).as_integer_ratio() - (10, 1) - >>> np.{ftype}(0.0).as_integer_ratio() - (0, 1) - >>> np.{ftype}(-.25).as_integer_ratio() - (-1, 4) - """.format(ftype=float_name))) diff --git a/numpy/core/_add_newdocs_scalars.py b/numpy/core/_add_newdocs_scalars.py new file mode 100644 index 000000000000..c367c18ed094 --- /dev/null +++ b/numpy/core/_add_newdocs_scalars.py @@ -0,0 +1,195 @@ +""" +This file is separate from ``_add_newdocs.py`` so that it can be mocked out by +our sphinx ``conf.py`` during doc builds, where we want to avoid showing +platform-dependent information. +""" +from numpy.core import dtype +from numpy.core import numerictypes as _numerictypes +from numpy.core.function_base import add_newdoc + +############################################################################## +# +# Documentation for concrete scalar classes +# +############################################################################## + +def numeric_type_aliases(aliases): + def type_aliases_gen(): + for alias, doc in aliases: + try: + alias_type = getattr(_numerictypes, alias) + except AttributeError: + # The set of aliases that actually exist varies between platforms + pass + else: + yield (alias_type, alias, doc) + return list(type_aliases_gen()) + + +possible_aliases = numeric_type_aliases([ + ('int8', '8-bit signed integer (``-128`` to ``127``)'), + ('int16', '16-bit signed integer (``-32_768`` to ``32_767``)'), + ('int32', '32-bit signed integer (``-2_147_483_648`` to ``2_147_483_647``)'), + ('int64', '64-bit signed integer (``-9_223_372_036_854_775_808`` to ``9_223_372_036_854_775_807``)'), + ('intp', 'Signed integer large enough to fit pointer, compatible with C ``intptr_t``'), + ('uint8', '8-bit unsigned integer (``0`` to ``255``)'), + ('uint16', '16-bit unsigned integer (``0`` to ``65_535``)'), + ('uint32', '32-bit unsigned integer (``0`` to ``4_294_967_295``)'), + ('uint64', '64-bit unsigned integer (``0`` to ``18_446_744_073_709_551_615``)'), + ('uintp', 'Unsigned integer large enough to fit pointer, compatible with C ``uintptr_t``'), + ('float16', '16-bit-precision floating-point number type: sign bit, 5 bits exponent, 10 bits mantissa'), + ('float32', '32-bit-precision floating-point number type: sign bit, 8 bits exponent, 23 bits mantissa'), + ('float64', '64-bit precision floating-point number type: sign bit, 11 bits exponent, 52 bits mantissa'), + ('float96', '96-bit extended-precision floating-point number type'), + ('float128', '128-bit extended-precision floating-point number type'), + ('complex64', 'Complex number type composed of 2 32-bit-precision floating-point numbers'), + ('complex128', 'Complex number type composed of 2 64-bit-precision floating-point numbers'), + ('complex192', 'Complex number type composed of 2 96-bit extended-precision floating-point numbers'), + ('complex256', 'Complex number type composed of 2 128-bit extended-precision floating-point numbers'), + ]) + + +def add_newdoc_for_scalar_type(obj, fixed_aliases, doc): + # note: `:field: value` is rST syntax which renders as field lists. + o = getattr(_numerictypes, obj) + + character_code = dtype(o).char + canonical_name_doc = "" if obj == o.__name__ else ":Canonical name: `numpy.{}`\n ".format(obj) + alias_doc = ''.join(":Alias: `numpy.{}`\n ".format(alias) for alias in fixed_aliases) + alias_doc += ''.join(":Alias on this platform: `numpy.{}`: {}.\n ".format(alias, doc) + for (alias_type, alias, doc) in possible_aliases if alias_type is o) + docstring = """ + {doc} + + :Character code: ``'{character_code}'`` + {canonical_name_doc}{alias_doc} + """.format(doc=doc.strip(), character_code=character_code, + canonical_name_doc=canonical_name_doc, alias_doc=alias_doc) + + add_newdoc('numpy.core.numerictypes', obj, docstring) + + +add_newdoc_for_scalar_type('bool_', ['bool8'], + """ + Boolean type (True or False), stored as a byte. + + .. warning:: + + The :class:`bool_` type is not a subclass of the :class:`int_` type + (the :class:`bool_` is not even a number type). This is different + than Python's default implementation of :class:`bool` as a + sub-class of :class:`int`. + """) + +add_newdoc_for_scalar_type('byte', [], + """ + Signed integer type, compatible with C ``char``. + """) + +add_newdoc_for_scalar_type('short', [], + """ + Signed integer type, compatible with C ``short``. + """) + +add_newdoc_for_scalar_type('intc', [], + """ + Signed integer type, compatible with C ``int``. + """) + +add_newdoc_for_scalar_type('int_', [], + """ + Signed integer type, compatible with Python `int` and C ``long``. + """) + +add_newdoc_for_scalar_type('longlong', [], + """ + Signed integer type, compatible with C ``long long``. + """) + +add_newdoc_for_scalar_type('ubyte', [], + """ + Unsigned integer type, compatible with C ``unsigned char``. + """) + +add_newdoc_for_scalar_type('ushort', [], + """ + Unsigned integer type, compatible with C ``unsigned short``. + """) + +add_newdoc_for_scalar_type('uintc', [], + """ + Unsigned integer type, compatible with C ``unsigned int``. + """) + +add_newdoc_for_scalar_type('uint', [], + """ + Unsigned integer type, compatible with C ``unsigned long``. + """) + +add_newdoc_for_scalar_type('ulonglong', [], + """ + Signed integer type, compatible with C ``unsigned long long``. + """) + +add_newdoc_for_scalar_type('half', [], + """ + Half-precision floating-point number type. + """) + +add_newdoc_for_scalar_type('single', [], + """ + Single-precision floating-point number type, compatible with C ``float``. + """) + +add_newdoc_for_scalar_type('double', ['float_'], + """ + Double-precision floating-point number type, compatible with Python `float` + and C ``double``. + """) + +add_newdoc_for_scalar_type('longdouble', ['longfloat'], + """ + Extended-precision floating-point number type, compatible with C + ``long double`` but not necessarily with IEEE 754 quadruple-precision. + """) + +add_newdoc_for_scalar_type('csingle', ['singlecomplex'], + """ + Complex number type composed of two single-precision floating-point + numbers. + """) + +add_newdoc_for_scalar_type('cdouble', ['cfloat', 'complex_'], + """ + Complex number type composed of two double-precision floating-point + numbers, compatible with Python `complex`. + """) + +add_newdoc_for_scalar_type('clongdouble', ['clongfloat', 'longcomplex'], + """ + Complex number type composed of two extended-precision floating-point + numbers. + """) + +add_newdoc_for_scalar_type('object_', [], + """ + Any Python object. + """) + +# TODO: work out how to put this on the base class, np.floating +for float_name in ('half', 'single', 'double', 'longdouble'): + add_newdoc('numpy.core.numerictypes', float_name, ('as_integer_ratio', + """ + {ftype}.as_integer_ratio() -> (int, int) + + Return a pair of integers, whose ratio is exactly equal to the original + floating point number, and with a positive denominator. + Raise OverflowError on infinities and a ValueError on NaNs. + + >>> np.{ftype}(10.0).as_integer_ratio() + (10, 1) + >>> np.{ftype}(0.0).as_integer_ratio() + (0, 1) + >>> np.{ftype}(-.25).as_integer_ratio() + (-1, 4) + """.format(ftype=float_name))) From 3edc19fabf6bbe6fcc8fb8155b3a9f72382a0372 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Sun, 4 Oct 2020 21:03:57 +0100 Subject: [PATCH 181/409] DOC: Overhaul the scalar type documentation to use `..autoclass` This remove the tables. since they only had three columns, and using the character code is advised against anyway. With this change, the individual scalar types as well as their aliases are now valid sphinx python domain targets. --- doc/source/reference/arrays.scalars.rst | 319 ++++++++++++++++-------- 1 file changed, 220 insertions(+), 99 deletions(-) diff --git a/doc/source/reference/arrays.scalars.rst b/doc/source/reference/arrays.scalars.rst index 46d2bb8faeef..174dd2b917a0 100644 --- a/doc/source/reference/arrays.scalars.rst +++ b/doc/source/reference/arrays.scalars.rst @@ -41,6 +41,13 @@ of the flexible itemsize array types (:class:`string`, pointer for the platform. All the number types can be obtained using bit-width names as well. + +.. TODO - use something like this instead of the diagram above, as it generates + links to the classes and is a vector graphic. Unfortunately it looks worse + and the html element providing the linked regions is misaligned. + + .. inheritance-diagram:: byte short intc int_ longlong ubyte ushort uintc uint ulonglong half single double longdouble csingle cdouble clongdouble bool_ datetime64 timedelta64 object_ bytes_ str_ void + .. [#] However, array scalars are immutable, so none of the array scalar attributes are settable. @@ -51,14 +58,8 @@ of the flexible itemsize array types (:class:`string`, Built-in scalar types ===================== -The built-in scalar types are shown below. Along with their (mostly) -C-derived names, the integer, float, and complex data-types are also -available using a bit-width convention so that an array of the right -size can always be ensured (e.g. :class:`int8`, :class:`float64`, -:class:`complex128`). Two aliases (:class:`intp` and :class:`uintp`) -pointing to the integer type that is sufficiently large to hold a C pointer -are also provided. The C-like names are associated with character codes, -which are shown in the table. Use of the character codes, however, +The built-in scalar types are shown below. The C-like names are associated with character codes, +which are shown in their descriptions. Use of the character codes, however, is discouraged. Some of the scalar types are essentially equivalent to fundamental @@ -84,13 +85,6 @@ The :class:`bool_` data type is very similar to the Python on the C-level the size of the actual bool data is not the same as a Python Boolean scalar. -.. warning:: - - The :class:`bool_` type is not a subclass of the :class:`int_` type - (the :class:`bool_` is not even a number type). This is different - than Python's default implementation of :class:`bool` as a - sub-class of :class:`int`. - .. warning:: The :class:`int_` type does **not** inherit from the @@ -99,84 +93,113 @@ Python Boolean scalar. .. tip:: The default data type in NumPy is :class:`float_`. -In the tables below, ``platform?`` means that the type may not be -available on all platforms. Compatibility with different C or Python -types is indicated: two types are compatible if their data is of the -same size and interpreted in the same way. - -Booleans: - -=================== ============================= =============== -Type Remarks Character code -=================== ============================= =============== -:class:`bool_` compatible: Python bool ``'?'`` -:class:`bool8` 8 bits -=================== ============================= =============== - -Integers: - -=================== ============================= =============== -:class:`byte` compatible: C ``char`` ``'b'`` -:class:`short` compatible: C ``short`` ``'h'`` -:class:`intc` compatible: C ``int`` ``'i'`` -:class:`int_` compatible: C ``long`` ``'l'`` -:class:`longlong` compatible: C ``long long`` ``'q'`` -:class:`intp` large enough to fit a pointer ``'p'`` -:class:`int8` 8 bits -:class:`int16` 16 bits -:class:`int32` 32 bits -:class:`int64` 64 bits -=================== ============================= =============== - -Unsigned integers: - -=================== ================================= =============== -:class:`ubyte` compatible: C ``unsigned char`` ``'B'`` -:class:`ushort` compatible: C ``unsigned short`` ``'H'`` -:class:`uintc` compatible: C ``unsigned int`` ``'I'`` -:class:`uint` compatible: C ``long`` ``'L'`` -:class:`ulonglong` compatible: C ``long long`` ``'Q'`` -:class:`uintp` large enough to fit a pointer ``'P'`` -:class:`uint8` 8 bits -:class:`uint16` 16 bits -:class:`uint32` 32 bits -:class:`uint64` 64 bits -=================== ================================= =============== - -Floating-point numbers: - -=================== ============================= =============== -:class:`half` ``'e'`` -:class:`single` compatible: C float ``'f'`` -:class:`double` compatible: C double -:class:`float_` compatible: Python float ``'d'`` -:class:`longfloat` compatible: C long float ``'g'`` -:class:`float16` 16 bits -:class:`float32` 32 bits -:class:`float64` 64 bits -:class:`float96` 96 bits, platform? -:class:`float128` 128 bits, platform? -=================== ============================= =============== - -Complex floating-point numbers: - -=================== ============================= =============== -:class:`csingle` ``'F'`` -:class:`complex_` compatible: Python complex ``'D'`` -:class:`clongfloat` ``'G'`` -:class:`complex64` two 32-bit floats -:class:`complex128` two 64-bit floats -:class:`complex192` two 96-bit floats, - platform? -:class:`complex256` two 128-bit floats, - platform? -=================== ============================= =============== - -Any Python object: - -=================== ============================= =============== -:class:`object_` any Python object ``'O'`` -=================== ============================= =============== +.. autoclass:: numpy.generic + :exclude-members: + +.. autoclass:: numpy.number + :exclude-members: + +Integer types +~~~~~~~~~~~~~ + +.. autoclass:: numpy.integer + :exclude-members: + +Signed integer types +++++++++++++++++++++ + +.. autoclass:: numpy.signedinteger + :exclude-members: + +.. autoclass:: numpy.byte + :exclude-members: + +.. autoclass:: numpy.short + :exclude-members: + +.. autoclass:: numpy.intc + :exclude-members: + +.. autoclass:: numpy.int_ + :exclude-members: + +.. autoclass:: numpy.longlong + :exclude-members: + +Unsigned integer types +++++++++++++++++++++++ + +.. autoclass:: numpy.unsignedinteger + :exclude-members: + +.. autoclass:: numpy.ubyte + :exclude-members: + +.. autoclass:: numpy.ushort + :exclude-members: + +.. autoclass:: numpy.uintc + :exclude-members: + +.. autoclass:: numpy.uint + :exclude-members: + +.. autoclass:: numpy.ulonglong + :exclude-members: + +Inexact types +~~~~~~~~~~~~~ + +.. autoclass:: numpy.inexact + :exclude-members: + +Floating-point types +++++++++++++++++++++ + +.. autoclass:: numpy.floating + :exclude-members: + +.. autoclass:: numpy.half + :exclude-members: + +.. autoclass:: numpy.single + :exclude-members: + +.. autoclass:: numpy.double + :exclude-members: + +.. autoclass:: numpy.longdouble + :exclude-members: + +Complex floating-point types +++++++++++++++++++++++++++++ + +.. autoclass:: numpy.complexfloating + :exclude-members: + +.. autoclass:: numpy.csingle + :exclude-members: + +.. autoclass:: numpy.cdouble + :exclude-members: + +.. autoclass:: numpy.clongdouble + :exclude-members: + +Other types +~~~~~~~~~~~ + +.. autoclass:: numpy.bool_ + :exclude-members: + +.. autoclass:: numpy.datetime64 + :exclude-members: + +.. autoclass:: numpy.timedelta64 + :exclude-members: + +.. autoclass:: numpy.object_ + :exclude-members: .. note:: @@ -198,11 +221,17 @@ size and the data they describe can be of different length in different arrays. (In the character codes ``#`` is an integer denoting how many elements the data type consists of.) -=================== ============================== ======== -:class:`bytes_` compatible: Python bytes ``'S#'`` -:class:`unicode_` compatible: Python unicode/str ``'U#'`` -:class:`void` ``'V#'`` -=================== ============================== ======== +.. autoclass:: numpy.flexible + :exclude-members: + +.. autoclass:: numpy.bytes_ + :exclude-members: + +.. autoclass:: numpy.str_ + :exclude-members: + +.. autoclass:: numpy.void + :exclude-members: .. warning:: @@ -217,6 +246,99 @@ elements the data type consists of.) convention more consistent with other Python modules such as the :mod:`struct` module. +Sized aliases +~~~~~~~~~~~~~ + +Along with their (mostly) +C-derived names, the integer, float, and complex data-types are also +available using a bit-width convention so that an array of the right +size can always be ensured. Two aliases (:class:`numpy.intp` and :class:`numpy.uintp`) +pointing to the integer type that is sufficiently large to hold a C pointer +are also provided. + +.. note that these are documented with ..attribute because that is what + autoclass does for aliases under the hood. + +.. autoclass:: numpy.bool8 + +.. attribute:: numpy.int8 + numpy.int16 + numpy.int32 + numpy.int64 + + Aliases for the signed integer types (one of `numpy.byte`, `numpy.short`, + `numpy.intc`, `numpy.int_` and `numpy.longlong`) with the specified number + of bits. + + Compatible with the C99 ``int8_t``, ``int16_t``, ``int32_t``, and + ``int64_t``, respectively. + +.. attribute:: numpy.uint8 + numpy.uint16 + numpy.uint32 + numpy.uint64 + + Alias for the unsigned integer types (one of `numpy.byte`, `numpy.short`, + `numpy.intc`, `numpy.int_` and `numpy.longlong`) with the specified number + of bits. + + Compatible with the C99 ``uint8_t``, ``uint16_t``, ``uint32_t``, and + ``uint64_t``, respectively. + +.. attribute:: numpy.intp + + Alias for the signed integer type (one of `numpy.byte`, `numpy.short`, + `numpy.intc`, `numpy.int_` and `np.longlong`) that is the same size as a + pointer. + + Compatible with the C ``intptr_t``. + + :Character code: ``'p'`` + +.. attribute:: numpy.uintp + + Alias for the unsigned integer type (one of `numpy.byte`, `numpy.short`, + `numpy.intc`, `numpy.int_` and `np.longlong`) that is the same size as a + pointer. + + Compatible with the C ``uintptr_t``. + + :Character code: ``'P'`` + +.. autoclass:: numpy.float16 + +.. autoclass:: numpy.float32 + +.. autoclass:: numpy.float64 + +.. attribute:: numpy.float96 + numpy.float128 + + Alias for `numpy.longdouble`, named after its size in bits. + The existance of these aliases depends on the platform. + +.. autoclass:: numpy.complex64 + +.. autoclass:: numpy.complex128 + +.. attribute:: numpy.complex192 + numpy.complex256 + + Alias for `numpy.clongdouble`, named after its size in bits. + The existance of these aliases depends on the platform. + +Other aliases +~~~~~~~~~~~~~ + +.. autoclass:: float_ + +.. autoclass:: complex_ + +.. autoclass:: longfloat + +.. autoclass:: clongfloat + +.. autoclass:: longcomplex Attributes ========== @@ -276,7 +398,6 @@ The exceptions to the above rules are given below: .. autosummary:: :toctree: generated/ - generic generic.__array__ generic.__array_wrap__ generic.squeeze From f16f01aa669756400dccf5b12ffc75faf2387638 Mon Sep 17 00:00:00 2001 From: takanori-pskq Date: Mon, 5 Oct 2020 10:10:27 +0900 Subject: [PATCH 182/409] Fix: Add the entry for `NPY_ARR_HAS_DESCR` --- doc/source/reference/arrays.interface.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/reference/arrays.interface.rst b/doc/source/reference/arrays.interface.rst index 6d12cf999790..49772a298772 100644 --- a/doc/source/reference/arrays.interface.rst +++ b/doc/source/reference/arrays.interface.rst @@ -238,6 +238,8 @@ interpreted. The data-bits are :c:macro:`NPY_ARRAY_C_CONTIGUOUS` (0x1), has the arrdescr field. The field should not be accessed unless this flag is present. + .. c:macro:: NPY_ARR_HAS_DESCR + .. admonition:: New since June 16, 2006: In the past most implementations used the ``desc`` member of the ``PyCObject`` From 37ff6fce0bfdb5df87aafcdd5f38e9ebc5ebe0e6 Mon Sep 17 00:00:00 2001 From: takanori-pskq Date: Mon, 5 Oct 2020 12:37:23 +0900 Subject: [PATCH 183/409] DOC: Fix some references for macros --- doc/source/reference/c-api/config.rst | 9 ++++++--- doc/source/reference/c-api/dtype.rst | 12 +++++++++--- doc/source/reference/c-api/iterator.rst | 2 +- doc/source/reference/c-api/types-and-structures.rst | 4 ++-- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/doc/source/reference/c-api/config.rst b/doc/source/reference/c-api/config.rst index 5fdbc0ff1ece..cec5b973a202 100644 --- a/doc/source/reference/c-api/config.rst +++ b/doc/source/reference/c-api/config.rst @@ -52,12 +52,15 @@ information is available to the pre-processor. .. c:macro:: NPY_SIZEOF_LONG_DOUBLE - sizeof(longdouble) (A macro defines **NPY_SIZEOF_LONGDOUBLE** as well.) +.. c:macro:: NPY_SIZEOF_LONGDOUBLE + + sizeof(longdouble) .. c:macro:: NPY_SIZEOF_PY_INTPTR_T - Size of a pointer on this platform (sizeof(void \*)) (A macro defines - NPY_SIZEOF_INTP as well.) +.. c:macro:: NPY_SIZEOF_INTP + + Size of a pointer on this platform (sizeof(void \*)) Platform information diff --git a/doc/source/reference/c-api/dtype.rst b/doc/source/reference/c-api/dtype.rst index a04d852123ed..47b998302e86 100644 --- a/doc/source/reference/c-api/dtype.rst +++ b/doc/source/reference/c-api/dtype.rst @@ -414,6 +414,12 @@ Printf Formatting For help in printing, the following strings are defined as the correct format specifier in printf and related commands. - :c:data:`NPY_LONGLONG_FMT`, :c:data:`NPY_ULONGLONG_FMT`, - :c:data:`NPY_INTP_FMT`, :c:data:`NPY_UINTP_FMT`, - :c:data:`NPY_LONGDOUBLE_FMT` +.. c:macro:: NPY_LONGLONG_FMT + +.. c:macro:: NPY_ULONGLONG_FMT + +.. c:macro:: NPY_INTP_FMT + +.. c:macro:: NPY_UINTP_FMT + +.. c:macro:: NPY_LONGDOUBLE_FMT diff --git a/doc/source/reference/c-api/iterator.rst b/doc/source/reference/c-api/iterator.rst index 7eac8c367dab..ae96bb3fb056 100644 --- a/doc/source/reference/c-api/iterator.rst +++ b/doc/source/reference/c-api/iterator.rst @@ -1264,7 +1264,7 @@ functions provide that information. NPY_MAX_INTP is placed in the stride. Once the iterator is prepared for iteration (after a reset if - :c:data:`NPY_DELAY_BUFALLOC` was used), call this to get the strides + :c:data:`NPY_ITER_DELAY_BUFALLOC` was used), call this to get the strides which may be used to select a fast inner loop function. For example, if the stride is 0, that means the inner loop can always load its value into a variable once, then use the variable throughout the loop, diff --git a/doc/source/reference/c-api/types-and-structures.rst b/doc/source/reference/c-api/types-and-structures.rst index ee57d4680cb1..cc961df3abf4 100644 --- a/doc/source/reference/c-api/types-and-structures.rst +++ b/doc/source/reference/c-api/types-and-structures.rst @@ -315,7 +315,7 @@ PyArrayDescr_Type and PyArray_Descr Bits set for the object data-type: ( :c:data:`NPY_LIST_PICKLE` \| :c:data:`NPY_USE_GETITEM` \| :c:data:`NPY_ITEM_IS_POINTER` \| - :c:data:`NPY_REFCOUNT` \| :c:data:`NPY_NEEDS_INIT` \| + :c:data:`NPY_ITEM_REFCOUNT` \| :c:data:`NPY_NEEDS_INIT` \| :c:data:`NPY_NEEDS_PYAPI`). .. c:function:: PyDataType_FLAGCHK(PyArray_Descr *dtype, int flags) @@ -1395,7 +1395,7 @@ PyArrayInterface as the *descr* key in :obj:`__array_interface__`). This can be ``NULL`` if *typekind* and *itemsize* provide enough information. This field is also ignored unless - :c:data:`ARR_HAS_DESCR` flag is on in *flags*. + :c:data:`NPY_ARR_HAS_DESCR` flag is on in *flags*. Internally used structures From bfd80ef54217c4d701163bf6ab09cc8629a9ca85 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2020 07:12:44 +0000 Subject: [PATCH 184/409] MAINT: Bump hypothesis from 5.36.1 to 5.37.0 Bumps [hypothesis](https://github.com/HypothesisWorks/hypothesis) from 5.36.1 to 5.37.0. - [Release notes](https://github.com/HypothesisWorks/hypothesis/releases) - [Commits](https://github.com/HypothesisWorks/hypothesis/compare/hypothesis-python-5.36.1...hypothesis-python-5.37.0) Signed-off-by: dependabot-preview[bot] --- test_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index c86d46eb879a..067dd07c888a 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,7 +1,7 @@ cython==0.29.21 wheel setuptools<49.2.0 -hypothesis==5.36.1 +hypothesis==5.37.0 pytest==6.0.2 pytz==2020.1 pytest-cov==2.10.1 From 9a9abf7a07271fed54dcba8ebadc3c725a63d916 Mon Sep 17 00:00:00 2001 From: Stuart Archibald Date: Mon, 5 Oct 2020 14:08:04 +0100 Subject: [PATCH 185/409] DOC: update code of conduct URL As title. Closes: #17458 --- .github/CODE_OF_CONDUCT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md index 57c98060eb3b..079098fae68a 100644 --- a/.github/CODE_OF_CONDUCT.md +++ b/.github/CODE_OF_CONDUCT.md @@ -1 +1 @@ -NumPy has a Code of Conduct, please see: https://www.numpy.org/devdocs/dev/conduct/code_of_conduct.html +NumPy has a Code of Conduct, please see: https://numpy.org/code-of-conduct From 5dbcbb79478a8eddf05a72f7030bdf29a93ff46c Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Thu, 17 Sep 2020 01:43:49 +0200 Subject: [PATCH 186/409] ENH: Add annotations for `generic` and `ndarray` bitwise operations --- numpy/__init__.pyi | 79 +++++++++++++++++++++++++++++++-------- numpy/typing/_callable.py | 29 +++++++++++++- 2 files changed, 91 insertions(+), 17 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 9966ef1999a5..1dd5d55528ce 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -7,12 +7,15 @@ from numpy.core._internal import _ctypes from numpy.typing import ArrayLike, DtypeLike, _Shape, _ShapeLike from numpy.typing._callable import ( _BoolOp, + _BoolBitOp, _BoolSub, _BoolTrueDiv, _TD64Div, _IntTrueDiv, _UnsignedIntOp, + _UnsignedIntBitOp, _SignedIntOp, + _SignedIntBitOp, _FloatOp, _ComplexOp, _NumberOp, @@ -677,20 +680,9 @@ class _ArrayOrScalarCommon( def __rmod__(self, other): ... def __divmod__(self, other): ... def __rdivmod__(self, other): ... - def __lshift__(self, other): ... - def __rlshift__(self, other): ... - def __rshift__(self, other): ... - def __rrshift__(self, other): ... - def __and__(self, other): ... - def __rand__(self, other): ... - def __xor__(self, other): ... - def __rxor__(self, other): ... - def __or__(self, other): ... - def __ror__(self, other): ... def __neg__(self: _ArraySelf) -> _ArraySelf: ... def __pos__(self: _ArraySelf) -> _ArraySelf: ... def __abs__(self: _ArraySelf) -> _ArraySelf: ... - def __invert__(self: _ArraySelf) -> _ArraySelf: ... def astype( self: _ArraySelf, dtype: DtypeLike, @@ -1257,6 +1249,17 @@ class ndarray(_ArrayOrScalarCommon, Iterable, Sized, Container): def __rpow__(self, other: ArrayLike) -> Union[ndarray, generic]: ... def __truediv__(self, other: ArrayLike) -> Union[ndarray, generic]: ... def __rtruediv__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __invert__(self: _ArraySelf) -> Union[_ArraySelf, integer, bool_]: ... + def __lshift__(self, other: ArrayLike) -> Union[ndarray, integer]: ... + def __rlshift__(self, other: ArrayLike) -> Union[ndarray, integer]: ... + def __rshift__(self, other: ArrayLike) -> Union[ndarray, integer]: ... + def __rrshift__(self, other: ArrayLike) -> Union[ndarray, integer]: ... + def __and__(self, other: ArrayLike) -> Union[ndarray, integer, bool_]: ... + def __rand__(self, other: ArrayLike) -> Union[ndarray, integer, bool_]: ... + def __xor__(self, other: ArrayLike) -> Union[ndarray, integer, bool_]: ... + def __rxor__(self, other: ArrayLike) -> Union[ndarray, integer, bool_]: ... + def __or__(self, other: ArrayLike) -> Union[ndarray, integer, bool_]: ... + def __ror__(self, other: ArrayLike) -> Union[ndarray, integer, bool_]: ... # `np.generic` does not support inplace operations def __iadd__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... def __isub__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... @@ -1265,11 +1268,11 @@ class ndarray(_ArrayOrScalarCommon, Iterable, Sized, Container): def __ifloordiv__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... def __ipow__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... def __imod__(self, other): ... - def __ilshift__(self, other): ... - def __irshift__(self, other): ... - def __iand__(self, other): ... - def __ixor__(self, other): ... - def __ior__(self, other): ... + def __ilshift__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... + def __irshift__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... + def __iand__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... + def __ixor__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... + def __ior__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... # NOTE: while `np.generic` is not technically an instance of `ABCMeta`, # the `@abstractmethod` decorator is herein used to (forcefully) deny @@ -1329,6 +1332,17 @@ class bool_(generic): __rpow__: _BoolOp[int8] __truediv__: _BoolTrueDiv __rtruediv__: _BoolTrueDiv + def __invert__(self) -> bool_: ... + __lshift__: _BoolBitOp[int8] + __rlshift__: _BoolBitOp[int8] + __rshift__: _BoolBitOp[int8] + __rrshift__: _BoolBitOp[int8] + __and__: _BoolBitOp[bool_] + __rand__: _BoolBitOp[bool_] + __xor__: _BoolBitOp[bool_] + __rxor__: _BoolBitOp[bool_] + __or__: _BoolBitOp[bool_] + __ror__: _BoolBitOp[bool_] class object_(generic): def __init__(self, __value: object = ...) -> None: ... @@ -1374,6 +1388,18 @@ class integer(number): # type: ignore def __index__(self) -> int: ... __truediv__: _IntTrueDiv __rtruediv__: _IntTrueDiv + def __invert__(self: _IntType) -> _IntType: ... + # Ensure that objects annotated as `integer` support bit-wise operations + def __lshift__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... + def __rlshift__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... + def __rshift__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... + def __rrshift__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... + def __and__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... + def __rand__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... + def __or__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... + def __ror__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... + def __xor__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... + def __rxor__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... class signedinteger(integer): # type: ignore __add__: _SignedIntOp @@ -1386,6 +1412,16 @@ class signedinteger(integer): # type: ignore __rfloordiv__: _SignedIntOp __pow__: _SignedIntOp __rpow__: _SignedIntOp + __lshift__: _SignedIntBitOp + __rlshift__: _SignedIntBitOp + __rshift__: _SignedIntBitOp + __rrshift__: _SignedIntBitOp + __and__: _SignedIntBitOp + __rand__: _SignedIntBitOp + __xor__: _SignedIntBitOp + __rxor__: _SignedIntBitOp + __or__: _SignedIntBitOp + __ror__: _SignedIntBitOp class int8(signedinteger): def __init__(self, __value: _IntValue = ...) -> None: ... @@ -1429,6 +1465,16 @@ class unsignedinteger(integer): # type: ignore __rfloordiv__: _UnsignedIntOp __pow__: _UnsignedIntOp __rpow__: _UnsignedIntOp + __lshift__: _UnsignedIntBitOp + __rlshift__: _UnsignedIntBitOp + __rshift__: _UnsignedIntBitOp + __rrshift__: _UnsignedIntBitOp + __and__: _UnsignedIntBitOp + __rand__: _UnsignedIntBitOp + __xor__: _UnsignedIntBitOp + __rxor__: _UnsignedIntBitOp + __or__: _UnsignedIntBitOp + __ror__: _UnsignedIntBitOp class uint8(unsignedinteger): def __init__(self, __value: _IntValue = ...) -> None: ... @@ -1458,6 +1504,7 @@ class floating(inexact): # type: ignore __pow__: _FloatOp __rpow__: _FloatOp +_IntType = TypeVar("_IntType", bound=integer) _FloatType = TypeVar('_FloatType', bound=floating) class float16(floating): diff --git a/numpy/typing/_callable.py b/numpy/typing/_callable.py index 5e14b708f1a6..943441cf494e 100644 --- a/numpy/typing/_callable.py +++ b/numpy/typing/_callable.py @@ -9,7 +9,7 @@ """ import sys -from typing import Union, TypeVar, overload, Any +from typing import Union, TypeVar, overload, Any, NoReturn from numpy import ( _BoolLike, @@ -26,6 +26,7 @@ signedinteger, int32, int64, + uint64, floating, float32, float64, @@ -45,6 +46,7 @@ HAVE_PROTOCOL = True if HAVE_PROTOCOL: + _IntType = TypeVar("_IntType", bound=integer) _NumberType = TypeVar("_NumberType", bound=number) _NumberType_co = TypeVar("_NumberType_co", covariant=True, bound=number) _GenericType_co = TypeVar("_GenericType_co", covariant=True, bound=generic) @@ -61,6 +63,14 @@ def __call__(self, __other: complex) -> complex128: ... @overload def __call__(self, __other: _NumberType) -> _NumberType: ... + class _BoolBitOp(Protocol[_GenericType_co]): + @overload + def __call__(self, __other: _BoolLike) -> _GenericType_co: ... + @overload # platform dependent + def __call__(self, __other: int) -> Union[int32, int64]: ... + @overload + def __call__(self, __other: _IntType) -> _IntType: ... + class _BoolSub(Protocol): # Note that `__other: bool_` is absent here @overload # platform dependent @@ -103,6 +113,17 @@ def __call__(self, __other: float) -> floating: ... @overload def __call__(self, __other: complex) -> complexfloating[floating]: ... + class _UnsignedIntBitOp(Protocol): + # The likes of `uint64 | np.signedinteger` will fail as there + # is no signed integer type large enough to hold a `uint64` + # See https://github.com/numpy/numpy/issues/2524 + @overload + def __call__(self, __other: Union[bool, unsignedinteger]) -> unsignedinteger: ... + @overload + def __call__(self: uint64, __other: Union[int, signedinteger]) -> NoReturn: ... + @overload + def __call__(self, __other: Union[int, signedinteger]) -> signedinteger: ... + class _SignedIntOp(Protocol): @overload def __call__(self, __other: Union[int, signedinteger]) -> signedinteger: ... @@ -111,6 +132,9 @@ def __call__(self, __other: float) -> floating: ... @overload def __call__(self, __other: complex) -> complexfloating[floating]: ... + class _SignedIntBitOp(Protocol): + def __call__(self, __other: Union[int, signedinteger]) -> signedinteger: ... + class _FloatOp(Protocol): @overload def __call__(self, __other: _FloatLike) -> floating: ... @@ -125,12 +149,15 @@ def __call__(self, __other: _NumberLike) -> number: ... else: _BoolOp = Any + _BoolBitOp = Any _BoolSub = Any _BoolTrueDiv = Any _TD64Div = Any _IntTrueDiv = Any _UnsignedIntOp = Any + _UnsignedIntBitOp = Any _SignedIntOp = Any + _SignedIntBitOp = Any _FloatOp = Any _ComplexOp = Any _NumberOp = Any From cb92195c59be15415ce795367aeac6619f069257 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Thu, 1 Oct 2020 01:26:11 +0200 Subject: [PATCH 187/409] TST: Added tests for bitwise operations --- numpy/typing/tests/data/fail/bitwise_ops.py | 19 +++ numpy/typing/tests/data/pass/bitwise_ops.py | 131 ++++++++++++++++++ numpy/typing/tests/data/reveal/bitwise_ops.py | 131 ++++++++++++++++++ 3 files changed, 281 insertions(+) create mode 100644 numpy/typing/tests/data/fail/bitwise_ops.py create mode 100644 numpy/typing/tests/data/pass/bitwise_ops.py create mode 100644 numpy/typing/tests/data/reveal/bitwise_ops.py diff --git a/numpy/typing/tests/data/fail/bitwise_ops.py b/numpy/typing/tests/data/fail/bitwise_ops.py new file mode 100644 index 000000000000..9be6869ad101 --- /dev/null +++ b/numpy/typing/tests/data/fail/bitwise_ops.py @@ -0,0 +1,19 @@ +import numpy as np + +i8 = np.int64() +i4 = np.int32() +u8 = np.uint64() +b_ = np.bool_() +i = int() + +f8 = np.float64() + +b_ >> f8 # E: No overload variant +i8 << f8 # E: incompatible type +i | f8 # E: Unsupported operand types +i8 ^ f8 # E: incompatible type +u8 & f8 # E: No overload variant +~f8 # E: Unsupported operand type + +# mypys' error message for `NoReturn` is unfortunately pretty nad +a = u8 | 0 # E: Need type annotation diff --git a/numpy/typing/tests/data/pass/bitwise_ops.py b/numpy/typing/tests/data/pass/bitwise_ops.py new file mode 100644 index 000000000000..67449e2c21d8 --- /dev/null +++ b/numpy/typing/tests/data/pass/bitwise_ops.py @@ -0,0 +1,131 @@ +import numpy as np + +i8 = np.int64(1) +u8 = np.uint64(1) + +i4 = np.int32(1) +u4 = np.uint32(1) + +b_ = np.bool_(1) + +b = bool(1) +i = int(1) + +AR = np.array([0, 1, 2], dtype=np.int32) +AR.setflags(write=False) + + +i8 << i8 +i8 >> i8 +i8 | i8 +i8 ^ i8 +i8 & i8 + +i8 << AR +i8 >> AR +i8 | AR +i8 ^ AR +i8 & AR + +i4 << i4 +i4 >> i4 +i4 | i4 +i4 ^ i4 +i4 & i4 + +i8 << i4 +i8 >> i4 +i8 | i4 +i8 ^ i4 +i8 & i4 + +i8 << i +i8 >> i +i8 | i +i8 ^ i +i8 & i + +i8 << b_ +i8 >> b_ +i8 | b_ +i8 ^ b_ +i8 & b_ + +i8 << b +i8 >> b +i8 | b +i8 ^ b +i8 & b + +u8 << u8 +u8 >> u8 +u8 | u8 +u8 ^ u8 +u8 & u8 + +u8 << AR +u8 >> AR +u8 | AR +u8 ^ AR +u8 & AR + +u4 << u4 +u4 >> u4 +u4 | u4 +u4 ^ u4 +u4 & u4 + +u4 << i4 +u4 >> i4 +u4 | i4 +u4 ^ i4 +u4 & i4 + +u4 << i +u4 >> i +u4 | i +u4 ^ i +u4 & i + +u8 << b_ +u8 >> b_ +u8 | b_ +u8 ^ b_ +u8 & b_ + +u8 << b +u8 >> b +u8 | b +u8 ^ b +u8 & b + +b_ << b_ +b_ >> b_ +b_ | b_ +b_ ^ b_ +b_ & b_ + +b_ << AR +b_ >> AR +b_ | AR +b_ ^ AR +b_ & AR + +b_ << b +b_ >> b +b_ | b +b_ ^ b +b_ & b + +b_ << i +b_ >> i +b_ | i +b_ ^ i +b_ & i + +~i8 +~i4 +~u8 +~u4 +~b_ +~AR diff --git a/numpy/typing/tests/data/reveal/bitwise_ops.py b/numpy/typing/tests/data/reveal/bitwise_ops.py new file mode 100644 index 000000000000..ae899f47b690 --- /dev/null +++ b/numpy/typing/tests/data/reveal/bitwise_ops.py @@ -0,0 +1,131 @@ +import numpy as np + +i8 = np.int64(1) +u8 = np.uint64(1) + +i4 = np.int32(1) +u4 = np.uint32(1) + +b_ = np.bool_(1) + +b = bool(1) +i = int(1) + +AR = np.array([0, 1, 2], dtype=np.int32) +AR.setflags(write=False) + + +reveal_type(i8 << i8) # E: numpy.signedinteger +reveal_type(i8 >> i8) # E: numpy.signedinteger +reveal_type(i8 | i8) # E: numpy.signedinteger +reveal_type(i8 ^ i8) # E: numpy.signedinteger +reveal_type(i8 & i8) # E: numpy.signedinteger + +reveal_type(i8 << AR) # E: Union[numpy.ndarray, numpy.integer] +reveal_type(i8 >> AR) # E: Union[numpy.ndarray, numpy.integer] +reveal_type(i8 | AR) # E: Union[numpy.ndarray, numpy.integer, numpy.bool_] +reveal_type(i8 ^ AR) # E: Union[numpy.ndarray, numpy.integer, numpy.bool_] +reveal_type(i8 & AR) # E: Union[numpy.ndarray, numpy.integer, numpy.bool_] + +reveal_type(i4 << i4) # E: numpy.signedinteger +reveal_type(i4 >> i4) # E: numpy.signedinteger +reveal_type(i4 | i4) # E: numpy.signedinteger +reveal_type(i4 ^ i4) # E: numpy.signedinteger +reveal_type(i4 & i4) # E: numpy.signedinteger + +reveal_type(i8 << i4) # E: numpy.signedinteger +reveal_type(i8 >> i4) # E: numpy.signedinteger +reveal_type(i8 | i4) # E: numpy.signedinteger +reveal_type(i8 ^ i4) # E: numpy.signedinteger +reveal_type(i8 & i4) # E: numpy.signedinteger + +reveal_type(i8 << i) # E: numpy.signedinteger +reveal_type(i8 >> i) # E: numpy.signedinteger +reveal_type(i8 | i) # E: numpy.signedinteger +reveal_type(i8 ^ i) # E: numpy.signedinteger +reveal_type(i8 & i) # E: numpy.signedinteger + +reveal_type(i8 << b_) # E: numpy.int64 +reveal_type(i8 >> b_) # E: numpy.int64 +reveal_type(i8 | b_) # E: numpy.int64 +reveal_type(i8 ^ b_) # E: numpy.int64 +reveal_type(i8 & b_) # E: numpy.int64 + +reveal_type(i8 << b) # E: numpy.signedinteger +reveal_type(i8 >> b) # E: numpy.signedinteger +reveal_type(i8 | b) # E: numpy.signedinteger +reveal_type(i8 ^ b) # E: numpy.signedinteger +reveal_type(i8 & b) # E: numpy.signedinteger + +reveal_type(u8 << u8) # E: numpy.unsignedinteger +reveal_type(u8 >> u8) # E: numpy.unsignedinteger +reveal_type(u8 | u8) # E: numpy.unsignedinteger +reveal_type(u8 ^ u8) # E: numpy.unsignedinteger +reveal_type(u8 & u8) # E: numpy.unsignedinteger + +reveal_type(u8 << AR) # E: Union[numpy.ndarray, numpy.integer] +reveal_type(u8 >> AR) # E: Union[numpy.ndarray, numpy.integer] +reveal_type(u8 | AR) # E: Union[numpy.ndarray, numpy.integer, numpy.bool_] +reveal_type(u8 ^ AR) # E: Union[numpy.ndarray, numpy.integer, numpy.bool_] +reveal_type(u8 & AR) # E: Union[numpy.ndarray, numpy.integer, numpy.bool_] + +reveal_type(u4 << u4) # E: numpy.unsignedinteger +reveal_type(u4 >> u4) # E: numpy.unsignedinteger +reveal_type(u4 | u4) # E: numpy.unsignedinteger +reveal_type(u4 ^ u4) # E: numpy.unsignedinteger +reveal_type(u4 & u4) # E: numpy.unsignedinteger + +reveal_type(u4 << i4) # E: numpy.signedinteger +reveal_type(u4 >> i4) # E: numpy.signedinteger +reveal_type(u4 | i4) # E: numpy.signedinteger +reveal_type(u4 ^ i4) # E: numpy.signedinteger +reveal_type(u4 & i4) # E: numpy.signedinteger + +reveal_type(u4 << i) # E: numpy.signedinteger +reveal_type(u4 >> i) # E: numpy.signedinteger +reveal_type(u4 | i) # E: numpy.signedinteger +reveal_type(u4 ^ i) # E: numpy.signedinteger +reveal_type(u4 & i) # E: numpy.signedinteger + +reveal_type(u8 << b_) # E: numpy.uint64 +reveal_type(u8 >> b_) # E: numpy.uint64 +reveal_type(u8 | b_) # E: numpy.uint64 +reveal_type(u8 ^ b_) # E: numpy.uint64 +reveal_type(u8 & b_) # E: numpy.uint64 + +reveal_type(u8 << b) # E: numpy.unsignedinteger +reveal_type(u8 >> b) # E: numpy.unsignedinteger +reveal_type(u8 | b) # E: numpy.unsignedinteger +reveal_type(u8 ^ b) # E: numpy.unsignedinteger +reveal_type(u8 & b) # E: numpy.unsignedinteger + +reveal_type(b_ << b_) # E: numpy.int8 +reveal_type(b_ >> b_) # E: numpy.int8 +reveal_type(b_ | b_) # E: numpy.bool_ +reveal_type(b_ ^ b_) # E: numpy.bool_ +reveal_type(b_ & b_) # E: numpy.bool_ + +reveal_type(b_ << AR) # E: Union[numpy.ndarray, numpy.integer] +reveal_type(b_ >> AR) # E: Union[numpy.ndarray, numpy.integer] +reveal_type(b_ | AR) # E: Union[numpy.ndarray, numpy.integer, numpy.bool_] +reveal_type(b_ ^ AR) # E: Union[numpy.ndarray, numpy.integer, numpy.bool_] +reveal_type(b_ & AR) # E: Union[numpy.ndarray, numpy.integer, numpy.bool_] + +reveal_type(b_ << b) # E: numpy.int8 +reveal_type(b_ >> b) # E: numpy.int8 +reveal_type(b_ | b) # E: numpy.bool_ +reveal_type(b_ ^ b) # E: numpy.bool_ +reveal_type(b_ & b) # E: numpy.bool_ + +reveal_type(b_ << i) # E: Union[numpy.int32, numpy.int64] +reveal_type(b_ >> i) # E: Union[numpy.int32, numpy.int64] +reveal_type(b_ | i) # E: Union[numpy.int32, numpy.int64] +reveal_type(b_ ^ i) # E: Union[numpy.int32, numpy.int64] +reveal_type(b_ & i) # E: Union[numpy.int32, numpy.int64] + +reveal_type(~i8) # E: numpy.int64 +reveal_type(~i4) # E: numpy.int32 +reveal_type(~u8) # E: numpy.uint64 +reveal_type(~u4) # E: numpy.uint32 +reveal_type(~b_) # E: numpy.bool_ +reveal_type(~AR) # E: Union[numpy.ndarray*, numpy.integer, numpy.bool_] From 225c3360721ba515de88a1e191ede58e8b95593d Mon Sep 17 00:00:00 2001 From: stuartarchibald Date: Mon, 5 Oct 2020 18:20:21 +0100 Subject: [PATCH 188/409] BUG: Fixes incorrect error message in numpy.ediff1d (#17457) Fixes the error message reported by ediff1d in the case of an invalid to_begin and updates tests to assert validity. Co-authored-by: Ross Barnowski --- numpy/lib/arraysetops.py | 2 +- numpy/lib/tests/test_arraysetops.py | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/numpy/lib/arraysetops.py b/numpy/lib/arraysetops.py index 9464692e0b2b..6c6c1ff809ce 100644 --- a/numpy/lib/arraysetops.py +++ b/numpy/lib/arraysetops.py @@ -93,7 +93,7 @@ def ediff1d(ary, to_end=None, to_begin=None): else: to_begin = np.asanyarray(to_begin) if not np.can_cast(to_begin, dtype_req, casting="same_kind"): - raise TypeError("dtype of `to_end` must be compatible " + raise TypeError("dtype of `to_begin` must be compatible " "with input `ary` under the `same_kind` rule.") to_begin = to_begin.ravel() diff --git a/numpy/lib/tests/test_arraysetops.py b/numpy/lib/tests/test_arraysetops.py index 81ba789e30d6..847e6cb8a3e7 100644 --- a/numpy/lib/tests/test_arraysetops.py +++ b/numpy/lib/tests/test_arraysetops.py @@ -125,32 +125,36 @@ def test_ediff1d(self): assert_array_equal([7, 1], ediff1d(two_elem, to_begin=7)) assert_array_equal([5, 6, 1], ediff1d(two_elem, to_begin=[5, 6])) - @pytest.mark.parametrize("ary, prepend, append", [ + @pytest.mark.parametrize("ary, prepend, append, expected", [ # should fail because trying to cast # np.nan standard floating point value # into an integer array: (np.array([1, 2, 3], dtype=np.int64), None, - np.nan), + np.nan, + 'to_end'), # should fail because attempting # to downcast to int type: (np.array([1, 2, 3], dtype=np.int64), np.array([5, 7, 2], dtype=np.float32), - None), + None, + 'to_begin'), # should fail because attempting to cast # two special floating point values - # to integers (on both sides of ary): + # to integers (on both sides of ary), + # `to_begin` is in the error message as the impl checks this first: (np.array([1., 3., 9.], dtype=np.int8), np.nan, - np.nan), + np.nan, + 'to_begin'), ]) - def test_ediff1d_forbidden_type_casts(self, ary, prepend, append): + def test_ediff1d_forbidden_type_casts(self, ary, prepend, append, expected): # verify resolution of gh-11490 # specifically, raise an appropriate # Exception when attempting to append or # prepend with an incompatible type - msg = 'must be compatible' + msg = 'dtype of `{}` must be compatible'.format(expected) with assert_raises_regex(TypeError, msg): ediff1d(ary=ary, to_end=append, From 72b9f4995045e73cf4431b08877f8d2dbb5fa0d9 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Mon, 5 Oct 2020 11:13:26 -0700 Subject: [PATCH 189/409] DOC: typo. --- doc/source/reference/arrays.scalars.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/reference/arrays.scalars.rst b/doc/source/reference/arrays.scalars.rst index 174dd2b917a0..3ab4d73013af 100644 --- a/doc/source/reference/arrays.scalars.rst +++ b/doc/source/reference/arrays.scalars.rst @@ -315,7 +315,7 @@ are also provided. numpy.float128 Alias for `numpy.longdouble`, named after its size in bits. - The existance of these aliases depends on the platform. + The existence of these aliases depends on the platform. .. autoclass:: numpy.complex64 From 1e0aa16427795bee76a0138115b93bebb95084a9 Mon Sep 17 00:00:00 2001 From: takanori-pskq Date: Tue, 6 Oct 2020 13:37:19 +0900 Subject: [PATCH 190/409] DOC: Add some entries for C types and macros --- .../reference/c-api/types-and-structures.rst | 39 +++-- doc/source/reference/c-api/ufunc.rst | 148 +++++++++++------- 2 files changed, 113 insertions(+), 74 deletions(-) diff --git a/doc/source/reference/c-api/types-and-structures.rst b/doc/source/reference/c-api/types-and-structures.rst index cc961df3abf4..45cc6725a27e 100644 --- a/doc/source/reference/c-api/types-and-structures.rst +++ b/doc/source/reference/c-api/types-and-structures.rst @@ -357,27 +357,25 @@ PyArrayDescr_Type and PyArray_Descr useful as the data-type descriptor for a field in another data-type descriptor. The fields member should be ``NULL`` if this is non- ``NULL`` (the fields member of the base descriptor can be - non- ``NULL`` however). The :c:type:`PyArray_ArrayDescr` structure is - defined using + non- ``NULL`` however). - .. code-block:: c - - typedef struct { - PyArray_Descr *base; - PyObject *shape; - } PyArray_ArrayDescr; + .. c:type:: PyArray_ArrayDescr - The elements of this structure are: + .. code-block:: c - .. c:member:: PyArray_Descr *PyArray_ArrayDescr.base + typedef struct { + PyArray_Descr *base; + PyObject *shape; + } PyArray_ArrayDescr; - The data-type-descriptor object of the base-type. + .. c:member:: PyArray_Descr *PyArray_ArrayDescr.base - .. c:member:: PyObject *PyArray_ArrayDescr.shape + The data-type-descriptor object of the base-type. - The shape (always C-style contiguous) of the sub-array as a Python - tuple. + .. c:member:: PyObject *PyArray_ArrayDescr.shape + The shape (always C-style contiguous) of the sub-array as a Python + tuple. .. c:member:: PyObject *PyArray_Descr.fields @@ -961,9 +959,14 @@ PyUFunc_Type and PyUFuncObject For each distinct core dimension, a set of ``UFUNC_CORE_DIM*`` flags - - :c:data:`UFUNC_CORE_DIM_CAN_IGNORE` if the dim name ends in ``?`` - - :c:data:`UFUNC_CORE_DIM_SIZE_INFERRED` if the dim size will be - determined from the operands and not from a :ref:`frozen ` signature + .. c:macro:: UFUNC_CORE_DIM_IGNORE + + if the dim name ends in ``?`` + + .. c:macro:: UFUNC_CORE_DIM_INFERRED + + if the dim size will be determined from the operands + and not from a :ref:`frozen ` signature PyArrayIter_Type and PyArrayIterObject -------------------------------------- @@ -1438,3 +1441,5 @@ for completeness and assistance in understanding the code. ``arrayobject.h`` header. This type is not exposed to Python and could be replaced with a C-structure. As a Python type it takes advantage of reference- counted memory management. + + .. c:type:: PyArrayMapIterObject diff --git a/doc/source/reference/c-api/ufunc.rst b/doc/source/reference/c-api/ufunc.rst index 50963c81f626..1b9b6864224d 100644 --- a/doc/source/reference/c-api/ufunc.rst +++ b/doc/source/reference/c-api/ufunc.rst @@ -12,12 +12,39 @@ Constants .. c:var:: UFUNC_ERR_{HANDLER} - ``{HANDLER}`` can be **IGNORE**, **WARN**, **RAISE**, or **CALL** + .. c:macro:: UFUNC_ERR_IGNORE + + .. c:macro:: UFUNC_ERR_WARN + + .. c:macro:: UFUNC_ERR_RAISE + + .. c:macro:: UFUNC_ERR_CALL .. c:var:: UFUNC_{THING}_{ERR} - ``{THING}`` can be **MASK**, **SHIFT**, or **FPE**, and ``{ERR}`` can - be **DIVIDEBYZERO**, **OVERFLOW**, **UNDERFLOW**, and **INVALID**. + .. c:macro:: UFUNC_MASK_DIVIDEBYZERO + + .. c:macro:: UFUNC_MASK_OVERFLOW + + .. c:macro:: UFUNC_MASK_UNDERFLOW + + .. c:macro:: UFUNC_MASK_INVALID + + .. c:macro:: UFUNC_SHIFT_DIVIDEBYZERO + + .. c:macro:: UFUNC_SHIFT_OVERFLOW + + .. c:macro:: UFUNC_SHIFT_UNDERFLOW + + .. c:macro:: UFUNC_SHIFT_INVALID + + .. c:macro:: UFUNC_FPE_DIVIDEBYZERO + + .. c:macro:: UFUNC_FPE_OVERFLOW + + .. c:macro:: UFUNC_FPE_UNDERFLOW + + .. c:macro:: UFUNC_FPE_INVALID .. c:var:: PyUFunc_{VALUE} @@ -50,6 +77,66 @@ Macros was released (because loop->obj was not true). +Types +----- + +.. c:type:: PyUFuncGenericFunction + + pointers to functions that actually implement the underlying + (element-by-element) function :math:`N` times with the following + signature: + + .. c:function:: void loopfunc( + char** args, npy_intp const *dimensions, npy_intp const *steps, void* data) + + *args* + + An array of pointers to the actual data for the input and output + arrays. The input arguments are given first followed by the output + arguments. + + *dimensions* + + A pointer to the size of the dimension over which this function is + looping. + + *steps* + + A pointer to the number of bytes to jump to get to the + next element in this dimension for each of the input and + output arguments. + + *data* + + Arbitrary data (extra arguments, function names, *etc.* ) + that can be stored with the ufunc and will be passed in + when it is called. + + This is an example of a func specialized for addition of doubles + returning doubles. + + .. code-block:: c + + static void + double_add(char **args, + npy_intp const *dimensions, + npy_intp const *steps, + void *extra) + { + npy_intp i; + npy_intp is1 = steps[0], is2 = steps[1]; + npy_intp os = steps[2], n = dimensions[0]; + char *i1 = args[0], *i2 = args[1], *op = args[2]; + for (i = 0; i < n; i++) { + *((double *)op) = *((double *)i1) + + *((double *)i2); + i1 += is1; + i2 += is2; + op += os; + } + } + + Functions --------- @@ -71,60 +158,7 @@ Functions :param func: Must to an array of length *ntypes* containing - :c:type:`PyUFuncGenericFunction` items. These items are pointers to - functions that actually implement the underlying - (element-by-element) function :math:`N` times with the following - signature: - - .. c:function:: void loopfunc( - char** args, npy_intp const *dimensions, npy_intp const *steps, void* data) - - *args* - - An array of pointers to the actual data for the input and output - arrays. The input arguments are given first followed by the output - arguments. - - *dimensions* - - A pointer to the size of the dimension over which this function is - looping. - - *steps* - - A pointer to the number of bytes to jump to get to the - next element in this dimension for each of the input and - output arguments. - - *data* - - Arbitrary data (extra arguments, function names, *etc.* ) - that can be stored with the ufunc and will be passed in - when it is called. - - This is an example of a func specialized for addition of doubles - returning doubles. - - .. code-block:: c - - static void - double_add(char **args, - npy_intp const *dimensions, - npy_intp const *steps, - void *extra) - { - npy_intp i; - npy_intp is1 = steps[0], is2 = steps[1]; - npy_intp os = steps[2], n = dimensions[0]; - char *i1 = args[0], *i2 = args[1], *op = args[2]; - for (i = 0; i < n; i++) { - *((double *)op) = *((double *)i1) + - *((double *)i2); - i1 += is1; - i2 += is2; - op += os; - } - } + :c:type:`PyUFuncGenericFunction` items. :param data: Should be ``NULL`` or a pointer to an array of size *ntypes* From 04ad2ce13c4b8a0557c08010d3121cbbaa1cad5a Mon Sep 17 00:00:00 2001 From: takanori-pskq Date: Tue, 6 Oct 2020 16:41:49 +0900 Subject: [PATCH 191/409] Fixup --- doc/source/reference/c-api/types-and-structures.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/reference/c-api/types-and-structures.rst b/doc/source/reference/c-api/types-and-structures.rst index 45cc6725a27e..2f4919f454b8 100644 --- a/doc/source/reference/c-api/types-and-structures.rst +++ b/doc/source/reference/c-api/types-and-structures.rst @@ -959,11 +959,11 @@ PyUFunc_Type and PyUFuncObject For each distinct core dimension, a set of ``UFUNC_CORE_DIM*`` flags - .. c:macro:: UFUNC_CORE_DIM_IGNORE + .. c:macro:: UFUNC_CORE_DIM_CAN_IGNORE if the dim name ends in ``?`` - .. c:macro:: UFUNC_CORE_DIM_INFERRED + .. c:macro:: UFUNC_CORE_DIM_SIZE_INFERRED if the dim size will be determined from the operands and not from a :ref:`frozen ` signature From 3c5de8a2bb6a75faef20f1b15985c03724271be5 Mon Sep 17 00:00:00 2001 From: takanori-pskq Date: Tue, 6 Oct 2020 17:49:08 +0900 Subject: [PATCH 192/409] Fix: Remove the link for `PyArrayMapIterObject` --- doc/source/reference/c-api/types-and-structures.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/doc/source/reference/c-api/types-and-structures.rst b/doc/source/reference/c-api/types-and-structures.rst index 2f4919f454b8..59361fd62818 100644 --- a/doc/source/reference/c-api/types-and-structures.rst +++ b/doc/source/reference/c-api/types-and-structures.rst @@ -1436,10 +1436,8 @@ for completeness and assistance in understanding the code. Advanced indexing is handled with this Python type. It is simply a loose wrapper around the C-structure containing the variables needed for advanced array indexing. The associated C-structure, - :c:type:`PyArrayMapIterObject`, is useful if you are trying to + ``PyArrayMapIterObject``, is useful if you are trying to understand the advanced-index mapping code. It is defined in the ``arrayobject.h`` header. This type is not exposed to Python and could be replaced with a C-structure. As a Python type it takes advantage of reference- counted memory management. - - .. c:type:: PyArrayMapIterObject From d934c2620439cbcf9b6f19b6037c1c67346dd71f Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Tue, 6 Oct 2020 13:40:39 +0200 Subject: [PATCH 193/409] REV: Remove the `uint64`/`signedinteger` specific overload Unfortunately the likes of `self: uint64` don't work with protocols. Revisit this once we add proper support for numerical precision. --- numpy/typing/_callable.py | 7 ++----- numpy/typing/tests/data/fail/bitwise_ops.py | 5 +++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/numpy/typing/_callable.py b/numpy/typing/_callable.py index 943441cf494e..194522977428 100644 --- a/numpy/typing/_callable.py +++ b/numpy/typing/_callable.py @@ -9,7 +9,7 @@ """ import sys -from typing import Union, TypeVar, overload, Any, NoReturn +from typing import Union, TypeVar, overload, Any from numpy import ( _BoolLike, @@ -26,7 +26,6 @@ signedinteger, int32, int64, - uint64, floating, float32, float64, @@ -114,14 +113,12 @@ def __call__(self, __other: float) -> floating: ... def __call__(self, __other: complex) -> complexfloating[floating]: ... class _UnsignedIntBitOp(Protocol): - # The likes of `uint64 | np.signedinteger` will fail as there + # TODO: The likes of `uint64 | np.signedinteger` will fail as there # is no signed integer type large enough to hold a `uint64` # See https://github.com/numpy/numpy/issues/2524 @overload def __call__(self, __other: Union[bool, unsignedinteger]) -> unsignedinteger: ... @overload - def __call__(self: uint64, __other: Union[int, signedinteger]) -> NoReturn: ... - @overload def __call__(self, __other: Union[int, signedinteger]) -> signedinteger: ... class _SignedIntOp(Protocol): diff --git a/numpy/typing/tests/data/fail/bitwise_ops.py b/numpy/typing/tests/data/fail/bitwise_ops.py index 9be6869ad101..3b5be95c0d96 100644 --- a/numpy/typing/tests/data/fail/bitwise_ops.py +++ b/numpy/typing/tests/data/fail/bitwise_ops.py @@ -15,5 +15,6 @@ u8 & f8 # E: No overload variant ~f8 # E: Unsupported operand type -# mypys' error message for `NoReturn` is unfortunately pretty nad -a = u8 | 0 # E: Need type annotation +# mypys' error message for `NoReturn` is unfortunately pretty bad +# TODO: Reenable this once we add support for numerical precision for `number`s +# a = u8 | 0 # E: Need type annotation From 96cb0963433d539c7a601edf4ab2261cbec0b897 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Tue, 6 Oct 2020 17:27:21 +0100 Subject: [PATCH 194/409] DOC: add some missing scalar aliases --- doc/source/reference/arrays.scalars.rst | 28 ++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/doc/source/reference/arrays.scalars.rst b/doc/source/reference/arrays.scalars.rst index 3ab4d73013af..d857f0855af4 100644 --- a/doc/source/reference/arrays.scalars.rst +++ b/doc/source/reference/arrays.scalars.rst @@ -330,15 +330,33 @@ are also provided. Other aliases ~~~~~~~~~~~~~ -.. autoclass:: float_ +The first two of these are conveniences which resemble the names of the +builtin types, in the same style as `bool_`, `int_`, `str_`, `bytes_`, and +`object_`: -.. autoclass:: complex_ +.. autoclass:: numpy.float_ -.. autoclass:: longfloat +.. autoclass:: numpy.complex_ -.. autoclass:: clongfloat +Some more use alternate naming conventions for extended-precision floats and +complex numbers: -.. autoclass:: longcomplex +.. autoclass:: numpy.longfloat + +.. autoclass:: numpy.singlecomplex + +.. autoclass:: numpy.cfloat + +.. autoclass:: numpy.longcomplex + +.. autoclass:: numpy.clongfloat + +The following aliases originate from Python 2, and it is recommended that they +not be used in new code. + +.. autoclass:: numpy.string_ + +.. autoclass:: numpy.unicode_ Attributes ========== From 3879e952dff2ff5a16decc997bf72716b8da5d17 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Tue, 6 Oct 2020 18:17:34 +0100 Subject: [PATCH 195/409] DOC: Fix broken references Unfortunately we can't use `np.` as a module name alias --- doc/source/user/basics.types.rst | 64 ++++++++++++++++---------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/doc/source/user/basics.types.rst b/doc/source/user/basics.types.rst index 3c39b35d0feb..ec2af409ae86 100644 --- a/doc/source/user/basics.types.rst +++ b/doc/source/user/basics.types.rst @@ -19,78 +19,78 @@ The primitive types supported are tied closely to those in C: - C type - Description - * - `np.bool_` + * - `numpy.bool_` - ``bool`` - Boolean (True or False) stored as a byte - * - `np.byte` + * - `numpy.byte` - ``signed char`` - Platform-defined - * - `np.ubyte` + * - `numpy.ubyte` - ``unsigned char`` - Platform-defined - * - `np.short` + * - `numpy.short` - ``short`` - Platform-defined - * - `np.ushort` + * - `numpy.ushort` - ``unsigned short`` - Platform-defined - * - `np.intc` + * - `numpy.intc` - ``int`` - Platform-defined - * - `np.uintc` + * - `numpy.uintc` - ``unsigned int`` - Platform-defined - * - `np.int_` + * - `numpy.int_` - ``long`` - Platform-defined - * - `np.uint` + * - `numpy.uint` - ``unsigned long`` - Platform-defined - * - `np.longlong` + * - `numpy.longlong` - ``long long`` - Platform-defined - * - `np.ulonglong` + * - `numpy.ulonglong` - ``unsigned long long`` - Platform-defined - * - `np.half` / `np.float16` + * - `numpy.half` / `numpy.float16` - - Half precision float: sign bit, 5 bits exponent, 10 bits mantissa - * - `np.single` + * - `numpy.single` - ``float`` - Platform-defined single precision float: typically sign bit, 8 bits exponent, 23 bits mantissa - * - `np.double` + * - `numpy.double` - ``double`` - Platform-defined double precision float: typically sign bit, 11 bits exponent, 52 bits mantissa. - * - `np.longdouble` + * - `numpy.longdouble` - ``long double`` - Platform-defined extended-precision float - * - `np.csingle` + * - `numpy.csingle` - ``float complex`` - Complex number, represented by two single-precision floats (real and imaginary components) - * - `np.cdouble` + * - `numpy.cdouble` - ``double complex`` - Complex number, represented by two double-precision floats (real and imaginary components). - * - `np.clongdouble` + * - `numpy.clongdouble` - ``long double complex`` - Complex number, represented by two extended-precision floats (real and imaginary components). @@ -105,59 +105,59 @@ aliases are provided: - C type - Description - * - `np.int8` + * - `numpy.int8` - ``int8_t`` - Byte (-128 to 127) - * - `np.int16` + * - `numpy.int16` - ``int16_t`` - Integer (-32768 to 32767) - * - `np.int32` + * - `numpy.int32` - ``int32_t`` - Integer (-2147483648 to 2147483647) - * - `np.int64` + * - `numpy.int64` - ``int64_t`` - Integer (-9223372036854775808 to 9223372036854775807) - * - `np.uint8` + * - `numpy.uint8` - ``uint8_t`` - Unsigned integer (0 to 255) - * - `np.uint16` + * - `numpy.uint16` - ``uint16_t`` - Unsigned integer (0 to 65535) - * - `np.uint32` + * - `numpy.uint32` - ``uint32_t`` - Unsigned integer (0 to 4294967295) - * - `np.uint64` + * - `numpy.uint64` - ``uint64_t`` - Unsigned integer (0 to 18446744073709551615) - * - `np.intp` + * - `numpy.intp` - ``intptr_t`` - Integer used for indexing, typically the same as ``ssize_t`` - * - `np.uintp` + * - `numpy.uintp` - ``uintptr_t`` - Integer large enough to hold a pointer - * - `np.float32` + * - `numpy.float32` - ``float`` - - * - `np.float64` / `np.float_` + * - `numpy.float64` / `numpy.float_` - ``double`` - Note that this matches the precision of the builtin python `float`. - * - `np.complex64` + * - `numpy.complex64` - ``float complex`` - Complex number, represented by two 32-bit floats (real and imaginary components) - * - `np.complex128` / `np.complex_` + * - `numpy.complex128` / `numpy.complex_` - ``double complex`` - Note that this matches the precision of the builtin python `complex`. From 11f051a37bed48c341849e79cac1863a03626f73 Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Tue, 6 Oct 2020 15:56:46 -0400 Subject: [PATCH 196/409] TST: Fix doctest for full_like --- numpy/core/numeric.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/numeric.py b/numpy/core/numeric.py index a023bf0da32e..25235f738bd2 100644 --- a/numpy/core/numeric.py +++ b/numpy/core/numeric.py @@ -409,7 +409,7 @@ def full_like(a, fill_value, dtype=None, order='K', subok=True, shape=None): >>> y = np.arange(6, dtype=np.double) >>> np.full_like(y, 0.1) - array([0.1, 0.1, 0.1, 0.1, 0.1, 0.1]) + array([0.1, 0.1, 0.1, 0.1, 0.1, 0.1]) """ res = empty_like(a, dtype=dtype, order=order, subok=subok, shape=shape) From dc3e276f0e777c8bc3a5b5cf76636999dde8f82e Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Tue, 6 Oct 2020 14:14:23 -0700 Subject: [PATCH 197/409] DOC: rm numpy. from type alias attr definitions. --- doc/source/reference/arrays.scalars.rst | 28 ++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/source/reference/arrays.scalars.rst b/doc/source/reference/arrays.scalars.rst index d857f0855af4..13d117af2432 100644 --- a/doc/source/reference/arrays.scalars.rst +++ b/doc/source/reference/arrays.scalars.rst @@ -261,10 +261,10 @@ are also provided. .. autoclass:: numpy.bool8 -.. attribute:: numpy.int8 - numpy.int16 - numpy.int32 - numpy.int64 +.. attribute:: int8 + int16 + int32 + int64 Aliases for the signed integer types (one of `numpy.byte`, `numpy.short`, `numpy.intc`, `numpy.int_` and `numpy.longlong`) with the specified number @@ -273,10 +273,10 @@ are also provided. Compatible with the C99 ``int8_t``, ``int16_t``, ``int32_t``, and ``int64_t``, respectively. -.. attribute:: numpy.uint8 - numpy.uint16 - numpy.uint32 - numpy.uint64 +.. attribute:: uint8 + uint16 + uint32 + uint64 Alias for the unsigned integer types (one of `numpy.byte`, `numpy.short`, `numpy.intc`, `numpy.int_` and `numpy.longlong`) with the specified number @@ -285,7 +285,7 @@ are also provided. Compatible with the C99 ``uint8_t``, ``uint16_t``, ``uint32_t``, and ``uint64_t``, respectively. -.. attribute:: numpy.intp +.. attribute:: intp Alias for the signed integer type (one of `numpy.byte`, `numpy.short`, `numpy.intc`, `numpy.int_` and `np.longlong`) that is the same size as a @@ -295,7 +295,7 @@ are also provided. :Character code: ``'p'`` -.. attribute:: numpy.uintp +.. attribute:: uintp Alias for the unsigned integer type (one of `numpy.byte`, `numpy.short`, `numpy.intc`, `numpy.int_` and `np.longlong`) that is the same size as a @@ -311,8 +311,8 @@ are also provided. .. autoclass:: numpy.float64 -.. attribute:: numpy.float96 - numpy.float128 +.. attribute:: float96 + float128 Alias for `numpy.longdouble`, named after its size in bits. The existence of these aliases depends on the platform. @@ -321,8 +321,8 @@ are also provided. .. autoclass:: numpy.complex128 -.. attribute:: numpy.complex192 - numpy.complex256 +.. attribute:: complex192 + complex256 Alias for `numpy.clongdouble`, named after its size in bits. The existance of these aliases depends on the platform. From 5014f7af75dee81d4636769de69969e8c97b5873 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Thu, 1 Oct 2020 18:32:39 +0200 Subject: [PATCH 198/409] MAINT: Move the `np.core.numeric` functions to their own stub file --- numpy/__init__.pyi | 146 +++++++++-------------------------------- numpy/core/numeric.pyi | 117 +++++++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+), 114 deletions(-) create mode 100644 numpy/core/numeric.pyi diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 9966ef1999a5..789bac896e56 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -54,10 +54,9 @@ else: def __index__(self) -> int: ... # Ensures that the stubs are picked up -from . import ( +from numpy import ( char, compat, - core, ctypeslib, emath, fft, @@ -127,6 +126,37 @@ from numpy.core._asarray import ( require as require, ) +from numpy.core.numeric import( + zeros_like as zeros_like, + ones as ones, + ones_like as ones_like, + empty_like as empty_like, + full as full, + full_like as full_like, + count_nonzero as count_nonzero, + isfortran as isfortran, + argwhere as argwhere, + flatnonzero as flatnonzero, + correlate as correlate, + convolve as convolve, + outer as outer, + tensordot as tensordot, + roll as roll, + rollaxis as rollaxis, + moveaxis as moveaxis, + cross as cross, + indices as indices, + fromfunction as fromfunction, + isscalar as isscalar, + binary_repr as binary_repr, + base_repr as base_repr, + identity as identity, + allclose as allclose, + isclose as isclose, + array_equal as array_equal, + array_equiv as array_equiv, +) + # Add an object to `__all__` if their stubs are defined in an external file; # their stubs will not be recognized otherwise. # NOTE: This is redundant for objects defined within this file. @@ -1552,13 +1582,6 @@ def zeros( *, like: ArrayLike = ..., ) -> ndarray: ... -def ones( - shape: _ShapeLike, - dtype: DtypeLike = ..., - order: _OrderCF = ..., - *, - like: ArrayLike = ..., -) -> ndarray: ... def empty( shape: _ShapeLike, dtype: DtypeLike = ..., @@ -1566,111 +1589,6 @@ def empty( *, like: ArrayLike = ..., ) -> ndarray: ... -def zeros_like( - a: ArrayLike, - dtype: DtypeLike = ..., - order: _OrderKACF = ..., - subok: bool = ..., - shape: Optional[Union[int, Sequence[int]]] = ..., -) -> ndarray: ... -def ones_like( - a: ArrayLike, - dtype: DtypeLike = ..., - order: _OrderKACF = ..., - subok: bool = ..., - shape: Optional[_ShapeLike] = ..., -) -> ndarray: ... -def empty_like( - a: ArrayLike, - dtype: DtypeLike = ..., - order: _OrderKACF = ..., - subok: bool = ..., - shape: Optional[_ShapeLike] = ..., -) -> ndarray: ... -def full( - shape: _ShapeLike, - fill_value: Any, - dtype: DtypeLike = ..., - order: _OrderCF = ..., - *, - like: ArrayLike = ..., -) -> ndarray: ... -def full_like( - a: ArrayLike, - fill_value: Any, - dtype: DtypeLike = ..., - order: _OrderKACF = ..., - subok: bool = ..., - shape: Optional[_ShapeLike] = ..., -) -> ndarray: ... -def count_nonzero( - a: ArrayLike, axis: Optional[Union[int, Tuple[int], Tuple[int, int]]] = ... -) -> Union[int, ndarray]: ... -def isfortran(a: ndarray) -> bool: ... -def argwhere(a: ArrayLike) -> ndarray: ... -def flatnonzero(a: ArrayLike) -> ndarray: ... - -_CorrelateMode = Literal["valid", "same", "full"] - -def correlate(a: ArrayLike, v: ArrayLike, mode: _CorrelateMode = ...) -> ndarray: ... -def convolve(a: ArrayLike, v: ArrayLike, mode: _CorrelateMode = ...) -> ndarray: ... -def outer(a: ArrayLike, b: ArrayLike, out: ndarray = ...) -> ndarray: ... -def tensordot( - a: ArrayLike, - b: ArrayLike, - axes: Union[ - int, Tuple[int, int], Tuple[Tuple[int, int], ...], Tuple[List[int, int], ...] - ] = ..., -) -> ndarray: ... -def roll( - a: ArrayLike, - shift: Union[int, Tuple[int, ...]], - axis: Optional[Union[int, Tuple[int, ...]]] = ..., -) -> ndarray: ... -def rollaxis(a: ArrayLike, axis: int, start: int = ...) -> ndarray: ... -def moveaxis( - a: ndarray, - source: Union[int, Sequence[int]], - destination: Union[int, Sequence[int]], -) -> ndarray: ... -def cross( - a: ArrayLike, - b: ArrayLike, - axisa: int = ..., - axisb: int = ..., - axisc: int = ..., - axis: Optional[int] = ..., -) -> ndarray: ... -def indices( - dimensions: Sequence[int], dtype: dtype = ..., sparse: bool = ... -) -> Union[ndarray, Tuple[ndarray, ...]]: ... -def fromfunction( - function: Callable, - shape: Tuple[int, int], - *, - like: ArrayLike = ..., - **kwargs, -) -> Any: ... -def isscalar(element: Any) -> bool: ... -def binary_repr(num: int, width: Optional[int] = ...) -> str: ... -def base_repr(number: int, base: int = ..., padding: int = ...) -> str: ... -def identity(n: int, dtype: DtypeLike = ..., *, like: ArrayLike = ...) -> ndarray: ... -def allclose( - a: ArrayLike, - b: ArrayLike, - rtol: float = ..., - atol: float = ..., - equal_nan: bool = ..., -) -> bool: ... -def isclose( - a: ArrayLike, - b: ArrayLike, - rtol: float = ..., - atol: float = ..., - equal_nan: bool = ..., -) -> Union[bool_, ndarray]: ... -def array_equal(a1: ArrayLike, a2: ArrayLike) -> bool: ... -def array_equiv(a1: ArrayLike, a2: ArrayLike) -> bool: ... # # Constants diff --git a/numpy/core/numeric.pyi b/numpy/core/numeric.pyi new file mode 100644 index 000000000000..19720fbdcc9b --- /dev/null +++ b/numpy/core/numeric.pyi @@ -0,0 +1,117 @@ +from typing import Any, Optional, Union, Sequence, Tuple + +from numpy import ndarray, dtype, bool_, _OrderKACF, _OrderCF +from numpy.typing import ArrayLike, DtypeLike, _ShapeLike + +def zeros_like( + a: ArrayLike, + dtype: DtypeLike = ..., + order: _OrderKACF = ..., + subok: bool = ..., + shape: Optional[Union[int, Sequence[int]]] = ..., +) -> ndarray: ... +def ones( + shape: _ShapeLike, + dtype: DtypeLike = ..., + order: _OrderCF = ..., + *, + like: ArrayLike = ..., +) -> ndarray: ... +def ones_like( + a: ArrayLike, + dtype: DtypeLike = ..., + order: _OrderKACF = ..., + subok: bool = ..., + shape: Optional[_ShapeLike] = ..., +) -> ndarray: ... +def empty_like( + a: ArrayLike, + dtype: DtypeLike = ..., + order: _OrderKACF = ..., + subok: bool = ..., + shape: Optional[_ShapeLike] = ..., +) -> ndarray: ... +def full( + shape: _ShapeLike, + fill_value: Any, + dtype: DtypeLike = ..., + order: _OrderCF = ..., + *, + like: ArrayLike = ..., +) -> ndarray: ... +def full_like( + a: ArrayLike, + fill_value: Any, + dtype: DtypeLike = ..., + order: _OrderKACF = ..., + subok: bool = ..., + shape: Optional[_ShapeLike] = ..., +) -> ndarray: ... +def count_nonzero( + a: ArrayLike, axis: Optional[Union[int, Tuple[int], Tuple[int, int]]] = ... +) -> Union[int, ndarray]: ... +def isfortran(a: ndarray) -> bool: ... +def argwhere(a: ArrayLike) -> ndarray: ... +def flatnonzero(a: ArrayLike) -> ndarray: ... + +_CorrelateMode = Literal["valid", "same", "full"] + +def correlate(a: ArrayLike, v: ArrayLike, mode: _CorrelateMode = ...) -> ndarray: ... +def convolve(a: ArrayLike, v: ArrayLike, mode: _CorrelateMode = ...) -> ndarray: ... +def outer(a: ArrayLike, b: ArrayLike, out: ndarray = ...) -> ndarray: ... +def tensordot( + a: ArrayLike, + b: ArrayLike, + axes: Union[ + int, Tuple[int, int], Tuple[Tuple[int, int], ...], Tuple[List[int, int], ...] + ] = ..., +) -> ndarray: ... +def roll( + a: ArrayLike, + shift: Union[int, Tuple[int, ...]], + axis: Optional[Union[int, Tuple[int, ...]]] = ..., +) -> ndarray: ... +def rollaxis(a: ArrayLike, axis: int, start: int = ...) -> ndarray: ... +def moveaxis( + a: ndarray, + source: Union[int, Sequence[int]], + destination: Union[int, Sequence[int]], +) -> ndarray: ... +def cross( + a: ArrayLike, + b: ArrayLike, + axisa: int = ..., + axisb: int = ..., + axisc: int = ..., + axis: Optional[int] = ..., +) -> ndarray: ... +def indices( + dimensions: Sequence[int], dtype: dtype = ..., sparse: bool = ... +) -> Union[ndarray, Tuple[ndarray, ...]]: ... +def fromfunction( + function: Callable, + shape: Tuple[int, int], + *, + like: ArrayLike = ..., + **kwargs, +) -> Any: ... +def isscalar(element: Any) -> bool: ... +def binary_repr(num: int, width: Optional[int] = ...) -> str: ... +def base_repr(number: int, base: int = ..., padding: int = ...) -> str: ... +def identity(n: int, dtype: DtypeLike = ..., *, like: ArrayLike = ...) -> ndarray: ... +def allclose( + a: ArrayLike, + b: ArrayLike, + rtol: float = ..., + atol: float = ..., + equal_nan: bool = ..., +) -> bool: ... +def isclose( + a: ArrayLike, + b: ArrayLike, + rtol: float = ..., + atol: float = ..., + equal_nan: bool = ..., +) -> Union[bool_, ndarray]: ... +def array_equal(a1: ArrayLike, a2: ArrayLike) -> bool: ... +def array_equiv(a1: ArrayLike, a2: ArrayLike) -> bool: ... From 789b21719fe89a3b38cae5a8800a4c0c3886a0f0 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Tue, 6 Oct 2020 16:08:06 -0700 Subject: [PATCH 199/409] NEP: update NEP 42 with discussion of type hinting applications (#17447) * NEP: update NEP 42 with discussion of type hinting applications As discussed in https://github.com/numpy/numpy/pull/16759, the new DType classes provide a good path forward for making `ndarray` generic over DType. Update NEP 42 to discuss those applications in more detail. * NEP: discuss typing for use scalar types in NEP 42 Also clean up the language a bit. --- doc/neps/nep-0042-new-dtypes.rst | 51 ++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 12 deletions(-) diff --git a/doc/neps/nep-0042-new-dtypes.rst b/doc/neps/nep-0042-new-dtypes.rst index 2d1e3a329e6c..99887a45100b 100644 --- a/doc/neps/nep-0042-new-dtypes.rst +++ b/doc/neps/nep-0042-new-dtypes.rst @@ -259,21 +259,48 @@ including the type hierarchy and the use of abstract DTypes. Class getter ============================================================================== -To create a dtype instance from a scalar type users now call ``np.dtype`` (for -instance, ``np.dtype(np.int64)``). - -To get the DType of a scalar type, we propose this getter syntax:: +To create a DType instance from a scalar type users now call +``np.dtype`` (for instance, ``np.dtype(np.int64)``). Sometimes it is +also necessary to access the underlying DType class; this comes up in +particular with type hinting because the "type" of a DType instance is +the DType class. Taking inspiration from type hinting, we propose the +following getter syntax:: np.dtype[np.int64] -The notation works equally well with built-in and user-defined DTypes -and is inspired by and potentially useful for type hinting. +to get the DType class corresponding to a scalar type. The notation +works equally well with built-in and user-defined DTypes. This getter eliminates the need to create an explicit name for every -DType, crowding the ``np`` namespace; the getter itself signifies the type. +DType, crowding the ``np`` namespace; the getter itself signifies the +type. It also opens the possibility of making ``np.ndarray`` generic +over DType class using annotations like:: + + np.ndarray[np.dtype[np.float64]] + +The above is fairly verbose, so it is possible that we will include +aliases like:: + + Float64 = np.dtype[np.float64] + +in ``numpy.typing``, thus keeping annotations concise but still +avoiding crowding the ``np`` namespace as discussed above. For a +user-defined DType:: + + class UserDtype(dtype): ... + +one can do ``np.ndarray[UserDtype]``, keeping annotations concise in +that case without introducing boilerplate in NumPy itself. For a user +user-defined scalar type:: + + class UserScalar(generic): ... + +we would need to add a typing overload to ``dtype``:: + + @overload + __new__(cls, dtype: Type[UserScalar], ...) -> UserDtype -Since getter calls won't be needed often, this is unlikely to be burdensome. -Classes can also offer concise alternatives. +to allow ``np.dtype[UserScalar]``. The initial implementation probably will return only concrete (not abstract) DTypes. @@ -393,7 +420,7 @@ casting and array coercion, which are described in detail below. sortfunction`` that must return ``NotImplemented`` if the given ``sortkind`` is not known. -* Functions that cannot be removed are implemented as special methods. +* Functions that cannot be removed are implemented as special methods. Many of these were previously defined part of the :c:type:`PyArray_ArrFuncs` slot of the dtype instance (``PyArray_Descr *``) and include functions such as ``nonzero``, ``fill`` (used for ``np.arange``), and @@ -408,7 +435,7 @@ casting and array coercion, which are described in detail below. object to ensure uniqueness for all DTypes. On the C side, ``kind`` and ``char`` are set to ``\0`` (NULL character). While ``kind`` will be discouraged, the current ``np.issubdtype`` - may remain the preferred method for this type of check. + may remain the preferred method for this type of check. * A method ``ensure_canonical(self) -> dtype`` returns a new dtype (or ``self``) with the ``canonical`` flag set. @@ -1229,7 +1256,7 @@ Non-parametric dtypes do not have to implement: * ``discover_descr_from_pyobject`` (uses ``default_descr`` instead) * ``common_instance`` (uses ``default_descr`` instead) -* ``ensure_canonical`` (uses ``default_descr`` instead). +* ``ensure_canonical`` (uses ``default_descr`` instead). Sorting is expected to be implemented using: From f249b1727c4b2b83c5ab524d44ec918a1882f435 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Wed, 7 Oct 2020 02:01:40 +0200 Subject: [PATCH 200/409] MAINT: Add an empty `__init__.pyi` file to `np.core` Prevents mypy crashes related to import cycles. See https://github.com/numpy/numpy/issues/17316 --- numpy/core/__init__.pyi | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 numpy/core/__init__.pyi diff --git a/numpy/core/__init__.pyi b/numpy/core/__init__.pyi new file mode 100644 index 000000000000..e69de29bb2d1 From 590201aa6ef313f9d34fa4136d6f0b3b3c36e951 Mon Sep 17 00:00:00 2001 From: Sahil Siddiq <39989901+valdaarhun@users.noreply.github.com> Date: Wed, 7 Oct 2020 10:38:29 +0530 Subject: [PATCH 201/409] MAINT: Chain exceptions in "_polybase.py" (#17453) * Removed "from" keyword and changed "raise e" to "raise" --- numpy/polynomial/_polybase.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/numpy/polynomial/_polybase.py b/numpy/polynomial/_polybase.py index f4a67a2227f9..59c380f10e3a 100644 --- a/numpy/polynomial/_polybase.py +++ b/numpy/polynomial/_polybase.py @@ -547,8 +547,8 @@ def __divmod__(self, other): othercoef = self._get_coefficients(other) try: quo, rem = self._div(self.coef, othercoef) - except ZeroDivisionError as e: - raise e + except ZeroDivisionError: + raise except Exception: return NotImplemented quo = self.__class__(quo, self.domain, self.window) @@ -605,8 +605,8 @@ def __rmod__(self, other): def __rdivmod__(self, other): try: quo, rem = self._div(other, self.coef) - except ZeroDivisionError as e: - raise e + except ZeroDivisionError: + raise except Exception: return NotImplemented quo = self.__class__(quo, self.domain, self.window) From 58fe5c6d371dd76e9163dff5467f9f3bc328ce4a Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Tue, 6 Oct 2020 22:26:25 -0700 Subject: [PATCH 202/409] ENH: type np.unicode_ as np.str_ The two are aliases; i.e. >>> np.unicode_ is np.str_ True so make them aliases on the typing level too. --- numpy/__init__.pyi | 3 ++- numpy/typing/tests/data/reveal/scalars.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 9966ef1999a5..1a3116d83dd1 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -468,7 +468,6 @@ uint0: Any uintc: Any uintp: Any ulonglong: Any -unicode_: Any union1d: Any unique: Any unpackbits: Any @@ -1527,6 +1526,8 @@ class str_(character, str): self, __value: bytes, encoding: str = ..., errors: str = ... ) -> None: ... +unicode_ = str_ + # TODO(alan): Platform dependent types # longcomplex, longdouble, longfloat # bytes, short, intc, intp, longlong diff --git a/numpy/typing/tests/data/reveal/scalars.py b/numpy/typing/tests/data/reveal/scalars.py index ec3713b0fc42..b0dd762dfc7a 100644 --- a/numpy/typing/tests/data/reveal/scalars.py +++ b/numpy/typing/tests/data/reveal/scalars.py @@ -14,3 +14,5 @@ reveal_type(np.complex64().real) # E: numpy.float32 reveal_type(np.complex128().imag) # E: numpy.float64 + +reveal_type(np.unicode_('foo')) # E: numpy.str_ From 2c0df1e6bf7da7c49f20622c5207e691aa078281 Mon Sep 17 00:00:00 2001 From: takanori-pskq Date: Wed, 7 Oct 2020 13:36:40 +0900 Subject: [PATCH 203/409] DOC: Fix the entries for members of structures --- .../reference/c-api/types-and-structures.rst | 502 +++++++++--------- 1 file changed, 251 insertions(+), 251 deletions(-) diff --git a/doc/source/reference/c-api/types-and-structures.rst b/doc/source/reference/c-api/types-and-structures.rst index 59361fd62818..8759af6a4638 100644 --- a/doc/source/reference/c-api/types-and-structures.rst +++ b/doc/source/reference/c-api/types-and-structures.rst @@ -93,84 +93,84 @@ PyArray_Type and PyArrayObject PyObject *weakreflist; } PyArrayObject; -.. c:macro:: PyArrayObject.PyObject_HEAD + .. c:macro:: PyObject_HEAD - This is needed by all Python objects. It consists of (at least) - a reference count member ( ``ob_refcnt`` ) and a pointer to the - typeobject ( ``ob_type`` ). (Other elements may also be present - if Python was compiled with special options see - Include/object.h in the Python source tree for more - information). The ob_type member points to a Python type - object. + This is needed by all Python objects. It consists of (at least) + a reference count member ( ``ob_refcnt`` ) and a pointer to the + typeobject ( ``ob_type`` ). (Other elements may also be present + if Python was compiled with special options see + Include/object.h in the Python source tree for more + information). The ob_type member points to a Python type + object. -.. c:member:: char *PyArrayObject.data + .. c:member:: char *data - Accessible via :c:data:`PyArray_DATA`, this data member is a - pointer to the first element of the array. This pointer can - (and normally should) be recast to the data type of the array. + Accessible via :c:data:`PyArray_DATA`, this data member is a + pointer to the first element of the array. This pointer can + (and normally should) be recast to the data type of the array. -.. c:member:: int PyArrayObject.nd + .. c:member:: int nd - An integer providing the number of dimensions for this - array. When nd is 0, the array is sometimes called a rank-0 - array. Such arrays have undefined dimensions and strides and - cannot be accessed. Macro :c:data:`PyArray_NDIM` defined in - ``ndarraytypes.h`` points to this data member. :c:data:`NPY_MAXDIMS` - is the largest number of dimensions for any array. + An integer providing the number of dimensions for this + array. When nd is 0, the array is sometimes called a rank-0 + array. Such arrays have undefined dimensions and strides and + cannot be accessed. Macro :c:data:`PyArray_NDIM` defined in + ``ndarraytypes.h`` points to this data member. :c:data:`NPY_MAXDIMS` + is the largest number of dimensions for any array. -.. c:member:: npy_intp PyArrayObject.dimensions + .. c:member:: npy_intp dimensions - An array of integers providing the shape in each dimension as - long as nd :math:`\geq` 1. The integer is always large enough - to hold a pointer on the platform, so the dimension size is - only limited by memory. :c:data:`PyArray_DIMS` is the macro - associated with this data member. + An array of integers providing the shape in each dimension as + long as nd :math:`\geq` 1. The integer is always large enough + to hold a pointer on the platform, so the dimension size is + only limited by memory. :c:data:`PyArray_DIMS` is the macro + associated with this data member. -.. c:member:: npy_intp *PyArrayObject.strides + .. c:member:: npy_intp *strides - An array of integers providing for each dimension the number of - bytes that must be skipped to get to the next element in that - dimension. Associated with macro :c:data:`PyArray_STRIDES`. + An array of integers providing for each dimension the number of + bytes that must be skipped to get to the next element in that + dimension. Associated with macro :c:data:`PyArray_STRIDES`. -.. c:member:: PyObject *PyArrayObject.base + .. c:member:: PyObject *base - Pointed to by :c:data:`PyArray_BASE`, this member is used to hold a - pointer to another Python object that is related to this array. - There are two use cases: + Pointed to by :c:data:`PyArray_BASE`, this member is used to hold a + pointer to another Python object that is related to this array. + There are two use cases: - - If this array does not own its own memory, then base points to the - Python object that owns it (perhaps another array object) - - If this array has the (deprecated) :c:data:`NPY_ARRAY_UPDATEIFCOPY` or - :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` flag set, then this array is a working - copy of a "misbehaved" array. + - If this array does not own its own memory, then base points to the + Python object that owns it (perhaps another array object) + - If this array has the (deprecated) :c:data:`NPY_ARRAY_UPDATEIFCOPY` or + :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` flag set, then this array is a working + copy of a "misbehaved" array. - When ``PyArray_ResolveWritebackIfCopy`` is called, the array pointed to - by base will be updated with the contents of this array. + When ``PyArray_ResolveWritebackIfCopy`` is called, the array pointed to + by base will be updated with the contents of this array. -.. c:member:: PyArray_Descr *PyArrayObject.descr + .. c:member:: PyArray_Descr *descr - A pointer to a data-type descriptor object (see below). The - data-type descriptor object is an instance of a new built-in - type which allows a generic description of memory. There is a - descriptor structure for each data type supported. This - descriptor structure contains useful information about the type - as well as a pointer to a table of function pointers to - implement specific functionality. As the name suggests, it is - associated with the macro :c:data:`PyArray_DESCR`. + A pointer to a data-type descriptor object (see below). The + data-type descriptor object is an instance of a new built-in + type which allows a generic description of memory. There is a + descriptor structure for each data type supported. This + descriptor structure contains useful information about the type + as well as a pointer to a table of function pointers to + implement specific functionality. As the name suggests, it is + associated with the macro :c:data:`PyArray_DESCR`. -.. c:member:: int PyArrayObject.flags + .. c:member:: int flags - Pointed to by the macro :c:data:`PyArray_FLAGS`, this data member represents - the flags indicating how the memory pointed to by data is to be - interpreted. Possible flags are :c:data:`NPY_ARRAY_C_CONTIGUOUS`, - :c:data:`NPY_ARRAY_F_CONTIGUOUS`, :c:data:`NPY_ARRAY_OWNDATA`, - :c:data:`NPY_ARRAY_ALIGNED`, :c:data:`NPY_ARRAY_WRITEABLE`, - :c:data:`NPY_ARRAY_WRITEBACKIFCOPY`, and :c:data:`NPY_ARRAY_UPDATEIFCOPY`. + Pointed to by the macro :c:data:`PyArray_FLAGS`, this data member represents + the flags indicating how the memory pointed to by data is to be + interpreted. Possible flags are :c:data:`NPY_ARRAY_C_CONTIGUOUS`, + :c:data:`NPY_ARRAY_F_CONTIGUOUS`, :c:data:`NPY_ARRAY_OWNDATA`, + :c:data:`NPY_ARRAY_ALIGNED`, :c:data:`NPY_ARRAY_WRITEABLE`, + :c:data:`NPY_ARRAY_WRITEBACKIFCOPY`, and :c:data:`NPY_ARRAY_UPDATEIFCOPY`. -.. c:member:: PyObject *PyArrayObject.weakreflist + .. c:member:: PyObject *weakreflist - This member allows array objects to have weak references (using the - weakref module). + This member allows array objects to have weak references (using the + weakref module). PyArrayDescr_Type and PyArray_Descr @@ -226,195 +226,195 @@ PyArrayDescr_Type and PyArray_Descr npy_hash_t hash; } PyArray_Descr; -.. c:member:: PyTypeObject *PyArray_Descr.typeobj + .. c:member:: PyTypeObject *typeobj - Pointer to a typeobject that is the corresponding Python type for - the elements of this array. For the builtin types, this points to - the corresponding array scalar. For user-defined types, this - should point to a user-defined typeobject. This typeobject can - either inherit from array scalars or not. If it does not inherit - from array scalars, then the :c:data:`NPY_USE_GETITEM` and - :c:data:`NPY_USE_SETITEM` flags should be set in the ``flags`` member. + Pointer to a typeobject that is the corresponding Python type for + the elements of this array. For the builtin types, this points to + the corresponding array scalar. For user-defined types, this + should point to a user-defined typeobject. This typeobject can + either inherit from array scalars or not. If it does not inherit + from array scalars, then the :c:data:`NPY_USE_GETITEM` and + :c:data:`NPY_USE_SETITEM` flags should be set in the ``flags`` member. -.. c:member:: char PyArray_Descr.kind + .. c:member:: char kind - A character code indicating the kind of array (using the array - interface typestring notation). A 'b' represents Boolean, a 'i' - represents signed integer, a 'u' represents unsigned integer, 'f' - represents floating point, 'c' represents complex floating point, 'S' - represents 8-bit zero-terminated bytes, 'U' represents 32-bit/character - unicode string, and 'V' represents arbitrary. + A character code indicating the kind of array (using the array + interface typestring notation). A 'b' represents Boolean, a 'i' + represents signed integer, a 'u' represents unsigned integer, 'f' + represents floating point, 'c' represents complex floating point, 'S' + represents 8-bit zero-terminated bytes, 'U' represents 32-bit/character + unicode string, and 'V' represents arbitrary. -.. c:member:: char PyArray_Descr.type + .. c:member:: char type - A traditional character code indicating the data type. + A traditional character code indicating the data type. -.. c:member:: char PyArray_Descr.byteorder + .. c:member:: char byteorder - A character indicating the byte-order: '>' (big-endian), '<' (little- - endian), '=' (native), '\|' (irrelevant, ignore). All builtin data- - types have byteorder '='. + A character indicating the byte-order: '>' (big-endian), '<' (little- + endian), '=' (native), '\|' (irrelevant, ignore). All builtin data- + types have byteorder '='. -.. c:member:: char PyArray_Descr.flags + .. c:member:: char flags - A data-type bit-flag that determines if the data-type exhibits object- - array like behavior. Each bit in this member is a flag which are named - as: + A data-type bit-flag that determines if the data-type exhibits object- + array like behavior. Each bit in this member is a flag which are named + as: - .. c:macro:: NPY_ITEM_REFCOUNT + .. c:macro:: NPY_ITEM_REFCOUNT - Indicates that items of this data-type must be reference - counted (using :c:func:`Py_INCREF` and :c:func:`Py_DECREF` ). + Indicates that items of this data-type must be reference + counted (using :c:func:`Py_INCREF` and :c:func:`Py_DECREF` ). - .. c:macro:: NPY_ITEM_HASOBJECT + .. c:macro:: NPY_ITEM_HASOBJECT - Same as :c:data:`NPY_ITEM_REFCOUNT`. + Same as :c:data:`NPY_ITEM_REFCOUNT`. - .. c:macro:: NPY_LIST_PICKLE + .. c:macro:: NPY_LIST_PICKLE - Indicates arrays of this data-type must be converted to a list - before pickling. + Indicates arrays of this data-type must be converted to a list + before pickling. - .. c:macro:: NPY_ITEM_IS_POINTER + .. c:macro:: NPY_ITEM_IS_POINTER - Indicates the item is a pointer to some other data-type + Indicates the item is a pointer to some other data-type - .. c:macro:: NPY_NEEDS_INIT + .. c:macro:: NPY_NEEDS_INIT - Indicates memory for this data-type must be initialized (set - to 0) on creation. + Indicates memory for this data-type must be initialized (set + to 0) on creation. - .. c:macro:: NPY_NEEDS_PYAPI + .. c:macro:: NPY_NEEDS_PYAPI - Indicates this data-type requires the Python C-API during - access (so don't give up the GIL if array access is going to - be needed). + Indicates this data-type requires the Python C-API during + access (so don't give up the GIL if array access is going to + be needed). - .. c:macro:: NPY_USE_GETITEM + .. c:macro:: NPY_USE_GETITEM - On array access use the ``f->getitem`` function pointer - instead of the standard conversion to an array scalar. Must - use if you don't define an array scalar to go along with - the data-type. + On array access use the ``f->getitem`` function pointer + instead of the standard conversion to an array scalar. Must + use if you don't define an array scalar to go along with + the data-type. - .. c:macro:: NPY_USE_SETITEM + .. c:macro:: NPY_USE_SETITEM - When creating a 0-d array from an array scalar use - ``f->setitem`` instead of the standard copy from an array - scalar. Must use if you don't define an array scalar to go - along with the data-type. + When creating a 0-d array from an array scalar use + ``f->setitem`` instead of the standard copy from an array + scalar. Must use if you don't define an array scalar to go + along with the data-type. - .. c:macro:: NPY_FROM_FIELDS + .. c:macro:: NPY_FROM_FIELDS - The bits that are inherited for the parent data-type if these - bits are set in any field of the data-type. Currently ( - :c:data:`NPY_NEEDS_INIT` \| :c:data:`NPY_LIST_PICKLE` \| - :c:data:`NPY_ITEM_REFCOUNT` \| :c:data:`NPY_NEEDS_PYAPI` ). + The bits that are inherited for the parent data-type if these + bits are set in any field of the data-type. Currently ( + :c:data:`NPY_NEEDS_INIT` \| :c:data:`NPY_LIST_PICKLE` \| + :c:data:`NPY_ITEM_REFCOUNT` \| :c:data:`NPY_NEEDS_PYAPI` ). - .. c:macro:: NPY_OBJECT_DTYPE_FLAGS + .. c:macro:: NPY_OBJECT_DTYPE_FLAGS - Bits set for the object data-type: ( :c:data:`NPY_LIST_PICKLE` - \| :c:data:`NPY_USE_GETITEM` \| :c:data:`NPY_ITEM_IS_POINTER` \| - :c:data:`NPY_ITEM_REFCOUNT` \| :c:data:`NPY_NEEDS_INIT` \| - :c:data:`NPY_NEEDS_PYAPI`). + Bits set for the object data-type: ( :c:data:`NPY_LIST_PICKLE` + \| :c:data:`NPY_USE_GETITEM` \| :c:data:`NPY_ITEM_IS_POINTER` \| + :c:data:`NPY_ITEM_REFCOUNT` \| :c:data:`NPY_NEEDS_INIT` \| + :c:data:`NPY_NEEDS_PYAPI`). - .. c:function:: PyDataType_FLAGCHK(PyArray_Descr *dtype, int flags) + .. c:function:: PyDataType_FLAGCHK(PyArray_Descr *dtype, int flags) - Return true if all the given flags are set for the data-type - object. + Return true if all the given flags are set for the data-type + object. - .. c:function:: PyDataType_REFCHK(PyArray_Descr *dtype) + .. c:function:: PyDataType_REFCHK(PyArray_Descr *dtype) - Equivalent to :c:func:`PyDataType_FLAGCHK` (*dtype*, - :c:data:`NPY_ITEM_REFCOUNT`). + Equivalent to :c:func:`PyDataType_FLAGCHK` (*dtype*, + :c:data:`NPY_ITEM_REFCOUNT`). -.. c:member:: int PyArray_Descr.type_num + .. c:member:: int type_num - A number that uniquely identifies the data type. For new data-types, - this number is assigned when the data-type is registered. + A number that uniquely identifies the data type. For new data-types, + this number is assigned when the data-type is registered. -.. c:member:: int PyArray_Descr.elsize + .. c:member:: int elsize - For data types that are always the same size (such as long), this - holds the size of the data type. For flexible data types where - different arrays can have a different elementsize, this should be - 0. + For data types that are always the same size (such as long), this + holds the size of the data type. For flexible data types where + different arrays can have a different elementsize, this should be + 0. -.. c:member:: int PyArray_Descr.alignment + .. c:member:: int alignment - A number providing alignment information for this data type. - Specifically, it shows how far from the start of a 2-element - structure (whose first element is a ``char`` ), the compiler - places an item of this type: ``offsetof(struct {char c; type v;}, - v)`` + A number providing alignment information for this data type. + Specifically, it shows how far from the start of a 2-element + structure (whose first element is a ``char`` ), the compiler + places an item of this type: ``offsetof(struct {char c; type v;}, + v)`` -.. c:member:: PyArray_ArrayDescr *PyArray_Descr.subarray + .. c:member:: PyArray_ArrayDescr *subarray - If this is non- ``NULL``, then this data-type descriptor is a - C-style contiguous array of another data-type descriptor. In - other-words, each element that this descriptor describes is - actually an array of some other base descriptor. This is most - useful as the data-type descriptor for a field in another - data-type descriptor. The fields member should be ``NULL`` if this - is non- ``NULL`` (the fields member of the base descriptor can be - non- ``NULL`` however). + If this is non- ``NULL``, then this data-type descriptor is a + C-style contiguous array of another data-type descriptor. In + other-words, each element that this descriptor describes is + actually an array of some other base descriptor. This is most + useful as the data-type descriptor for a field in another + data-type descriptor. The fields member should be ``NULL`` if this + is non- ``NULL`` (the fields member of the base descriptor can be + non- ``NULL`` however). - .. c:type:: PyArray_ArrayDescr + .. c:type:: PyArray_ArrayDescr - .. code-block:: c + .. code-block:: c - typedef struct { - PyArray_Descr *base; - PyObject *shape; - } PyArray_ArrayDescr; + typedef struct { + PyArray_Descr *base; + PyObject *shape; + } PyArray_ArrayDescr; - .. c:member:: PyArray_Descr *PyArray_ArrayDescr.base + .. c:member:: PyArray_Descr *base - The data-type-descriptor object of the base-type. + The data-type-descriptor object of the base-type. - .. c:member:: PyObject *PyArray_ArrayDescr.shape + .. c:member:: PyObject *shape - The shape (always C-style contiguous) of the sub-array as a Python - tuple. + The shape (always C-style contiguous) of the sub-array as a Python + tuple. -.. c:member:: PyObject *PyArray_Descr.fields + .. c:member:: PyObject *fields - If this is non-NULL, then this data-type-descriptor has fields - described by a Python dictionary whose keys are names (and also - titles if given) and whose values are tuples that describe the - fields. Recall that a data-type-descriptor always describes a - fixed-length set of bytes. A field is a named sub-region of that - total, fixed-length collection. A field is described by a tuple - composed of another data- type-descriptor and a byte - offset. Optionally, the tuple may contain a title which is - normally a Python string. These tuples are placed in this - dictionary keyed by name (and also title if given). + If this is non-NULL, then this data-type-descriptor has fields + described by a Python dictionary whose keys are names (and also + titles if given) and whose values are tuples that describe the + fields. Recall that a data-type-descriptor always describes a + fixed-length set of bytes. A field is a named sub-region of that + total, fixed-length collection. A field is described by a tuple + composed of another data- type-descriptor and a byte + offset. Optionally, the tuple may contain a title which is + normally a Python string. These tuples are placed in this + dictionary keyed by name (and also title if given). -.. c:member:: PyObject *PyArray_Descr.names + .. c:member:: PyObject *names - An ordered tuple of field names. It is NULL if no field is - defined. + An ordered tuple of field names. It is NULL if no field is + defined. -.. c:member:: PyArray_ArrFuncs *PyArray_Descr.f + .. c:member:: PyArray_ArrFuncs *f - A pointer to a structure containing functions that the type needs - to implement internal features. These functions are not the same - thing as the universal functions (ufuncs) described later. Their - signatures can vary arbitrarily. + A pointer to a structure containing functions that the type needs + to implement internal features. These functions are not the same + thing as the universal functions (ufuncs) described later. Their + signatures can vary arbitrarily. -.. c:member:: PyObject *PyArray_Descr.metadata + .. c:member:: PyObject *metadata - Metadata about this dtype. + Metadata about this dtype. -.. c:member:: NpyAuxData *PyArray_Descr.c_metadata + .. c:member:: NpyAuxData *c_metadata - Metadata specific to the C implementation - of the particular dtype. Added for NumPy 1.7.0. + Metadata specific to the C implementation + of the particular dtype. Added for NumPy 1.7.0. -.. c:member:: Npy_hash_t *PyArray_Descr.hash + .. c:member:: Npy_hash_t *hash - Currently unused. Reserved for future use in caching - hash values. + Currently unused. Reserved for future use in caching + hash values. .. c:type:: PyArray_ArrFuncs @@ -793,31 +793,31 @@ PyUFunc_Type and PyUFuncObject } PyUFuncObject; - .. c:macro: PyUFuncObject.PyObject_HEAD + .. c:macro: PyObject_HEAD required for all Python objects. - .. c:member:: int PyUFuncObject.nin + .. c:member:: int nin The number of input arguments. - .. c:member:: int PyUFuncObject.nout + .. c:member:: int nout The number of output arguments. - .. c:member:: int PyUFuncObject.nargs + .. c:member:: int nargs The total number of arguments (*nin* + *nout*). This must be less than :c:data:`NPY_MAXARGS`. - .. c:member:: int PyUFuncObject.identity + .. c:member:: int identity Either :c:data:`PyUFunc_One`, :c:data:`PyUFunc_Zero`, :c:data:`PyUFunc_None` or :c:data:`PyUFunc_AllOnes` to indicate the identity for this operation. It is only used for a reduce-like call on an empty array. - .. c:member:: void PyUFuncObject.functions( \ + .. c:member:: void functions( \ char** args, npy_intp* dims, npy_intp* steps, void* extradata) An array of function pointers --- one for each data type @@ -835,7 +835,7 @@ PyUFunc_Type and PyUFuncObject passed in as *extradata*. The size of this function pointer array is ntypes. - .. c:member:: void **PyUFuncObject.data + .. c:member:: void **data Extra data to be passed to the 1-d vector loops or ``NULL`` if no extra-data is needed. This C-array must be the same size ( @@ -844,22 +844,22 @@ PyUFunc_Type and PyUFuncObject just 1-d vector loops that make use of this extra data to receive a pointer to the actual function to call. - .. c:member:: int PyUFuncObject.ntypes + .. c:member:: int ntypes The number of supported data types for the ufunc. This number specifies how many different 1-d loops (of the builtin data types) are available. - .. c:member:: int PyUFuncObject.reserved1 + .. c:member:: int reserved1 Unused. - .. c:member:: char *PyUFuncObject.name + .. c:member:: char *name A string name for the ufunc. This is used dynamically to build the __doc\__ attribute of ufuncs. - .. c:member:: char *PyUFuncObject.types + .. c:member:: char *types An array of :math:`nargs \times ntypes` 8-bit type_numbers which contains the type signature for the function for each of @@ -869,24 +869,24 @@ PyUFunc_Type and PyUFuncObject vector loop. These type numbers do not have to be the same type and mixed-type ufuncs are supported. - .. c:member:: char *PyUFuncObject.doc + .. c:member:: char *doc Documentation for the ufunc. Should not contain the function signature as this is generated dynamically when __doc\__ is retrieved. - .. c:member:: void *PyUFuncObject.ptr + .. c:member:: void *ptr Any dynamically allocated memory. Currently, this is used for dynamic ufuncs created from a python function to store room for the types, data, and name members. - .. c:member:: PyObject *PyUFuncObject.obj + .. c:member:: PyObject *obj For ufuncs dynamically created from python functions, this member holds a reference to the underlying Python function. - .. c:member:: PyObject *PyUFuncObject.userloops + .. c:member:: PyObject *userloops A dictionary of user-defined 1-d vector loops (stored as CObject ptrs) for user-defined types. A loop may be registered by the @@ -894,68 +894,68 @@ PyUFunc_Type and PyUFuncObject User defined type numbers are always larger than :c:data:`NPY_USERDEF`. - .. c:member:: int PyUFuncObject.core_enabled + .. c:member:: int core_enabled 0 for scalar ufuncs; 1 for generalized ufuncs - .. c:member:: int PyUFuncObject.core_num_dim_ix + .. c:member:: int core_num_dim_ix Number of distinct core dimension names in the signature - .. c:member:: int *PyUFuncObject.core_num_dims + .. c:member:: int *core_num_dims Number of core dimensions of each argument - .. c:member:: int *PyUFuncObject.core_dim_ixs + .. c:member:: int *core_dim_ixs Dimension indices in a flattened form; indices of argument ``k`` are stored in ``core_dim_ixs[core_offsets[k] : core_offsets[k] + core_numdims[k]]`` - .. c:member:: int *PyUFuncObject.core_offsets + .. c:member:: int *core_offsets Position of 1st core dimension of each argument in ``core_dim_ixs``, equivalent to cumsum(``core_num_dims``) - .. c:member:: char *PyUFuncObject.core_signature + .. c:member:: char *core_signature Core signature string - .. c:member:: PyUFunc_TypeResolutionFunc *PyUFuncObject.type_resolver + .. c:member:: PyUFunc_TypeResolutionFunc *type_resolver A function which resolves the types and fills an array with the dtypes for the inputs and outputs - .. c:member:: PyUFunc_LegacyInnerLoopSelectionFunc *PyUFuncObject.legacy_inner_loop_selector + .. c:member:: PyUFunc_LegacyInnerLoopSelectionFunc *legacy_inner_loop_selector A function which returns an inner loop. The ``legacy`` in the name arises because for NumPy 1.6 a better variant had been planned. This variant has not yet come about. - .. c:member:: void *PyUFuncObject.reserved2 + .. c:member:: void *reserved2 For a possible future loop selector with a different signature. - .. c:member:: PyUFunc_MaskedInnerLoopSelectionFunc *PyUFuncObject.masked_inner_loop_selector + .. c:member:: PyUFunc_MaskedInnerLoopSelectionFunc *masked_inner_loop_selector Function which returns a masked inner loop for the ufunc - .. c:member:: npy_uint32 PyUFuncObject.op_flags + .. c:member:: npy_uint32 op_flags Override the default operand flags for each ufunc operand. - .. c:member:: npy_uint32 PyUFuncObject.iter_flags + .. c:member:: npy_uint32 iter_flags Override the default nditer flags for the ufunc. Added in API version 0x0000000D - .. c:member:: npy_intp *PyUFuncObject.core_dim_sizes + .. c:member:: npy_intp *core_dim_sizes For each distinct core dimension, the possible :ref:`frozen ` size if :c:data:`UFUNC_CORE_DIM_SIZE_INFERRED` is 0 - .. c:member:: npy_uint32 *PyUFuncObject.core_dim_flags + .. c:member:: npy_uint32 *core_dim_flags For each distinct core dimension, a set of ``UFUNC_CORE_DIM*`` flags @@ -1015,54 +1015,54 @@ PyArrayIter_Type and PyArrayIterObject Bool contiguous; } PyArrayIterObject; - .. c:member:: int PyArrayIterObject.nd_m1 + .. c:member:: int nd_m1 :math:`N-1` where :math:`N` is the number of dimensions in the underlying array. - .. c:member:: npy_intp PyArrayIterObject.index + .. c:member:: npy_intp index The current 1-d index into the array. - .. c:member:: npy_intp PyArrayIterObject.size + .. c:member:: npy_intp size The total size of the underlying array. - .. c:member:: npy_intp *PyArrayIterObject.coordinates + .. c:member:: npy_intp *coordinates An :math:`N` -dimensional index into the array. - .. c:member:: npy_intp *PyArrayIterObject.dims_m1 + .. c:member:: npy_intp *dims_m1 The size of the array minus 1 in each dimension. - .. c:member:: npy_intp *PyArrayIterObject.strides + .. c:member:: npy_intp *strides The strides of the array. How many bytes needed to jump to the next element in each dimension. - .. c:member:: npy_intp *PyArrayIterObject.backstrides + .. c:member:: npy_intp *backstrides How many bytes needed to jump from the end of a dimension back to its beginning. Note that ``backstrides[k] == strides[k] * dims_m1[k]``, but it is stored here as an optimization. - .. c:member:: npy_intp *PyArrayIterObject.factors + .. c:member:: npy_intp *factors This array is used in computing an N-d index from a 1-d index. It contains needed products of the dimensions. - .. c:member:: PyArrayObject *PyArrayIterObject.ao + .. c:member:: PyArrayObject *ao A pointer to the underlying ndarray this iterator was created to represent. - .. c:member:: char *PyArrayIterObject.dataptr + .. c:member:: char *dataptr This member points to an element in the ndarray indicated by the index. - .. c:member:: Bool PyArrayIterObject.contiguous + .. c:member:: Bool contiguous This flag is true if the underlying array is :c:data:`NPY_ARRAY_C_CONTIGUOUS`. It is used to simplify @@ -1109,32 +1109,32 @@ PyArrayMultiIter_Type and PyArrayMultiIterObject PyArrayIterObject *iters[NPY_MAXDIMS]; } PyArrayMultiIterObject; - .. c:macro: PyArrayMultiIterObject.PyObject_HEAD + .. c:macro: PyObject_HEAD Needed at the start of every Python object (holds reference count and type identification). - .. c:member:: int PyArrayMultiIterObject.numiter + .. c:member:: int numiter The number of arrays that need to be broadcast to the same shape. - .. c:member:: npy_intp PyArrayMultiIterObject.size + .. c:member:: npy_intp size The total broadcasted size. - .. c:member:: npy_intp PyArrayMultiIterObject.index + .. c:member:: npy_intp index The current (1-d) index into the broadcasted result. - .. c:member:: int PyArrayMultiIterObject.nd + .. c:member:: int nd The number of dimensions in the broadcasted result. - .. c:member:: npy_intp *PyArrayMultiIterObject.dimensions + .. c:member:: npy_intp *dimensions The shape of the broadcasted result (only ``nd`` slots are used). - .. c:member:: PyArrayIterObject **PyArrayMultiIterObject.iters + .. c:member:: PyArrayIterObject **iters An array of iterator objects that holds the iterators for the arrays to be broadcast together. On return, the iterators are @@ -1252,12 +1252,12 @@ PyArray_Dims The members of this structure are - .. c:member:: npy_intp *PyArray_Dims.ptr + .. c:member:: npy_intp *ptr A pointer to a list of (:c:type:`npy_intp`) integers which usually represent array shape or array strides. - .. c:member:: int PyArray_Dims.len + .. c:member:: int len The length of the list of integers. It is assumed safe to access *ptr* [0] to *ptr* [len-1]. @@ -1286,26 +1286,26 @@ PyArray_Chunk The members are - .. c:macro: PyArray_Chunk.PyObject_HEAD + .. c:macro: PyObject_HEAD Necessary for all Python objects. Included here so that the :c:type:`PyArray_Chunk` structure matches that of the buffer object (at least to the len member). - .. c:member:: PyObject *PyArray_Chunk.base + .. c:member:: PyObject *base The Python object this chunk of memory comes from. Needed so that memory can be accounted for properly. - .. c:member:: void *PyArray_Chunk.ptr + .. c:member:: void *ptr A pointer to the start of the single-segment chunk of memory. - .. c:member:: npy_intp PyArray_Chunk.len + .. c:member:: npy_intp len The length of the segment in bytes. - .. c:member:: int PyArray_Chunk.flags + .. c:member:: int flags Any data flags (*e.g.* :c:data:`NPY_ARRAY_WRITEABLE` ) that should be used to interpret the memory. @@ -1345,15 +1345,15 @@ PyArrayInterface PyObject *descr; } PyArrayInterface; - .. c:member:: int PyArrayInterface.two + .. c:member:: int two the integer 2 as a sanity check. - .. c:member:: int PyArrayInterface.nd + .. c:member:: int nd the number of dimensions in the array. - .. c:member:: char PyArrayInterface.typekind + .. c:member:: char typekind A character indicating what kind of array is present according to the typestring convention with 't' -> bitfield, 'b' -> Boolean, 'i' -> @@ -1361,11 +1361,11 @@ PyArrayInterface complex floating point, 'O' -> object, 'S' -> (byte-)string, 'U' -> unicode, 'V' -> void. - .. c:member:: int PyArrayInterface.itemsize + .. c:member:: int itemsize The number of bytes each item in the array requires. - .. c:member:: int PyArrayInterface.flags + .. c:member:: int flags Any of the bits :c:data:`NPY_ARRAY_C_CONTIGUOUS` (1), :c:data:`NPY_ARRAY_F_CONTIGUOUS` (2), :c:data:`NPY_ARRAY_ALIGNED` (0x100), @@ -1379,20 +1379,20 @@ PyArrayInterface structure is present (it will be ignored by objects consuming version 2 of the array interface). - .. c:member:: npy_intp *PyArrayInterface.shape + .. c:member:: npy_intp *shape An array containing the size of the array in each dimension. - .. c:member:: npy_intp *PyArrayInterface.strides + .. c:member:: npy_intp *strides An array containing the number of bytes to jump to get to the next element in each dimension. - .. c:member:: void *PyArrayInterface.data + .. c:member:: void *data A pointer *to* the first element of the array. - .. c:member:: PyObject *PyArrayInterface.descr + .. c:member:: PyObject *descr A Python object describing the data-type in more detail (same as the *descr* key in :obj:`__array_interface__`). This can be From a818ad5ffddcbea16dafd01c4cc5d2b82e86215d Mon Sep 17 00:00:00 2001 From: mattip Date: Wed, 7 Oct 2020 12:08:11 +0300 Subject: [PATCH 204/409] BLD: circleCI- merge before build, add -n to sphinx --- .circleci/config.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f4536cca5525..d694230a581f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,7 +13,9 @@ jobs: working_directory: ~/repo steps: - - checkout + - checkout: + post: + git pull --ff-only origin "refs/pull/${CI_PULL_REQUEST//*pull\//}/merge" - run: name: create virtual environment, install dependencies @@ -48,11 +50,23 @@ jobs: (cd doc ; git submodule update --init) python tools/refguide_check.py --rst + - run: + name: build devdocs w/ref warnings + command: | + set +e + # allow this to fail for now: issue 13114 + . venv/bin/activate + cd doc + SPHINXOPTS="-q -n" make -e html + # clear the error + echo ok + - run: name: build devdocs command: | . venv/bin/activate cd doc + make clean SPHINXOPTS=-q make -e html - run: From a7a7b6ba1833f7d5a6acd2e3cc63bfd29cc49005 Mon Sep 17 00:00:00 2001 From: Matti Picus Date: Wed, 7 Oct 2020 12:16:00 +0300 Subject: [PATCH 205/409] Update .circleci/config.yml Co-authored-by: Eric Wieser --- .circleci/config.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d694230a581f..bcebb58b872f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -53,13 +53,9 @@ jobs: - run: name: build devdocs w/ref warnings command: | - set +e - # allow this to fail for now: issue 13114 . venv/bin/activate cd doc - SPHINXOPTS="-q -n" make -e html - # clear the error - echo ok + SPHINXOPTS="-q -n" make -e html || echo "ignoring errors for now, see gh-13114" - run: name: build devdocs From 2b05b8f5e82c309858ae23aceddfe15c80e43f0a Mon Sep 17 00:00:00 2001 From: mattip Date: Wed, 7 Oct 2020 12:55:23 +0300 Subject: [PATCH 206/409] BLD: try merging in a separate step --- .circleci/config.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bcebb58b872f..7fd46a931185 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,8 +14,7 @@ jobs: steps: - checkout: - post: - git pull --ff-only origin "refs/pull/${CI_PULL_REQUEST//*pull\//}/merge" + - run: git pull --ff-only origin "refs/pull/${CI_PULL_REQUEST//*pull\//}/merge" - run: name: create virtual environment, install dependencies @@ -53,9 +52,13 @@ jobs: - run: name: build devdocs w/ref warnings command: | + set +e + # allow this to fail for now: issue 13114 . venv/bin/activate cd doc - SPHINXOPTS="-q -n" make -e html || echo "ignoring errors for now, see gh-13114" + SPHINXOPTS="-q -n" make -e html + # clear the error + echo ok - run: name: build devdocs From 16934bee9bcb15f4a6a9946dc710a8a8fd535286 Mon Sep 17 00:00:00 2001 From: mattip Date: Wed, 7 Oct 2020 13:18:08 +0300 Subject: [PATCH 207/409] BLD: don't use -q in sphinx build --- .circleci/config.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7fd46a931185..d77cde59b85c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,13 +52,10 @@ jobs: - run: name: build devdocs w/ref warnings command: | - set +e - # allow this to fail for now: issue 13114 . venv/bin/activate cd doc - SPHINXOPTS="-q -n" make -e html - # clear the error - echo ok + # Don't use -q, show warning summary" + SPHINXOPTS="-n" make -e html || echo "ignoring errors for now, see gh-13114" - run: name: build devdocs From 248eee330407fc4b36375c0fb42098e403cd2d68 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Wed, 7 Oct 2020 12:11:16 +0200 Subject: [PATCH 208/409] MAINT: Removed a number of duplicate placeholders --- numpy/__init__.pyi | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 789bac896e56..76d1b14e3135 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -56,7 +56,6 @@ else: # Ensures that the stubs are picked up from numpy import ( char, - compat, ctypeslib, emath, fft, @@ -248,14 +247,12 @@ can_cast: Any cast: Any cdouble: Any cfloat: Any -char: Any chararray: Any clongdouble: Any clongfloat: Any column_stack: Any common_type: Any compare_chararrays: Any -compat: Any complex256: Any complex_: Any concatenate: Any @@ -265,7 +262,6 @@ copyto: Any corrcoef: Any cov: Any csingle: Any -ctypeslib: Any cumproduct: Any datetime_as_string: Any datetime_data: Any @@ -287,12 +283,10 @@ dstack: Any ediff1d: Any einsum: Any einsum_path: Any -emath: Any errstate: Any expand_dims: Any extract: Any eye: Any -fft: Any fill_diagonal: Any finfo: Any fix: Any @@ -311,7 +305,6 @@ frompyfunc: Any fromregex: Any fromstring: Any genfromtxt: Any -geomspace: Any get_include: Any get_printoptions: Any getbufsize: Any @@ -355,25 +348,18 @@ ix_: Any kaiser: Any kron: Any lexsort: Any -lib: Any -linalg: Any -linspace: Any load: Any loads: Any loadtxt: Any -logspace: Any longcomplex: Any longdouble: Any longfloat: Any longlong: Any lookfor: Any -ma: Any mafromtxt: Any mask_indices: Any mat: Any -math: Any matrix: Any -matrixlib: Any max: Any may_share_memory: Any median: Any @@ -423,7 +409,6 @@ polydiv: Any polyfit: Any polyint: Any polymul: Any -polynomial: Any polysub: Any polyval: Any printoptions: Any @@ -433,11 +418,9 @@ put_along_axis: Any putmask: Any quantile: Any r_: Any -random: Any ravel_multi_index: Any real: Any real_if_close: Any -rec: Any recarray: Any recfromcsv: Any recfromtxt: Any @@ -476,9 +459,7 @@ split: Any stack: Any str0: Any string_: Any -sys: Any take_along_axis: Any -testing: Any tile: Any trapz: Any tri: Any @@ -508,7 +489,6 @@ ushort: Any vander: Any vdot: Any vectorize: Any -version: Any void0: Any vsplit: Any vstack: Any From 065b300e1280b139f7463fa33dc4ddc32744ec0c Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Wed, 7 Oct 2020 12:27:17 +0200 Subject: [PATCH 209/409] MAINT: Ignore three builtin modules present in the numpy namespace --- tools/functions_missing_types.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/functions_missing_types.py b/tools/functions_missing_types.py index a32e72dad8b1..9df0c6a51d97 100755 --- a/tools/functions_missing_types.py +++ b/tools/functions_missing_types.py @@ -27,6 +27,9 @@ "division", "print_function", "warnings", + "sys", + "os", + "math", # Accidentally public, deprecated, or shouldn't be used "Tester", "alen", From 30e037cf51ff476ef249926f365212e991886a27 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Wed, 7 Oct 2020 12:52:44 +0200 Subject: [PATCH 210/409] MAINT: Ignore the private `np.compat` module --- tools/functions_missing_types.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/functions_missing_types.py b/tools/functions_missing_types.py index 9df0c6a51d97..c2fe156f0716 100755 --- a/tools/functions_missing_types.py +++ b/tools/functions_missing_types.py @@ -37,6 +37,7 @@ "add_newdoc", "add_newdoc_ufunc", "core", + "compat", "fastCopyAndTranspose", "get_array_wrap", "int_asbuffer", From c5fd38a38b931bd74d217663d35e7b8a507df2bc Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Wed, 7 Oct 2020 12:41:45 +0200 Subject: [PATCH 211/409] TST: Updated the module-related tests --- numpy/typing/tests/data/fail/modules.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/numpy/typing/tests/data/fail/modules.py b/numpy/typing/tests/data/fail/modules.py index be031e6e15e5..5e2d820abc85 100644 --- a/numpy/typing/tests/data/fail/modules.py +++ b/numpy/typing/tests/data/fail/modules.py @@ -1,4 +1,10 @@ import numpy as np np.testing.bob # E: Module has no attribute -np.bob # E: Module has no attribute \ No newline at end of file +np.bob # E: Module has no attribute + +# Stdlib modules in the namespace by accident +np.warnings # E: Module has no attribute +np.sys # E: Module has no attribute +np.os # E: Module has no attribute +np.math # E: Module has no attribute From e48ffc616eb7aa4caeb9df9b3b2318a0c49b0020 Mon Sep 17 00:00:00 2001 From: Ashutosh singh <55102089+Ashutosh619-sudo@users.noreply.github.com> Date: Wed, 7 Oct 2020 17:33:52 +0530 Subject: [PATCH 212/409] MAINT: Chaining exception in numpy\numpy\ma\mrecords.py (#17438) * fix runF2c exception --- numpy/ma/mrecords.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/ma/mrecords.py b/numpy/ma/mrecords.py index cd93a9a14313..c017bee95684 100644 --- a/numpy/ma/mrecords.py +++ b/numpy/ma/mrecords.py @@ -198,8 +198,8 @@ def __getattribute__(self, attr): fielddict = ndarray.__getattribute__(self, 'dtype').fields try: res = fielddict[attr][:2] - except (TypeError, KeyError): - raise AttributeError("record array has no attribute %s" % attr) + except (TypeError, KeyError) as e: + raise AttributeError("record array has no attribute %s" % attr) from e # So far, so good _localdict = ndarray.__getattribute__(self, '__dict__') _data = ndarray.view(self, _localdict['_baseclass']) From fd01786ea4c7dde540cede258ad11d08d25bacfc Mon Sep 17 00:00:00 2001 From: Bas van Beek <43369155+BvB93@users.noreply.github.com> Date: Wed, 7 Oct 2020 18:53:33 +0200 Subject: [PATCH 213/409] MAINT: Move aliases for common scalar unions to `numpy.typing` (#17429) * MAINT: Move the `Like` unions to `numpy.typing` --- numpy/__init__.pyi | 20 ++++++++++++-------- numpy/core/fromnumeric.pyi | 13 +++++++++---- numpy/core/function_base.pyi | 4 ++-- numpy/typing/__init__.py | 11 ++++++++++- numpy/typing/_array_like.py | 8 +++++++- numpy/typing/_callable.py | 12 +++++++----- numpy/typing/_scalars.py | 26 ++++++++++++++++++++++++++ 7 files changed, 73 insertions(+), 21 deletions(-) create mode 100644 numpy/typing/_scalars.py diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 789bac896e56..3d40682e7432 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -4,7 +4,18 @@ import datetime as dt from abc import abstractmethod from numpy.core._internal import _ctypes -from numpy.typing import ArrayLike, DtypeLike, _Shape, _ShapeLike +from numpy.typing import ( + ArrayLike, + DtypeLike, + _Shape, + _ShapeLike, + _CharLike, + _BoolLike, + _IntLike, + _FloatLike, + _ComplexLike, + _NumberLike, +) from numpy.typing._callable import ( _BoolOp, _BoolSub, @@ -1309,13 +1320,6 @@ class ndarray(_ArrayOrScalarCommon, Iterable, Sized, Container): # See https://github.com/numpy/numpy-stubs/pull/80 for more details. -_CharLike = Union[str, bytes] -_BoolLike = Union[bool, bool_] -_IntLike = Union[int, integer] -_FloatLike = Union[_IntLike, float, floating] -_ComplexLike = Union[_FloatLike, complex, complexfloating] -_NumberLike = Union[int, float, complex, number, bool_] - class generic(_ArrayOrScalarCommon): @abstractmethod def __init__(self, *args: Any, **kwargs: Any) -> None: ... diff --git a/numpy/core/fromnumeric.pyi b/numpy/core/fromnumeric.pyi index 6b3d2268f35e..3167d12b98c6 100644 --- a/numpy/core/fromnumeric.pyi +++ b/numpy/core/fromnumeric.pyi @@ -10,9 +10,6 @@ from numpy import ( generic, _OrderKACF, _OrderACF, - _IntLike, - _BoolLike, - _NumberLike, _ArrayLikeBool, _ArrayLikeIntOrBool, _ModeKind, @@ -20,7 +17,15 @@ from numpy import ( _SortKind, _SortSide, ) -from numpy.typing import DtypeLike, ArrayLike, _ShapeLike, _Shape +from numpy.typing import ( + DtypeLike, + ArrayLike, + _ShapeLike, + _Shape, + _IntLike, + _BoolLike, + _NumberLike, +) if sys.version_info >= (3, 8): from typing import Literal diff --git a/numpy/core/function_base.pyi b/numpy/core/function_base.pyi index c6ebbd5f5db4..7057122531fd 100644 --- a/numpy/core/function_base.pyi +++ b/numpy/core/function_base.pyi @@ -1,8 +1,8 @@ import sys from typing import overload, Tuple, Union, Sequence, Any -from numpy import ndarray, inexact, _NumberLike -from numpy.typing import ArrayLike, DtypeLike, _SupportsArray +from numpy import ndarray, inexact +from numpy.typing import ArrayLike, DtypeLike, _SupportsArray, _NumberLike if sys.version_info >= (3, 8): from typing import SupportsIndex, Literal diff --git a/numpy/typing/__init__.py b/numpy/typing/__init__.py index 86fd5e7870e2..987aa39aa1d5 100644 --- a/numpy/typing/__init__.py +++ b/numpy/typing/__init__.py @@ -90,6 +90,16 @@ Please see : https://numpy.org/devdocs/reference/arrays.dtypes.html """ +from ._scalars import ( + _CharLike, + _BoolLike, + _IntLike, + _FloatLike, + _ComplexLike, + _NumberLike, + _ScalarLike, + _VoidLike, +) from ._array_like import _SupportsArray, ArrayLike from ._shape import _Shape, _ShapeLike from ._dtype_like import DtypeLike @@ -97,4 +107,3 @@ from numpy._pytesttester import PytestTester test = PytestTester(__name__) del PytestTester - diff --git a/numpy/typing/_array_like.py b/numpy/typing/_array_like.py index 76c0c839c567..1c00b200f4ed 100644 --- a/numpy/typing/_array_like.py +++ b/numpy/typing/_array_like.py @@ -2,6 +2,7 @@ from typing import Any, overload, Sequence, TYPE_CHECKING, Union from numpy import ndarray +from ._scalars import _ScalarLike from ._dtype_like import DtypeLike if sys.version_info >= (3, 8): @@ -31,4 +32,9 @@ def __array__(self, dtype: DtypeLike = ...) -> ndarray: ... # is resolved. See also the mypy issue: # # https://github.com/python/typing/issues/593 -ArrayLike = Union[bool, int, float, complex, _SupportsArray, Sequence] +ArrayLike = Union[ + _ScalarLike, + Sequence[_ScalarLike], + Sequence[Sequence[Any]], # TODO: Wait for support for recursive types + _SupportsArray, +] diff --git a/numpy/typing/_callable.py b/numpy/typing/_callable.py index 5e14b708f1a6..0d876ae8d76a 100644 --- a/numpy/typing/_callable.py +++ b/numpy/typing/_callable.py @@ -12,11 +12,6 @@ from typing import Union, TypeVar, overload, Any from numpy import ( - _BoolLike, - _IntLike, - _FloatLike, - _ComplexLike, - _NumberLike, generic, bool_, timedelta64, @@ -32,6 +27,13 @@ complexfloating, complex128, ) +from ._scalars import ( + _BoolLike, + _IntLike, + _FloatLike, + _ComplexLike, + _NumberLike, +) if sys.version_info >= (3, 8): from typing import Protocol diff --git a/numpy/typing/_scalars.py b/numpy/typing/_scalars.py new file mode 100644 index 000000000000..e4fc28b074ac --- /dev/null +++ b/numpy/typing/_scalars.py @@ -0,0 +1,26 @@ +from typing import Union, Tuple, Any + +import numpy as np + +# NOTE: `_StrLike` and `_BytesLike` are pointless, as `np.str_` and `np.bytes_` +# are already subclasses of their builtin counterpart + +_CharLike = Union[str, bytes] + +_BoolLike = Union[bool, np.bool_] +_IntLike = Union[int, np.integer] +_FloatLike = Union[_IntLike, float, np.floating] +_ComplexLike = Union[_FloatLike, complex, np.complexfloating] +_NumberLike = Union[int, float, complex, np.number, np.bool_] + +_ScalarLike = Union[ + int, + float, + complex, + str, + bytes, + np.generic, +] + +# `_VoidLike` is technically not a scalar, but it's close enough +_VoidLike = Union[Tuple[Any, ...], np.void] From 2a267e6a49ed68da01761c92deb7c90be207660d Mon Sep 17 00:00:00 2001 From: Takanori H Date: Thu, 8 Oct 2020 02:34:34 +0900 Subject: [PATCH 214/409] DOC: Fixing references in ma docstrings (#16300) Fix broken links in see also section of some ma functions. --- doc/source/reference/routines.ma.rst | 2 +- numpy/ma/core.py | 2 +- numpy/ma/extras.py | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/source/reference/routines.ma.rst b/doc/source/reference/routines.ma.rst index 97859ac67a24..18248fe099e5 100644 --- a/doc/source/reference/routines.ma.rst +++ b/doc/source/reference/routines.ma.rst @@ -272,7 +272,7 @@ Filling a masked array ma.common_fill_value ma.default_fill_value ma.maximum_fill_value - ma.maximum_fill_value + ma.minimum_fill_value ma.set_fill_value ma.MaskedArray.get_fill_value diff --git a/numpy/ma/core.py b/numpy/ma/core.py index 4e320576bdb3..313d9e0b9b16 100644 --- a/numpy/ma/core.py +++ b/numpy/ma/core.py @@ -5389,7 +5389,7 @@ def round(self, decimals=0, out=None): See Also -------- - numpy.ndarray.around : corresponding function for ndarrays + numpy.ndarray.round : corresponding function for ndarrays numpy.around : equivalent function """ result = self._data.round(decimals=decimals, out=out).view(type(self)) diff --git a/numpy/ma/extras.py b/numpy/ma/extras.py index 613bcb550dfd..1bf03e966326 100644 --- a/numpy/ma/extras.py +++ b/numpy/ma/extras.py @@ -901,11 +901,11 @@ def compress_rows(a): Suppress whole rows of a 2-D array that contain masked values. This is equivalent to ``np.ma.compress_rowcols(a, 0)``, see - `extras.compress_rowcols` for details. + `compress_rowcols` for details. See Also -------- - extras.compress_rowcols + compress_rowcols """ a = asarray(a) @@ -918,11 +918,11 @@ def compress_cols(a): Suppress whole columns of a 2-D array that contain masked values. This is equivalent to ``np.ma.compress_rowcols(a, 1)``, see - `extras.compress_rowcols` for details. + `compress_rowcols` for details. See Also -------- - extras.compress_rowcols + compress_rowcols """ a = asarray(a) From d868325d62b707ccaa3b14a10b6d9897f2956592 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Wed, 7 Oct 2020 18:46:42 -0400 Subject: [PATCH 215/409] DOC: Use consistent lowercase on docs landing page Corrects titles to all use sentence case, per style guide. --- doc/source/_templates/indexcontent.html | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/source/_templates/indexcontent.html b/doc/source/_templates/indexcontent.html index 6633aa9bef90..2dba16ce8a16 100644 --- a/doc/source/_templates/indexcontent.html +++ b/doc/source/_templates/indexcontent.html @@ -12,21 +12,21 @@

{{ docstitle|e }}

For users:

-

Setting Up
+

Setting up
Learn about what NumPy is and how to install it

-

Quickstart Tutorial
+

Quickstart
Aimed at domain experts or people migrating to NumPy

-

Absolute Beginners Tutorial
+

Absolute beginner's guide
Start here for an overview of NumPy features and syntax

Tutorials
Learn about concepts and submodules

-

How Tos
+

How-tos
How to do common tasks with NumPy

-

NumPy API Reference
+

NumPy API reference
Automatically generated reference documentation

Explanations
In depth explanation of concepts, best practices and techniques

-

F2Py Guide
+

F2Py guide
Documentation for the f2py module (Fortran extensions for Python)

Glossary
List of the most important terms

@@ -36,11 +36,11 @@

{{ docstitle|e }}

For developers/contributors:

-

NumPy Contributor Guide
+

NumPy contributor guide
Contributing to NumPy

Under-the-hood docs
Specialized, in-depth documentation

-

Building and Extending the Documentation
+

Building and extending the documentation
How to contribute to this documentation (user and API)

The numpydoc docstring guide
How to write docstrings in the numpydoc format

@@ -54,7 +54,7 @@

{{ docstitle|e }}

Reporting bugs

-

Release Notes

+

Release notes

Document conventions

License of NumPy

@@ -65,13 +65,13 @@

Acknowledgements

Large parts of this manual originate from Travis E. Oliphant's book "Guide to NumPy" - (which generously entered Public Domain in August 2008). The reference + (which generously entered public domain in August 2008). The reference documentation for many of the functions are written by numerous contributors and developers of NumPy.

The preferred way to update the documentation is by submitting a pull - request on Github (see the Documentation Index). + request on GitHub (see the Documentation index). Please help us to further improve the NumPy documentation!

{% endblock %} From fd0f3dd2723ed7effde52bf31a673c9128a0a28a Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Sun, 5 Jul 2020 15:32:09 -0700 Subject: [PATCH 216/409] ENH: make dtype generic over scalar type This allows representing dtype subclasses via constructs like `np.dtype[np.float64]`. --- numpy/__init__.pyi | 108 ++++++++++++++++++++++-- numpy/typing/__init__.py | 2 +- numpy/typing/_dtype_like.py | 30 ++++--- numpy/typing/tests/data/fail/dtype.py | 9 +- numpy/typing/tests/data/reveal/dtype.py | 24 ++++++ 5 files changed, 151 insertions(+), 22 deletions(-) create mode 100644 numpy/typing/tests/data/reveal/dtype.py diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 3d40682e7432..139f2a1bc580 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -15,6 +15,8 @@ from numpy.typing import ( _FloatLike, _ComplexLike, _NumberLike, + _SupportsDtype, + _VoidDtypeLike, ) from numpy.typing._callable import ( _BoolOp, @@ -527,16 +529,112 @@ where: Any who: Any _NdArraySubClass = TypeVar("_NdArraySubClass", bound=ndarray) +_DTypeScalar = TypeVar("_DTypeScalar", bound=generic) _ByteOrder = Literal["S", "<", ">", "=", "|", "L", "B", "N", "I"] -class dtype: +class dtype(Generic[_DTypeScalar]): names: Optional[Tuple[str, ...]] - def __init__( - self, - dtype: DtypeLike, + # Overload for subclass of generic + @overload + def __new__( + cls, + dtype: Type[_DTypeScalar], align: bool = ..., copy: bool = ..., - ) -> None: ... + ) -> dtype[_DTypeScalar]: ... + # Overloads for string aliases + @overload + def __new__( + cls, + dtype: Literal["float64", "f8", "f8", "float", "double", "float_", "d"], + align: bool = ..., + copy: bool = ..., + ) -> dtype[float64]: ... + @overload + def __new__( + cls, + dtype: Literal["float32", "f4", "f4", "single"], + align: bool = ..., + copy: bool = ..., + ) -> dtype[float32]: ... + @overload + def __new__( + cls, + dtype: Literal["int64", "i8", "i8"], + align: bool = ..., + copy: bool = ..., + ) -> dtype[int64]: ... + @overload + def __new__( + cls, + dtype: Literal["int32", "i4", "i4"], + align: bool = ..., + copy: bool = ..., + ) -> dtype[int32]: ... + # "int" resolves to int_, which is system dependent, and as of now + # untyped. Long-term we'll do something fancier here. + @overload + def __new__( + cls, + dtype: Literal["int"], + align: bool = ..., + copy: bool = ..., + ) -> dtype: ... + # Overloads for Python types. Order is important here. + @overload + def __new__( + cls, + dtype: Type[bool], + align: bool = ..., + copy: bool = ..., + ) -> dtype[bool_]: ... + # See the notes for "int" + @overload + def __new__( + cls, + dtype: Type[int], + align: bool = ..., + copy: bool = ..., + ) -> dtype[Any]: ... + @overload + def __new__( + cls, + dtype: Type[float], + align: bool = ..., + copy: bool = ..., + ) -> dtype[float64]: ... + # None is a special case + @overload + def __new__( + cls, + dtype: None, + align: bool = ..., + copy: bool = ..., + ) -> dtype[float64]: ... + # dtype of a dtype is the same dtype + @overload + def __new__( + cls, + dtype: dtype[_DTypeScalar], + align: bool = ..., + copy: bool = ..., + ) -> dtype[_DTypeScalar]: ... + # TODO: handle _SupportsDtype better + @overload + def __new__( + cls, + dtype: _SupportsDtype, + align: bool = ..., + copy: bool = ..., + ) -> dtype[Any]: ... + # Catchall overload + @overload + def __new__( + cls, + dtype: _VoidDtypeLike, + align: bool = ..., + copy: bool = ..., + ) -> dtype[void]: ... def __eq__(self, other: DtypeLike) -> bool: ... def __ne__(self, other: DtypeLike) -> bool: ... def __gt__(self, other: DtypeLike) -> bool: ... diff --git a/numpy/typing/__init__.py b/numpy/typing/__init__.py index 987aa39aa1d5..dafabd95aefd 100644 --- a/numpy/typing/__init__.py +++ b/numpy/typing/__init__.py @@ -102,7 +102,7 @@ ) from ._array_like import _SupportsArray, ArrayLike from ._shape import _Shape, _ShapeLike -from ._dtype_like import DtypeLike +from ._dtype_like import _SupportsDtype, _VoidDtypeLike, DtypeLike from numpy._pytesttester import PytestTester test = PytestTester(__name__) diff --git a/numpy/typing/_dtype_like.py b/numpy/typing/_dtype_like.py index 7c1946a3e8b5..5bfd8ffdc0d6 100644 --- a/numpy/typing/_dtype_like.py +++ b/numpy/typing/_dtype_like.py @@ -38,18 +38,9 @@ class _SupportsDtype(Protocol): _DtypeDict = Any _SupportsDtype = Any -# Anything that can be coerced into numpy.dtype. -# Reference: https://docs.scipy.org/doc/numpy/reference/arrays.dtypes.html -DtypeLike = Union[ - dtype, - # default data type (float64) - None, - # array-scalar types and generic types - type, # TODO: enumerate these when we add type hints for numpy scalars - # anything with a dtype attribute - _SupportsDtype, - # character codes, type strings or comma-separated fields, e.g., 'float64' - str, + +# Would create a dtype[np.void] +_VoidDtypeLike = Union[ # (flexible_dtype, itemsize) Tuple[_DtypeLikeNested, int], # (fixed_dtype, shape) @@ -67,6 +58,21 @@ class _SupportsDtype(Protocol): Tuple[_DtypeLikeNested, _DtypeLikeNested], ] +# Anything that can be coerced into numpy.dtype. +# Reference: https://docs.scipy.org/doc/numpy/reference/arrays.dtypes.html +DtypeLike = Union[ + dtype, + # default data type (float64) + None, + # array-scalar types and generic types + type, # TODO: enumerate these when we add type hints for numpy scalars + # anything with a dtype attribute + _SupportsDtype, + # character codes, type strings or comma-separated fields, e.g., 'float64' + str, + _VoidDtypeLike, +] + # NOTE: while it is possible to provide the dtype as a dict of # dtype-like objects (e.g. `{'field1': ..., 'field2': ..., ...}`), # this syntax is officially discourged and diff --git a/numpy/typing/tests/data/fail/dtype.py b/numpy/typing/tests/data/fail/dtype.py index 3dc027daf243..7d4783d8f651 100644 --- a/numpy/typing/tests/data/fail/dtype.py +++ b/numpy/typing/tests/data/fail/dtype.py @@ -1,15 +1,16 @@ import numpy as np - class Test: not_dtype = float -np.dtype(Test()) # E: Argument 1 to "dtype" has incompatible type +np.dtype(Test()) # E: No overload variant of "dtype" matches -np.dtype( - { # E: Argument 1 to "dtype" has incompatible type +np.dtype( # E: No overload variant of "dtype" matches + { "field1": (float, 1), "field2": (int, 3), } ) + +np.dtype[np.float64](np.int64) # E: Argument 1 to "dtype" has incompatible type diff --git a/numpy/typing/tests/data/reveal/dtype.py b/numpy/typing/tests/data/reveal/dtype.py new file mode 100644 index 000000000000..aca7e8a5e983 --- /dev/null +++ b/numpy/typing/tests/data/reveal/dtype.py @@ -0,0 +1,24 @@ +import numpy as np + +reveal_type(np.dtype(np.float64)) # E: numpy.dtype[numpy.float64*] +reveal_type(np.dtype(np.int64)) # E: numpy.dtype[numpy.int64*] + +# String aliases +reveal_type(np.dtype("float64")) # E: numpy.dtype[numpy.float64] +reveal_type(np.dtype("float32")) # E: numpy.dtype[numpy.float32] +reveal_type(np.dtype("int64")) # E: numpy.dtype[numpy.int64] +reveal_type(np.dtype("int32")) # E: numpy.dtype[numpy.int32] + +# Python types +reveal_type(np.dtype(float)) # E: numpy.dtype[numpy.float64] +reveal_type(np.dtype(int)) # E: numpy.dtype +reveal_type(np.dtype(bool)) # E: numpy.dtype[numpy.bool_] + +# Special case for None +reveal_type(np.dtype(None)) # E: numpy.dtype[numpy.float64] + +# Dtypes of dtypes +reveal_type(np.dtype(np.dtype(np.float64))) # E: numpy.dtype[numpy.float64*] + +# Void +reveal_type(np.dtype(("U", 10))) # E: numpy.dtype[numpy.void] From 02688c220591250082d4ce109eb51421d8412099 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Tue, 6 Oct 2020 22:20:47 -0700 Subject: [PATCH 217/409] MAINT: add more dtype __new__ overloads for missing scalar types --- numpy/__init__.pyi | 282 ++++++++++++++++++++++-- numpy/typing/tests/data/reveal/dtype.py | 9 + 2 files changed, 270 insertions(+), 21 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 139f2a1bc580..f4caaab7ccdc 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -542,75 +542,307 @@ class dtype(Generic[_DTypeScalar]): align: bool = ..., copy: bool = ..., ) -> dtype[_DTypeScalar]: ... - # Overloads for string aliases + # Overloads for string aliases, Python types, and some assorted + # other special cases. Order is sometimes important because of the + # subtype relationships + # + # bool < int < float < complex + # + # so we have to make sure the overloads for the narrowest type is + # first. @overload def __new__( cls, - dtype: Literal["float64", "f8", "f8", "float", "double", "float_", "d"], + dtype: Union[ + Type[bool], + Literal[ + "?", + "=?", + "?", + "bool", + "bool_", + ], + ], align: bool = ..., copy: bool = ..., - ) -> dtype[float64]: ... + ) -> dtype[bool_]: ... @overload def __new__( cls, - dtype: Literal["float32", "f4", "f4", "single"], + dtype: Literal[ + "uint8", + "u1", + "=u1", + "u1", + "B", + "=B", + "B", + ], align: bool = ..., copy: bool = ..., - ) -> dtype[float32]: ... + ) -> dtype[uint8]: ... @overload def __new__( cls, - dtype: Literal["int64", "i8", "i8"], + dtype: Literal[ + "uint16", + "u2", + "=u2", + "u2", + "h", + "=h", + "h", + ], align: bool = ..., copy: bool = ..., - ) -> dtype[int64]: ... + ) -> dtype[uint16]: ... @overload def __new__( cls, - dtype: Literal["int32", "i4", "i4"], + dtype: Literal[ + "uint32", + "u4", + "=u4", + "u4", + "I", + "=I", + "I", + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[uint32]: ... + @overload + def __new__( + cls, + dtype: Literal[ + "uint64", + "u8", + "=u8", + "u8", + "L", + "=L", + "L", + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[uint64]: ... + @overload + def __new__( + cls, + dtype: Literal[ + "int8", + "i1", + "=i1", + "i1", + "b", + "=b", + "b", + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[int8]: ... + @overload + def __new__( + cls, + dtype: Literal[ + "int16", + "i2", + "=i2", + "i2", + "h", + "=h", + "h", + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[int16]: ... + @overload + def __new__( + cls, + dtype: Literal[ + "int32", + "i4", + "=i4", + "i4", + "i", + "=i", + "i", + ], align: bool = ..., copy: bool = ..., ) -> dtype[int32]: ... - # "int" resolves to int_, which is system dependent, and as of now - # untyped. Long-term we'll do something fancier here. @overload def __new__( cls, - dtype: Literal["int"], + dtype: Literal[ + "int64", + "i8", + "=i8", + "i8", + "l", + "=l", + "l", + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[int64]: ... + # "int"/int resolve to int_, which is system dependent and as of + # now untyped. Long-term we'll do something fancier here. + @overload + def __new__( + cls, + dtype: Union[Type[int], Literal["int"]], align: bool = ..., copy: bool = ..., ) -> dtype: ... - # Overloads for Python types. Order is important here. @overload def __new__( cls, - dtype: Type[bool], + dtype: Literal[ + "float16", + "f4", + "=f4", + "f4", + "e", + "=e", + "e", + "half", + ], align: bool = ..., copy: bool = ..., - ) -> dtype[bool_]: ... - # See the notes for "int" + ) -> dtype[float16]: ... @overload def __new__( cls, - dtype: Type[int], + dtype: Literal[ + "float32", + "f4", + "=f4", + "f4", + "f", + "=f", + "f", + "single", + ], align: bool = ..., copy: bool = ..., - ) -> dtype[Any]: ... + ) -> dtype[float32]: ... @overload def __new__( cls, - dtype: Type[float], + dtype: Union[ + None, + Type[float], + Literal[ + "float64", + "f8", + "=f8", + "f8", + "d", + "d", + "float", + "double", + "float_", + ], + ], align: bool = ..., copy: bool = ..., ) -> dtype[float64]: ... - # None is a special case @overload def __new__( cls, - dtype: None, + dtype: Literal[ + "complex64", + "c8", + "=c8", + "c8", + "F", + "=F", + "F", + ], align: bool = ..., copy: bool = ..., - ) -> dtype[float64]: ... + ) -> dtype[complex128]: ... + @overload + def __new__( + cls, + dtype: Union[ + Type[complex], + Literal[ + "complex128", + "c16", + "=c16", + "c16", + "D", + "=D", + "D", + ], + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[complex128]: ... + @overload + def __new__( + cls, + dtype: Union[ + Type[bytes], + Literal[ + "S", + "=S", + "S", + "bytes", + "bytes_", + ], + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[bytes_]: ... + @overload + def __new__( + cls, + dtype: Union[ + Type[str], + Literal[ + "U", + "=U", + # U intentionally not included; they are not + # the same dtype and which one dtype("U") translates + # to is platform-dependent. + "str", + "str_", + ], + ], + align: bool = ..., + copy: bool = ..., + ) -> dtype[str_]: ... # dtype of a dtype is the same dtype @overload def __new__( @@ -627,6 +859,14 @@ class dtype(Generic[_DTypeScalar]): align: bool = ..., copy: bool = ..., ) -> dtype[Any]: ... + # Handle strings that can't be expressed as literals; i.e. s1, s2, ... + @overload + def __new__( + cls, + dtype: str, + align: bool = ..., + copy: bool = ..., + ) -> dtype[Any]: ... # Catchall overload @overload def __new__( diff --git a/numpy/typing/tests/data/reveal/dtype.py b/numpy/typing/tests/data/reveal/dtype.py index aca7e8a5e983..e0802299e973 100644 --- a/numpy/typing/tests/data/reveal/dtype.py +++ b/numpy/typing/tests/data/reveal/dtype.py @@ -8,11 +8,17 @@ reveal_type(np.dtype("float32")) # E: numpy.dtype[numpy.float32] reveal_type(np.dtype("int64")) # E: numpy.dtype[numpy.int64] reveal_type(np.dtype("int32")) # E: numpy.dtype[numpy.int32] +reveal_type(np.dtype("bool")) # E: numpy.dtype[numpy.bool_] +reveal_type(np.dtype("bytes")) # E: numpy.dtype[numpy.bytes_] +reveal_type(np.dtype("str")) # E: numpy.dtype[numpy.str_] # Python types +reveal_type(np.dtype(complex)) # E: numpy.dtype[numpy.complex128] reveal_type(np.dtype(float)) # E: numpy.dtype[numpy.float64] reveal_type(np.dtype(int)) # E: numpy.dtype reveal_type(np.dtype(bool)) # E: numpy.dtype[numpy.bool_] +reveal_type(np.dtype(str)) # E: numpy.dtype[numpy.str_] +reveal_type(np.dtype(bytes)) # E: numpy.dtype[numpy.bytes_] # Special case for None reveal_type(np.dtype(None)) # E: numpy.dtype[numpy.float64] @@ -20,5 +26,8 @@ # Dtypes of dtypes reveal_type(np.dtype(np.dtype(np.float64))) # E: numpy.dtype[numpy.float64*] +# Parameterized dtypes +reveal_type(np.dtype("S8")) # E: numpy.dtype + # Void reveal_type(np.dtype(("U", 10))) # E: numpy.dtype[numpy.void] From 12e3e1a33ebbef9f3b263b258e3eabab6de5f6a5 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Wed, 7 Oct 2020 21:03:30 -0700 Subject: [PATCH 218/409] MAINT: fix complex64 overload; add str0/bytes to literals --- numpy/__init__.pyi | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index f4caaab7ccdc..c51141d5de85 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -787,7 +787,7 @@ class dtype(Generic[_DTypeScalar]): ], align: bool = ..., copy: bool = ..., - ) -> dtype[complex128]: ... + ) -> dtype[complex64]: ... @overload def __new__( cls, @@ -820,6 +820,7 @@ class dtype(Generic[_DTypeScalar]): ">S", "bytes", "bytes_", + "bytes0", ], ], align: bool = ..., @@ -838,6 +839,7 @@ class dtype(Generic[_DTypeScalar]): # to is platform-dependent. "str", "str_", + "str0", ], ], align: bool = ..., From 980d52eeaf0498f49dfef61f2a6cba6b78100d83 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 7 Oct 2020 23:04:17 -0500 Subject: [PATCH 219/409] BUG: Accept legacy user-dtypes if not isinstance(dtype.type, np.generic) These are now accepted (as a no-op) in `_PyArray_MapPyTypeToDType` retaining previous behaviour (as lookup for a registered user-type was only done if the instance check suceeded. Retain the check for future user DTypes, as it sould be a conscience choice to relax this requirement. --- numpy/core/src/multiarray/array_coercion.c | 36 ++++++++++++++-------- numpy/core/tests/test_dtype.py | 4 --- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/numpy/core/src/multiarray/array_coercion.c b/numpy/core/src/multiarray/array_coercion.c index 64a06d58b12a..4831dfca6552 100644 --- a/numpy/core/src/multiarray/array_coercion.c +++ b/numpy/core/src/multiarray/array_coercion.c @@ -128,7 +128,9 @@ _prime_global_pytype_to_type_dict(void) /** - * Add a new mapping from a python type to the DType class. + * Add a new mapping from a python type to the DType class. For a user + * defined legacy dtype, this function does nothing unless the pytype + * subclass from `np.generic`. * * This assumes that the DType class is guaranteed to hold on the * python type (this assumption is guaranteed). @@ -145,21 +147,29 @@ _PyArray_MapPyTypeToDType( { PyObject *Dtype_obj = (PyObject *)DType; - if (userdef) { + if (userdef && !PyObject_IsSubclass( + (PyObject *)pytype, (PyObject *)&PyGenericArrType_Type)) { /* - * It seems we did not strictly enforce this in the legacy dtype - * API, but assume that it is always true. Further, this could be - * relaxed in the future. In particular we should have a new - * superclass of ``np.generic`` in order to note enforce the array - * scalar behaviour. + * We expect that user dtypes (for now) will subclass some numpy + * scalar class to allow automatic discovery. */ - if (!PyObject_IsSubclass((PyObject *)pytype, (PyObject *)&PyGenericArrType_Type)) { - PyErr_Format(PyExc_RuntimeError, - "currently it is only possible to register a DType " - "for scalars deriving from `np.generic`, got '%S'.", - (PyObject *)pytype); - return -1; + if (DType->legacy) { + /* + * For legacy user dtypes, discovery relied on subclassing, but + * arbitrary type objects are supported, so do nothing. + */ + return 0; } + /* + * We currently enforce that user DTypes subclass from `np.generic` + * (this should become a `np.generic` base class and may be lifted + * entirely). + */ + PyErr_Format(PyExc_RuntimeError, + "currently it is only possible to register a DType " + "for scalars deriving from `np.generic`, got '%S'.", + (PyObject *)pytype); + return -1; } /* Create the global dictionary if it does not exist */ diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index 898ceebcd45b..45cc0b8b30cd 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -1355,10 +1355,6 @@ class mytype: # unnecessary restriction, but one that has been around forever: assert np.dtype(mytype) == np.dtype("O") - with pytest.raises(RuntimeError): - # Registering a second time should fail - create_custom_field_dtype(blueprint, mytype, 0) - def test_custom_structured_dtype_errors(self): class mytype: pass From 6681c87e08a4e9bbd4d7a2aa7fb49e1f5852e359 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 7 Oct 2020 22:43:26 -0500 Subject: [PATCH 220/409] BUG: Fix usertype typenumber during registration This slightly rearranges the code to ensure that the type number set for a user type is -1 (invalid) if it does not succeed. (This should be unnecessary, but might be relevant in the unlikely event that a usertype does not check the error return correctly.) --- numpy/core/src/multiarray/usertypes.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/numpy/core/src/multiarray/usertypes.c b/numpy/core/src/multiarray/usertypes.c index f8bb5ece7049..1404c9b68c41 100644 --- a/numpy/core/src/multiarray/usertypes.c +++ b/numpy/core/src/multiarray/usertypes.c @@ -251,12 +251,16 @@ PyArray_RegisterDataType(PyArray_Descr *descr) PyErr_SetString(PyExc_MemoryError, "RegisterDataType"); return -1; } + userdescrs[NPY_NUMUSERTYPES++] = descr; + descr->type_num = typenum; if (dtypemeta_wrap_legacy_descriptor(descr) < 0) { + descr->type_num = -1; + NPY_NUMUSERTYPES--; return -1; } - descr->type_num = typenum; + return typenum; } From 7e952c739ca9ac123f2e8568c5ded7693be058da Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Wed, 7 Oct 2020 21:14:06 -0700 Subject: [PATCH 221/409] MAINT: add str0 as an alias of str_ on the typing level --- numpy/__init__.pyi | 3 +-- numpy/typing/tests/data/reveal/scalars.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 1a3116d83dd1..b8c881fd128a 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -444,7 +444,6 @@ sort_complex: Any source: Any split: Any stack: Any -str0: Any string_: Any sys: Any take_along_axis: Any @@ -1526,7 +1525,7 @@ class str_(character, str): self, __value: bytes, encoding: str = ..., errors: str = ... ) -> None: ... -unicode_ = str_ +unicode_ = str0 = str_ # TODO(alan): Platform dependent types # longcomplex, longdouble, longfloat diff --git a/numpy/typing/tests/data/reveal/scalars.py b/numpy/typing/tests/data/reveal/scalars.py index b0dd762dfc7a..0168ebed8c6a 100644 --- a/numpy/typing/tests/data/reveal/scalars.py +++ b/numpy/typing/tests/data/reveal/scalars.py @@ -16,3 +16,4 @@ reveal_type(np.complex128().imag) # E: numpy.float64 reveal_type(np.unicode_('foo')) # E: numpy.str_ +reveal_type(np.str0('foo')) # E: numpy.str_ From be9b5759ef0893b6e8bf03ca48a5ed83d7b9dc75 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Wed, 7 Oct 2020 21:17:26 -0700 Subject: [PATCH 222/409] BUG: remove `sys` from the type stubs It is the builtin sys module, which people should not be accessing through the NumPy namespace. --- numpy/__init__.pyi | 1 - 1 file changed, 1 deletion(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 3d40682e7432..26673d95556a 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -487,7 +487,6 @@ split: Any stack: Any str0: Any string_: Any -sys: Any take_along_axis: Any testing: Any tile: Any From 86472322736260e7c2fb2ef72e7d01925adc9de0 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Thu, 8 Oct 2020 00:39:04 -0400 Subject: [PATCH 223/409] DOC: New round of NEP 42 edits (gh-17493) * DOC: Revise NEP 42 "common DType" section * DOC: Eliminate duplicate doc in NEP42 * DOC: Fix missing 'not' in NEP42 edit * DOC: Minor typos in NEP42 edit * DOC: Reword opening sentence of NEP 42 revision Also clarify a wording. * DOC: Correct :class: reference in NEP42 edit * DOC: Fix Windows line-enders on NEP42 edit * DOC: New round of common DType edits of NEP 42 * DOC: Uniquify anchors in NEP 42 * DOC: Typo in NEP42 edit * DOC: Last push for NEP 42 edit * Small fixups and moving example One paragraph on `__common_dtype` in the alternatives was already before very confusing. This makes it hopefully more clear. * NEP: Change title to "New and extensible DTypes" Co-authored-by: Sebastian Berg --- doc/neps/nep-0042-new-dtypes.rst | 864 ++++++++++++++++--------------- 1 file changed, 439 insertions(+), 425 deletions(-) diff --git a/doc/neps/nep-0042-new-dtypes.rst b/doc/neps/nep-0042-new-dtypes.rst index 99887a45100b..6f457a9d4984 100644 --- a/doc/neps/nep-0042-new-dtypes.rst +++ b/doc/neps/nep-0042-new-dtypes.rst @@ -1,10 +1,10 @@ .. _NEP42: ============================================================================== -NEP 42 — User-extensible dtypes +NEP 42 — New and extensible DTypes ============================================================================== -:title: User-extensible dtypes +:title: New and extensible DTypes :Author: Sebastian Berg :Author: Ben Nathanson :Author: Marten van Kerkwijk @@ -30,14 +30,15 @@ NEP 42 — User-extensible dtypes Abstract ****************************************************************************** -NumPy's dtype architecture is monolithic, built around a single class that -handles each dtype as an instance. There's no principled way to expand it to -new dtypes, and the code is difficult to read and maintain. +NumPy's dtype architecture is monolithic -- each dtype is an instance of a +single class. There's no principled way to expand it for new dtypes, and the +code is difficult to read and maintain. -As NEP 41 explains, we are proposing a new architecture that is modular and -open to user additions. dtypes will derive from a new ``DType`` class serving -as the extension point for new types. ``np.dtype("float64")`` will return an -instance of a ``Float64`` class, a subclass of root class ``np.dtype``. +As :ref:`NEP 41 ` explains, we are proposing a new architecture that is +modular and open to user additions. dtypes will derive from a new ``DType`` +class serving as the extension point for new types. ``np.dtype("float64")`` +will return an instance of a ``Float64`` class, a subclass of root class +``np.dtype``. This NEP is one of two that lay out the design and API of this new architecture. This NEP addresses dtype implementation; NEP 43 addresses @@ -56,8 +57,8 @@ Motivation and scope Our goal is to allow user code to create fully featured dtypes for a broad variety of uses, from physical units (such as meters) to domain-specific -representations of geometric objects. NEP 41 describes a number of these new -dtypes and their benefits. +representations of geometric objects. :ref:`NEP 41 ` describes a number +of these new dtypes and their benefits. Any design supporting dtypes must consider: @@ -77,21 +78,21 @@ And to provide this, All these are the subjects of this NEP. - The class hierarchy, its relation to the Python scalar types, and its - important attributes are described in `DType class`_. + important attributes are described in `nep42_DType class`_. - The functionality that will support dtype casting is described in `Casting`_. - The implementation of item access and storage, and the way shape and dtype - are determined when creating an array, are described in `Array coercion`_. + are determined when creating an array, are described in :ref:`nep42_array_coercion`. - The functionality for users to define their own DTypes is described in `Public C-API`_. The API here and in NEP 43 is entirely on the C side. A Python-side version -will be proposed in a future NEP. -A future Python API is expected to be similar, but provide a more convenient -API to reuse the functionality of existing DTypes. -It could also provide shorthands to create structured DTypes similar to python's +will be proposed in a future NEP. A future Python API is expected to be +similar, but provide a more convenient API to reuse the functionality of +existing DTypes. It could also provide shorthands to create structured DTypes +similar to Python's `dataclasses `_. @@ -117,6 +118,9 @@ release. ``np.array([np.array(array_like)])`` - array operations may or may not preserve dtype metadata +- Documentation that describes the internal structure of dtypes will need + to be updated. + The new code must pass NumPy's regular test suite, giving some assurance that the changes are compatible with existing code. @@ -134,7 +138,7 @@ future NEP will tackle the Python API. After implementing this NEP, creating a DType will be possible by implementing the following outlined DType base class, -that is further described in `DType class`_: +that is further described in `nep42_DType class`_: .. code-block:: python :dedent: 0 @@ -206,9 +210,9 @@ Other elements of the casting implementation is the ``CastingImpl``: def _get_loop(...) -> lowlevel_C_loop: raise NotImplementedError -which describes the casting from one DType to another. -In NEP 43 this ``CastingImpl`` object is used unchanged to support -universal functions. +which describes the casting from one DType to another. In +NEP 43 this ``CastingImpl`` object is used unchanged to +support universal functions. ****************************************************************************** @@ -229,6 +233,12 @@ Definitions cast Conversion of an array to a different dtype. + parametric type + A dtype whose representation can change based on a parameter value, + like a string dtype with a length parameter. All members of the current + ``flexible`` dtype class are parametric. See + :ref:`NEP 40 `. + promotion Finding a dtype that can perform an operation on a mix of dtypes without loss of information. @@ -241,13 +251,13 @@ On the C level we use ``descriptor`` or ``descr`` to mean dtype instances from DType classes. .. note:: - Perhaps confusingly, NumPy already has a class hierarchy for numeric types, as - seen :ref:`in the figure ` of NEP 40, and the new - DType hierarchy will resemble it. But the existing hierarchy is for scalar - types, not DTypes, and its existence is largely irrelevant here, as NEP 40 and - 41 explain. + NumPy has an existing class hierarchy for scalar types, as + seen :ref:`in the figure ` of + :ref:`NEP 40 `, and the new DType hierarchy will resemble it. The + types are used as an attribute of the single dtype class in the current + NumPy; they're not dtype classes. They neither harm nor help this work. -.. _DType class: +.. _nep42_DType class: ****************************************************************************** The DType class @@ -322,7 +332,7 @@ hierarchy. of any precision; the precision of the results is determined by the precision of the arguments. -3. There is room for user-created families of DTypes. We can envision an +3. There's room for user-created families of DTypes. We can envision an abstract ``Unit`` class for physical units, with a concrete subclass like ``Float64Unit``. Calling ``Unit(np.float64, "m")`` (``m`` for meters) would be equivalent to ``Float64Unit("m")``. @@ -381,7 +391,7 @@ Miscellaneous methods and attributes This section collects definitions in the DType class that are not used in casting and array coercion, which are described in detail below. -* Existing dtype methods and C-side fields are preserved. +* Existing dtype methods (:class:`numpy.dtype`) and C-side fields are preserved. * ``DType.type`` replaces ``dtype.type``. Unless a use case arises, ``dtype.type`` will be deprecated. @@ -396,19 +406,11 @@ casting and array coercion, which are described in detail below. existing code, "canonical" will just signify native byte order, but it can take on new meanings in new DTypes -- for instance, to distinguish a complex-conjugated instance of Complex which stores ``real - imag`` instead - of ``real + imag`` and is thus not the canonical storage. The ISNBO ("is + of ``real + imag``. The ISNBO ("is native byte order") flag might be repurposed as the canonical flag. -* Support is included for parametric DTypes. As explained in - :ref:`NEP 40 `, parametric types have a - value associated with them. A DType will be deemed parametric if it - inherits from ParametricDType. - - Strings are one example of a parametric type -- ``S8`` is different from - ``S4`` because ``S4`` cannot store a length 8 string such as ``"length 8"`` - while ``S8`` can. - Similarly, the ``datetime64`` DType is parametric, since its unit must be specified. - The associated ``type`` is the ``np.datetime64`` scalar. +* Support is included for parametric DTypes. A DType will be deemed parametric + if it inherits from ParametricDType. * DType methods may resemble or even reuse existing Python slots. Thus Python special slots are off-limits for user-defined DTypes (for instance, defining @@ -444,7 +446,7 @@ casting and array coercion, which are described in detail below. functions like sorting that will be implemented in DTypes might eventually be reimplemented as generalized ufuncs. -.. _casting: +.. _nep_42_casting: ****************************************************************************** Casting @@ -452,82 +454,59 @@ Casting We review here the operations related to casting arrays: -- Finding the "common dtype," currently exposed by ``np.promote_types`` or - ``np.result_type`` +- Finding the "common dtype," returned by :func:`numpy.promote_types` and + :func:`numpy.result_type` -- The result of calling ``np.can_cast`` +- The result of calling :func:`numpy.can_cast` -We show how casting arrays with ``arr.astype(new_dtype)`` will be implemented. +We show how casting arrays with ``astype(new_dtype)`` will be implemented. `Common DType` operations ============================================================================== -Common-type operations are vital for array coercion when input types are -mixed. They determine the output dtype of ``np.concatenate()`` and are useful -in themselves. +When input types are mixed, a first step is to find a DType that can hold +the result without loss of information -- a "common DType." -NumPy provides ``np.result_type`` and -``np.promote_types``. -These differ in that ``np.result_type`` can take arrays and scalars as input -and implements value-based promotion [1]_. +Array coercion and concatenation both return a common dtype instance. Most +universal functions use the common DType for dispatching, though they might +not use it for a result (for instance, the result of a comparison is always +bool). -To distinguish between the promotion occurring during universal function -application, we will call it "common type" operation here. +We propose the following implementation: -**Motivation:** +- For two DType classes:: -Furthermore, common type operations may be used to find the correct dtype -to use for functions with different inputs (including universal functions). -This includes an interesting distinction: + __common_dtype__(cls, other : DTypeMeta) -> DTypeMeta -1. Universal functions use the DType classes for dispatching, they thus - require the common DType class (as a first step). - While this can help with finding the correct loop to execute, the loop - may not need the actual common dtype instance. - (Hypothetical example: - ``float_arr + string_arr -> string``, but the output string length is - not the same as ``np.concatenate(float_arr, string_arr)).dtype``.) + Returns a new DType, often one of the inputs, which can represent values + of both input DTypes. This should usually be minimal: + the common DType of ``Int16`` and ``Uint16`` is ``Int32`` and not ``Int64``. + ``__common_dtype__`` may return NotImplemented to defer to other and, + like Python operators, subclasses take precedence (their + ``__common_dtype__`` method is tried first). -2. Array coercion and concatenation require the common dtype *instance*. +- For two instances of the same DType:: -**Implementation:** The implementation of the common dtype (instance) -determination has some overlap with casting. Casting from a specific dtype -(Float64) to a String needs to find the correct string length (a step that is -mainly necessary for parametric dtypes). + __common_instance__(self: SelfT, other : SelfT) -> SelfT -We propose the following implementation: + For nonparametric built-in dtypes, this returns a canonicalized copy of + ``self``, preserving metadata. For nonparametric user types, this provides + a default implementation. -1. ``__common_dtype__(cls, other : DTypeMeta) -> DTypeMeta`` answers what the - common DType class is, given two DType class objects. It may return - ``NotImplemented`` to defer to ``other``. (For abstract DTypes, subclasses - get precedence, concrete types are never superclasses, so always get preference - or are tried from left to right). - -2. ``__common_instance__(self: SelfT, other : SelfT) -> SelfT`` is used when - two instances of the same DType are given. - For built-in dtypes (that are not parametric), this - currently always returns ``self`` (but ensures canonical representation). - This is to preserve metadata. We can thus provide a default implementation - for non-parametric user dtypes. - -These two cases do *not* cover the case where two different dtype instances -need to be promoted. For example `">float64"` and `"S8"`. The solution is -partially "outsourced" to the casting machinery by splitting the operation up -into three steps: - -1. ``Float64.__common_dtype__(type(>float64), type(S8))`` - returns `String` (or defers to ``String.__common_dtype__``). -2. The casting machinery provides the information that `">float64"` casts - to `"S32"` (see below for how casting will be defined). -3. ``String.__common_instance__("S8", "S32")`` returns the final `"S32"`. - -The main reason for this is to avoid the need to implement identical -functionality multiple times. The design (together with casting) naturally -separates the concerns of different Datatypes. In the above example, Float64 -does not need to know about the cast. While the casting machinery -(``CastingImpl[Float64, String]``) could include the third step, it is not -required to do so and the string can always be extended (e.g. with new -encodings) without extending the ``CastingImpl[Float64, String]``. +- For instances of different DTypes, for example ``>float64`` and ``S8``, + the operation is done in three steps: + + 1. ``Float64.__common_dtype__(type(>float64), type(S8))`` + returns ``String`` (or defers to ``String.__common_dtype__``). + + 2. The casting machinery (explained in detail below) provides the + information that ``">float64"`` casts to ``"S32"`` + + 3. ``String.__common_instance__("S8", "S32")`` returns the final ``"S32"``. + +The benefit of this handoff is to reduce duplicated code and keep concerns +separate. DType implementations don't need to know how to cast, and the +results of casting can be extended to new types, such as a new string encoding. This means the implementation will work like this:: @@ -539,7 +518,7 @@ This means the implementation will work like this:: raise TypeError("no common dtype") return common_dtype - def promote_types(dtype1, dtyp2): + def promote_types(dtype1, dtype2): common = common_dtype(type(dtype1), type(dtype2)) if type(dtype1) is not common: @@ -555,269 +534,248 @@ This means the implementation will work like this:: if dtype1 is not dtype2: return common.__common_instance__(dtype1, dtype2) -Some of these steps may be optimized for non-parametric DTypes. - -**Note:** A currently implemented fallback for the ``__common_dtype__`` -operation is to use the "safe" casting logic. Since ``int16`` can safely cast -to ``int64``, it is clear that ``np.promote_types(int16, int64)`` should be -``int64``. +Some of these steps may be optimized for nonparametric DTypes. -However, this cannot define all such operations, and will fail for example for:: +Since the type returned by ``__common_dtype__`` is not necessarily one of the +two arguments, it's not equivalent to NumPy's "safe" casting. +Safe casting works for ``np.promote_types(int16, int64)``, which returns +``int64``, but fails for:: np.promote_types("int64", "float32") -> np.dtype("float64") -In this design, it is the responsibility of the DType author to ensure that -in most cases a safe-cast implies that this will be the result of the -``__common_dtype__`` method. +It is the responsibility of the DType author to ensure that the inputs +can be safely cast to the ``__common_dtype__``. -Note that some exceptions may apply. For example casting ``int32`` to +Exceptions may apply. For example, casting ``int32`` to a (long enough) string is at least at this time considered "safe". However ``np.promote_types(int32, String)`` will *not* be defined. -**Alternatives:** The use of casting for common dtype (instance) determination -neatly separates the concerns and allows for a minimal set of duplicate -functionality being implemented. In cases of mixed DType (classes), it also -adds an additional step to finding the common dtype. The common dtype (of two -instances) could thus be implemented explicitly to avoid this indirection, -potentially only as a fast-path. The above suggestion assumes that this is, -however, not a speed relevant path, since in most cases, e.g. in array -coercion, only a single Python type (and thus dtype) is involved. The proposed -design hinges in the implementation of casting to be separated into its own -ufunc-like object as described below. - -In principle common DType could be defined only based on "safe casting" rules, -if we order all DTypes and find the first one both can cast to safely. -However, the issue with this approach is that a newly added DType can change -the behaviour of an existing program. For example, a new ``int24`` would be -the first valid common type for ``int16`` and ``uint16``, demoting the -currently defined behavior of ``int32``. -Both, the need of a linear type hierarchy and the potential of changing -existing behaviour by adding a new DType, are a downside to using a generic -rule based on "safe casting". -However, a more generic common DType could be implemented in the future, since -``__common_dtype__`` can in principle use casting information internally. - -**Example:** ``object`` always chooses ``object`` as the common DType. For +**Example:** + +``object`` always chooses ``object`` as the common DType. For ``datetime64`` type promotion is defined with no other datatype, but if someone were to implement a new higher precision datetime, then:: - HighPrecisionDatetime.__common_dtype__(np.dtype[np.datetime64]) + HighPrecisionDatetime.__common_dtype__(np.dtype[np.datetime64]) -would return ``HighPrecisionDatetime``, and the below casting may need to -decide how to handle the datetime unit. +would return ``HighPrecisionDatetime``, and the casting implementation, +as described below, may need to decide how to handle the datetime unit. + + +**Alternatives:** + +- We're pushing the decision on common DTypes to the DType classes. Suppose + instead we could turn to a universal algorithm based on safe casting, + imposing a total order on DTypes and returning the first type that both + arguments could cast to safely. + + It would be difficult to devise a reasonable total order, and it would have + to accept new entries. Beyond that, the approach is flawed because + importing a type can change the behavior of a program. For example, a + program requiring the common DType of ``int16`` and ``uint16`` would + ordinarily get the built-in type ``int32`` as the first match; if the + program adds ``import int24``, the first match becomes ``int24`` and the + smaller type might make the program overflow for the first time. [1]_ + +- A more flexible common DType could be implemented in the future where + ``__common_dtype__`` relies on information from the casting logic. + Since ``__commond_dtype__`` is a method a such a default implementation + could be added at a later time. + +- The three-step handling of differing dtypes could, of course, be coalesced. + It would lose the value of splitting in return for a possibly faster + execution. But few cases would benefit. Most cases, such as array coercion, + involve a single Python type (and thus dtype). The cast operation ============================================================================== -Perhaps the most complex and interesting DType operation is casting. Casting +Casting is perhaps the most complex and interesting DType operation. It is much like a typical universal function on arrays, converting one input to a -new output. There are two key distinctions: +new output, with two distinctions: -1. Casting always requires an explicit output datatype. -2. The NumPy iterator API requires access to functions that are lower-level - than what universal functions currently need. +- Casting always requires an explicit output datatype. +- The NumPy iterator API requires access to functions that are lower-level + than what universal functions currently need. -Casting can be complex, and may not implement all details of each input -datatype (such as non-native byte order or unaligned access). Thus casting -naturally is performed in up to three steps: +Casting can be complex and may not implement all details of each input +datatype (such as non-native byte order or unaligned access). So a complex +type conversion might entail 3 steps: -1. The given datatype is normalized and prepared for the actual cast. +1. The input datatype is normalized and prepared for the cast. 2. The cast is performed. -3. The cast result, which is in a normalized form, is cast to the requested +3. The result, which is in a normalized form, is cast to the requested form (non-native byte order). -Often only step 2 is required. - Further, NumPy provides different casting kinds or safety specifiers: -* "equivalent" -* "safe" -* "same_kind" -* "unsafe" +* ``equivalent``, allowing only byte-order changes +* ``safe``, requiring a type large enough to preserve value +* ``same_kind``, requiring a safe cast or one within a kind, like float64 to float32 +* ``unsafe``, allowing any data conversion + +and in some cases a cast may be just a view. + +We need to support the two current signatures of ``arr.astype``: -and in some cases a cast may even be represented as a simple view. +- For DTypes: ``arr.astype(np.String)`` + - current spelling ``arr.astype("S")`` + - ``np.String`` can be an abstract DType -**Motivation:** Similar to the common dtype/DType operation above, we again -have two use cases: +- For dtypes: ``arr.astype(np.dtype("S8"))`` -1. ``arr.astype(np.String)`` (current spelling ``arr.astype("S")``) -2. ``arr.astype(np.dtype("S8"))`` -where the first case is also noted in NEP 40 and 41 as a design goal, since -``np.String`` could also be an abstract DType as mentioned above. +We also have two signatures of ``np.can_cast``: -The implementation of casting should also come with as little duplicate -implementation as necessary, i.e. to avoid unnecessary methods on the DTypes. -Furthermore, it is desirable that casting is implemented similar to universal -functions. +- Instance to class: ``np.can_cast(dtype, DType, "safe")`` +- Instance to instance: ``np.can_cast(dtype, other_dtype, "safe")`` -Analogous to the above, the following also need to be defined: +On the Python level ``dtype`` is overloaded to mean class or instance. -1. ``np.can_cast(dtype, DType, "safe")`` (instance to class) -2. ``np.can_cast(dtype, other_dtype, "safe")`` (casting an instance to another - instance) +A third ``can_cast`` signature, ``np.can_cast(DType, OtherDType, "safe")``,may be used +internally but need not be exposed to Python. -overloading the meaning of ``dtype`` to mean either class or instance (on the -Python level). The question of ``np.can_cast(DType, OtherDType, "safe")`` is -also a possibility and may be used internally. However, it is initially not -necessary to expose to Python. +During DType creation, DTypes will be able to pass a list of ``CastingImpl`` +objects, which can define casting to and from the DType. +One of them should define the cast between instances of that DType. It can be +omitted if the DType has only a single implementation and is nonparametric. -**Implementation:** During DType creation, DTypes will have the ability to -pass a list of ``CastingImpl`` objects, which can define casting to and from -the DType. One of these ``CastingImpl`` objects is special because it should -define the cast within the same DType (from one instance to another). A DType -which does not define this, must have only a single implementation and not be -parametric. +Each ``CastingImpl`` has a distinct DType signature: + + ``CastingImpl[InputDtype, RequestedDtype]`` -Each ``CastingImpl`` has a specific DType signature: -``CastingImpl[InputDtype, RequestedDtype]`` and implements the following methods and attributes: -* ``resolve_descriptors(self, Tuple[DType] : input) -> casting, Tuple[DType]``. - Here ``casting`` signals the casting safeness (safe, unsafe, or same-kind) - and the output dtype tuple is used for more multi-step casting (see below). -* ``get_transferfunction(...) -> function handling cast`` (signature to be decided). - This function returns a low-level implementation of a strided casting function - ("transfer function"). -* ``casting`` attribute with one of equivalent, safe, unsafe, or same-kind. Used to - quickly decide casting safety when this is relevant. -``resolve_descriptors`` provides information about whether or -not a cast is safe and is of importance mainly for parametric DTypes. -``get_transferfunction`` provides NumPy with a function capable of performing -the actual cast. Initially the implementation of ``get_transferfunction`` -will be *private*, and users will only be able to provide strided loops -with the signature. +* To report safeness, + + ``resolve_descriptors(self, Tuple[DType] : input) -> casting, Tuple[DType]``. + + The ``casting`` output reports safeness (safe, unsafe, or same-kind), and + the tuple is used for more multistep casting, as in the example below. -**Performing the cast** +* To get a casting function, -.. _cast_figure: + ``get_loop(...) -> function_to_handle_cast (signature to be decided)`` + + returns a low-level implementation of a strided casting function + ("transfer function") capable of performing the + cast. + + Initially the implementation will be *private*, and users will only be + able to provide strided loops with the signature. + +* For performance, a ``casting`` attribute taking a value of ``equivalent``, ``safe``, + ``unsafe``, or ``same-kind``. + + +**Performing a cast** + +.. _nep42_cast_figure: .. figure:: _static/casting_flow.svg :figclass: align-center -`The above figure `_ illustrates the multi-step logic necessary to -cast for example an ``int24`` with a value of ``42`` to a string of length 20 +The above figure illustrates a multistep +cast of an ``int24`` with a value of ``42`` to a string of length 20 (``"S20"``). -In this example, the implementer only provided the functionality of casting -an ``int24`` to an ``S8`` string (which can hold all 24bit integers). -Due to this limited implementation, the full cast has to do multiple -conversions. The full process is: -1. Call ``CastingImpl[Int24, String].resolve_descriptors((int24, "S20"))``. +We've picked an example where the implementer has only provided limited +functionality: a function to cast an ``int24`` to an ``S8`` string (which can +hold all 24-bit integers). This means multiple conversions are needed. + +The full process is: + +1. Call + + ``CastingImpl[Int24, String].resolve_descriptors((int24, "S20"))``. + This provides the information that ``CastingImpl[Int24, String]`` only implements the cast of ``int24`` to ``"S8"``. + 2. Since ``"S8"`` does not match ``"S20"``, use - ``CastingImpl[String, String].get_transferfunction()`` + + ``CastingImpl[String, String].get_loop()`` + to find the transfer (casting) function to convert an ``"S8"`` into an ``"S20"`` + 3. Fetch the transfer function to convert an ``int24`` to an ``"S8"`` using - ``CastingImpl[Int24, String].get_transferfunction()`` + + ``CastingImpl[Int24, String].get_loop()`` + 4. Perform the actual cast using the two transfer functions: + ``int24(42) -> S8("42") -> S20("42")``. -Note that in this example the ``resolve_descriptors`` function plays a less -central role. It becomes more important for ``np.can_cast``. + ``resolve_descriptors`` allows the implementation for + + ``np.array(42, dtype=int24).astype(String)`` -Further, ``resolve_descriptors`` allows the implementation for -``np.array(42, dtype=int24).astype(String)`` to call -``CastingImpl[Int24, String].resolve_descriptors((int24, None))``. -In this case the result of ``(int24, "S8")`` defines the correct cast: -``np.array(42, dtype=int24),astype(String) == np.array("42", dtype="S8")``. + to call + + ``CastingImpl[Int24, String].resolve_descriptors((int24, None))``. + + In this case the result of ``(int24, "S8")`` defines the correct cast: + + ``np.array(42, dtype=int24).astype(String) == np.array("42", dtype="S8")``. **Casting safety** -To answer the question of casting safety ``np.can_cast(int24, "S20", -casting="safe")``, only the ``resolve_descriptors`` function is required and -is called in the same way as in `the figure describing a cast `_. +To compute ``np.can_cast(int24, "S20", casting="safe")``, only the +``resolve_descriptors`` function is required and +is called in the same way as in :ref:`the figure describing a cast `. + In this case, the calls to ``resolve_descriptors``, will also provide the information that ``int24 -> "S8"`` as well as ``"S8" -> "S20"`` are safe casts, and thus also the ``int24 -> "S20"`` is a safe cast. In some cases, no cast is necessary. For example, on most Linux systems ``np.dtype("long")`` and ``np.dtype("longlong")`` are different dtypes but are -both 64bit integers. -In this case, the cast can be performed using ``long_arr.view("longlong")``. -The information that a cast is a -"view" will be handled by an additional flag. Thus the ``casting`` -can have the 8 values in total: equivalent, safe, unsafe, same-kind as well as equivalent+view, safe+view, -unsafe+view, and same-kind+view. -NumPy currently defines ``dtype1 == dtype2`` to be True only if byte order matches. -This functionality can be replaced with the combination of "equivalent" casting -and the "view" flag. +both 64-bit integers. In this case, the cast can be performed using +``long_arr.view("longlong")``. The information that a cast is a view will be +handled by an additional flag. Thus the ``casting`` can have the 8 values in +total: the original 4 of ``equivalent``, ``safe``, ``unsafe``, and ``same-kind``, +plus ``equivalent+view``, ``safe+view``, ``unsafe+view``, and +``same-kind+view``. NumPy currently defines ``dtype1 == dtype2`` to be True +only if byte order matches. This functionality can be replaced with the +combination of "equivalent" casting and the "view" flag. -(For more information on the ``resolve_descriptors`` signature see the C-API -section below and NEP 43.) +(For more information on the ``resolve_descriptors`` signature see the +:ref:`nep42_C-API` section below and NEP 43.) **Casting between instances of the same DType** -In general one of the casting implementations defined by the DType implementor -must be ``CastingImpl[DType, DType]`` (unless there is only a singleton -instance). To keep the casting to as few steps as possible, this -implementation must initially be capable of any conversions between all instances of this -DType. +To keep down the number of casting +steps, CastingImpl must be capable of any conversion between all instances +of this DType. +In general the DType implementer must include ``CastingImpl[DType, DType]`` +unless there is only a singleton instance. **General multistep casting** -In general we could implement certain casts, such as ``int8`` to ``int24`` -even if the user only provides an ``int16 -> int24`` cast. This proposal -currently does not provide this functionality. However, it could be extended -in the future to either find such casts dynamically, or at least allow -``resolve_descriptors`` to return arbitrary ``dtypes``. If ``CastingImpl[Int8, -Int24].resolve_descriptors((int8, int24))`` returns ``(int16, int24)``, the -actual casting process could be extended to include the ``int8 -> int16`` -cast. This adds an additional step to the casting process. - - -**Alternatives:** The choice of using only the DType classes in the first step -of finding the correct ``CastingImpl`` means that the default implementation -of ``__common_dtype__`` has a reasonable definition of "safe casting" between -DTypes classes (although e.g. the concatenate operation using it may still -fail when attempting to find the actual common instance or cast). - -The split into multiple steps may seem to add complexity rather than reduce -it, however, it consolidates that we have the two distinct signatures of -``np.can_cast(dtype, DTypeClass)`` and ``np.can_cast(dtype, other_dtype)``. -Further, the above API guarantees the separation of concerns for user DTypes. -The user ``Int24`` dtype does not have to handle all string lengths if it does -not wish to do so. Further, if an encoding was added to the ``String`` DType, -this does not affect the overall cast. The ``resolve_descriptors`` function can -keep returning the default encoding and the ``CastingImpl[String, String]`` -can take care of any necessary encoding changes. - -The main alternative to the proposed design is to move most of the information -which is here pushed into the ``CastingImpl`` directly into methods on the -DTypes. This, however, will not allow the close similarity between casting and -universal functions. On the up side, it reduces the necessary indirection as -noted below. - -An initial proposal defined two methods ``__can_cast_to__(self, other)`` to -dynamically return ``CastingImpl``. The advantage of this addition is that it -removes the requirement to define all possible casts at DType creation time (of -one of the involved DTypes). -Such API could be added at a later time. This is similar to Python which -provides ``__getattr__`` for additional control over attribute lookup. - -**Notes:** The proposed ``CastingImpl`` is designed to be identical to the -``PyArrayMethod`` proposed in NEP 43 as part of restructuring ufuncs to handle -new DTypes. +We could implement certain casts, such as ``int8`` to ``int24``, +even if the user provides only an ``int16 -> int24`` cast. This proposal does +not provide that, but future work might find such casts dynamically, or at least +allow ``resolve_descriptors`` to return arbitrary ``dtypes``. + +If ``CastingImpl[Int8, Int24].resolve_descriptors((int8, int24))`` returns +``(int16, int24)``, the actual casting process could be extended to include +the ``int8 -> int16`` cast. This adds a step. -The way dispatching works for ``CastingImpl`` is planned to be limited -initially and fully opaque. In the future, it may or may not be moved into a -special UFunc, or behave more like a universal function. +**Example:** -**Example:** The implementation for casting integers to datetime would generally +The implementation for casting integers to datetime would generally say that this cast is unsafe (because it is always an unsafe cast). Its ``resolve_descriptors`` function may look like:: - def resolve_descriptors(self, given_dtypes): + def resolve_descriptors(self, given_dtypes): from_dtype, to_dtype = given_dtypes - from_dtype = from_dtype.ensure_canonical() # ensure not byte-swapped if to_dtype is None: raise TypeError("Cannot convert to a NumPy datetime without a unit") @@ -831,39 +789,87 @@ Its ``resolve_descriptors`` function may look like:: .. note:: - While NumPy currently defines integer to datetime casts, with the possible + While NumPy currently defines integer-to-datetime casts, with the possible exception of the unit-less ``timedelta64`` it may be better to not define these casts at all. In general we expect that user defined DTypes will be using custom methods such as ``unit.drop_unit(arr)`` or ``arr * unit.seconds``. -****************************************************************************** -Array coercion -****************************************************************************** +**Alternatives:** + +- Our design objectives are: + - Minimize the number of DType methods and avoid code duplication. + - Mirror the implementation of universal functions. + +- The decision to use only the DType classes in the first step of finding the + correct ``CastingImpl`` in addition to defining ``CastingImpl.casting``, + allows to retain the current default implementation of + ``__common_dtype__`` for existing user defined dtypes, which could be + expanded in the future. + +- The split into multiple steps may seem to add complexity rather than reduce + it, but it consolidates the signatures of ``np.can_cast(dtype, DTypeClass)`` + and ``np.can_cast(dtype, other_dtype)``. + + Further, the API guarantees separation of concerns for user DTypes. The user + ``Int24`` dtype does not have to handle all string lengths if it does not + wish to do so. Further, an encoding added to the ``String`` DType would + not affect the overall cast. The ``resolve_descriptors`` function + can keep returning the default encoding and the ``CastingImpl[String, + String]`` can take care of any necessary encoding changes. + +- The main alternative is moving most of the information that is here pushed + into the ``CastingImpl`` directly into methods on the DTypes. But this + obscures the similarity between casting and universal functions. It does + reduce indirection, as noted below. + +- An earlier proposal defined two methods ``__can_cast_to__(self, other)`` to + dynamically return ``CastingImpl``. This + removes the requirement to define all possible casts at DType creation + (of one of the involved DTypes). + + Such an API could be added later. It resembles Python's ``__getattr__`` in + providing additional control over attribute lookup. + + +**Notes:** + +The proposed ``CastingImpl`` is designed to be identical to the +``PyArrayMethod`` proposed in NEP43 as part of restructuring ufuncs to handle +new DTypes. + +The way dispatching works for ``CastingImpl`` is planned to be limited +initially and fully opaque. In the future, it may or may not be moved into a +special UFunc, or behave more like a universal function. + + +.. _nep42_array_coercion: -The following sections discuss the two aspects related to creating an array from -arbitrary python objects. This requires a defined protocol to store data -inside the array. Further, it requires the ability to find the correct dtype -when a user does not provide the dtype explicitly. Coercion to and from Python objects ============================================================================== -**Motivation:** When storing a single value in an array or taking it out, it -is necessary to coerce (convert) it to and from the low-level representation +When storing a single value in an array or taking it out, it is necessary to +coerce it -- that is, convert it -- to and from the low-level representation inside the array. -**Description:** Coercing to and from Python scalars requires two to three -methods: +Coercion is slightly more complex than typical casts. One reason is that a +Python object could itself be a 0-dimensional array or scalar with an +associated DType. + +Coercing to and from Python scalars requires two to three +methods that largely correspond to the current definitions: 1. ``__dtype_setitem__(self, item_pointer, value)`` + 2. ``__dtype_getitem__(self, item_pointer, base_obj) -> object``; ``base_obj`` is for memory management and usually ignored; it points to an object owning the data. Its only role is to support structured datatypes with subarrays within NumPy, which currently return views into the array. The function returns an equivalent Python scalar (i.e. typically a NumPy scalar). + 3. ``__dtype_get_pyitem__(self, item_pointer, base_obj) -> object`` (initially hidden for new-style user-defined datatypes, may be exposed on user request). This corresponds to the ``arr.item()`` method also used by @@ -873,18 +879,20 @@ methods: (The above is meant for C-API. A Python-side API would have to use byte buffers or similar to implement this, which may be useful for prototyping.) -These largely correspond to the current definitions. When a certain scalar +When a certain scalar has a known (different) dtype, NumPy may in the future use casting instead of -``__dtype_setitem__``. A user datatype is (initially) expected to implement +``__dtype_setitem__``. + +A user datatype is (initially) expected to implement ``__dtype_setitem__`` for its own ``DType.type`` and all basic Python scalars it wishes to support (e.g. ``int`` and ``float``). In the future a -function "``known_scalartype``" may be made public to allow a user dtype to signal +function ``known_scalar_type`` may be made public to allow a user dtype to signal which Python scalars it can store directly. **Implementation:** The pseudocode implementation for setting a single item in -an array from an arbitrary Python object ``value`` is (note that some -functions are only defined below):: +an array from an arbitrary Python object ``value`` is (some +functions here are defined later):: def PyArray_Pack(dtype, item_pointer, value): DType = type(dtype) @@ -930,28 +938,35 @@ would require the user to take to duplicate or fall back to existing casting functionality. It is certainly possible to describe the coercion to and from Python objects -using the general casting machinery, -but the ``object`` dtype is special and important enough to be handled by NumPy -using the presented methods. - -**Further Issues and Discussion:** The ``__dtype_setitem__`` function currently duplicates -some code, such as coercion from a string. ``datetime64`` allows assignment -from string, but the same conversion also occurs for casting from the string -dtype to ``datetime64``. In the future, we may expose the ``known_scalartype`` -function to allow the user to implement such duplication. -For example, NumPy would normally use ``np.array(np.string_("2019")).astype(datetime64)``, -but ``datetime64`` could choose to use its ``__dtype_setitem__`` instead, -e.g. for performance reasons. - -There is an issue about how subclasses of scalars should be handled. -We anticipate to stop automatically detecting the dtype for -``np.array(float64_subclass)`` to be float64. -The user can still provide ``dtype=np.float64``. -However, the above automatic casting using ``np.array(scalar_subclass).astype(requested_dtype)`` -will fail. -In many cases, this is not an issue, since the Python ``__float__`` protocol -can be used instead. But in some cases, this will mean that subclasses of -Python scalars will behave differently. +using the general casting machinery, but the ``object`` dtype is special and +important enough to be handled by NumPy using the presented methods. + +**Further issues and discussion:** + +- The ``__dtype_setitem__`` function duplicates some code, such as coercion + from a string. + + ``datetime64`` allows assignment from string, but the same conversion also + occurs for casting from the string dtype to ``datetime64``. + + We may in the future expose the ``known_scalartype`` function to allow the + user to implement such duplication. + + For example, NumPy would normally use + + ``np.array(np.string_("2019")).astype(datetime64)`` + + but ``datetime64`` could choose to use its ``__dtype_setitem__`` instead + for performance reasons. + +- There is an issue about how subclasses of scalars should be handled. We + anticipate to stop automatically detecting the dtype for + ``np.array(float64_subclass)`` to be float64. The user can still provide + ``dtype=np.float64``. However, the above automatic casting using + ``np.array(scalar_subclass).astype(requested_dtype)`` will fail. In many + cases, this is not an issue, since the Python ``__float__`` protocol can be + used instead. But in some cases, this will mean that subclasses of Python + scalars will behave differently. .. note:: @@ -968,8 +983,8 @@ Python scalars will behave differently. DType discovery during array coercion ============================================================================== -An important step in the use of NumPy arrays is creation of the array -from collections of generic Python objects. +An important step in the use of NumPy arrays is creation of the array from +collections of generic Python objects. **Motivation:** Although the distinction is not clear currently, there are two main needs:: @@ -998,10 +1013,12 @@ There are three further issues to consider: 1. It may be desirable to create datatypes associated with normal Python scalars (such as ``datetime.datetime``) that do not have a ``dtype`` attribute already. + 2. In general, a datatype could represent a sequence, however, NumPy currently assumes that sequences are always collections of elements (the sequence cannot be an element itself). An example would be a ``vector`` DType. + 3. An array may itself contain arrays with a specific dtype (even general Python objects). For example: ``np.array([np.array(None, dtype=object)], dtype=np.String)`` @@ -1013,6 +1030,7 @@ of the output array and finding the correct datatype are closely related. **Implementation:** There are two distinct cases above: 1. The user has provided no dtype information. + 2. The user provided a DType class -- as represented, for example, by ``"S"`` representing a string of any length. @@ -1026,10 +1044,11 @@ These two cases shall be implemented by leveraging two pieces of information: 1. ``DType.type``: The current type attribute to indicate which Python scalar type is associated with the DType class (this is a *class* attribute that always exists for any datatype and is not limited to array coercion). + 2. ``__discover_descr_from_pyobject__(cls, obj) -> dtype``: A classmethod that returns the correct descriptor given the input object. Note that only parametric DTypes have to implement this. - For non-parametric DTypes using the default instance will always be acceptable. + For nonparametric DTypes using the default instance will always be acceptable. The Python scalar type which is already associated with a DType through the ``DType.type`` attribute maps from the DType to the Python scalar type. @@ -1066,7 +1085,7 @@ subclass. dispatching/promotion in ufuncs, it will also be necessary to dynamically create ``AbstractPyInt[value]`` classes (creation can be cached), so that they can provide the current value based promotion functionality provided - by ``np.result_type(python_integer, array)`` [1]_. + by ``np.result_type(python_integer, array)`` [2]_ . To allow for a DType to accept inputs as scalars that are not basic Python types or instances of ``DType.type``, we use ``known_scalar_type`` method. @@ -1086,7 +1105,7 @@ operation, this allows it to automatically find that the datetime64 unit should be "minutes". -**NumPy Internal Implementation:** The implementation to find the correct dtype +**NumPy internal implementation:** The implementation to find the correct dtype will work similar to the following pseudocode:: def find_dtype(array_like): @@ -1150,31 +1169,30 @@ should only be implemented for types defined within the same library to avoid the potential for conflicts. It will be the DType implementor's responsibility to be careful about this and use avoid registration when in doubt. -**Alternatives:** Instead of a global mapping, we could rely on the scalar -attribute ``scalar.__associated_array_dtype__``. -This only creates a difference in behaviour for subclasses and the exact -implementation can be undefined initially. -Scalars will be expected to derive from a NumPy scalar. -In principle NumPy could, for a time, still choose to rely on the attribute. - -An earlier proposal for the ``dtype`` discovery algorithm, -was to use a two-pass approach. -First finding only the correct ``DType`` class and only then discovering the parametric -``dtype`` instance. -This was rejected for unnecessary complexity. -The main advantage of this method is that it would have enabled value -based promotion in universal functions, allowing:: +**Alternatives:** + +- Instead of a global mapping, we could rely on the scalar attribute + ``scalar.__associated_array_dtype__``. This only creates a difference in + behavior for subclasses, and the exact implementation can be undefined + initially. Scalars will be expected to derive from a NumPy scalar. In + principle NumPy could, for a time, still choose to rely on the attribute. + +- An earlier proposal for the ``dtype`` discovery algorithm used a two-pass + approach, first finding the correct ``DType`` class and only then + discovering the parametric ``dtype`` instance. It was rejected as + needlessly complex. But it would have enabled value-based promotion + in universal functions, allowing:: np.add(np.array([8], dtype="uint8"), [4]) -to return a ``uint8`` result (instead of ``int16``), which currently happens for:: + to return a ``uint8`` result (instead of ``int16``), which currently happens for:: np.add(np.array([8], dtype="uint8"), 4) -(note the list ``[4]`` instead of scalar ``4``). -This is not a feature NumPy currently has or desires to support. + (note the list ``[4]`` instead of scalar ``4``). + This is not a feature NumPy currently has or desires to support. -**Further Issues and Discussion:** It is possible to create a DType +**Further issues and discussion:** It is possible to create a DType such as Categorical, array, or vector which can only be used if ``dtype=DType`` is provided. Such DTypes cannot roundtrip correctly. For example:: @@ -1186,29 +1204,25 @@ This is a general limitation, but round-tripping is always possible if ``dtype=original_arr.dtype`` is passed. -.. _c-api: +.. _nep42_c-api: ****************************************************************************** Public C-API ****************************************************************************** -A Python side API shall not be defined here. This is a general side approach. - - DType creation ============================================================================== -To create a new DType the user will need to define all the methods and -attributes as presented above and outlined in the `Usage and impact`_ -section. -Some additional methods similar to those currently defined as part of -:c:type:`PyArray_ArrFuncs` will be necessary and part of the slots struct -below. +To create a new DType the user will need to define the methods and attributes +outlined in the `Usage and impact`_ section and detailed throughout this +proposal. + +In addition, some methods similar to those in :c:type:`PyArray_ArrFuncs` will +be needed for the slots struct below. -As already mentioned in NEP 41, the interface to define this DType class in C is -modeled after the `Python limited API `_: -the above-mentioned slots and some additional necessary information will -thus be passed within a slots struct and identified by ``ssize_t`` integers:: +As mentioned in :ref:`NEP 41 `, the interface to define this DType +class in C is modeled after :PEP:`384`: Slots and some additional information +will be passed in a slots struct and identified by ``ssize_t`` integers:: static struct PyArrayMethodDef slots[] = { {NPY_dt_method, method_implementation}, @@ -1227,16 +1241,15 @@ thus be passed within a slots struct and identified by ``ssize_t`` integers:: PyObject* PyArray_InitDTypeMetaFromSpec(PyArrayDTypeMeta_Spec *dtype_spec); -All of this information will be copied. - -**TODO:** The DType author should be able to define new methods for their -DType, up to defining a full type object and in the future possibly even -extending the ``PyArrayDTypeMeta_Type`` struct. We have to decide on how (and -what) to make available to the user initially. A possible initial solution may -be to only allow inheriting from an existing class: ``class MyDType(np.dtype, -MyBaseclass)``. If ``np.dtype`` is first in the method resolution order, this -also prevents overriding some slots, such as ``==`` which may not be desirable. +All of this is passed by copying. +**TODO:** The DType author should be able to define new methods for the +DType, up to defining a full object, and, in the future, possibly even +extending the ``PyArrayDTypeMeta_Type`` struct. We have to decide what to make +available initially. A solution may be to allow inheriting only from an +existing class: ``class MyDType(np.dtype, MyBaseclass)``. If ``np.dtype`` is +first in the method resolution order, this also prevents an undesirable +override of slots like ``==``. The ``slots`` will be identified by names which are prefixed with ``NPY_dt_`` and are: @@ -1251,8 +1264,7 @@ and are: * ``common_instance(self, other) -> dtype or NULL`` Where possible, a default implementation will be provided if the slot is -ommitted or set to ``NULL``. -Non-parametric dtypes do not have to implement: +omitted or set to ``NULL``. Nonparametric dtypes do not have to implement: * ``discover_descr_from_pyobject`` (uses ``default_descr`` instead) * ``common_instance`` (uses ``default_descr`` instead) @@ -1262,17 +1274,16 @@ Sorting is expected to be implemented using: * ``get_sort_function(self, NPY_SORTKIND sort_kind) -> {out_sortfunction, NotImplemented, NULL}``. -Although for convenience, it will be sufficient if the user implements only: +For convenience, it will be sufficient if the user implements only: * ``compare(self, char *item_ptr1, char *item_ptr2, int *res) -> {-1, 0, 1}`` -**Limitations:** Using the above ``PyArrayDTypeMeta_Spec`` struct, the -structure itself can only be extended clumsily (e.g. by adding a version tag -to the ``slots`` to indicate a new, longer version of the struct). We could -also provide the struct using a function, which however will require memory -management but would allow ABI-compatible extension (the struct is freed again -when the DType is created). +**Limitations:** The ``PyArrayDTypeMeta_Spec`` struct is clumsy to extend (for +instance, by adding a version tag to the ``slots`` to indicate a new, longer +version). We could use a function to provide the struct; it would require +memory management but would allow ABI-compatible extension (the struct is +freed again when the DType is created). CastingImpl @@ -1281,24 +1292,24 @@ CastingImpl The external API for ``CastingImpl`` will be limited initially to defining: * ``casting`` attribute, which can be one of the supported casting kinds. - This is the safest cast possible. For example casting between two NumPy + This is the safest cast possible. For example, casting between two NumPy strings is of course "safe" in general, but may be "same kind" in a specific instance if the second string is shorter. If neither type is parametric the ``resolve_descriptors`` must use it. -* ``resolve_descriptors(self, given_descrs[2], loop_descrs[2]) -> int {casting, -1}``: - The ``loop_descrs`` must be set correctly to dtypes which the strided loop +* ``resolve_descriptors(dtypes_in[2], dtypes_out[2], casting_out) -> int {0, + -1}`` The out + dtypes must be set correctly to dtypes which the strided loop (transfer function) can handle. Initially the result must have instances of the same DType class as the ``CastingImpl`` is defined for. The ``casting`` will be set to ``NPY_EQUIV_CASTING``, ``NPY_SAFE_CASTING``, ``NPY_UNSAFE_CASTING``, or ``NPY_SAME_KIND_CASTING``. - A new, additional flag, ``NPY_CAST_IS_VIEW``, can be set to indicate that - no cast is necessary and a view is sufficient to perform the cast. - The return value shall be ``-1`` to indicate that the cast is not possible. - If no error is set, a generic error message will be given. If an error is - already set it will be chained and may provide additional information. - Note that ``self`` represents additional call information; details are given - in NEP 43. + A new, additional flag, + ``NPY_CAST_IS_VIEW``, can be set to indicate that no cast is necessary and a + view is sufficient to perform the cast. The cast should return + ``-1`` when a custom error is set and ``NPY_NO_CASTING`` to indicate + that a generic casting error should be set (this is in most cases + preferable). * ``strided_loop(char **args, npy_intp *dimensions, npy_intp *strides, ...) -> int {0, -1}`` (signature will be fully defined in NEP 43) @@ -1308,7 +1319,7 @@ part of the signature will include information such as the two ``dtype``\s. More optimized loops are in use internally, and will be made available to users in the future (see notes). -Although verbose, the API shall mimic the one for creating a new DType: +Although verbose, the API will mimic the one for creating a new DType: .. code-block:: C @@ -1321,56 +1332,53 @@ Although verbose, the API shall mimic the one for creating a new DType: PyType_Slot *slots; } PyArrayMethod_Spec; -The focus differs between casting and general ufuncs. For example for casts +The focus differs between casting and general ufuncs. For example, for casts ``nin == nout == 1`` is always correct, while for ufuncs ``casting`` is expected to be usually `"safe"`. -**Notes:** We may initially allow users to define only a single loop. However, -internally NumPy optimizes far more, and this should be made public -incrementally, either by allowing multiple versions, such as: +**Notes:** We may initially allow users to define only a single loop. +Internally NumPy optimizes far more, and this should be made public +incrementally in one of two ways: -* contiguous inner loop -* strided inner loop -* scalar inner loop +* Allow multiple versions, such as: -or more likely through exposure of the ``get_loop`` function which is passed -additional information, such as the fixed strides (similar to our internal -API). + * contiguous inner loop + * strided inner loop + * scalar inner loop -The above example does not yet include potential setup and error handling -requirements. Since these are similar to the UFunc machinery, this will be -defined in detail in NEP 43 and then incorporated identically into casting. +* Or, more likely, expose the ``get_loop`` function which is passed additional + information, such as the fixed strides (similar to our internal API). -The slots/methods used will be prefixed ``NPY_uf_`` for similarity to the -ufunc machinery. +The example does not yet include setup and error handling. Since these are +similar to the UFunc machinery, they will be defined in NEP 43 and then +incorporated identically into casting. +The slots/methods used will be prefixed with ``NPY_meth_``. -**Alternatives:** Aside from name changes, and possible signature tweaks, -there seem to be few alternatives to the above structure. -The proposed API using ``*_FromSpec`` function is a good way to achieve a stable -and extensible API. The slots design is extensible and can be -changed without breaking binary compatibility. -Convenience functions can still be provided to allow creation with less code. +**Alternatives:** -One downside of this approach is that compilers cannot warn about function pointer -incompatibilities. +- Aside from name changes and signature tweaks, there seem to be few + alternatives to the above structure. The proposed API using ``*_FromSpec`` + function is a good way to achieve a stable and extensible API. The slots + design is extensible and can be changed without breaking binary + compatibility. Convenience functions can still be provided to allow creation + with less code. + +- One downside is that compilers cannot warn about function-pointer + incompatibilities. ****************************************************************************** Implementation ****************************************************************************** -Steps for implementation are outlined in :ref:`NEP 41 `. This includes -internal restructuring for the new casting and array-coercion. -First, the NumPy will internally be rewritten using the above methods for -casting and array-coercion. - -After that, the new public API will be added incrementally. -We plan to expose it in a preliminary state initially to allow modification -after some experience can be gained. -In addition to the features presented in detail in this NEP, all functionality -currently implemented on the dtypes will be replaced systematically. +Steps for implementation are outlined in the Implementation section of +:ref:`NEP 41 `. In brief, we first will rewrite the internals of +casting and array coercion. After that, the new public API will be added +incrementally. We plan to expose it in a preliminary state initially to gain +experience. All functionality currently implemented on the dtypes will be +replaced systematically as new features are added. ****************************************************************************** @@ -1378,16 +1386,22 @@ Alternatives ****************************************************************************** The space of possible implementations is large, so there have been many -discussions, conceptions, and design documents. These are listed in NEP 40. -Since this NEP encompasses multiple individual decisions, alternatives -are discussed in the above individual sections. +discussions, conceptions, and design documents. These are listed in +:ref:`NEP 40 `. Alternatives were also been discussed in the +relevant sections above. ****************************************************************************** References ****************************************************************************** -.. [1] NumPy currently inspects the value to allow the operations:: +.. [1] To be clear, the program is broken: It should not have stored a value + in the common DType that was below the lowest int16 or above the highest + uint16. It avoided overflow earlier by an accident of implementation. + Nonetheless, we insist that program behavior not be altered just by + importing a type. + +.. [2] NumPy currently inspects the value to allow the operations:: np.array([1], dtype=np.uint8) + 1 np.array([1.2], dtype=np.float32) + 1. From f39453eae98b92efaee39bb855c6bcdf3905e3d6 Mon Sep 17 00:00:00 2001 From: Royston E Tauro <54945757+lucasace@users.noreply.github.com> Date: Thu, 8 Oct 2020 22:13:27 +0530 Subject: [PATCH 224/409] Update __init__.pyi --- numpy/__init__.pyi | 85 +++++++++++++++++++++++----------------------- 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 26673d95556a..154fc8722dab 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -36,6 +36,7 @@ from typing import ( Container, Callable, Dict, + Final, Generic, IO, Iterable, @@ -1597,48 +1598,48 @@ def empty( # Constants # -Inf: float -Infinity: float -NAN: float -NINF: float -NZERO: float -NaN: float -PINF: float -PZERO: float -e: float -euler_gamma: float -inf: float -infty: float -nan: float -pi: float - -ALLOW_THREADS: int -BUFSIZE: int -CLIP: int -ERR_CALL: int -ERR_DEFAULT: int -ERR_IGNORE: int -ERR_LOG: int -ERR_PRINT: int -ERR_RAISE: int -ERR_WARN: int -FLOATING_POINT_SUPPORT: int -FPE_DIVIDEBYZERO: int -FPE_INVALID: int -FPE_OVERFLOW: int -FPE_UNDERFLOW: int -MAXDIMS: int -MAY_SHARE_BOUNDS: int -MAY_SHARE_EXACT: int -RAISE: int -SHIFT_DIVIDEBYZERO: int -SHIFT_INVALID: int -SHIFT_OVERFLOW: int -SHIFT_UNDERFLOW: int -UFUNC_BUFSIZE_DEFAULT: int -WRAP: int -little_endian: int -tracemalloc_domain: int +Inf: Final[float] +Infinity: Final[float] +NAN: Final[float] +NINF: Final[float] +NZERO: Final[float] +NaN: Final[float] +PINF: Final[float] +PZERO: Final[float] +e: Final[float] +euler_gamma: FInal[float] +inf: Final[float] +infty: Final[float] +nan: Final[float] +pi: Final[float] + +ALLOW_THREADS: Final[int] +BUFSIZE: Final[int] +CLIP: Final[int] +ERR_CALL: Final[int] +ERR_DEFAULT: Final[int] +ERR_IGNORE: Final[int] +ERR_LOG: Final[int] +ERR_PRINT: Final[int] +ERR_RAISE: Final[int] +ERR_WARN: Final[int] +FLOATING_POINT_SUPPORT: Final[int] +FPE_DIVIDEBYZERO: Final[int] +FPE_INVALID: Final[int] +FPE_OVERFLOW: Final[int] +FPE_UNDERFLOW: Final[int] +MAXDIMS: Final[int] +MAY_SHARE_BOUNDS: Final[int] +MAY_SHARE_EXACT: Final[int] +RAISE: Final[int] +SHIFT_DIVIDEBYZERO: Final[int] +SHIFT_INVALID: Final[int] +SHIFT_OVERFLOW: Final[int] +SHIFT_UNDERFLOW: Final[int] +UFUNC_BUFSIZE_DEFAULT: Final[int] +WRAP: Final[int] +little_endian: Final[int] +tracemalloc_domain: Final[int] class ufunc: @property From ff81db7831432ef94e11ecc0cb014260ff55b3d4 Mon Sep 17 00:00:00 2001 From: Royston E Tauro <54945757+lucasace@users.noreply.github.com> Date: Thu, 8 Oct 2020 22:23:13 +0530 Subject: [PATCH 225/409] ENH: Marked all constants as `Final --- numpy/__init__.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 154fc8722dab..9d5b9d438237 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -1612,7 +1612,7 @@ inf: Final[float] infty: Final[float] nan: Final[float] pi: Final[float] - + ALLOW_THREADS: Final[int] BUFSIZE: Final[int] CLIP: Final[int] From 45e04d904099f824f860c5426039e2f4099a714c Mon Sep 17 00:00:00 2001 From: Royston E Tauro <54945757+lucasace@users.noreply.github.com> Date: Thu, 8 Oct 2020 23:03:54 +0530 Subject: [PATCH 226/409] Minor bug fix --- numpy/__init__.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 9d5b9d438237..0c023caf0bbf 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -1607,7 +1607,7 @@ NaN: Final[float] PINF: Final[float] PZERO: Final[float] e: Final[float] -euler_gamma: FInal[float] +euler_gamma: Final[float] inf: Final[float] infty: Final[float] nan: Final[float] From 688202aeb9db604cd3ace797b76291281bc7f97e Mon Sep 17 00:00:00 2001 From: Royston E Tauro <54945757+lucasace@users.noreply.github.com> Date: Thu, 8 Oct 2020 23:38:22 +0530 Subject: [PATCH 227/409] Adjusted typing.Final's python 3.8 compatibility --- numpy/__init__.pyi | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 0c023caf0bbf..1fbdb259b2a1 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -36,7 +36,6 @@ from typing import ( Container, Callable, Dict, - Final, Generic, IO, Iterable, @@ -59,9 +58,9 @@ from typing import ( ) if sys.version_info >= (3, 8): - from typing import Literal, Protocol, SupportsIndex + from typing import Literal, Protocol, SupportsIndex,Final else: - from typing_extensions import Literal, Protocol + from typing_extensions import Literal, Protocol, Final class SupportsIndex(Protocol): def __index__(self) -> int: ... From 055afcfaaf0004c537cb03de2520d38bbbb19e67 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Thu, 8 Oct 2020 15:28:28 -0400 Subject: [PATCH 228/409] DOC: Fewer blank lines in PR template Reduce the need to scroll, per @mattip suggestion in #17440 --- .github/PULL_REQUEST_TEMPLATE.md | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index dee33ee5fd13..508c8c034869 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,22 +1,16 @@ - From 20d5999e2f01c32876368b97797047c3adc536bd Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Thu, 8 Oct 2020 17:01:02 -0400 Subject: [PATCH 229/409] DOC: Rename 'Quickstart tutorial' (#17504) Remove tutorial wording in favor of article, minor wording updates. --- doc/source/user/quickstart.rst | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/doc/source/user/quickstart.rst b/doc/source/user/quickstart.rst index 8e38234c598f..b675204e0ad5 100644 --- a/doc/source/user/quickstart.rst +++ b/doc/source/user/quickstart.rst @@ -1,5 +1,5 @@ =================== -Quickstart tutorial +NumPy quickstart =================== .. currentmodule:: numpy @@ -12,26 +12,24 @@ Quickstart tutorial Prerequisites ============= -Before reading this tutorial you should know a bit of Python. If you -would like to refresh your memory, take a look at the `Python +You'll need to know a bit of Python. For a refresher, see the `Python tutorial `__. -If you wish to work the examples in this tutorial, you must also have -some software installed on your computer. Please see -https://scipy.org/install.html for instructions. +To work the examples, you'll need `matplotlib` installed +in addition to NumPy. **Learner profile** -This tutorial is intended as a quick overview of +This is a quick overview of algebra and arrays in NumPy. It demonstrates how n-dimensional (:math:`n>=2`) arrays are represented and can be manipulated. In particular, if you don't know how to apply common functions to n-dimensional arrays (without using for-loops), or if you want to understand axis and shape properties for -n-dimensional arrays, this tutorial might be of help. +n-dimensional arrays, this article might be of help. **Learning Objectives** -After this tutorial, you should be able to: +After reading, you should be able to: - Understand the difference between one-, two- and n-dimensional arrays in NumPy; From 7396e03df3085b96fa4a99de5aaab0499670dd6e Mon Sep 17 00:00:00 2001 From: Takanori H Date: Fri, 9 Oct 2020 07:48:32 +0900 Subject: [PATCH 230/409] DOC: Fix random links in release note (#17483) Co-authored-by: Eric Wieser --- doc/source/release/1.17.0-notes.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/doc/source/release/1.17.0-notes.rst b/doc/source/release/1.17.0-notes.rst index a93eb21863e5..4bdc6105fc1b 100644 --- a/doc/source/release/1.17.0-notes.rst +++ b/doc/source/release/1.17.0-notes.rst @@ -171,15 +171,15 @@ The functions `load`, and ``lib.format.read_array`` take an `CVE-2019-6446 `_. -.. currentmodule:: numpy.random.mtrand +.. currentmodule:: numpy.random Potential changes to the random stream in old random module ----------------------------------------------------------- Due to bugs in the application of ``log`` to random floating point numbers, the stream may change when sampling from `~RandomState.beta`, `~RandomState.binomial`, `~RandomState.laplace`, `~RandomState.logistic`, `~RandomState.logseries` or -`~RandomState.multinomial` if a ``0`` is generated in the underlying `MT19937 -<~numpy.random.mt11937.MT19937>` random stream. There is a ``1`` in +`~RandomState.multinomial` if a ``0`` is generated in the underlying `MT19937` +random stream. There is a ``1`` in :math:`10^{53}` chance of this occurring, so the probability that the stream changes for any given seed is extremely small. If a ``0`` is encountered in the underlying generator, then the incorrect value produced (either `numpy.inf` or @@ -559,4 +559,3 @@ Structured arrays indexed with non-existent fields raise ``KeyError`` not ``Valu ---------------------------------------------------------------------------------------- ``arr['bad_field']`` on a structured type raises ``KeyError``, for consistency with ``dict['bad_field']``. - From 1b8637d4af9b27735629d5371b9d92a0fd1c147d Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Thu, 17 Sep 2020 12:11:07 +0200 Subject: [PATCH 231/409] ENH, SIMD: Add partial/non-contig load and store intrinsics for 32/64-bit This patch improves the implementation of memory load/store for VSX --- numpy/core/src/common/simd/avx2/avx2.h | 2 + numpy/core/src/common/simd/avx2/memory.h | 286 ++++++++++++++ numpy/core/src/common/simd/avx512/avx512.h | 3 + numpy/core/src/common/simd/avx512/memory.h | 238 ++++++++++++ numpy/core/src/common/simd/neon/memory.h | 287 ++++++++++++++ numpy/core/src/common/simd/simd.h | 49 +++ numpy/core/src/common/simd/sse/memory.h | 424 ++++++++++++++++++++ numpy/core/src/common/simd/vsx/memory.h | 432 +++++++++++++++------ 8 files changed, 1603 insertions(+), 118 deletions(-) diff --git a/numpy/core/src/common/simd/avx2/avx2.h b/numpy/core/src/common/simd/avx2/avx2.h index c99d628ee408..0641f2314ec6 100644 --- a/numpy/core/src/common/simd/avx2/avx2.h +++ b/numpy/core/src/common/simd/avx2/avx2.h @@ -5,6 +5,8 @@ #define NPY_SIMD 256 #define NPY_SIMD_WIDTH 32 #define NPY_SIMD_F64 1 +// Enough limit to allow us to use _mm256_i32gather_* +#define NPY_SIMD_MAXLOAD_STRIDE32 (0x7fffffff / 8) typedef __m256i npyv_u8; typedef __m256i npyv_s8; diff --git a/numpy/core/src/common/simd/avx2/memory.h b/numpy/core/src/common/simd/avx2/memory.h index 5ea7414fdf5c..e27bf15fec2e 100644 --- a/numpy/core/src/common/simd/avx2/memory.h +++ b/numpy/core/src/common/simd/avx2/memory.h @@ -2,6 +2,8 @@ #error "Not a standalone header" #endif +#include "misc.h" + #ifndef _NPY_SIMD_AVX2_MEMORY_H #define _NPY_SIMD_AVX2_MEMORY_H @@ -66,5 +68,289 @@ NPYV_IMPL_AVX2_MEM_INT(npy_int64, s64) // store higher part #define npyv_storeh_f32(PTR, VEC) _mm_storeu_ps(PTR, _mm256_extractf128_ps(VEC, 1)) #define npyv_storeh_f64(PTR, VEC) _mm_storeu_pd(PTR, _mm256_extractf128_pd(VEC, 1)) +/*************************** + * Non-contiguous Load + ***************************/ +//// 32 +NPY_FINLINE npyv_u32 npyv_loadn_u32(const npy_uint32 *ptr, npy_intp stride) +{ + assert(llabs(stride) <= NPY_SIMD_MAXLOAD_STRIDE32); + const __m256i steps = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7); + const __m256i idx = _mm256_mullo_epi32(_mm256_set1_epi32((int)stride), steps); + return _mm256_i32gather_epi32((const int*)ptr, idx, 4); +} +NPY_FINLINE npyv_s32 npyv_loadn_s32(const npy_int32 *ptr, npy_intp stride) +{ return npyv_loadn_u32((const npy_uint32*)ptr, stride); } +NPY_FINLINE npyv_f32 npyv_loadn_f32(const float *ptr, npy_intp stride) +{ return _mm256_castsi256_ps(npyv_loadn_u32((const npy_uint32*)ptr, stride)); } +//// 64 +#if 0 // slower +NPY_FINLINE npyv_u64 npyv_loadn_u64(const npy_uint64 *ptr, npy_intp stride) +{ + const __m256i idx = _mm256_setr_epi64x(0, 1*stride, 2*stride, 3*stride); + return _mm256_i64gather_epi64((const void*)ptr, idx, 8); +} +NPY_FINLINE npyv_s64 npyv_loadn_s64(const npy_int64 *ptr, npy_intp stride) +{ return npyv_loadn_u64((const npy_uint64*)ptr, stride); } +NPY_FINLINE npyv_f64 npyv_loadn_f64(const double *ptr, npy_intp stride) +{ return _mm256_castsi256_pd(npyv_loadn_u64((const npy_uint64*)ptr, stride)); } +#endif +NPY_FINLINE npyv_f64 npyv_loadn_f64(const double *ptr, npy_intp stride) +{ + __m128d a0 = _mm_castsi128_pd(_mm_loadl_epi64((const __m128i*)ptr)); + __m128d a2 = _mm_castsi128_pd(_mm_loadl_epi64((const __m128i*)(ptr + stride*2))); + __m128d a01 = _mm_loadh_pd(a0, ptr + stride); + __m128d a23 = _mm_loadh_pd(a2, ptr + stride*3); + return _mm256_insertf128_pd(_mm256_castpd128_pd256(a01), a23, 1); +} +NPY_FINLINE npyv_u64 npyv_loadn_u64(const npy_uint64 *ptr, npy_intp stride) +{ return _mm256_castpd_si256(npyv_loadn_f64((const double*)ptr, stride)); } +NPY_FINLINE npyv_s64 npyv_loadn_s64(const npy_int64 *ptr, npy_intp stride) +{ return _mm256_castpd_si256(npyv_loadn_f64((const double*)ptr, stride)); } +/*************************** + * Non-contiguous Store + ***************************/ +//// 32 +NPY_FINLINE void npyv_storen_s32(npy_int32 *ptr, npy_intp stride, npyv_s32 a) +{ + __m128i a0 = _mm256_castsi256_si128(a); + __m128i a1 = _mm256_extracti128_si256(a, 1); + ptr[stride * 0] = _mm_cvtsi128_si32(a0); + ptr[stride * 1] = _mm_extract_epi32(a0, 1); + ptr[stride * 2] = _mm_extract_epi32(a0, 2); + ptr[stride * 3] = _mm_extract_epi32(a0, 3); + ptr[stride * 4] = _mm_cvtsi128_si32(a1); + ptr[stride * 5] = _mm_extract_epi32(a1, 1); + ptr[stride * 6] = _mm_extract_epi32(a1, 2); + ptr[stride * 7] = _mm_extract_epi32(a1, 3); +} +NPY_FINLINE void npyv_storen_u32(npy_uint32 *ptr, npy_intp stride, npyv_u32 a) +{ npyv_storen_s32((npy_int32*)ptr, stride, a); } +NPY_FINLINE void npyv_storen_f32(float *ptr, npy_intp stride, npyv_f32 a) +{ npyv_storen_s32((npy_int32*)ptr, stride, _mm256_castps_si256(a)); } +//// 64 +NPY_FINLINE void npyv_storen_f64(double *ptr, npy_intp stride, npyv_f64 a) +{ + __m128d a0 = _mm256_castpd256_pd128(a); + __m128d a1 = _mm256_extractf128_pd(a, 1); + _mm_storel_pd(ptr + stride * 0, a0); + _mm_storeh_pd(ptr + stride * 1, a0); + _mm_storel_pd(ptr + stride * 2, a1); + _mm_storeh_pd(ptr + stride * 3, a1); +} +NPY_FINLINE void npyv_storen_u64(npy_uint64 *ptr, npy_intp stride, npyv_u64 a) +{ npyv_storen_f64((double*)ptr, stride, _mm256_castsi256_pd(a)); } +NPY_FINLINE void npyv_storen_s64(npy_int64 *ptr, npy_intp stride, npyv_s64 a) +{ npyv_storen_f64((double*)ptr, stride, _mm256_castsi256_pd(a)); } + +/********************************* + * Partial Load + *********************************/ +//// 32 +NPY_FINLINE npyv_s32 npyv_load_till_s32(const npy_int32 *ptr, npy_uintp nlane, npy_int32 fill) +{ + assert(nlane > 0); + const __m256i vfill = _mm256_set1_epi32(fill); + const __m256i steps = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7); + __m256i vnlane = _mm256_set1_epi32(nlane > 8 ? 8 : (int)nlane); + __m256i mask = _mm256_cmpgt_epi32(vnlane, steps); + __m256i payload = _mm256_maskload_epi32((const int*)ptr, mask); + return _mm256_blendv_epi8(vfill, payload, mask); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s32 npyv_load_tillz_s32(const npy_int32 *ptr, npy_uintp nlane) +{ + assert(nlane > 0); + const __m256i steps = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7); + __m256i vnlane = _mm256_set1_epi32(nlane > 8 ? 8 : (int)nlane); + __m256i mask = _mm256_cmpgt_epi32(vnlane, steps); + return _mm256_maskload_epi32((const int*)ptr, mask); +} +//// 64 +NPY_FINLINE npyv_s64 npyv_load_till_s64(const npy_int64 *ptr, npy_uintp nlane, npy_int64 fill) +{ + assert(nlane > 0); + const __m256i vfill = _mm256_set1_epi64x(fill); + const __m256i steps = _mm256_setr_epi64x(0, 1, 2, 3); + __m256i vnlane = _mm256_set1_epi64x(nlane > 4 ? 4 : (int)nlane); + __m256i mask = _mm256_cmpgt_epi64(vnlane, steps); + __m256i payload = _mm256_maskload_epi64((const void*)ptr, mask); + return _mm256_blendv_epi8(vfill, payload, mask); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s64 npyv_load_tillz_s64(const npy_int64 *ptr, npy_uintp nlane) +{ + assert(nlane > 0); + const __m256i steps = _mm256_setr_epi64x(0, 1, 2, 3); + __m256i vnlane = _mm256_set1_epi64x(nlane > 4 ? 4 : (int)nlane); + __m256i mask = _mm256_cmpgt_epi64(vnlane, steps); + return _mm256_maskload_epi64((const void*)ptr, mask); +} +/********************************* + * Non-contiguous partial load + *********************************/ +//// 32 +NPY_FINLINE npyv_s32 +npyv_loadn_till_s32(const npy_int32 *ptr, npy_intp stride, npy_uintp nlane, npy_int32 fill) +{ + assert(nlane > 0); + assert(llabs(stride) <= NPY_SIMD_MAXLOAD_STRIDE32); + const __m256i vfill = _mm256_set1_epi32(fill); + const __m256i steps = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7); + const __m256i idx = _mm256_mullo_epi32(_mm256_set1_epi32((int)stride), steps); + __m256i vnlane = _mm256_set1_epi32(nlane > 8 ? 8 : (int)nlane); + __m256i mask = _mm256_cmpgt_epi32(vnlane, steps); + return _mm256_mask_i32gather_epi32(vfill, (const int*)ptr, idx, mask, 4); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s32 +npyv_loadn_tillz_s32(const npy_int32 *ptr, npy_intp stride, npy_uintp nlane) +{ return npyv_loadn_till_s32(ptr, stride, nlane, 0); } +//// 64 +NPY_FINLINE npyv_s64 +npyv_loadn_till_s64(const npy_int64 *ptr, npy_intp stride, npy_uintp nlane, npy_int64 fill) +{ + assert(nlane > 0); + const __m256i vfill = _mm256_set1_epi64x(fill); + const __m256i idx = _mm256_setr_epi64x(0, 1*stride, 2*stride, 3*stride); + const __m256i steps = _mm256_setr_epi64x(0, 1, 2, 3); + __m256i vnlane = _mm256_set1_epi64x(nlane > 4 ? 4 : (int)nlane); + __m256i mask = _mm256_cmpgt_epi64(vnlane, steps); + return _mm256_mask_i64gather_epi64(vfill, (const void*)ptr, idx, mask, 8); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s64 +npyv_loadn_tillz_s64(const npy_int64 *ptr, npy_intp stride, npy_uintp nlane) +{ return npyv_loadn_till_s64(ptr, stride, nlane, 0); } +/********************************* + * Partial store + *********************************/ +//// 32 +NPY_FINLINE void npyv_store_till_s32(npy_int32 *ptr, npy_uintp nlane, npyv_s32 a) +{ + assert(nlane > 0); + const __m256i steps = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7); + __m256i vnlane = _mm256_set1_epi32(nlane > 8 ? 8 : (int)nlane); + __m256i mask = _mm256_cmpgt_epi32(vnlane, steps); + _mm256_maskstore_epi32((int*)ptr, mask, a); +} +//// 64 +NPY_FINLINE void npyv_store_till_s64(npy_int64 *ptr, npy_uintp nlane, npyv_s64 a) +{ + assert(nlane > 0); + const __m256i steps = _mm256_setr_epi64x(0, 1, 2, 3); + __m256i vnlane = _mm256_set1_epi64x(nlane > 8 ? 8 : (int)nlane); + __m256i mask = _mm256_cmpgt_epi64(vnlane, steps); + _mm256_maskstore_epi64((void*)ptr, mask, a); +} +/********************************* + * Non-contiguous partial store + *********************************/ +//// 32 +NPY_FINLINE void npyv_storen_till_s32(npy_int32 *ptr, npy_intp stride, npy_uintp nlane, npyv_s32 a) +{ + assert(nlane > 0); + __m128i a0 = _mm256_castsi256_si128(a); + __m128i a1 = _mm256_extracti128_si256(a, 1); + switch(nlane) { + default: + ptr[stride*7] = _mm_extract_epi32(a1, 3); + case 7: + ptr[stride*6] = _mm_extract_epi32(a1, 2); + case 6: + ptr[stride*5] = _mm_extract_epi32(a1, 1); + case 5: + ptr[stride*4] = _mm_extract_epi32(a1, 0); + case 4: + ptr[stride*3] = _mm_extract_epi32(a0, 3); + case 3: + ptr[stride*2] = _mm_extract_epi32(a0, 2); + case 2: + ptr[stride*1] = _mm_extract_epi32(a0, 1); + case 1: + ptr[stride*0] = _mm_extract_epi32(a0, 0); + } +} +//// 64 +NPY_FINLINE void npyv_storen_till_s64(npy_int64 *ptr, npy_intp stride, npy_uintp nlane, npyv_s64 a) +{ + assert(nlane > 0); + __m128d a0 = _mm256_castpd256_pd128(_mm256_castsi256_pd(a)); + __m128d a1 = _mm256_extractf128_pd(_mm256_castsi256_pd(a), 1); + double *dptr = (double*)ptr; + switch(nlane) { + default: + _mm_storeh_pd(dptr + stride * 3, a1); + case 3: + _mm_storel_pd(dptr + stride * 2, a1); + case 2: + _mm_storeh_pd(dptr + stride * 1, a0); + case 1: + _mm_storel_pd(dptr + stride * 0, a0); + } +} + +/***************************************************************************** + * Implement partial load/store for u32/f32/u64/f64... via reinterpret cast + *****************************************************************************/ +#define NPYV_IMPL_AVX2_REST_PARTIAL_TYPES(F_SFX, T_SFX) \ + NPY_FINLINE npyv_##F_SFX npyv_load_till_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_uintp nlane, npyv_lanetype_##F_SFX fill) \ + { \ + union { \ + npyv_lanetype_##F_SFX from_##F_SFX; \ + npyv_lanetype_##T_SFX to_##T_SFX; \ + } pun = {.from_##F_SFX = fill}; \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_load_till_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, nlane, pun.to_##T_SFX \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_loadn_till_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane, \ + npyv_lanetype_##F_SFX fill) \ + { \ + union { \ + npyv_lanetype_##F_SFX from_##F_SFX; \ + npyv_lanetype_##T_SFX to_##T_SFX; \ + } pun = {.from_##F_SFX = fill}; \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_loadn_till_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, stride, nlane, pun.to_##T_SFX \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_load_tillz_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_uintp nlane) \ + { \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_load_tillz_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, nlane \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_loadn_tillz_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane) \ + { \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_loadn_tillz_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, stride, nlane \ + )); \ + } \ + NPY_FINLINE void npyv_store_till_##F_SFX \ + (npyv_lanetype_##F_SFX *ptr, npy_uintp nlane, npyv_##F_SFX a) \ + { \ + npyv_store_till_##T_SFX( \ + (npyv_lanetype_##T_SFX *)ptr, nlane, \ + npyv_reinterpret_##T_SFX##_##F_SFX(a) \ + ); \ + } \ + NPY_FINLINE void npyv_storen_till_##F_SFX \ + (npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane, npyv_##F_SFX a) \ + { \ + npyv_storen_till_##T_SFX( \ + (npyv_lanetype_##T_SFX *)ptr, stride, nlane, \ + npyv_reinterpret_##T_SFX##_##F_SFX(a) \ + ); \ + } + +NPYV_IMPL_AVX2_REST_PARTIAL_TYPES(u32, s32) +NPYV_IMPL_AVX2_REST_PARTIAL_TYPES(f32, s32) +NPYV_IMPL_AVX2_REST_PARTIAL_TYPES(u64, s64) +NPYV_IMPL_AVX2_REST_PARTIAL_TYPES(f64, s64) #endif // _NPY_SIMD_AVX2_MEMORY_H diff --git a/numpy/core/src/common/simd/avx512/avx512.h b/numpy/core/src/common/simd/avx512/avx512.h index 96fdf72b91f1..b09d772f2fec 100644 --- a/numpy/core/src/common/simd/avx512/avx512.h +++ b/numpy/core/src/common/simd/avx512/avx512.h @@ -4,6 +4,9 @@ #define NPY_SIMD 512 #define NPY_SIMD_WIDTH 64 #define NPY_SIMD_F64 1 +// Enough limit to allow us to use _mm512_i32gather_* and _mm512_i32scatter_* +#define NPY_SIMD_MAXLOAD_STRIDE32 (0x7fffffff / 16) +#define NPY_SIMD_MAXSTORE_STRIDE32 (0x7fffffff / 16) typedef __m512i npyv_u8; typedef __m512i npyv_s8; diff --git a/numpy/core/src/common/simd/avx512/memory.h b/numpy/core/src/common/simd/avx512/memory.h index e212c4555270..bffd6e907246 100644 --- a/numpy/core/src/common/simd/avx512/memory.h +++ b/numpy/core/src/common/simd/avx512/memory.h @@ -90,5 +90,243 @@ NPYV_IMPL_AVX512_MEM_INT(npy_int64, s64) // store higher part #define npyv_storeh_f32(PTR, VEC) _mm256_storeu_ps(PTR, npyv512_higher_ps256(VEC)) #define npyv_storeh_f64(PTR, VEC) _mm256_storeu_pd(PTR, npyv512_higher_pd256(VEC)) +/*************************** + * Non-contiguous Load + ***************************/ +//// 32 +NPY_FINLINE npyv_u32 npyv_loadn_u32(const npy_uint32 *ptr, npy_intp stride) +{ + assert(llabs(stride) <= NPY_SIMD_MAXLOAD_STRIDE32); + const __m512i steps = npyv_set_s32( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + ); + const __m512i idx = _mm512_mullo_epi32(steps, _mm512_set1_epi32((int)stride)); + return _mm512_i32gather_epi32(idx, (const __m512i*)ptr, 4); +} +NPY_FINLINE npyv_s32 npyv_loadn_s32(const npy_int32 *ptr, npy_intp stride) +{ return npyv_loadn_u32((const npy_uint32*)ptr, stride); } +NPY_FINLINE npyv_f32 npyv_loadn_f32(const float *ptr, npy_intp stride) +{ return _mm512_castsi512_ps(npyv_loadn_u32((const npy_uint32*)ptr, stride)); } +//// 64 +NPY_FINLINE npyv_u64 npyv_loadn_u64(const npy_uint64 *ptr, npy_intp stride) +{ + const __m512i idx = _mm512_setr_epi64( + 0*stride, 1*stride, 2*stride, 3*stride, + 4*stride, 5*stride, 6*stride, 7*stride + ); + return _mm512_i64gather_epi64(idx, (const __m512i*)ptr, 8); +} +NPY_FINLINE npyv_s64 npyv_loadn_s64(const npy_int64 *ptr, npy_intp stride) +{ return npyv_loadn_u64((const npy_uint64*)ptr, stride); } +NPY_FINLINE npyv_f64 npyv_loadn_f64(const double *ptr, npy_intp stride) +{ return _mm512_castsi512_pd(npyv_loadn_u64((const npy_uint64*)ptr, stride)); } +/*************************** + * Non-contiguous Store + ***************************/ +//// 32 +NPY_FINLINE void npyv_storen_u32(npy_uint32 *ptr, npy_intp stride, npyv_u32 a) +{ + assert(llabs(stride) <= NPY_SIMD_MAXSTORE_STRIDE32); + const __m512i steps = _mm512_setr_epi32( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + ); + const __m512i idx = _mm512_mullo_epi32(steps, _mm512_set1_epi32((int)stride)); + _mm512_i32scatter_epi32((__m512i*)ptr, idx, a, 4); +} +NPY_FINLINE void npyv_storen_s32(npy_int32 *ptr, npy_intp stride, npyv_s32 a) +{ npyv_storen_u32((npy_uint32*)ptr, stride, a); } +NPY_FINLINE void npyv_storen_f32(float *ptr, npy_intp stride, npyv_f32 a) +{ npyv_storen_u32((npy_uint32*)ptr, stride, _mm512_castps_si512(a)); } +//// 64 +NPY_FINLINE void npyv_storen_u64(npy_uint64 *ptr, npy_intp stride, npyv_u64 a) +{ + const __m512i idx = _mm512_setr_epi64( + 0*stride, 1*stride, 2*stride, 3*stride, + 4*stride, 5*stride, 6*stride, 7*stride + ); + _mm512_i64scatter_epi64((__m512i*)ptr, idx, a, 8); +} +NPY_FINLINE void npyv_storen_s64(npy_int64 *ptr, npy_intp stride, npyv_s64 a) +{ npyv_storen_u64((npy_uint64*)ptr, stride, a); } +NPY_FINLINE void npyv_storen_f64(double *ptr, npy_intp stride, npyv_f64 a) +{ npyv_storen_u64((npy_uint64*)ptr, stride, _mm512_castpd_si512(a)); } + +/********************************* + * Partial Load + *********************************/ +//// 32 +NPY_FINLINE npyv_s32 npyv_load_till_s32(const npy_int32 *ptr, npy_uintp nlane, npy_int32 fill) +{ + assert(nlane > 0); + const __m512i vfill = _mm512_set1_epi32(fill); + const __mmask16 mask = nlane > 31 ? -1 : (1 << nlane) - 1; + return _mm512_mask_loadu_epi32(vfill, mask, (const __m512i*)ptr); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s32 npyv_load_tillz_s32(const npy_int32 *ptr, npy_uintp nlane) +{ + assert(nlane > 0); + const __mmask16 mask = nlane > 31 ? -1 : (1 << nlane) - 1; + return _mm512_maskz_loadu_epi32(mask, (const __m512i*)ptr); +} +//// 64 +NPY_FINLINE npyv_s64 npyv_load_till_s64(const npy_int64 *ptr, npy_uintp nlane, npy_int64 fill) +{ + assert(nlane > 0); + const __m512i vfill = _mm512_set1_epi64(fill); + const __mmask8 mask = nlane > 31 ? -1 : (1 << nlane) - 1; + return _mm512_mask_loadu_epi64(vfill, mask, (const __m512i*)ptr); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s64 npyv_load_tillz_s64(const npy_int64 *ptr, npy_uintp nlane) +{ + assert(nlane > 0); + const __mmask8 mask = nlane > 15 ? -1 : (1 << nlane) - 1; + return _mm512_maskz_loadu_epi64(mask, (const __m512i*)ptr); +} +/********************************* + * Non-contiguous partial load + *********************************/ +//// 32 +NPY_FINLINE npyv_s32 +npyv_loadn_till_s32(const npy_int32 *ptr, npy_intp stride, npy_uintp nlane, npy_int32 fill) +{ + assert(nlane > 0); + assert(llabs(stride) <= NPY_SIMD_MAXLOAD_STRIDE32); + const __m512i steps = npyv_set_s32( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + ); + const __m512i idx = _mm512_mullo_epi32(steps, _mm512_set1_epi32((int)stride)); + const __m512i vfill = _mm512_set1_epi32(fill); + const __mmask16 mask = nlane > 31 ? -1 : (1 << nlane) - 1; + return _mm512_mask_i32gather_epi32(vfill, mask, idx, (const __m512i*)ptr, 4); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s32 +npyv_loadn_tillz_s32(const npy_int32 *ptr, npy_intp stride, npy_uintp nlane) +{ return npyv_loadn_till_s32(ptr, stride, nlane, 0); } +//// 64 +NPY_FINLINE npyv_s64 +npyv_loadn_till_s64(const npy_int64 *ptr, npy_intp stride, npy_uintp nlane, npy_int64 fill) +{ + assert(nlane > 0); + const __m512i idx = _mm512_setr_epi64( + 0*stride, 1*stride, 2*stride, 3*stride, + 4*stride, 5*stride, 6*stride, 7*stride + ); + const __m512i vfill = _mm512_set1_epi64(fill); + const __mmask8 mask = nlane > 31 ? -1 : (1 << nlane) - 1; + return _mm512_mask_i64gather_epi64(vfill, mask, idx, (const __m512i*)ptr, 8); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s64 +npyv_loadn_tillz_s64(const npy_int64 *ptr, npy_intp stride, npy_uintp nlane) +{ return npyv_loadn_till_s64(ptr, stride, nlane, 0); } +/********************************* + * Partial store + *********************************/ +//// 32 +NPY_FINLINE void npyv_store_till_s32(npy_int32 *ptr, npy_uintp nlane, npyv_s32 a) +{ + assert(nlane > 0); + const __mmask16 mask = nlane > 31 ? -1 : (1 << nlane) - 1; + _mm512_mask_storeu_epi32((__m512i*)ptr, mask, a); +} +//// 64 +NPY_FINLINE void npyv_store_till_s64(npy_int64 *ptr, npy_uintp nlane, npyv_s64 a) +{ + assert(nlane > 0); + const __mmask8 mask = nlane > 15 ? -1 : (1 << nlane) - 1; + _mm512_mask_storeu_epi64((__m512i*)ptr, mask, a); +} +/********************************* + * Non-contiguous partial store + *********************************/ +//// 32 +NPY_FINLINE void npyv_storen_till_s32(npy_int32 *ptr, npy_intp stride, npy_uintp nlane, npyv_s32 a) +{ + assert(nlane > 0); + assert(llabs(stride) <= NPY_SIMD_MAXSTORE_STRIDE32); + const __m512i steps = _mm512_setr_epi32( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + ); + const __m512i idx = _mm512_mullo_epi32(steps, _mm512_set1_epi32((int)stride)); + const __mmask16 mask = nlane > 31 ? -1 : (1 << nlane) - 1; + _mm512_mask_i32scatter_epi32((__m512i*)ptr, mask, idx, a, 4); +} +//// 64 +NPY_FINLINE void npyv_storen_till_s64(npy_int64 *ptr, npy_intp stride, npy_uintp nlane, npyv_s64 a) +{ + assert(nlane > 0); + const __m512i idx = _mm512_setr_epi64( + 0*stride, 1*stride, 2*stride, 3*stride, + 4*stride, 5*stride, 6*stride, 7*stride + ); + const __mmask8 mask = nlane > 15 ? -1 : (1 << nlane) - 1; + _mm512_mask_i64scatter_epi64((__m512i*)ptr, mask, idx, a, 8); +} + +/***************************************************************************** + * Implement partial load/store for u32/f32/u64/f64... via reinterpret cast + *****************************************************************************/ +#define NPYV_IMPL_AVX512_REST_PARTIAL_TYPES(F_SFX, T_SFX) \ + NPY_FINLINE npyv_##F_SFX npyv_load_till_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_uintp nlane, npyv_lanetype_##F_SFX fill) \ + { \ + union { \ + npyv_lanetype_##F_SFX from_##F_SFX; \ + npyv_lanetype_##T_SFX to_##T_SFX; \ + } pun = {.from_##F_SFX = fill}; \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_load_till_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, nlane, pun.to_##T_SFX \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_loadn_till_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane, \ + npyv_lanetype_##F_SFX fill) \ + { \ + union { \ + npyv_lanetype_##F_SFX from_##F_SFX; \ + npyv_lanetype_##T_SFX to_##T_SFX; \ + } pun = {.from_##F_SFX = fill}; \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_loadn_till_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, stride, nlane, pun.to_##T_SFX \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_load_tillz_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_uintp nlane) \ + { \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_load_tillz_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, nlane \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_loadn_tillz_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane) \ + { \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_loadn_tillz_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, stride, nlane \ + )); \ + } \ + NPY_FINLINE void npyv_store_till_##F_SFX \ + (npyv_lanetype_##F_SFX *ptr, npy_uintp nlane, npyv_##F_SFX a) \ + { \ + npyv_store_till_##T_SFX( \ + (npyv_lanetype_##T_SFX *)ptr, nlane, \ + npyv_reinterpret_##T_SFX##_##F_SFX(a) \ + ); \ + } \ + NPY_FINLINE void npyv_storen_till_##F_SFX \ + (npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane, npyv_##F_SFX a) \ + { \ + npyv_storen_till_##T_SFX( \ + (npyv_lanetype_##T_SFX *)ptr, stride, nlane, \ + npyv_reinterpret_##T_SFX##_##F_SFX(a) \ + ); \ + } + +NPYV_IMPL_AVX512_REST_PARTIAL_TYPES(u32, s32) +NPYV_IMPL_AVX512_REST_PARTIAL_TYPES(f32, s32) +NPYV_IMPL_AVX512_REST_PARTIAL_TYPES(u64, s64) +NPYV_IMPL_AVX512_REST_PARTIAL_TYPES(f64, s64) #endif // _NPY_SIMD_AVX512_MEMORY_H diff --git a/numpy/core/src/common/simd/neon/memory.h b/numpy/core/src/common/simd/neon/memory.h index afa703584dac..1e258f1bcbef 100644 --- a/numpy/core/src/common/simd/neon/memory.h +++ b/numpy/core/src/common/simd/neon/memory.h @@ -5,6 +5,8 @@ #ifndef _NPY_SIMD_NEON_MEMORY_H #define _NPY_SIMD_NEON_MEMORY_H +#include "misc.h" + /*************************** * load/store ***************************/ @@ -45,5 +47,290 @@ NPYV_IMPL_NEON_MEM(f32, float) #if NPY_SIMD_F64 NPYV_IMPL_NEON_MEM(f64, double) #endif +/*************************** + * Non-contiguous Load + ***************************/ +NPY_FINLINE npyv_s32 npyv_loadn_s32(const npy_int32 *ptr, npy_intp stride) +{ + switch (stride) { + case 2: + return vld2q_s32((const int32_t*)ptr).val[0]; + case 3: + return vld3q_s32((const int32_t*)ptr).val[0]; + case 4: + return vld4q_s32((const int32_t*)ptr).val[0]; + default:; + int32x2_t ax = vcreate_s32(*ptr); + int32x4_t a = vcombine_s32(ax, ax); + a = vld1q_lane_s32((const int32_t*)ptr + stride, a, 1); + a = vld1q_lane_s32((const int32_t*)ptr + stride*2, a, 2); + a = vld1q_lane_s32((const int32_t*)ptr + stride*3, a, 3); + return a; + } +} + +NPY_FINLINE npyv_u32 npyv_loadn_u32(const npy_uint32 *ptr, npy_intp stride) +{ + return npyv_reinterpret_u32_s32( + npyv_loadn_s32((const npy_int32*)ptr, stride) + ); +} +NPY_FINLINE npyv_f32 npyv_loadn_f32(const float *ptr, npy_intp stride) +{ + return npyv_reinterpret_f32_s32( + npyv_loadn_s32((const npy_int32*)ptr, stride) + ); +} +//// 64 +NPY_FINLINE npyv_s64 npyv_loadn_s64(const npy_int64 *ptr, npy_intp stride) +{ + return vcombine_s64( + vld1_s64((const int64_t*)ptr), vld1_s64((const int64_t*)ptr + stride) + ); +} +NPY_FINLINE npyv_u64 npyv_loadn_u64(const npy_uint64 *ptr, npy_intp stride) +{ + return npyv_reinterpret_u64_s64( + npyv_loadn_s64((const npy_int64*)ptr, stride) + ); +} +#if NPY_SIMD_F64 +NPY_FINLINE npyv_f64 npyv_loadn_f64(const double *ptr, npy_intp stride) +{ + return npyv_reinterpret_f64_s64( + npyv_loadn_s64((const npy_int64*)ptr, stride) + ); +} +#endif +/*************************** + * Non-contiguous Store + ***************************/ +//// 32 +NPY_FINLINE void npyv_storen_s32(npy_int32 *ptr, npy_intp stride, npyv_s32 a) +{ + vst1q_lane_s32((int32_t*)ptr, a, 0); + vst1q_lane_s32((int32_t*)ptr + stride, a, 1); + vst1q_lane_s32((int32_t*)ptr + stride*2, a, 2); + vst1q_lane_s32((int32_t*)ptr + stride*3, a, 3); +} +NPY_FINLINE void npyv_storen_u32(npy_uint32 *ptr, npy_intp stride, npyv_u32 a) +{ npyv_storen_s32((npy_int32*)ptr, stride, npyv_reinterpret_s32_u32(a)); } +NPY_FINLINE void npyv_storen_f32(float *ptr, npy_intp stride, npyv_f32 a) +{ npyv_storen_s32((npy_int32*)ptr, stride, npyv_reinterpret_s32_f32(a)); } +//// 64 +NPY_FINLINE void npyv_storen_s64(npy_int64 *ptr, npy_intp stride, npyv_s64 a) +{ + vst1q_lane_s64((int64_t*)ptr, a, 0); + vst1q_lane_s64((int64_t*)ptr + stride, a, 1); +} +NPY_FINLINE void npyv_storen_u64(npy_uint64 *ptr, npy_intp stride, npyv_u64 a) +{ npyv_storen_s64((npy_int64*)ptr, stride, npyv_reinterpret_s64_u64(a)); } + +#if NPY_SIMD_F64 +NPY_FINLINE void npyv_storen_f64(double *ptr, npy_intp stride, npyv_f64 a) +{ npyv_storen_s64((npy_int64*)ptr, stride, npyv_reinterpret_s64_f64(a)); } +#endif + +/********************************* + * Partial Load + *********************************/ +//// 32 +NPY_FINLINE npyv_s32 npyv_load_till_s32(const npy_int32 *ptr, npy_uintp nlane, npy_int32 fill) +{ + assert(nlane > 0); + switch(nlane) { + case 1: + return vld1q_lane_s32((const int32_t*)ptr, vdupq_n_s32(fill), 0); + case 2: + return vcombine_s32(vld1_s32((const int32_t*)ptr), vdup_n_s32(fill)); + case 3: + return vcombine_s32( + vld1_s32((const int32_t*)ptr), + vld1_lane_s32((const int32_t*)ptr + 2, vdup_n_s32(fill), 0) + ); + default: + return npyv_load_s32(ptr); + } +} +// fill zero to rest lanes +NPY_FINLINE npyv_s32 npyv_load_tillz_s32(const npy_int32 *ptr, npy_uintp nlane) +{ return npyv_load_till_s32(ptr, nlane, 0); } +//// 64 +NPY_FINLINE npyv_s64 npyv_load_till_s64(const npy_int64 *ptr, npy_uintp nlane, npy_int64 fill) +{ + assert(nlane > 0); + if (nlane == 1) { + return vcombine_s64(vld1_s64((const int64_t*)ptr), vdup_n_s64(fill)); + } + return npyv_load_s64(ptr); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s64 npyv_load_tillz_s64(const npy_int64 *ptr, npy_uintp nlane) +{ return npyv_load_till_s64(ptr, nlane, 0); } + +/********************************* + * Non-contiguous partial load + *********************************/ +//// 32 +NPY_FINLINE npyv_s32 +npyv_loadn_till_s32(const npy_int32 *ptr, npy_intp stride, npy_uintp nlane, npy_int32 fill) +{ + assert(nlane > 0); + int32x4_t vfill = vdupq_n_s32(fill); + switch(nlane) { + case 3: + vfill = vld1q_lane_s32((const int32_t*)ptr + stride*2, vfill, 2); + case 2: + vfill = vld1q_lane_s32((const int32_t*)ptr + stride, vfill, 1); + case 1: + vfill = vld1q_lane_s32((const int32_t*)ptr, vfill, 0); + return vfill; + default: + return npyv_loadn_s32(ptr, stride); + } +} +NPY_FINLINE npyv_s32 +npyv_loadn_tillz_s32(const npy_int32 *ptr, npy_intp stride, npy_uintp nlane) +{ return npyv_loadn_till_s32(ptr, stride, nlane, 0); } + +NPY_FINLINE npyv_s64 +npyv_loadn_till_s64(const npy_int64 *ptr, npy_intp stride, npy_uintp nlane, npy_int64 fill) +{ + assert(nlane > 0); + if (nlane == 1) { + return vcombine_s64(vld1_s64((const int64_t*)ptr), vdup_n_s64(fill)); + } + return npyv_loadn_s64(ptr, stride); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s64 npyv_loadn_tillz_s64(const npy_int64 *ptr, npy_intp stride, npy_uintp nlane) +{ return npyv_loadn_till_s64(ptr, stride, nlane, 0); } + +/********************************* + * Partial store + *********************************/ +//// 32 +NPY_FINLINE void npyv_store_till_s32(npy_int32 *ptr, npy_uintp nlane, npyv_s32 a) +{ + assert(nlane > 0); + switch(nlane) { + case 1: + vst1q_lane_s32((int32_t*)ptr, a, 0); + break; + case 2: + vst1_s32((int32_t*)ptr, vget_low_s32(a)); + break; + case 3: + vst1_s32((int32_t*)ptr, vget_low_s32(a)); + vst1q_lane_s32((int32_t*)ptr + 2, a, 2); + break; + default: + npyv_store_s32(ptr, a); + } +} +//// 64 +NPY_FINLINE void npyv_store_till_s64(npy_int64 *ptr, npy_uintp nlane, npyv_s64 a) +{ + assert(nlane > 0); + if (nlane == 1) { + vst1q_lane_s64((int64_t*)ptr, a, 0); + return; + } + npyv_store_s64(ptr, a); +} +/********************************* + * Non-contiguous partial store + *********************************/ +//// 32 +NPY_FINLINE void npyv_storen_till_s32(npy_int32 *ptr, npy_intp stride, npy_uintp nlane, npyv_s32 a) +{ + assert(nlane > 0); + switch(nlane) { + default: + vst1q_lane_s32((int32_t*)ptr + stride*3, a, 3); + case 3: + vst1q_lane_s32((int32_t*)ptr + stride*2, a, 2); + case 2: + vst1q_lane_s32((int32_t*)ptr + stride, a, 1); + case 1: + vst1q_lane_s32((int32_t*)ptr, a, 0); + break; + } +} +//// 64 +NPY_FINLINE void npyv_storen_till_s64(npy_int64 *ptr, npy_intp stride, npy_uintp nlane, npyv_s64 a) +{ + assert(nlane > 0); + if (nlane == 1) { + vst1q_lane_s64((int64_t*)ptr, a, 0); + return; + } + npyv_storen_s64(ptr, stride, a); +} + +/***************************************************************** + * Implement partial load/store for u32/f32/u64/f64... via casting + *****************************************************************/ +#define NPYV_IMPL_NEON_REST_PARTIAL_TYPES(F_SFX, T_SFX) \ + NPY_FINLINE npyv_##F_SFX npyv_load_till_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_uintp nlane, npyv_lanetype_##F_SFX fill) \ + { \ + union { \ + npyv_lanetype_##F_SFX from_##F_SFX; \ + npyv_lanetype_##T_SFX to_##T_SFX; \ + } pun = {.from_##F_SFX = fill}; \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_load_till_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, nlane, pun.to_##T_SFX \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_loadn_till_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane, \ + npyv_lanetype_##F_SFX fill) \ + { \ + union { \ + npyv_lanetype_##F_SFX from_##F_SFX; \ + npyv_lanetype_##T_SFX to_##T_SFX; \ + } pun = {.from_##F_SFX = fill}; \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_loadn_till_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, stride, nlane, pun.to_##T_SFX \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_load_tillz_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_uintp nlane) \ + { \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_load_tillz_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, nlane \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_loadn_tillz_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane) \ + { \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_loadn_tillz_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, stride, nlane \ + )); \ + } \ + NPY_FINLINE void npyv_store_till_##F_SFX \ + (npyv_lanetype_##F_SFX *ptr, npy_uintp nlane, npyv_##F_SFX a) \ + { \ + npyv_store_till_##T_SFX( \ + (npyv_lanetype_##T_SFX *)ptr, nlane, \ + npyv_reinterpret_##T_SFX##_##F_SFX(a) \ + ); \ + } \ + NPY_FINLINE void npyv_storen_till_##F_SFX \ + (npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane, npyv_##F_SFX a) \ + { \ + npyv_storen_till_##T_SFX( \ + (npyv_lanetype_##T_SFX *)ptr, stride, nlane, \ + npyv_reinterpret_##T_SFX##_##F_SFX(a) \ + ); \ + } + +NPYV_IMPL_NEON_REST_PARTIAL_TYPES(u32, s32) +NPYV_IMPL_NEON_REST_PARTIAL_TYPES(f32, s32) +NPYV_IMPL_NEON_REST_PARTIAL_TYPES(u64, s64) +#if NPY_SIMD_F64 +NPYV_IMPL_NEON_REST_PARTIAL_TYPES(f64, s64) +#endif #endif // _NPY_SIMD_NEON_MEMORY_H diff --git a/numpy/core/src/common/simd/simd.h b/numpy/core/src/common/simd/simd.h index 2f39c8427b5d..8804223c9fef 100644 --- a/numpy/core/src/common/simd/simd.h +++ b/numpy/core/src/common/simd/simd.h @@ -49,6 +49,55 @@ typedef double npyv_lanetype_f64; #define NPY_SIMD_WIDTH 0 #define NPY_SIMD_F64 0 #endif +/** + * Some SIMD extensions currently(AVX2, AVX512F) require (de facto) + * a maximum number of strides sizes when dealing with non-contiguous memory access. + * + * Therefore the following functions must be used to check the maximum + * acceptable limit of strides before using any of non-contiguous load/store intrinsics. + * + * For instance: + * npy_intp ld_stride = step[0] / sizeof(float); + * npy_intp st_stride = step[1] / sizeof(float); + * + * if (npyv_loadable_stride_f32(ld_stride) && npyv_storable_stride_f32(st_stride)) { + * for (;;) + * npyv_f32 a = npyv_loadn_f32(ld_pointer, ld_stride); + * // ... + * npyv_storen_f32(st_pointer, st_stride, a); + * } + * else { + * for (;;) + * // C scalars + * } + */ +#ifndef NPY_SIMD_MAXLOAD_STRIDE32 + #define NPY_SIMD_MAXLOAD_STRIDE32 0 +#endif +#ifndef NPY_SIMD_MAXSTORE_STRIDE32 + #define NPY_SIMD_MAXSTORE_STRIDE32 0 +#endif +#ifndef NPY_SIMD_MAXLOAD_STRIDE64 + #define NPY_SIMD_MAXLOAD_STRIDE64 0 +#endif +#ifndef NPY_SIMD_MAXSTORE_STRIDE64 + #define NPY_SIMD_MAXSTORE_STRIDE64 0 +#endif +#define NPYV_IMPL_MAXSTRIDE(SFX, MAXLOAD, MAXSTORE) \ + NPY_FINLINE int npyv_loadable_stride_##SFX(npy_intp stride) \ + { return MAXLOAD > 0 ? llabs(stride) <= MAXLOAD : 1; } \ + NPY_FINLINE int npyv_storable_stride_##SFX(npy_intp stride) \ + { return MAXSTORE > 0 ? llabs(stride) <= MAXSTORE : 1; } +#if NPY_SIMD + NPYV_IMPL_MAXSTRIDE(u32, NPY_SIMD_MAXLOAD_STRIDE32, NPY_SIMD_MAXSTORE_STRIDE32) + NPYV_IMPL_MAXSTRIDE(s32, NPY_SIMD_MAXLOAD_STRIDE32, NPY_SIMD_MAXSTORE_STRIDE32) + NPYV_IMPL_MAXSTRIDE(f32, NPY_SIMD_MAXLOAD_STRIDE32, NPY_SIMD_MAXSTORE_STRIDE32) + NPYV_IMPL_MAXSTRIDE(u64, NPY_SIMD_MAXLOAD_STRIDE64, NPY_SIMD_MAXSTORE_STRIDE64) + NPYV_IMPL_MAXSTRIDE(s64, NPY_SIMD_MAXLOAD_STRIDE64, NPY_SIMD_MAXSTORE_STRIDE64) +#endif +#if NPY_SIMD_F64 + NPYV_IMPL_MAXSTRIDE(f64, NPY_SIMD_MAXLOAD_STRIDE64, NPY_SIMD_MAXSTORE_STRIDE64) +#endif #ifdef __cplusplus } diff --git a/numpy/core/src/common/simd/sse/memory.h b/numpy/core/src/common/simd/sse/memory.h index 1a555d6f03de..1074c3b02efe 100644 --- a/numpy/core/src/common/simd/sse/memory.h +++ b/numpy/core/src/common/simd/sse/memory.h @@ -5,6 +5,8 @@ #ifndef _NPY_SIMD_SSE_MEMORY_H #define _NPY_SIMD_SSE_MEMORY_H +#include "misc.h" + /*************************** * load/store ***************************/ @@ -70,5 +72,427 @@ NPYV_IMPL_SSE_MEM_INT(npy_int64, s64) // store higher part #define npyv_storeh_f32(PTR, VEC) npyv_storeh_u32((npy_uint32*)(PTR), _mm_castps_si128(VEC)) #define npyv_storeh_f64(PTR, VEC) npyv_storeh_u32((npy_uint32*)(PTR), _mm_castpd_si128(VEC)) +/*************************** + * Non-contiguous Load + ***************************/ +//// 32 +NPY_FINLINE npyv_s32 npyv_loadn_s32(const npy_int32 *ptr, npy_intp stride) +{ + __m128i a = _mm_cvtsi32_si128(*ptr); +#ifdef NPY_HAVE_SSE41 + a = _mm_insert_epi32(a, ptr[stride], 1); + a = _mm_insert_epi32(a, ptr[stride*2], 2); + a = _mm_insert_epi32(a, ptr[stride*3], 3); +#else + __m128i a1 = _mm_cvtsi32_si128(ptr[stride]); + __m128i a2 = _mm_cvtsi32_si128(ptr[stride*2]); + __m128i a3 = _mm_cvtsi32_si128(ptr[stride*3]); + a = _mm_unpacklo_epi32(a, a1); + a = _mm_unpacklo_epi64(a, _mm_unpacklo_epi32(a2, a3)); +#endif + return a; +} +NPY_FINLINE npyv_u32 npyv_loadn_u32(const npy_uint32 *ptr, npy_intp stride) +{ return npyv_loadn_s32((const npy_int32*)ptr, stride); } +NPY_FINLINE npyv_f32 npyv_loadn_f32(const float *ptr, npy_intp stride) +{ return _mm_castsi128_ps(npyv_loadn_s32((const npy_int32*)ptr, stride)); } +//// 64 +NPY_FINLINE npyv_f64 npyv_loadn_f64(const double *ptr, npy_intp stride) +{ return _mm_loadh_pd(npyv_loadl_f64(ptr), ptr + stride); } +NPY_FINLINE npyv_u64 npyv_loadn_u64(const npy_uint64 *ptr, npy_intp stride) +{ return _mm_castpd_si128(npyv_loadn_f64((const double*)ptr, stride)); } +NPY_FINLINE npyv_s64 npyv_loadn_s64(const npy_int64 *ptr, npy_intp stride) +{ return _mm_castpd_si128(npyv_loadn_f64((const double*)ptr, stride)); } +/*************************** + * Non-contiguous Store + ***************************/ +//// 32 +NPY_FINLINE void npyv_storen_s32(npy_int32 *ptr, npy_intp stride, npyv_s32 a) +{ + ptr[stride * 0] = _mm_cvtsi128_si32(a); +#ifdef NPY_HAVE_SSE41 + ptr[stride * 1] = _mm_extract_epi32(a, 1); + ptr[stride * 2] = _mm_extract_epi32(a, 2); + ptr[stride * 3] = _mm_extract_epi32(a, 3); +#else + ptr[stride * 1] = _mm_cvtsi128_si32(_mm_shuffle_epi32(a, _MM_SHUFFLE(0, 0, 0, 1))); + ptr[stride * 2] = _mm_cvtsi128_si32(_mm_shuffle_epi32(a, _MM_SHUFFLE(0, 0, 0, 2))); + ptr[stride * 3] = _mm_cvtsi128_si32(_mm_shuffle_epi32(a, _MM_SHUFFLE(0, 0, 0, 3))); +#endif +} +NPY_FINLINE void npyv_storen_u32(npy_uint32 *ptr, npy_intp stride, npyv_u32 a) +{ npyv_storen_s32((npy_int32*)ptr, stride, a); } +NPY_FINLINE void npyv_storen_f32(float *ptr, npy_intp stride, npyv_f32 a) +{ npyv_storen_s32((npy_int32*)ptr, stride, _mm_castps_si128(a)); } +//// 64 +NPY_FINLINE void npyv_storen_f64(double *ptr, npy_intp stride, npyv_f64 a) +{ + _mm_storel_pd(ptr, a); + _mm_storeh_pd(ptr + stride, a); +} +NPY_FINLINE void npyv_storen_u64(npy_uint64 *ptr, npy_intp stride, npyv_u64 a) +{ npyv_storen_f64((double*)ptr, stride, _mm_castsi128_pd(a)); } +NPY_FINLINE void npyv_storen_s64(npy_int64 *ptr, npy_intp stride, npyv_s64 a) +{ npyv_storen_f64((double*)ptr, stride, _mm_castsi128_pd(a)); } + +/********************************* + * Partial Load + *********************************/ +#if defined(__clang__) && __clang_major__ > 7 + /** + * Clang >=8 perform aggressive optimization that tends to + * zero the bits of upper half part of vectors even + * when we try to fill it up with certain scalars, + * which my lead to zero division errors. + */ + #define NPYV__CLANG_ZEROUPPER +#endif +//// 32 +NPY_FINLINE npyv_s32 npyv_load_till_s32(const npy_int32 *ptr, npy_uintp nlane, npy_int32 fill) +{ + assert(nlane > 0); +#ifdef NPYV__CLANG_ZEROUPPER + if (nlane > 3) { + return npyv_load_s32(ptr); + } + npy_int32 NPY_DECL_ALIGNED(16) data[4] = {fill, fill, fill, fill}; + for (npy_uint64 i = 0; i < nlane; ++i) { + data[i] = ptr[i]; + } + return npyv_loada_s32(data); +#else + #ifndef NPY_HAVE_SSE41 + const short *wptr = (const short*)ptr; + #endif + const __m128i vfill = npyv_setall_s32(fill); + __m128i a; + switch(nlane) { + case 2: + return _mm_castpd_si128( + _mm_loadl_pd(_mm_castsi128_pd(vfill), (double*)ptr) + ); + #ifdef NPY_HAVE_SSE41 + case 1: + return _mm_insert_epi32(vfill, ptr[0], 0); + case 3: + a = _mm_loadl_epi64((const __m128i*)ptr); + a = _mm_insert_epi32(a, ptr[2], 2); + a = _mm_insert_epi32(a, fill, 3); + return a; + #else + case 1: + a = _mm_insert_epi16(vfill, wptr[0], 0); + return _mm_insert_epi16(a, wptr[1], 1); + case 3: + a = _mm_loadl_epi64((const __m128i*)ptr); + a = _mm_unpacklo_epi64(a, vfill); + a = _mm_insert_epi16(a, wptr[4], 4); + a = _mm_insert_epi16(a, wptr[5], 5); + return a; + #endif // NPY_HAVE_SSE41 + default: + return npyv_load_s32(ptr); + } +#endif +} +// fill zero to rest lanes +NPY_FINLINE npyv_s32 npyv_load_tillz_s32(const npy_int32 *ptr, npy_uintp nlane) +{ + assert(nlane > 0); + switch(nlane) { + case 1: + return _mm_cvtsi32_si128(*ptr); + case 2: + return _mm_loadl_epi64((const __m128i*)ptr); + case 3:; + npyv_s32 a = _mm_loadl_epi64((const __m128i*)ptr); + #ifdef NPY_HAVE_SSE41 + return _mm_insert_epi32(a, ptr[2], 2); + #else + return _mm_unpacklo_epi64(a, _mm_cvtsi32_si128(ptr[2])); + #endif + default: + return npyv_load_s32(ptr); + } +} +//// 64 +NPY_FINLINE npyv_s64 npyv_load_till_s64(const npy_int64 *ptr, npy_uintp nlane, npy_int64 fill) +{ + assert(nlane > 0); +#ifdef NPYV__CLANG_ZEROUPPER + if (nlane <= 2) { + npy_int64 NPY_DECL_ALIGNED(16) data[2] = {fill, fill}; + for (npy_uint64 i = 0; i < nlane; ++i) { + data[i] = ptr[i]; + } + return npyv_loada_s64(data); + } +#else + if (nlane == 1) { + const __m128i vfill = npyv_setall_s64(fill); + return _mm_castpd_si128( + _mm_loadl_pd(_mm_castsi128_pd(vfill), (double*)ptr) + ); + } +#endif + return npyv_load_s64(ptr); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s64 npyv_load_tillz_s64(const npy_int64 *ptr, npy_uintp nlane) +{ + assert(nlane > 0); + if (nlane == 1) { + return _mm_loadl_epi64((const __m128i*)ptr); + } + return npyv_load_s64(ptr); +} +/********************************* + * Non-contiguous partial load + *********************************/ +//// 32 +NPY_FINLINE npyv_s32 +npyv_loadn_till_s32(const npy_int32 *ptr, npy_intp stride, npy_uintp nlane, npy_int32 fill) +{ + assert(nlane > 0); +#ifdef NPYV__CLANG_ZEROUPPER + if (nlane > 3) { + return npyv_loadn_s32(ptr, stride); + } + npy_int32 NPY_DECL_ALIGNED(16) data[4] = {fill, fill, fill, fill}; + for (npy_uint64 i = 0; i < nlane; ++i) { + data[i] = ptr[stride*i]; + } + return npyv_loada_s32(data); +#else + __m128i vfill = npyv_setall_s32(fill); + #ifndef NPY_HAVE_SSE41 + const short *wptr = (const short*)ptr; + #endif + switch(nlane) { + #ifdef NPY_HAVE_SSE41 + case 3: + vfill = _mm_insert_epi32(vfill, ptr[stride*2], 2); + case 2: + vfill = _mm_insert_epi32(vfill, ptr[stride], 1); + case 1: + vfill = _mm_insert_epi32(vfill, ptr[0], 0); + break; + #else + case 3: + vfill = _mm_unpacklo_epi32(_mm_cvtsi32_si128(ptr[stride*2]), vfill); + case 2: + vfill = _mm_unpacklo_epi64(_mm_unpacklo_epi32( + _mm_cvtsi32_si128(*ptr), _mm_cvtsi32_si128(ptr[stride]) + ), vfill); + break; + case 1: + vfill = _mm_insert_epi16(vfill, wptr[0], 0); + vfill = _mm_insert_epi16(vfill, wptr[1], 1); + break; + #endif // NPY_HAVE_SSE41 + default: + return npyv_loadn_s32(ptr, stride); + } // switch + return vfill; +#endif +} +// fill zero to rest lanes +NPY_FINLINE npyv_s32 +npyv_loadn_tillz_s32(const npy_int32 *ptr, npy_intp stride, npy_uintp nlane) +{ + assert(nlane > 0); + switch(nlane) { + case 1: + return _mm_cvtsi32_si128(ptr[0]); + case 2:; + npyv_s32 a = _mm_cvtsi32_si128(ptr[0]); +#ifdef NPY_HAVE_SSE41 + return _mm_insert_epi32(a, ptr[stride], 1); +#else + return _mm_unpacklo_epi32(a, _mm_cvtsi32_si128(ptr[stride])); +#endif // NPY_HAVE_SSE41 + case 3:; + a = _mm_cvtsi32_si128(ptr[0]); +#ifdef NPY_HAVE_SSE41 + a = _mm_insert_epi32(a, ptr[stride], 1); + a = _mm_insert_epi32(a, ptr[stride*2], 2); + return a; +#else + a = _mm_unpacklo_epi32(a, _mm_cvtsi32_si128(ptr[stride])); + a = _mm_unpacklo_epi64(a, _mm_cvtsi32_si128(ptr[stride*2])); + return a; +#endif // NPY_HAVE_SSE41 + default: + return npyv_loadn_s32(ptr, stride); + } +} +//// 64 +NPY_FINLINE npyv_s64 +npyv_loadn_till_s64(const npy_int64 *ptr, npy_intp stride, npy_uintp nlane, npy_int64 fill) +{ + assert(nlane > 0); +#ifdef NPYV__CLANG_ZEROUPPER + if (nlane <= 2) { + npy_int64 NPY_DECL_ALIGNED(16) data[2] = {fill, fill}; + for (npy_uint64 i = 0; i < nlane; ++i) { + data[i] = ptr[i*stride]; + } + return npyv_loada_s64(data); + } +#else + if (nlane == 1) { + const __m128i vfill = npyv_setall_s64(fill); + return _mm_castpd_si128( + _mm_loadl_pd(_mm_castsi128_pd(vfill), (double*)ptr) + ); + } +#endif + return npyv_loadn_s64(ptr, stride); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s64 npyv_loadn_tillz_s64(const npy_int64 *ptr, npy_intp stride, npy_uintp nlane) +{ + assert(nlane > 0); + if (nlane == 1) { + return _mm_loadl_epi64((const __m128i*)ptr); + } + return npyv_loadn_s64(ptr, stride); +} +/********************************* + * Partial store + *********************************/ +//// 32 +NPY_FINLINE void npyv_store_till_s32(npy_int32 *ptr, npy_uintp nlane, npyv_s32 a) +{ + assert(nlane > 0); + switch(nlane) { + case 1: + *ptr = _mm_cvtsi128_si32(a); + break; + case 2: + _mm_storel_epi64((__m128i *)ptr, a); + break; + case 3: + _mm_storel_epi64((__m128i *)ptr, a); + #ifdef NPY_HAVE_SSE41 + ptr[2] = _mm_extract_epi32(a, 2); + #else + ptr[2] = _mm_cvtsi128_si32(_mm_shuffle_epi32(a, _MM_SHUFFLE(0, 0, 0, 2))); + #endif + break; + default: + npyv_store_s32(ptr, a); + } +} +//// 64 +NPY_FINLINE void npyv_store_till_s64(npy_int64 *ptr, npy_uintp nlane, npyv_s64 a) +{ + assert(nlane > 0); + if (nlane == 1) { + _mm_storel_epi64((__m128i *)ptr, a); + return; + } + npyv_store_s64(ptr, a); +} +/********************************* + * Non-contiguous partial store + *********************************/ +//// 32 +NPY_FINLINE void npyv_storen_till_s32(npy_int32 *ptr, npy_intp stride, npy_uintp nlane, npyv_s32 a) +{ + assert(nlane > 0); + switch(nlane) { +#ifdef NPY_HAVE_SSE41 + default: + ptr[stride*3] = _mm_extract_epi32(a, 3); + case 3: + ptr[stride*2] = _mm_extract_epi32(a, 2); + case 2: + ptr[stride*1] = _mm_extract_epi32(a, 1); +#else + default: + ptr[stride*3] = _mm_cvtsi128_si32(_mm_shuffle_epi32(a, _MM_SHUFFLE(0, 0, 0, 3))); + case 3: + ptr[stride*2] = _mm_cvtsi128_si32(_mm_shuffle_epi32(a, _MM_SHUFFLE(0, 0, 0, 2))); + case 2: + ptr[stride*1] = _mm_cvtsi128_si32(_mm_shuffle_epi32(a, _MM_SHUFFLE(0, 0, 0, 1))); +#endif + case 1: + ptr[stride*0] = _mm_cvtsi128_si32(a); + break; + } +} +//// 64 +NPY_FINLINE void npyv_storen_till_s64(npy_int64 *ptr, npy_intp stride, npy_uintp nlane, npyv_s64 a) +{ + assert(nlane > 0); + if (nlane == 1) { + _mm_storel_epi64((__m128i *)ptr, a); + return; + } + npyv_storen_s64(ptr, stride, a); +} +/***************************************************************** + * Implement partial load/store for u32/f32/u64/f64... via casting + *****************************************************************/ +#define NPYV_IMPL_SSE_REST_PARTIAL_TYPES(F_SFX, T_SFX) \ + NPY_FINLINE npyv_##F_SFX npyv_load_till_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_uintp nlane, npyv_lanetype_##F_SFX fill) \ + { \ + union { \ + npyv_lanetype_##F_SFX from_##F_SFX; \ + npyv_lanetype_##T_SFX to_##T_SFX; \ + } pun = {.from_##F_SFX = fill}; \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_load_till_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, nlane, pun.to_##T_SFX \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_loadn_till_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane, \ + npyv_lanetype_##F_SFX fill) \ + { \ + union { \ + npyv_lanetype_##F_SFX from_##F_SFX; \ + npyv_lanetype_##T_SFX to_##T_SFX; \ + } pun = {.from_##F_SFX = fill}; \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_loadn_till_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, stride, nlane, pun.to_##T_SFX \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_load_tillz_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_uintp nlane) \ + { \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_load_tillz_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, nlane \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_loadn_tillz_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane) \ + { \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_loadn_tillz_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, stride, nlane \ + )); \ + } \ + NPY_FINLINE void npyv_store_till_##F_SFX \ + (npyv_lanetype_##F_SFX *ptr, npy_uintp nlane, npyv_##F_SFX a) \ + { \ + npyv_store_till_##T_SFX( \ + (npyv_lanetype_##T_SFX *)ptr, nlane, \ + npyv_reinterpret_##T_SFX##_##F_SFX(a) \ + ); \ + } \ + NPY_FINLINE void npyv_storen_till_##F_SFX \ + (npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane, npyv_##F_SFX a) \ + { \ + npyv_storen_till_##T_SFX( \ + (npyv_lanetype_##T_SFX *)ptr, stride, nlane, \ + npyv_reinterpret_##T_SFX##_##F_SFX(a) \ + ); \ + } + +NPYV_IMPL_SSE_REST_PARTIAL_TYPES(u32, s32) +NPYV_IMPL_SSE_REST_PARTIAL_TYPES(f32, s32) +NPYV_IMPL_SSE_REST_PARTIAL_TYPES(u64, s64) +NPYV_IMPL_SSE_REST_PARTIAL_TYPES(f64, s64) #endif // _NPY_SIMD_SSE_MEMORY_H diff --git a/numpy/core/src/common/simd/vsx/memory.h b/numpy/core/src/common/simd/vsx/memory.h index e0d908bf906b..08a0a9276cc6 100644 --- a/numpy/core/src/common/simd/vsx/memory.h +++ b/numpy/core/src/common/simd/vsx/memory.h @@ -4,147 +4,343 @@ #ifndef _NPY_SIMD_VSX_MEMORY_H #define _NPY_SIMD_VSX_MEMORY_H + +#include "misc.h" + /**************************** - * load/store + * Private utilities ****************************/ // TODO: test load by cast #define VSX__CAST_lOAD 0 #if VSX__CAST_lOAD - #define npyv__load(PTR, T_VEC) (*((T_VEC*)(PTR))) + #define npyv__load(T_VEC, PTR) (*((T_VEC*)(PTR))) #else /** * CLANG fails to load unaligned addresses via vec_xl, vec_xst * so we failback to vec_vsx_ld, vec_vsx_st */ #if (defined(__GNUC__) && !defined(vec_xl)) || (defined(__clang__) && !defined(__IBMC__)) - #define npyv__load(PTR, T_VEC) vec_vsx_ld(0, PTR) + #define npyv__load(T_VEC, PTR) vec_vsx_ld(0, PTR) #else - #define npyv__load(PTR, T_VEC) vec_xl(0, PTR) + #define npyv__load(T_VEC, PTR) vec_xl(0, PTR) #endif #endif -// unaligned load -#define npyv_load_u8(PTR) npyv__load(PTR, npyv_u8) -#define npyv_load_s8(PTR) npyv__load(PTR, npyv_s8) -#define npyv_load_u16(PTR) npyv__load(PTR, npyv_u16) -#define npyv_load_s16(PTR) npyv__load(PTR, npyv_s16) -#define npyv_load_u32(PTR) npyv__load(PTR, npyv_u32) -#define npyv_load_s32(PTR) npyv__load(PTR, npyv_s32) -#define npyv_load_f32(PTR) npyv__load(PTR, npyv_f32) -#define npyv_load_f64(PTR) npyv__load(PTR, npyv_f64) -#if VSX__CAST_lOAD - #define npyv_load_u64(PTR) npyv__load(PTR, npyv_u64) - #define npyv_load_s64(PTR) npyv__load(PTR, npyv_s64) +// unaligned store +#if (defined(__GNUC__) && !defined(vec_xl)) || (defined(__clang__) && !defined(__IBMC__)) + #define npyv__store(PTR, VEC) vec_vsx_st(VEC, 0, PTR) #else - #define npyv_load_u64(PTR) ((npyv_u64)npyv_load_u32((const unsigned int*)PTR)) - #define npyv_load_s64(PTR) ((npyv_s64)npyv_load_s32((const unsigned int*)PTR)) + #define npyv__store(PTR, VEC) vec_xst(VEC, 0, PTR) #endif -// aligned load -#define npyv_loada_u8(PTR) vec_ld(0, PTR) -#define npyv_loada_s8 npyv_loada_u8 -#define npyv_loada_u16 npyv_loada_u8 -#define npyv_loada_s16 npyv_loada_u8 -#define npyv_loada_u32 npyv_loada_u8 -#define npyv_loada_s32 npyv_loada_u8 -#define npyv_loada_u64 npyv_load_u64 -#define npyv_loada_s64 npyv_load_s64 -#define npyv_loada_f32 npyv_loada_u8 -#define npyv_loada_f64 npyv_load_f64 -// stream load -#define npyv_loads_u8 npyv_loada_u8 -#define npyv_loads_s8 npyv_loada_s8 -#define npyv_loads_u16 npyv_loada_u16 -#define npyv_loads_s16 npyv_loada_s16 -#define npyv_loads_u32 npyv_loada_u32 -#define npyv_loads_s32 npyv_loada_s32 -#define npyv_loads_u64 npyv_loada_u64 -#define npyv_loads_s64 npyv_loada_s64 -#define npyv_loads_f32 npyv_loada_f32 -#define npyv_loads_f64 npyv_loada_f64 -// load lower part + // avoid aliasing rules #ifdef __cplusplus template - NPY_FINLINE npy_uint64 *npyv__ptr2u64(T_PTR *ptr) - { return npy_uint64 *ptr64 = (npy_uint64*)ptr; return ptr; } + NPY_FINLINE npy_uint64 *npyv__ptr2u64(const T_PTR *ptr) + { npy_uint64 *ptr64 = (npy_uint64*)ptr; return ptr64; } #else - NPY_FINLINE npy_uint64 *npyv__ptr2u64(void *ptr) - { npy_uint64 *ptr64 = ptr; return ptr64; } + NPY_FINLINE npy_uint64 *npyv__ptr2u64(const void *ptr) + { npy_uint64 *ptr64 = (npy_uint64*)ptr; return ptr64; } #endif // __cplusplus -#if defined(__clang__) && !defined(__IBMC__) - // vec_promote doesn't support doubleword on clang - #define npyv_loadl_u64(PTR) npyv_setall_u64(*npyv__ptr2u64(PTR)) -#else - #define npyv_loadl_u64(PTR) vec_promote(*npyv__ptr2u64(PTR), 0) -#endif -#define npyv_loadl_u8(PTR) ((npyv_u8)npyv_loadl_u64(PTR)) -#define npyv_loadl_s8(PTR) ((npyv_s8)npyv_loadl_u64(PTR)) -#define npyv_loadl_u16(PTR) ((npyv_u16)npyv_loadl_u64(PTR)) -#define npyv_loadl_s16(PTR) ((npyv_s16)npyv_loadl_u64(PTR)) -#define npyv_loadl_u32(PTR) ((npyv_u32)npyv_loadl_u64(PTR)) -#define npyv_loadl_s32(PTR) ((npyv_s32)npyv_loadl_u64(PTR)) -#define npyv_loadl_s64(PTR) ((npyv_s64)npyv_loadl_u64(PTR)) -#define npyv_loadl_f32(PTR) ((npyv_f32)npyv_loadl_u64(PTR)) -#define npyv_loadl_f64(PTR) ((npyv_f64)npyv_loadl_u64(PTR)) -// unaligned store -#if (defined(__GNUC__) && !defined(vec_xl)) || (defined(__clang__) && !defined(__IBMC__)) - #define npyv_store_u8(PTR, VEC) vec_vsx_st(VEC, 0, PTR) -#else - #define npyv_store_u8(PTR, VEC) vec_xst(VEC, 0, PTR) -#endif -#define npyv_store_s8 npyv_store_u8 -#define npyv_store_u16 npyv_store_u8 -#define npyv_store_s16 npyv_store_u8 -#define npyv_store_u32 npyv_store_u8 -#define npyv_store_s32 npyv_store_u8 -#define npyv_store_u64(PTR, VEC) npyv_store_u8((unsigned int*)PTR, (npyv_u32)VEC) -#define npyv_store_s64(PTR, VEC) npyv_store_u8((unsigned int*)PTR, (npyv_u32)VEC) -#define npyv_store_f32 npyv_store_u8 -#define npyv_store_f64 npyv_store_u8 -// aligned store -#define npyv_storea_u8(PTR, VEC) vec_st(VEC, 0, PTR) -#define npyv_storea_s8 npyv_storea_u8 -#define npyv_storea_u16 npyv_storea_u8 -#define npyv_storea_s16 npyv_storea_u8 -#define npyv_storea_u32 npyv_storea_u8 -#define npyv_storea_s32 npyv_storea_u8 -#define npyv_storea_u64 npyv_store_u64 -#define npyv_storea_s64 npyv_store_s64 -#define npyv_storea_f32 npyv_storea_u8 -#define npyv_storea_f64 npyv_store_f64 -// stream store -#define npyv_stores_u8 npyv_storea_u8 -#define npyv_stores_s8 npyv_storea_s8 -#define npyv_stores_u16 npyv_storea_u16 -#define npyv_stores_s16 npyv_storea_s16 -#define npyv_stores_u32 npyv_storea_u32 -#define npyv_stores_s32 npyv_storea_s32 -#define npyv_stores_u64 npyv_storea_u64 -#define npyv_stores_s64 npyv_storea_s64 -#define npyv_stores_f32 npyv_storea_f32 -#define npyv_stores_f64 npyv_storea_f64 + +// load lower part +NPY_FINLINE npyv_u64 npyv__loadl(const void *ptr) +{ + #if defined(__clang__) && !defined(__IBMC__) + // vec_promote doesn't support doubleword on clang + return npyv_setall_u64(*npyv__ptr2u64(ptr)); + #else + return vec_promote(*npyv__ptr2u64(ptr), 0); + #endif +} // store lower part -#define npyv_storel_u8(PTR, VEC) \ +#define npyv__storel(PTR, VEC) \ *npyv__ptr2u64(PTR) = vec_extract(((npyv_u64)VEC), 0) -#define npyv_storel_s8 npyv_storel_u8 -#define npyv_storel_u16 npyv_storel_u8 -#define npyv_storel_s16 npyv_storel_u8 -#define npyv_storel_u32 npyv_storel_u8 -#define npyv_storel_s32 npyv_storel_u8 -#define npyv_storel_s64 npyv_storel_u8 -#define npyv_storel_u64 npyv_storel_u8 -#define npyv_storel_f32 npyv_storel_u8 -#define npyv_storel_f64 npyv_storel_u8 -// store higher part -#define npyv_storeh_u8(PTR, VEC) \ + +#define npyv__storeh(PTR, VEC) \ *npyv__ptr2u64(PTR) = vec_extract(((npyv_u64)VEC), 1) -#define npyv_storeh_s8 npyv_storeh_u8 -#define npyv_storeh_u16 npyv_storeh_u8 -#define npyv_storeh_s16 npyv_storeh_u8 -#define npyv_storeh_u32 npyv_storeh_u8 -#define npyv_storeh_s32 npyv_storeh_u8 -#define npyv_storeh_s64 npyv_storeh_u8 -#define npyv_storeh_u64 npyv_storeh_u8 -#define npyv_storeh_f32 npyv_storeh_u8 -#define npyv_storeh_f64 npyv_storeh_u8 + +/**************************** + * load/store + ****************************/ +#define NPYV_IMPL_VSX_MEM(SFX, DW_CAST) \ + NPY_FINLINE npyv_##SFX npyv_load_##SFX(const npyv_lanetype_##SFX *ptr) \ + { return (npyv_##SFX)npyv__load(npyv_##SFX, (const npyv_lanetype_##DW_CAST*)ptr); } \ + NPY_FINLINE npyv_##SFX npyv_loada_##SFX(const npyv_lanetype_##SFX *ptr) \ + { return (npyv_##SFX)vec_ld(0, (const npyv_lanetype_u32*)ptr); } \ + NPY_FINLINE npyv_##SFX npyv_loads_##SFX(const npyv_lanetype_##SFX *ptr) \ + { return npyv_loada_##SFX(ptr); } \ + NPY_FINLINE npyv_##SFX npyv_loadl_##SFX(const npyv_lanetype_##SFX *ptr) \ + { return (npyv_##SFX)npyv__loadl(ptr); } \ + NPY_FINLINE void npyv_store_##SFX(npyv_lanetype_##SFX *ptr, npyv_##SFX vec) \ + { npyv__store((npyv_lanetype_##DW_CAST*)ptr, (npyv_##DW_CAST)vec); } \ + NPY_FINLINE void npyv_storea_##SFX(npyv_lanetype_##SFX *ptr, npyv_##SFX vec) \ + { vec_st((npyv_u32)vec, 0, (npyv_lanetype_u32*)ptr); } \ + NPY_FINLINE void npyv_stores_##SFX(npyv_lanetype_##SFX *ptr, npyv_##SFX vec) \ + { npyv_storea_##SFX(ptr, vec); } \ + NPY_FINLINE void npyv_storel_##SFX(npyv_lanetype_##SFX *ptr, npyv_##SFX vec) \ + { npyv__storel(ptr, vec); } \ + NPY_FINLINE void npyv_storeh_##SFX(npyv_lanetype_##SFX *ptr, npyv_##SFX vec) \ + { npyv__storeh(ptr, vec); } + +NPYV_IMPL_VSX_MEM(u8, u8) +NPYV_IMPL_VSX_MEM(s8, s8) +NPYV_IMPL_VSX_MEM(u16, u16) +NPYV_IMPL_VSX_MEM(s16, s16) +NPYV_IMPL_VSX_MEM(u32, u32) +NPYV_IMPL_VSX_MEM(s32, s32) +NPYV_IMPL_VSX_MEM(u64, f64) +NPYV_IMPL_VSX_MEM(s64, f64) +NPYV_IMPL_VSX_MEM(f32, f32) +NPYV_IMPL_VSX_MEM(f64, f64) + +/*************************** + * Non-contiguous Load + ***************************/ +//// 32 +NPY_FINLINE npyv_u32 npyv_loadn_u32(const npy_uint32 *ptr, npy_intp stride) +{ + return npyv_set_u32( + ptr[stride * 0], ptr[stride * 1], + ptr[stride * 2], ptr[stride * 3] + ); +} +NPY_FINLINE npyv_s32 npyv_loadn_s32(const npy_int32 *ptr, npy_intp stride) +{ return (npyv_s32)npyv_loadn_u32((const npy_uint32*)ptr, stride); } +NPY_FINLINE npyv_f32 npyv_loadn_f32(const float *ptr, npy_intp stride) +{ return (npyv_f32)npyv_loadn_u32((const npy_uint32*)ptr, stride); } +//// 64 +NPY_FINLINE npyv_u64 npyv_loadn_u64(const npy_uint64 *ptr, npy_intp stride) +{ return npyv_set_u64(ptr[0], ptr[stride]); } +NPY_FINLINE npyv_s64 npyv_loadn_s64(const npy_int64 *ptr, npy_intp stride) +{ return npyv_set_s64(ptr[0], ptr[stride]); } +NPY_FINLINE npyv_f64 npyv_loadn_f64(const double *ptr, npy_intp stride) +{ return npyv_set_f64(ptr[0], ptr[stride]); } +/*************************** + * Non-contiguous Store + ***************************/ +//// 32 +NPY_FINLINE void npyv_storen_u32(npy_uint32 *ptr, npy_intp stride, npyv_u32 a) +{ + ptr[stride * 0] = vec_extract(a, 0); + ptr[stride * 1] = vec_extract(a, 1); + ptr[stride * 2] = vec_extract(a, 2); + ptr[stride * 3] = vec_extract(a, 3); +} +NPY_FINLINE void npyv_storen_s32(npy_int32 *ptr, npy_intp stride, npyv_s32 a) +{ npyv_storen_u32((npy_uint32*)ptr, stride, (npyv_u32)a); } +NPY_FINLINE void npyv_storen_f32(float *ptr, npy_intp stride, npyv_f32 a) +{ npyv_storen_u32((npy_uint32*)ptr, stride, (npyv_u32)a); } +//// 64 +NPY_FINLINE void npyv_storen_u64(npy_uint64 *ptr, npy_intp stride, npyv_u64 a) +{ + ptr[stride * 0] = vec_extract(a, 0); + ptr[stride * 1] = vec_extract(a, 1); +} +NPY_FINLINE void npyv_storen_s64(npy_int64 *ptr, npy_intp stride, npyv_s64 a) +{ npyv_storen_u64((npy_uint64*)ptr, stride, (npyv_u64)a); } +NPY_FINLINE void npyv_storen_f64(double *ptr, npy_intp stride, npyv_f64 a) +{ npyv_storen_u64((npy_uint64*)ptr, stride, (npyv_u64)a); } + +/********************************* + * Partial Load + *********************************/ +//// 32 +NPY_FINLINE npyv_s32 npyv_load_till_s32(const npy_int32 *ptr, npy_uintp nlane, npy_int32 fill) +{ + assert(nlane > 0); + npyv_s32 vfill = npyv_setall_s32(fill); + switch(nlane) { + case 1: + return vec_insert(ptr[0], vfill, 0); + case 2: + return (npyv_s32)vec_insert( + *npyv__ptr2u64(ptr), (npyv_u64)vfill, 0 + ); + case 3: + vfill = vec_insert(ptr[2], vfill, 2); + return (npyv_s32)vec_insert( + *npyv__ptr2u64(ptr), (npyv_u64)vfill, 0 + ); + default: + return npyv_load_s32(ptr); + } +} +// fill zero to rest lanes +NPY_FINLINE npyv_s32 npyv_load_tillz_s32(const npy_int32 *ptr, npy_uintp nlane) +{ return npyv_load_till_s32(ptr, nlane, 0); } +//// 64 +NPY_FINLINE npyv_s64 npyv_load_till_s64(const npy_int64 *ptr, npy_uintp nlane, npy_int64 fill) +{ + assert(nlane > 0); + if (nlane == 1) { + return npyv_set_s64(ptr[0], fill); + } + return npyv_load_s64(ptr); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s64 npyv_load_tillz_s64(const npy_int64 *ptr, npy_uintp nlane) +{ return npyv_load_till_s64(ptr, nlane, 0); } +/********************************* + * Non-contiguous partial load + *********************************/ +//// 32 +NPY_FINLINE npyv_s32 +npyv_loadn_till_s32(const npy_int32 *ptr, npy_intp stride, npy_uintp nlane, npy_int32 fill) +{ + assert(nlane > 0); + npyv_s32 vfill = npyv_setall_s32(fill); + switch(nlane) { + case 3: + vfill = vec_insert(ptr[stride*2], vfill, 2); + case 2: + vfill = vec_insert(ptr[stride], vfill, 1); + case 1: + vfill = vec_insert(*ptr, vfill, 0); + break; + default: + return npyv_loadn_s32(ptr, stride); + } // switch + return vfill; +} +// fill zero to rest lanes +NPY_FINLINE npyv_s32 +npyv_loadn_tillz_s32(const npy_int32 *ptr, npy_intp stride, npy_uintp nlane) +{ return npyv_loadn_till_s32(ptr, stride, nlane, 0); } +//// 64 +NPY_FINLINE npyv_s64 +npyv_loadn_till_s64(const npy_int64 *ptr, npy_intp stride, npy_uintp nlane, npy_int64 fill) +{ + assert(nlane > 0); + if (nlane == 1) { + return npyv_set_s64(*ptr, fill); + } + return npyv_loadn_s64(ptr, stride); +} +// fill zero to rest lanes +NPY_FINLINE npyv_s64 npyv_loadn_tillz_s64(const npy_int64 *ptr, npy_intp stride, npy_uintp nlane) +{ return npyv_loadn_till_s64(ptr, stride, nlane, 0); } +/********************************* + * Partial store + *********************************/ +//// 32 +NPY_FINLINE void npyv_store_till_s32(npy_int32 *ptr, npy_uintp nlane, npyv_s32 a) +{ + assert(nlane > 0); + switch(nlane) { + case 1: + *ptr = vec_extract(a, 0); + break; + case 2: + npyv_storel_s32(ptr, a); + break; + case 3: + npyv_storel_s32(ptr, a); + ptr[2] = vec_extract(a, 2); + break; + default: + npyv_store_s32(ptr, a); + } +} +//// 64 +NPY_FINLINE void npyv_store_till_s64(npy_int64 *ptr, npy_uintp nlane, npyv_s64 a) +{ + assert(nlane > 0); + if (nlane == 1) { + npyv_storel_s64(ptr, a); + return; + } + npyv_store_s64(ptr, a); +} +/********************************* + * Non-contiguous partial store + *********************************/ +//// 32 +NPY_FINLINE void npyv_storen_till_s32(npy_int32 *ptr, npy_intp stride, npy_uintp nlane, npyv_s32 a) +{ + assert(nlane > 0); + switch(nlane) { + default: + ptr[stride*3] = vec_extract(a, 3); + case 3: + ptr[stride*2] = vec_extract(a, 2); + case 2: + ptr[stride*1] = vec_extract(a, 1); + case 1: + ptr[stride*0] = vec_extract(a, 0); + break; + } +} +//// 64 +NPY_FINLINE void npyv_storen_till_s64(npy_int64 *ptr, npy_intp stride, npy_uintp nlane, npyv_s64 a) +{ + assert(nlane > 0); + if (nlane == 1) { + npyv_storel_s64(ptr, a); + return; + } + npyv_storen_s64(ptr, stride, a); +} +/***************************************************************** + * Implement partial load/store for u32/f32/u64/f64... via casting + *****************************************************************/ +#define NPYV_IMPL_VSX_REST_PARTIAL_TYPES(F_SFX, T_SFX) \ + NPY_FINLINE npyv_##F_SFX npyv_load_till_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_uintp nlane, npyv_lanetype_##F_SFX fill) \ + { \ + union { \ + npyv_lanetype_##F_SFX from_##F_SFX; \ + npyv_lanetype_##T_SFX to_##T_SFX; \ + } pun = {.from_##F_SFX = fill}; \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_load_till_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, nlane, pun.to_##T_SFX \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_loadn_till_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane, \ + npyv_lanetype_##F_SFX fill) \ + { \ + union { \ + npyv_lanetype_##F_SFX from_##F_SFX; \ + npyv_lanetype_##T_SFX to_##T_SFX; \ + } pun = {.from_##F_SFX = fill}; \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_loadn_till_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, stride, nlane, pun.to_##T_SFX \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_load_tillz_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_uintp nlane) \ + { \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_load_tillz_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, nlane \ + )); \ + } \ + NPY_FINLINE npyv_##F_SFX npyv_loadn_tillz_##F_SFX \ + (const npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane) \ + { \ + return npyv_reinterpret_##F_SFX##_##T_SFX(npyv_loadn_tillz_##T_SFX( \ + (const npyv_lanetype_##T_SFX *)ptr, stride, nlane \ + )); \ + } \ + NPY_FINLINE void npyv_store_till_##F_SFX \ + (npyv_lanetype_##F_SFX *ptr, npy_uintp nlane, npyv_##F_SFX a) \ + { \ + npyv_store_till_##T_SFX( \ + (npyv_lanetype_##T_SFX *)ptr, nlane, \ + npyv_reinterpret_##T_SFX##_##F_SFX(a) \ + ); \ + } \ + NPY_FINLINE void npyv_storen_till_##F_SFX \ + (npyv_lanetype_##F_SFX *ptr, npy_intp stride, npy_uintp nlane, npyv_##F_SFX a) \ + { \ + npyv_storen_till_##T_SFX( \ + (npyv_lanetype_##T_SFX *)ptr, stride, nlane, \ + npyv_reinterpret_##T_SFX##_##F_SFX(a) \ + ); \ + } + +NPYV_IMPL_VSX_REST_PARTIAL_TYPES(u32, s32) +NPYV_IMPL_VSX_REST_PARTIAL_TYPES(f32, s32) +NPYV_IMPL_VSX_REST_PARTIAL_TYPES(u64, s64) +NPYV_IMPL_VSX_REST_PARTIAL_TYPES(f64, s64) #endif // _NPY_SIMD_VSX_MEMORY_H From 2fc25ea5c13667c9f09912815a2bdf9c326cf3c8 Mon Sep 17 00:00:00 2001 From: Royston E Tauro <54945757+lucasace@users.noreply.github.com> Date: Fri, 9 Oct 2020 08:07:57 +0530 Subject: [PATCH 232/409] [CODE] Made code cleaner Co-authored-by: Bas van Beek <43369155+BvB93@users.noreply.github.com> --- numpy/__init__.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 1fbdb259b2a1..aacae14309c2 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -58,7 +58,7 @@ from typing import ( ) if sys.version_info >= (3, 8): - from typing import Literal, Protocol, SupportsIndex,Final + from typing import Literal, Protocol, SupportsIndex, Final else: from typing_extensions import Literal, Protocol, Final class SupportsIndex(Protocol): From 3f56ca70e286d8d7eacbe997dbcc5113a1c091b3 Mon Sep 17 00:00:00 2001 From: Royston E Tauro <54945757+lucasace@users.noreply.github.com> Date: Fri, 9 Oct 2020 08:08:39 +0530 Subject: [PATCH 233/409] [CODE]Redundant spaces removed Co-authored-by: Bas van Beek <43369155+BvB93@users.noreply.github.com> --- numpy/__init__.pyi | 1 - 1 file changed, 1 deletion(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index aacae14309c2..0bb44ed20e01 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -1611,7 +1611,6 @@ inf: Final[float] infty: Final[float] nan: Final[float] pi: Final[float] - ALLOW_THREADS: Final[int] BUFSIZE: Final[int] CLIP: Final[int] From 55ed4981998ce78496e30752192a9732c3570888 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Fri, 9 Oct 2020 10:37:46 -0600 Subject: [PATCH 234/409] BUG: Fix indentation. --- numpy/distutils/fcompiler/nv.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/numpy/distutils/fcompiler/nv.py b/numpy/distutils/fcompiler/nv.py index c5f203ee8ddb..8e9f1683558a 100644 --- a/numpy/distutils/fcompiler/nv.py +++ b/numpy/distutils/fcompiler/nv.py @@ -5,14 +5,14 @@ compilers = ['NVHPCFCompiler'] class NVHPCFCompiler(FCompiler): - """ NVIDIA High Performance Computing (HPC) SDK Fortran Compiler + """ NVIDIA High Performance Computing (HPC) SDK Fortran Compiler - https://developer.nvidia.com/hpc-sdk + https://developer.nvidia.com/hpc-sdk - Since august 2020 the NVIDIA HPC SDK includes the compilers formerly known as The Portland Group compilers, - https://www.pgroup.com/index.htm. - See also `numpy.distutils.fcompiler.pg`. - """ + Since august 2020 the NVIDIA HPC SDK includes the compilers formerly known as The Portland Group compilers, + https://www.pgroup.com/index.htm. + See also `numpy.distutils.fcompiler.pg`. + """ compiler_type = 'nv' description = 'NVIDIA HPC SDK' From 73c555fd5ef964896113dc40b09b21fd381085b2 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Fri, 9 Oct 2020 14:25:44 -0400 Subject: [PATCH 235/409] DOC: Display real license on license page Changes license.html to include LICENSE.txt, the real license file. Was showing an out-of-date copy. --- doc/source/license.rst | 35 +++-------------------------------- 1 file changed, 3 insertions(+), 32 deletions(-) diff --git a/doc/source/license.rst b/doc/source/license.rst index 8f360af8830e..beea023ce05a 100644 --- a/doc/source/license.rst +++ b/doc/source/license.rst @@ -1,35 +1,6 @@ ************* -NumPy License +NumPy license ************* -Copyright (c) 2005, NumPy Developers - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -* Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - -* Neither the name of the NumPy Developers nor the names of any - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +.. include:: ../../LICENSE.txt + :literal: From 4f8c5cae3c98c2a9e9934d30e26f10dfcd1b1ed1 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Fri, 9 Oct 2020 20:07:40 +0100 Subject: [PATCH 236/409] DOC: Add docstrings for some scalar types --- numpy/core/_add_newdocs_scalars.py | 40 ++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/numpy/core/_add_newdocs_scalars.py b/numpy/core/_add_newdocs_scalars.py index c367c18ed094..113f84eb4ffa 100644 --- a/numpy/core/_add_newdocs_scalars.py +++ b/numpy/core/_add_newdocs_scalars.py @@ -176,6 +176,46 @@ def add_newdoc_for_scalar_type(obj, fixed_aliases, doc): Any Python object. """) +add_newdoc_for_scalar_type('str_', ['str0', 'unicode_'], + r""" + A unicode string. + + Unlike the builtin `str`, this supports the buffer protocol, exposing its + contents as UCS4: + + >>> memoryview(np.str_("abcd")) + b'a\x00\x00\x00b\x00\x00\x00c\x00\x00\x00d\x00\x00\x00' + + When used in array, this type cannot contain trailing nulls. + """) + +add_newdoc_for_scalar_type('bytes_', ['bytes0'], + r""" + A byte string. + + When used in array, this type cannot contain trailing nulls. + """) + +add_newdoc_for_scalar_type('void', ['void0'], + """ + Either an opaque sequence of bytes, or a structure. + """) + +add_newdoc_for_scalar_type('datetime64', [], + """ + A datetime stored as a 64-bit integer, counting from ``1970-01-01T00:00:00``. + + >>> np.datetime64(10, 'Y') + numpy.datetime64('1980') + >>> np.datetime64(10, 'D') + numpy.datetime64('1970-01-11') + """) + +add_newdoc_for_scalar_type('timedelta64', [], + """ + A timedelta stored as a 64-bit integer. + """) + # TODO: work out how to put this on the base class, np.floating for float_name in ('half', 'single', 'double', 'longdouble'): add_newdoc('numpy.core.numerictypes', float_name, ('as_integer_ratio', From 84a4fcb5af5f9c056e82893223944820af35df18 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Fri, 9 Oct 2020 15:27:34 -0400 Subject: [PATCH 237/409] DOC: Fix empty 'C style guide' page (#17503) Add external link to NEP45 in tocs of dev/index Co-authored-by: Ross Barnowski --- doc/source/dev/index.rst | 4 ++-- doc/source/dev/style_guide.rst | 8 -------- 2 files changed, 2 insertions(+), 10 deletions(-) delete mode 100644 doc/source/dev/style_guide.rst diff --git a/doc/source/dev/index.rst b/doc/source/dev/index.rst index 020df0b2bd29..d2c162c9a4ec 100644 --- a/doc/source/dev/index.rst +++ b/doc/source/dev/index.rst @@ -13,7 +13,7 @@ Contributing to NumPy development_environment development_workflow ../benchmarking - style_guide + NumPy C style guide releasing governance/index howto-docs @@ -296,7 +296,7 @@ The rest of the story development_environment development_workflow ../benchmarking - style_guide + NumPy C style guide releasing governance/index howto-docs diff --git a/doc/source/dev/style_guide.rst b/doc/source/dev/style_guide.rst deleted file mode 100644 index bede3424d0af..000000000000 --- a/doc/source/dev/style_guide.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. _style_guide: - -=================== -NumPy C Style Guide -=================== - -.. include:: ../../C_STYLE_GUIDE.rst.txt - :start-line: 4 From 156cd054e007b05d4ac4829e10a369d19dd2b0b1 Mon Sep 17 00:00:00 2001 From: Lisa Schwetlick Date: Fri, 9 Oct 2020 21:35:47 +0200 Subject: [PATCH 238/409] ENH: add dtype option to cov and corrcoef (#17456) Adds a keyword-only dtype parameter to correlate and coerrcoef to allow user to specify the dtype of the output. Co-authored-by: Eric Wieser Co-authored-by: Ross Barnowski --- .../upcoming_changes/17456.new_feature.rst | 5 +++ numpy/lib/function_base.py | 33 ++++++++++++++----- numpy/lib/tests/test_function_base.py | 12 +++++++ 3 files changed, 41 insertions(+), 9 deletions(-) create mode 100644 doc/release/upcoming_changes/17456.new_feature.rst diff --git a/doc/release/upcoming_changes/17456.new_feature.rst b/doc/release/upcoming_changes/17456.new_feature.rst new file mode 100644 index 000000000000..7ab014e77df9 --- /dev/null +++ b/doc/release/upcoming_changes/17456.new_feature.rst @@ -0,0 +1,5 @@ +``dtype`` option for `cov` and `corrcoef` +---------------------------------------------------- +The ``dtype`` option is now available for `numpy.cov` and `numpy.corrcoef`. +It specifies which data-type the returned result should have. +By default the functions still return a `numpy.float64` result. diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index c7ddbdb8de67..984f3086e5b7 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -2268,13 +2268,13 @@ def _vectorize_call_with_signature(self, func, args): def _cov_dispatcher(m, y=None, rowvar=None, bias=None, ddof=None, - fweights=None, aweights=None): + fweights=None, aweights=None, *, dtype=None): return (m, y, fweights, aweights) @array_function_dispatch(_cov_dispatcher) def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None, - aweights=None): + aweights=None, *, dtype=None): """ Estimate a covariance matrix, given data and weights. @@ -2325,6 +2325,11 @@ def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None, weights can be used to assign probabilities to observation vectors. .. versionadded:: 1.10 + dtype : data-type, optional + Data-type of the result. By default, the return data-type will have + at least `numpy.float64` precision. + + .. versionadded:: 1.20 Returns ------- @@ -2400,13 +2405,16 @@ def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None, if m.ndim > 2: raise ValueError("m has more than 2 dimensions") - if y is None: - dtype = np.result_type(m, np.float64) - else: + if y is not None: y = np.asarray(y) if y.ndim > 2: raise ValueError("y has more than 2 dimensions") - dtype = np.result_type(m, y, np.float64) + + if dtype is None: + if y is None: + dtype = np.result_type(m, np.float64) + else: + dtype = np.result_type(m, y, np.float64) X = array(m, ndmin=2, dtype=dtype) if not rowvar and X.shape[0] != 1: @@ -2486,12 +2494,14 @@ def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None, return c.squeeze() -def _corrcoef_dispatcher(x, y=None, rowvar=None, bias=None, ddof=None): +def _corrcoef_dispatcher(x, y=None, rowvar=None, bias=None, ddof=None, *, + dtype=None): return (x, y) @array_function_dispatch(_corrcoef_dispatcher) -def corrcoef(x, y=None, rowvar=True, bias=np._NoValue, ddof=np._NoValue): +def corrcoef(x, y=None, rowvar=True, bias=np._NoValue, ddof=np._NoValue, *, + dtype=None): """ Return Pearson product-moment correlation coefficients. @@ -2525,6 +2535,11 @@ def corrcoef(x, y=None, rowvar=True, bias=np._NoValue, ddof=np._NoValue): Has no effect, do not use. .. deprecated:: 1.10.0 + dtype : data-type, optional + Data-type of the result. By default, the return data-type will have + at least `numpy.float64` precision. + + .. versionadded:: 1.20 Returns ------- @@ -2616,7 +2631,7 @@ def corrcoef(x, y=None, rowvar=True, bias=np._NoValue, ddof=np._NoValue): # 2015-03-15, 1.10 warnings.warn('bias and ddof have no effect and are deprecated', DeprecationWarning, stacklevel=3) - c = cov(x, y, rowvar) + c = cov(x, y, rowvar, dtype=dtype) try: d = diag(c) except ValueError: diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index 7bddb941c5c8..4c7c0480c106 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -2023,6 +2023,12 @@ def test_extreme(self): assert_array_almost_equal(c, np.array([[1., -1.], [-1., 1.]])) assert_(np.all(np.abs(c) <= 1.0)) + @pytest.mark.parametrize("test_type", [np.half, np.single, np.double, np.longdouble]) + def test_corrcoef_dtype(self, test_type): + cast_A = self.A.astype(test_type) + res = corrcoef(cast_A, dtype=test_type) + assert test_type == res.dtype + class TestCov: x1 = np.array([[0, 2], [1, 1], [2, 0]]).T @@ -2123,6 +2129,12 @@ def test_unit_fweights_and_aweights(self): aweights=self.unit_weights), self.res1) + @pytest.mark.parametrize("test_type", [np.half, np.single, np.double, np.longdouble]) + def test_cov_dtype(self, test_type): + cast_x1 = self.x1.astype(test_type) + res = cov(cast_x1, dtype=test_type) + assert test_type == res.dtype + class Test_I0: From c6dc9d66ebc7dca16eb83686e74673668b276440 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Fri, 9 Oct 2020 15:40:49 -0400 Subject: [PATCH 239/409] DOC: Update top links in landing page Same intent as PR #17365 -- point "Installation" links to Hugo, start page with "What is NumPy", let Hugo page link to "Troubleshooting ImportError". --- doc/source/_templates/indexcontent.html | 5 +++-- doc/source/user/index.rst | 11 +++++------ doc/source/user/install.rst | 17 +++++++---------- doc/source/user/troubleshooting-importerror.rst | 4 +++- 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/doc/source/_templates/indexcontent.html b/doc/source/_templates/indexcontent.html index 2dba16ce8a16..6dd6bf9b0851 100644 --- a/doc/source/_templates/indexcontent.html +++ b/doc/source/_templates/indexcontent.html @@ -12,8 +12,9 @@

{{ docstitle|e }}

For users:

-

Setting up
- Learn about what NumPy is and how to install it

+

What is NumPy?
+ Who uses it and why

+

Installation

Quickstart
Aimed at domain experts or people migrating to NumPy

Absolute beginner's guide
diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst index 11a019b489b8..28297d9eaa9b 100644 --- a/doc/source/user/index.rst +++ b/doc/source/user/index.rst @@ -3,18 +3,17 @@ .. _user: ################ -NumPy User Guide +NumPy user guide ################ -This guide is intended as an introductory overview of NumPy and -explains how to install and make use of the most important features of -NumPy. For detailed reference documentation of the functions and -classes contained in the package, see the :ref:`reference`. +This guide is an overview and explains the important features; +details are found in :ref:`reference`. .. toctree:: :maxdepth: 1 - setting-up + whatisnumpy + Installation quickstart absolute_beginners basics diff --git a/doc/source/user/install.rst b/doc/source/user/install.rst index 52586f3d7d6a..e05cee2f130d 100644 --- a/doc/source/user/install.rst +++ b/doc/source/user/install.rst @@ -1,10 +1,7 @@ -**************** -Installing NumPy -**************** - -In most use cases the best way to install NumPy on your system is by using a -pre-built package for your operating system. Please see -https://scipy.org/install.html for links to available options. - -For instructions on building for source package, see -:doc:`building`. This information is useful mainly for advanced users. +:orphan: + +**************** +Installing NumPy +**************** + +See `Installing NumPy `_. \ No newline at end of file diff --git a/doc/source/user/troubleshooting-importerror.rst b/doc/source/user/troubleshooting-importerror.rst index 7d4846f7794f..d124e657a64b 100644 --- a/doc/source/user/troubleshooting-importerror.rst +++ b/doc/source/user/troubleshooting-importerror.rst @@ -1,3 +1,5 @@ +:orphan: + *************************** Troubleshooting ImportError *************************** @@ -69,7 +71,7 @@ or conda. Using Eclipse/PyDev with Anaconda/conda Python (or environments) ---------------------------------------------------------------- -Please see the +Please see the `Anaconda Documentation `_ on how to properly configure Eclipse/PyDev to use Anaconda Python with specific conda environments. From eed58620a2a4ae2bfc8c1d4b6a0c7b878927d7e9 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Fri, 9 Oct 2020 13:25:21 -0700 Subject: [PATCH 240/409] CI: modify circleci merge ref grabbing. Only attempt to pull from refs/pull/xxxx/merge when pr is still active. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d77cde59b85c..11288999c3d5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,7 @@ jobs: steps: - checkout: - - run: git pull --ff-only origin "refs/pull/${CI_PULL_REQUEST//*pull\//}/merge" + - run: [[ ! -z $CI_PULL_REQUEST ]] git pull --ff-only origin "refs/pull/${CI_PULL_REQUEST//*pull\//}/merge" - run: name: create virtual environment, install dependencies From 0485b2395704ba9847a87a159d49bfecbf4ba1b5 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Fri, 9 Oct 2020 13:32:56 -0700 Subject: [PATCH 241/409] Fix bash syntax --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 11288999c3d5..ca75580547fe 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,7 @@ jobs: steps: - checkout: - - run: [[ ! -z $CI_PULL_REQUEST ]] git pull --ff-only origin "refs/pull/${CI_PULL_REQUEST//*pull\//}/merge" + - run: [[ ! -z $CI_PULL_REQUEST ]] && git pull --ff-only origin "refs/pull/${CI_PULL_REQUEST//*pull\//}/merge" - run: name: create virtual environment, install dependencies From 7f1d6358129b38f82d638310754d51ff7db79ab4 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Fri, 9 Oct 2020 13:42:57 -0700 Subject: [PATCH 242/409] Fix yaml parsing error. --- .circleci/config.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ca75580547fe..dc18757d213a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,10 @@ jobs: steps: - checkout: - - run: [[ ! -z $CI_PULL_REQUEST ]] && git pull --ff-only origin "refs/pull/${CI_PULL_REQUEST//*pull\//}/merge" + - run: + name: pull changes from merge + command: | + [[ ! -z $CI_PULL_REQUEST ]] && git pull --ff-only origin "refs/pull/${CI_PULL_REQUEST//*pull\//}/merge" - run: name: create virtual environment, install dependencies From 997d6ec1a13d24da29ead73e89ad55a40ad8e9d3 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Fri, 9 Oct 2020 17:03:55 -0400 Subject: [PATCH 243/409] DOC: Delete unused file in PR #17523 To fix CircleCI fail; missed it in the c6dc9d6 commit. --- doc/source/user/setting-up.rst | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 doc/source/user/setting-up.rst diff --git a/doc/source/user/setting-up.rst b/doc/source/user/setting-up.rst deleted file mode 100644 index 7ca3a365c4e7..000000000000 --- a/doc/source/user/setting-up.rst +++ /dev/null @@ -1,10 +0,0 @@ -********** -Setting up -********** - -.. toctree:: - :maxdepth: 1 - - whatisnumpy - install - troubleshooting-importerror From 9c2ae9e7147abf2d50d4555a580ec539330a3af3 Mon Sep 17 00:00:00 2001 From: takanori-pskq Date: Sat, 10 Oct 2020 10:21:56 +0900 Subject: [PATCH 244/409] DOC: Fix Boolean types in C functions --- doc/source/reference/c-api/array.rst | 12 ++++++------ doc/source/reference/c-api/types-and-structures.rst | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/source/reference/c-api/array.rst b/doc/source/reference/c-api/array.rst index 9fe45d2def92..7360c0985ec9 100644 --- a/doc/source/reference/c-api/array.rst +++ b/doc/source/reference/c-api/array.rst @@ -999,7 +999,7 @@ argument must be a :c:type:`PyObject *` that can be directly interpret Evaluates true if the data area of the ndarray *m* is **not** in machine byte-order according to the array's data-type descriptor. -.. c:function:: Bool PyArray_EquivTypes( \ +.. c:function:: npy_bool PyArray_EquivTypes( \ PyArray_Descr* type1, PyArray_Descr* type2) Return :c:data:`NPY_TRUE` if *type1* and *type2* actually represent @@ -1008,13 +1008,13 @@ argument must be a :c:type:`PyObject *` that can be directly interpret :c:data:`NPY_LONG` and :c:data:`NPY_INT` are equivalent. Otherwise return :c:data:`NPY_FALSE`. -.. c:function:: Bool PyArray_EquivArrTypes( \ +.. c:function:: npy_bool PyArray_EquivArrTypes( \ PyArrayObject* a1, PyArrayObject * a2) Return :c:data:`NPY_TRUE` if *a1* and *a2* are arrays with equivalent types for this platform. -.. c:function:: Bool PyArray_EquivTypenums(int typenum1, int typenum2) +.. c:function:: npy_bool PyArray_EquivTypenums(int typenum1, int typenum2) Special case of :c:func:`PyArray_EquivTypes` (...) that does not accept flexible data types but may be easier to call. @@ -1650,7 +1650,7 @@ Conversion destination must be an integer multiple of the number of elements in *val*. -.. c:function:: PyObject* PyArray_Byteswap(PyArrayObject* self, Bool inplace) +.. c:function:: PyObject* PyArray_Byteswap(PyArrayObject* self, npy_bool inplace) Equivalent to :meth:`ndarray.byteswap` (*self*, *inplace*). Return an array whose data area is byteswapped. If *inplace* is non-zero, then do @@ -2309,7 +2309,7 @@ Array Functions Other functions ^^^^^^^^^^^^^^^ -.. c:function:: Bool PyArray_CheckStrides( \ +.. c:function:: npy_bool PyArray_CheckStrides( \ int elsize, int nd, npy_intp numbytes, npy_intp const* dims, \ npy_intp const* newstrides) @@ -2959,7 +2959,7 @@ to. :c:data:`NPY_MAXDIMS` which is interpreted correctly by the C-API functions that take axis arguments. -.. c:function:: int PyArray_BoolConverter(PyObject* obj, Bool* value) +.. c:function:: int PyArray_BoolConverter(PyObject* obj, npy_bool* value) Convert any Python object, *obj*, to :c:data:`NPY_TRUE` or :c:data:`NPY_FALSE`, and place the result in *value*. diff --git a/doc/source/reference/c-api/types-and-structures.rst b/doc/source/reference/c-api/types-and-structures.rst index 8759af6a4638..d22a3f03faa8 100644 --- a/doc/source/reference/c-api/types-and-structures.rst +++ b/doc/source/reference/c-api/types-and-structures.rst @@ -566,7 +566,7 @@ PyArrayDescr_Type and PyArray_Descr This function should be called without holding the Python GIL, and has to grab it for error reporting. - .. c:member:: Bool nonzero(void* data, void* arr) + .. c:member:: npy_bool nonzero(void* data, void* arr) A pointer to a function that returns TRUE if the item of ``arr`` pointed to by ``data`` is nonzero. This function can @@ -1012,7 +1012,7 @@ PyArrayIter_Type and PyArrayIterObject npy_intp factors[NPY_MAXDIMS]; PyArrayObject *ao; char *dataptr; - Bool contiguous; + npy_bool contiguous; } PyArrayIterObject; .. c:member:: int nd_m1 @@ -1062,7 +1062,7 @@ PyArrayIter_Type and PyArrayIterObject This member points to an element in the ndarray indicated by the index. - .. c:member:: Bool contiguous + .. c:member:: npy_bool contiguous This flag is true if the underlying array is :c:data:`NPY_ARRAY_C_CONTIGUOUS`. It is used to simplify From dbb8e993da05b4f12f41cdbc68dbd9733cdd65ab Mon Sep 17 00:00:00 2001 From: Chunlin Date: Sat, 10 Oct 2020 16:26:30 +0800 Subject: [PATCH 245/409] DOC: Fix some links and typos (#17528) * Fix some links and typos * Use the official website --- INSTALL.rst.txt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/INSTALL.rst.txt b/INSTALL.rst.txt index 1c33060a625a..5ee97d7901a0 100644 --- a/INSTALL.rst.txt +++ b/INSTALL.rst.txt @@ -30,8 +30,8 @@ Building NumPy requires the following installed software: This is required for testing NumPy, but not for using it. -Python__ http://www.python.org -pytest__ http://pytest.readthedocs.io +Python__ https://www.python.org/ +pytest__ https://docs.pytest.org/en/stable/ Hypothesis__ https://hypothesis.readthedocs.io/en/latest/ @@ -39,12 +39,12 @@ Hypothesis__ https://hypothesis.readthedocs.io/en/latest/ If you want to build NumPy in order to work on NumPy itself, use ``runtests.py``. For more details, see - https://docs.scipy.org/doc/numpy/dev/development_environment.html + https://numpy.org/devdocs/dev/development_environment.html .. note:: - More extensive information on building NumPy (and SciPy) is maintained at - https://scipy.github.io/devdocs/building/ + More extensive information on building NumPy is maintained at + https://numpy.org/devdocs/user/building.html#building-from-source Basic Installation @@ -69,7 +69,7 @@ NPY_NUM_BUILD_JOBS. Choosing compilers ================== -NumPy needs a C compiler, and for development versions also Cython. A Fortran +NumPy needs a C compiler, and for development versions also needs Cython. A Fortran compiler isn't needed to build NumPy itself; the ``numpy.f2py`` tests will be skipped when running the test suite if no Fortran compiler is available. For building Scipy a Fortran compiler is needed though, so we include some details @@ -87,7 +87,7 @@ Windows ------- On Windows, building from source can be difficult (in particular if you need to -build SciPy as well, because that requires a Fortran compiler). Currently, the +build SciPy as well, because that requires a Fortran compiler). Currently, the most robust option is to use MSVC (for NumPy only). If you also need SciPy, you can either use MSVC + Intel Fortran or the Intel compiler suite. Intel itself maintains a good `application note @@ -151,7 +151,7 @@ Or by preloading a specific BLAS library with:: Build issues ============ -If you run into build issues and need help, the NumPy +If you run into build issues and need help, the NumPy and SciPy `mailing list `_ is the best -place to ask. If the issue is clearly a bug in NumPy, please file an issue (or +place to ask. If the issue is clearly a bug in NumPy, please file an issue (or even better, a pull request) at https://github.com/numpy/numpy. From aeb23741b1186e6ae525c4dd7f46ce9e1378e670 Mon Sep 17 00:00:00 2001 From: Steve Joachim Date: Sat, 10 Oct 2020 05:47:20 -0400 Subject: [PATCH 246/409] MAINT: Do not emit empty Methods heading in np.info (#17498) Fixes the incompatible type comparison found in #17490. This also corrects the logic to not print the heading when only private/magic methods are present. --- numpy/lib/tests/test_utils.py | 19 +++++++++++++++++++ numpy/lib/utils.py | 8 ++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/numpy/lib/tests/test_utils.py b/numpy/lib/tests/test_utils.py index 261cfef5d680..33951b92ade9 100644 --- a/numpy/lib/tests/test_utils.py +++ b/numpy/lib/tests/test_utils.py @@ -140,3 +140,22 @@ def test_strided(self): def test_assert_raises_regex_context_manager(): with assert_raises_regex(ValueError, 'no deprecation warning'): raise ValueError('no deprecation warning') + + +def test_info_method_heading(): + # info(class) should only print "Methods:" heading if methods exist + + class NoPublicMethods: + pass + + class WithPublicMethods: + def first_method(): + pass + + def _has_method_heading(cls): + out = StringIO() + utils.info(cls, output=out) + return 'Methods:' in out.getvalue() + + assert _has_method_heading(WithPublicMethods) + assert not _has_method_heading(NoPublicMethods) diff --git a/numpy/lib/utils.py b/numpy/lib/utils.py index d511c2a400e9..5447608bf5bb 100644 --- a/numpy/lib/utils.py +++ b/numpy/lib/utils.py @@ -587,11 +587,11 @@ def info(object=None, maxwidth=76, output=sys.stdout, toplevel='numpy'): print(inspect.getdoc(object), file=output) methods = pydoc.allmethods(object) - if methods != []: + + public_methods = [meth for meth in methods if meth[0] != '_'] + if public_methods: print("\n\nMethods:\n", file=output) - for meth in methods: - if meth[0] == '_': - continue + for meth in public_methods: thisobj = getattr(object, meth, None) if thisobj is not None: methstr, other = pydoc.splitdoc( From c63dd1316067f8d0ebe99f4f61cf9853b77e937d Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Sat, 10 Oct 2020 11:00:55 +0100 Subject: [PATCH 247/409] DOC: Add cross-links and `void` example --- numpy/core/_add_newdocs_scalars.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/numpy/core/_add_newdocs_scalars.py b/numpy/core/_add_newdocs_scalars.py index 113f84eb4ffa..2339c0d1746b 100644 --- a/numpy/core/_add_newdocs_scalars.py +++ b/numpy/core/_add_newdocs_scalars.py @@ -180,25 +180,34 @@ def add_newdoc_for_scalar_type(obj, fixed_aliases, doc): r""" A unicode string. + When used in array, this type strips trailing nulls. + Unlike the builtin `str`, this supports the buffer protocol, exposing its contents as UCS4: >>> memoryview(np.str_("abcd")) b'a\x00\x00\x00b\x00\x00\x00c\x00\x00\x00d\x00\x00\x00' - - When used in array, this type cannot contain trailing nulls. """) add_newdoc_for_scalar_type('bytes_', ['bytes0'], r""" A byte string. - When used in array, this type cannot contain trailing nulls. + When used in array, this type strips trailing nulls. """) add_newdoc_for_scalar_type('void', ['void0'], """ Either an opaque sequence of bytes, or a structure. + + >>> np.void(b'abcd') + void(b'\x61\x62\x63\x64') + + Structured `void` scalars can only be constructed via extraction from `structured_arrays`: + + >>> arr = np.array((1, 2), dtype=[('x', np.int8), ('y', np.int8)]) + >>> arr[()] + (1, 2) # looks like a tuple, but is `np.void` """) add_newdoc_for_scalar_type('datetime64', [], @@ -209,11 +218,15 @@ def add_newdoc_for_scalar_type(obj, fixed_aliases, doc): numpy.datetime64('1980') >>> np.datetime64(10, 'D') numpy.datetime64('1970-01-11') + + See `arrays.datetime` for more information. """) add_newdoc_for_scalar_type('timedelta64', [], """ A timedelta stored as a 64-bit integer. + + See `arrays.datetime` for more information. """) # TODO: work out how to put this on the base class, np.floating From 4ce3b27846d6c79a36616ecc8a875178d6cb7d39 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Sat, 10 Oct 2020 11:16:19 +0100 Subject: [PATCH 248/409] DOC: remove 0 aliases, fix references --- numpy/core/_add_newdocs_scalars.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/numpy/core/_add_newdocs_scalars.py b/numpy/core/_add_newdocs_scalars.py index 2339c0d1746b..12f30712ac3b 100644 --- a/numpy/core/_add_newdocs_scalars.py +++ b/numpy/core/_add_newdocs_scalars.py @@ -176,11 +176,11 @@ def add_newdoc_for_scalar_type(obj, fixed_aliases, doc): Any Python object. """) -add_newdoc_for_scalar_type('str_', ['str0', 'unicode_'], +add_newdoc_for_scalar_type('str_', ['unicode_'], r""" A unicode string. - When used in array, this type strips trailing nulls. + When used in arrays, this type strips trailing null codepoints. Unlike the builtin `str`, this supports the buffer protocol, exposing its contents as UCS4: @@ -189,21 +189,21 @@ def add_newdoc_for_scalar_type(obj, fixed_aliases, doc): b'a\x00\x00\x00b\x00\x00\x00c\x00\x00\x00d\x00\x00\x00' """) -add_newdoc_for_scalar_type('bytes_', ['bytes0'], +add_newdoc_for_scalar_type('bytes_', ['string_'], r""" A byte string. - When used in array, this type strips trailing nulls. + When used in arrays, this type strips trailing null bytes. """) -add_newdoc_for_scalar_type('void', ['void0'], +add_newdoc_for_scalar_type('void', [], """ Either an opaque sequence of bytes, or a structure. >>> np.void(b'abcd') void(b'\x61\x62\x63\x64') - Structured `void` scalars can only be constructed via extraction from `structured_arrays`: + Structured `void` scalars can only be constructed via extraction from :ref:`structured_arrays`: >>> arr = np.array((1, 2), dtype=[('x', np.int8), ('y', np.int8)]) >>> arr[()] @@ -219,14 +219,14 @@ def add_newdoc_for_scalar_type(obj, fixed_aliases, doc): >>> np.datetime64(10, 'D') numpy.datetime64('1970-01-11') - See `arrays.datetime` for more information. + See :ref:`arrays.datetime` for more information. """) add_newdoc_for_scalar_type('timedelta64', [], """ A timedelta stored as a 64-bit integer. - See `arrays.datetime` for more information. + See :ref:`arrays.datetime` for more information. """) # TODO: work out how to put this on the base class, np.floating @@ -237,7 +237,7 @@ def add_newdoc_for_scalar_type(obj, fixed_aliases, doc): Return a pair of integers, whose ratio is exactly equal to the original floating point number, and with a positive denominator. - Raise OverflowError on infinities and a ValueError on NaNs. + Raise `OverflowError` on infinities and a `ValueError` on NaNs. >>> np.{ftype}(10.0).as_integer_ratio() (10, 1) From 50811dc06a3aab1909931fd0e1e55c3effeb90a5 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Sat, 10 Oct 2020 06:29:36 -0400 Subject: [PATCH 249/409] DOC: Explain :orphan: in PR #17523 --- doc/source/user/troubleshooting-importerror.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/source/user/troubleshooting-importerror.rst b/doc/source/user/troubleshooting-importerror.rst index d124e657a64b..1f99491a1e95 100644 --- a/doc/source/user/troubleshooting-importerror.rst +++ b/doc/source/user/troubleshooting-importerror.rst @@ -1,5 +1,11 @@ :orphan: +.. Reason for orphan: This page is referenced by the installation + instructions, which have moved from Sphinx to https://numpy.org/install. + All install links in Sphinx now point there, leaving no Sphinx references + to this page. + + *************************** Troubleshooting ImportError *************************** From 4afd82d8dd2c1ef48c62bdf875a194e407f2d2d3 Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Sat, 10 Oct 2020 12:56:06 +0200 Subject: [PATCH 250/409] MAINT: py3k: remove os.fspath and os.PathLike backports (#17473) Since we no longer support Python 3.5 and below, this code is unreachable. As with the rest of `py3k.py`, we leave behind the aliases to prevent breaking downstream code. --- numpy/compat/py3k.py | 53 ++------------------------------------------ 1 file changed, 2 insertions(+), 51 deletions(-) diff --git a/numpy/compat/py3k.py b/numpy/compat/py3k.py index fd9f8bd4217e..5e742bf6cf29 100644 --- a/numpy/compat/py3k.py +++ b/numpy/compat/py3k.py @@ -133,54 +133,5 @@ def npy_load_module(name, fn, info=None): # Backport os.fs_path, os.PathLike, and PurePath.__fspath__ -if sys.version_info[:2] >= (3, 6): - os_fspath = os.fspath - os_PathLike = os.PathLike -else: - def _PurePath__fspath__(self): - return str(self) - - class os_PathLike(abc_ABC): - """Abstract base class for implementing the file system path protocol.""" - - @abc.abstractmethod - def __fspath__(self): - """Return the file system path representation of the object.""" - raise NotImplementedError - - @classmethod - def __subclasshook__(cls, subclass): - if PurePath is not None and issubclass(subclass, PurePath): - return True - return hasattr(subclass, '__fspath__') - - - def os_fspath(path): - """Return the path representation of a path-like object. - If str or bytes is passed in, it is returned unchanged. Otherwise the - os.PathLike interface is used to get the path representation. If the - path representation is not str or bytes, TypeError is raised. If the - provided path is not str, bytes, or os.PathLike, TypeError is raised. - """ - if isinstance(path, (str, bytes)): - return path - - # Work from the object's type to match method resolution of other magic - # methods. - path_type = type(path) - try: - path_repr = path_type.__fspath__(path) - except AttributeError: - if hasattr(path_type, '__fspath__'): - raise - elif PurePath is not None and issubclass(path_type, PurePath): - return _PurePath__fspath__(path) - else: - raise TypeError("expected str, bytes or os.PathLike object, " - "not " + path_type.__name__) - if isinstance(path_repr, (str, bytes)): - return path_repr - else: - raise TypeError("expected {}.__fspath__() to return str or bytes, " - "not {}".format(path_type.__name__, - type(path_repr).__name__)) +os_fspath = os.fspath +os_PathLike = os.PathLike From fd2749e3eeb4ca8e0bc118ec28022a921f411997 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Sat, 10 Oct 2020 12:02:05 +0100 Subject: [PATCH 251/409] MAINT: Cleanup compatibility code for pathlib `Path` can now never be none, and `PurePath` is not used and not in `__all__`. --- numpy/compat/py3k.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/numpy/compat/py3k.py b/numpy/compat/py3k.py index 5e742bf6cf29..f36aaca170ff 100644 --- a/numpy/compat/py3k.py +++ b/numpy/compat/py3k.py @@ -18,7 +18,7 @@ import sys import os -from pathlib import Path, PurePath +from pathlib import Path import io import abc @@ -78,11 +78,11 @@ def asunicode_nested(x): def is_pathlib_path(obj): """ - Check whether obj is a pathlib.Path object. + Check whether obj is a `pathlib.Path` object. - Prefer using `isinstance(obj, os_PathLike)` instead of this function. + Prefer using ``isinstance(obj, os.PathLike)`` instead of this function. """ - return Path is not None and isinstance(obj, Path) + return isinstance(obj, Path) # from Python 3.7 class contextlib_nullcontext: @@ -132,6 +132,5 @@ def npy_load_module(name, fn, info=None): return SourceFileLoader(name, fn).load_module() -# Backport os.fs_path, os.PathLike, and PurePath.__fspath__ os_fspath = os.fspath os_PathLike = os.PathLike From 6eb16bc6ca0c51aa43931fdcddec7632c6d67378 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Sat, 10 Oct 2020 12:08:20 +0100 Subject: [PATCH 252/409] DOC: fix refguide --- numpy/core/_add_newdocs_scalars.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/numpy/core/_add_newdocs_scalars.py b/numpy/core/_add_newdocs_scalars.py index 12f30712ac3b..1cf82a65f0de 100644 --- a/numpy/core/_add_newdocs_scalars.py +++ b/numpy/core/_add_newdocs_scalars.py @@ -182,11 +182,14 @@ def add_newdoc_for_scalar_type(obj, fixed_aliases, doc): When used in arrays, this type strips trailing null codepoints. - Unlike the builtin `str`, this supports the buffer protocol, exposing its + Unlike the builtin `str`, this supports the :ref:`python:bufferobjects`, exposing its contents as UCS4: - >>> memoryview(np.str_("abcd")) - b'a\x00\x00\x00b\x00\x00\x00c\x00\x00\x00d\x00\x00\x00' + >>> m = memoryview(np.str_("abc")) + >>> m.format + '3w' + >>> m.tobytes() + b'a\x00\x00\x00b\x00\x00\x00c\x00\x00\x00' """) add_newdoc_for_scalar_type('bytes_', ['string_'], From 588a517b68d9f273b607622dc854b1b3d014ac58 Mon Sep 17 00:00:00 2001 From: Takanori H Date: Sun, 11 Oct 2020 21:00:44 +0900 Subject: [PATCH 253/409] DOC: Fix typo (#17534) --- doc/source/user/numpy-for-matlab-users.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/user/numpy-for-matlab-users.rst b/doc/source/user/numpy-for-matlab-users.rst index eb011de633c4..ed0be82a0851 100644 --- a/doc/source/user/numpy-for-matlab-users.rst +++ b/doc/source/user/numpy-for-matlab-users.rst @@ -578,7 +578,7 @@ indices using the ``ix_`` command. E.g., for 2D array ``a``, one might do: ``ind=[1, 3]; a[np.ix_(ind, ind)] += 100``. \ **HELP**: There is no direct equivalent of MATLAB's ``which`` command, -but the commands :func:`help`` and :func:`numpy.source` will usually list the filename +but the commands :func:`help` and :func:`numpy.source` will usually list the filename where the function is located. Python also has an ``inspect`` module (do ``import inspect``) which provides a ``getfile`` that often works. From a6fb91670017abd2c9378e5f1df3174a6e5a505d Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Sun, 11 Oct 2020 13:01:53 +0100 Subject: [PATCH 254/409] DOC: Fix escaping --- numpy/core/_add_newdocs_scalars.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/_add_newdocs_scalars.py b/numpy/core/_add_newdocs_scalars.py index 1cf82a65f0de..b9b151224e61 100644 --- a/numpy/core/_add_newdocs_scalars.py +++ b/numpy/core/_add_newdocs_scalars.py @@ -200,7 +200,7 @@ def add_newdoc_for_scalar_type(obj, fixed_aliases, doc): """) add_newdoc_for_scalar_type('void', [], - """ + r""" Either an opaque sequence of bytes, or a structure. >>> np.void(b'abcd') From b81ab444c0e56011e96c8895a19e18906ab4e731 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Sun, 11 Oct 2020 14:30:32 -0700 Subject: [PATCH 255/409] MAINT: remove character code literals for integer types The various character codes are for C types, which are all platform-dependent. --- numpy/__init__.pyi | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index c51141d5de85..be4f55a32116 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -576,10 +576,6 @@ class dtype(Generic[_DTypeScalar]): "=u1", "u1", - "B", - "=B", - "B", ], align: bool = ..., copy: bool = ..., @@ -593,10 +589,6 @@ class dtype(Generic[_DTypeScalar]): "=u2", "u2", - "h", - "=h", - "h", ], align: bool = ..., copy: bool = ..., @@ -610,10 +602,6 @@ class dtype(Generic[_DTypeScalar]): "=u4", "u4", - "I", - "=I", - "I", ], align: bool = ..., copy: bool = ..., @@ -627,10 +615,6 @@ class dtype(Generic[_DTypeScalar]): "=u8", "u8", - "L", - "=L", - "L", ], align: bool = ..., copy: bool = ..., @@ -644,10 +628,6 @@ class dtype(Generic[_DTypeScalar]): "=i1", "i1", - "b", - "=b", - "b", ], align: bool = ..., copy: bool = ..., @@ -661,10 +641,6 @@ class dtype(Generic[_DTypeScalar]): "=i2", "i2", - "h", - "=h", - "h", ], align: bool = ..., copy: bool = ..., @@ -678,10 +654,6 @@ class dtype(Generic[_DTypeScalar]): "=i4", "i4", - "i", - "=i", - "i", ], align: bool = ..., copy: bool = ..., @@ -695,10 +667,6 @@ class dtype(Generic[_DTypeScalar]): "=i8", "i8", - "l", - "=l", - "l", ], align: bool = ..., copy: bool = ..., From 4ad5551c59595fd9adeb528ef0f7d27313d99b2c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 12 Oct 2020 07:14:31 +0000 Subject: [PATCH 256/409] MAINT: Bump hypothesis from 5.37.0 to 5.37.1 Bumps [hypothesis](https://github.com/HypothesisWorks/hypothesis) from 5.37.0 to 5.37.1. - [Release notes](https://github.com/HypothesisWorks/hypothesis/releases) - [Commits](https://github.com/HypothesisWorks/hypothesis/compare/hypothesis-python-5.37.0...hypothesis-python-5.37.1) Signed-off-by: dependabot-preview[bot] --- test_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index 067dd07c888a..4b4bf2c8c05c 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,7 +1,7 @@ cython==0.29.21 wheel setuptools<49.2.0 -hypothesis==5.37.0 +hypothesis==5.37.1 pytest==6.0.2 pytz==2020.1 pytest-cov==2.10.1 From a5fc6e73895de1d76929a0970c4d40eaa189b36f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 12 Oct 2020 07:14:53 +0000 Subject: [PATCH 257/409] MAINT: Bump pydata-sphinx-theme from 0.4.0 to 0.4.1 Bumps [pydata-sphinx-theme](https://github.com/pandas-dev/pydata-sphinx-theme) from 0.4.0 to 0.4.1. - [Release notes](https://github.com/pandas-dev/pydata-sphinx-theme/releases) - [Changelog](https://github.com/pandas-dev/pydata-sphinx-theme/blob/master/docs/changelog.rst) - [Commits](https://github.com/pandas-dev/pydata-sphinx-theme/compare/v0.4.0...v0.4.1) Signed-off-by: dependabot-preview[bot] --- doc_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc_requirements.txt b/doc_requirements.txt index 36b651c6465f..f69cc5dcb406 100644 --- a/doc_requirements.txt +++ b/doc_requirements.txt @@ -4,4 +4,4 @@ ipython scipy matplotlib pandas -pydata-sphinx-theme==0.4.0 +pydata-sphinx-theme==0.4.1 From 7ffd3a7fc2878229e285990d7e018fc11c743cb8 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 12 Oct 2020 07:15:14 +0000 Subject: [PATCH 258/409] MAINT: Bump mypy from 0.782 to 0.790 Bumps [mypy](https://github.com/python/mypy) from 0.782 to 0.790. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.782...v0.790) Signed-off-by: dependabot-preview[bot] --- test_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index 067dd07c888a..0f828cf43b20 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -13,5 +13,5 @@ cffi # - Mypy relies on C API features not present in PyPy # - Mypy doesn't currently work on Python 3.9 # - Python 3.6 doesn't work because it doesn't understand py.typed -mypy==0.782; platform_python_implementation != "PyPy" and python_version > "3.6" +mypy==0.790; platform_python_implementation != "PyPy" and python_version > "3.6" typing_extensions From b6d95828dfc61d7530c39c958e4a1e9c8d884393 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Mon, 12 Oct 2020 09:42:32 -0700 Subject: [PATCH 259/409] CI: fix conditional for PR merge cmd Co-authored-by: Matti Picus --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dc18757d213a..9324de943607 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,7 +17,7 @@ jobs: - run: name: pull changes from merge command: | - [[ ! -z $CI_PULL_REQUEST ]] && git pull --ff-only origin "refs/pull/${CI_PULL_REQUEST//*pull\//}/merge" + if [[ -v CI_PULL_REQUEST ]] ; then git pull --ff-only origin "refs/pull/${CI_PULL_REQUEST//*pull\//}/merge" ; fi - run: name: create virtual environment, install dependencies From 2fce5499d2ee1e79dd24bf16a078b592e653e33d Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Wed, 16 Sep 2020 14:36:49 +0200 Subject: [PATCH 260/409] MAINT: Explicit disabling `CCompilerOpt` in F2PY This patch also cleans up `CCompilerOpt` calls in build_ext and build_clib. --- numpy/distutils/command/build_clib.py | 67 +++++++++++++-------------- numpy/distutils/command/build_ext.py | 59 +++++++++++------------ numpy/f2py/f2py2e.py | 4 +- 3 files changed, 64 insertions(+), 66 deletions(-) diff --git a/numpy/distutils/command/build_clib.py b/numpy/distutils/command/build_clib.py index 87345adbc2a0..a0db6f31f7e5 100644 --- a/numpy/distutils/command/build_clib.py +++ b/numpy/distutils/command/build_clib.py @@ -259,57 +259,56 @@ def build_a_library(self, build_info, lib_name, libraries): if requiref90: self.mkpath(module_build_dir) - dispatch_objects = [] - if not self.disable_optimization: - dispatch_sources = [ - c_sources.pop(c_sources.index(src)) - for src in c_sources[:] if src.endswith(".dispatch.c") - ] - if dispatch_sources: - if not self.inplace: - build_src = self.get_finalized_command("build_src").build_src - else: - build_src = None - dispatch_objects = self.compiler_opt.try_dispatch( - dispatch_sources, - output_dir=self.build_temp, - src_dir=build_src, - macros=macros, - include_dirs=include_dirs, - debug=self.debug, - extra_postargs=extra_postargs - ) - extra_args_baseopt = extra_postargs + self.compiler_opt.cpu_baseline_flags() - else: - extra_args_baseopt = extra_postargs - macros.append(("NPY_DISABLE_OPTIMIZATION", 1)) - if compiler.compiler_type == 'msvc': # this hack works around the msvc compiler attributes # problem, msvc uses its own convention :( c_sources += cxx_sources cxx_sources = [] + # filtering C dispatch-table sources when optimization is not disabled, + # otherwise treated as normal sources. + copt_c_sources = [] + copt_baseline_flags = [] + copt_macros = [] + if not self.disable_optimization: + copt_build_src = None if self.inplace else self.get_finalized_command("build_src").build_src + copt_c_sources = [ + c_sources.pop(c_sources.index(src)) + for src in c_sources[:] if src.endswith(".dispatch.c") + ] + copt_baseline_flags = self.compiler_opt.cpu_baseline_flags() + else: + copt_macros.append(("NPY_DISABLE_OPTIMIZATION", 1)) + objects = [] + if copt_c_sources: + log.info("compiling C dispatch-able sources") + objects += self.compiler_opt.try_dispatch(copt_c_sources, + output_dir=self.build_temp, + src_dir=copt_build_src, + macros=macros + copt_macros, + include_dirs=include_dirs, + debug=self.debug, + extra_postargs=extra_postargs) + if c_sources: log.info("compiling C sources") - objects = compiler.compile(c_sources, - output_dir=self.build_temp, - macros=macros, - include_dirs=include_dirs, - debug=self.debug, - extra_postargs=extra_args_baseopt) - objects.extend(dispatch_objects) + objects += compiler.compile(c_sources, + output_dir=self.build_temp, + macros=macros + copt_macros, + include_dirs=include_dirs, + debug=self.debug, + extra_postargs=extra_postargs + copt_baseline_flags) if cxx_sources: log.info("compiling C++ sources") cxx_compiler = compiler.cxx_compiler() cxx_objects = cxx_compiler.compile(cxx_sources, output_dir=self.build_temp, - macros=macros, + macros=macros + copt_macros, include_dirs=include_dirs, debug=self.debug, - extra_postargs=extra_postargs) + extra_postargs=extra_postargs + copt_baseline_flags) objects.extend(cxx_objects) if f_sources or fmodule_sources: diff --git a/numpy/distutils/command/build_ext.py b/numpy/distutils/command/build_ext.py index c7502d3e6daa..1a881c56a2aa 100644 --- a/numpy/distutils/command/build_ext.py +++ b/numpy/distutils/command/build_ext.py @@ -406,52 +406,49 @@ def build_extension(self, ext): include_dirs = ext.include_dirs + get_numpy_include_dirs() - dispatch_objects = [] + # filtering C dispatch-table sources when optimization is not disabled, + # otherwise treated as normal sources. + copt_c_sources = [] + copt_baseline_flags = [] + copt_macros = [] if not self.disable_optimization: - dispatch_sources = [ + copt_build_src = None if self.inplace else self.get_finalized_command("build_src").build_src + copt_c_sources = [ c_sources.pop(c_sources.index(src)) for src in c_sources[:] if src.endswith(".dispatch.c") ] - if dispatch_sources: - if not self.inplace: - build_src = self.get_finalized_command("build_src").build_src - else: - build_src = None - dispatch_objects = self.compiler_opt.try_dispatch( - dispatch_sources, - output_dir=output_dir, - src_dir=build_src, - macros=macros, - include_dirs=include_dirs, - debug=self.debug, - extra_postargs=extra_args, - **kws - ) - extra_args_baseopt = extra_args + self.compiler_opt.cpu_baseline_flags() + copt_baseline_flags = self.compiler_opt.cpu_baseline_flags() else: - extra_args_baseopt = extra_args - macros.append(("NPY_DISABLE_OPTIMIZATION", 1)) + copt_macros.append(("NPY_DISABLE_OPTIMIZATION", 1)) c_objects = [] + if copt_c_sources: + log.info("compiling C dispatch-able sources") + c_objects += self.compiler_opt.try_dispatch(copt_c_sources, + output_dir=output_dir, + src_dir=copt_build_src, + macros=macros + copt_macros, + include_dirs=include_dirs, + debug=self.debug, + extra_postargs=extra_args, + **kws) if c_sources: log.info("compiling C sources") - c_objects = self.compiler.compile(c_sources, - output_dir=output_dir, - macros=macros, - include_dirs=include_dirs, - debug=self.debug, - extra_postargs=extra_args_baseopt, - **kws) - c_objects.extend(dispatch_objects) - + c_objects += self.compiler.compile(c_sources, + output_dir=output_dir, + macros=macros + copt_macros, + include_dirs=include_dirs, + debug=self.debug, + extra_postargs=extra_args + copt_baseline_flags, + **kws) if cxx_sources: log.info("compiling C++ sources") c_objects += cxx_compiler.compile(cxx_sources, output_dir=output_dir, - macros=macros, + macros=macros + copt_macros, include_dirs=include_dirs, debug=self.debug, - extra_postargs=extra_args, + extra_postargs=extra_args + copt_baseline_flags, **kws) extra_postargs = [] diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index 71a049e410f6..e68b2e01aceb 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -650,7 +650,9 @@ def run_compile(): sys.argv.extend(['build', '--build-temp', build_dir, '--build-base', build_dir, - '--build-platlib', '.']) + '--build-platlib', '.', + # disable CCompilerOpt + '--disable-optimization']) if fc_flags: sys.argv.extend(['config_fc'] + fc_flags) if flib_flags: From 9cc9f776cebf8207c0811ce600b7a76d9f54afa4 Mon Sep 17 00:00:00 2001 From: "E. Madison Bray" Date: Tue, 13 Oct 2020 14:20:31 +0200 Subject: [PATCH 261/409] BLD: add new check_compiler_gcc and check_gcc_version_at_least configure commands replaces obsolete (no longer used anywhere AFICT check_compiler_gcc4 with some more general utilities for checking GCC version --- numpy/distutils/command/autodist.py | 33 +++++++++++++++++++++++++---- numpy/distutils/command/config.py | 14 ++++++++---- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/numpy/distutils/command/autodist.py b/numpy/distutils/command/autodist.py index 8f6436004a0a..b72d0cab1a7d 100644 --- a/numpy/distutils/command/autodist.py +++ b/numpy/distutils/command/autodist.py @@ -46,15 +46,16 @@ def check_restrict(cmd): return '' -def check_compiler_gcc4(cmd): - """Return True if the C compiler is GCC 4.x.""" +def check_compiler_gcc(cmd): + """Check if the compiler is GCC.""" + cmd._check_compiler() body = textwrap.dedent(""" int main() { - #if (! defined __GNUC__) || (__GNUC__ < 4) - #error gcc >= 4 required + #if (! defined __GNUC__) + #error gcc required #endif return 0; } @@ -62,6 +63,30 @@ def check_compiler_gcc4(cmd): return cmd.try_compile(body, None, None) +def check_gcc_version_at_least(cmd, major, minor=0, patchlevel=0): + """ + Check that the gcc version is at least the specified version.""" + + cmd._check_compiler() + version = '.'.join([str(major), str(minor), str(patchlevel)]) + body = textwrap.dedent(""" + int + main() + { + #if (! defined __GNUC__) || (__GNUC__ < %(major)d) || \\ + (__GNUC_MINOR__ < %(minor)d) || \\ + (__GNUC_PATCHLEVEL__ < %(patchlevel)d) + #error gcc >= %(version)s required + #endif + return 0; + } + """) + kw = {'version': version, 'major': major, 'minor': minor, + 'patchlevel': patchlevel} + + return cmd.try_compile(body % kw, None, None) + + def check_gcc_function_attribute(cmd, attribute, name): """Return True if the given function attribute is supported.""" cmd._check_compiler() diff --git a/numpy/distutils/command/config.py b/numpy/distutils/command/config.py index e54a5444995b..60881f4a3a4a 100644 --- a/numpy/distutils/command/config.py +++ b/numpy/distutils/command/config.py @@ -20,9 +20,10 @@ from numpy.distutils.command.autodist import (check_gcc_function_attribute, check_gcc_function_attribute_with_intrinsics, check_gcc_variable_attribute, + check_gcc_version_at_least, check_inline, check_restrict, - check_compiler_gcc4) + check_compiler_gcc) LANG_EXT['f77'] = '.f' LANG_EXT['f90'] = '.f90' @@ -416,9 +417,9 @@ def check_restrict(self): otherwise.""" return check_restrict(self) - def check_compiler_gcc4(self): - """Return True if the C compiler is gcc >= 4.""" - return check_compiler_gcc4(self) + def check_compiler_gcc(self): + """Return True if the C compiler is gcc""" + return check_compiler_gcc(self) def check_gcc_function_attribute(self, attribute, name): return check_gcc_function_attribute(self, attribute, name) @@ -431,6 +432,11 @@ def check_gcc_function_attribute_with_intrinsics(self, attribute, name, def check_gcc_variable_attribute(self, attribute): return check_gcc_variable_attribute(self, attribute) + def check_gcc_version_at_least(self, major, minor=0, patchlevel=0): + """Return True if the GCC version is greater than or equal to the + specified version.""" + return check_gcc_version_at_least(self, major, minor, patchlevel) + def get_output(self, body, headers=None, include_dirs=None, libraries=None, library_dirs=None, lang="c", use_tee=None): From 3c118aab05ca06f30b2eec8057e2e46db98ad745 Mon Sep 17 00:00:00 2001 From: "E. Madison Bray" Date: Tue, 13 Oct 2020 14:32:15 +0200 Subject: [PATCH 262/409] BUG: Workaround for #14787: on Windows/Cygwin add extra compiler flags when building the numpy.core._multiarray_umath module This adds the extra compile args to the extension needed to work around the bug but only in the known case where the bug is relevant. --- numpy/core/setup.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/numpy/core/setup.py b/numpy/core/setup.py index 92dcacede412..b3e17baed8df 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -102,7 +102,7 @@ def win32_checks(deflist): if a == "Intel" or a == "AMD64": deflist.append('FORCE_NO_LONG_DOUBLE_FORMATTING') -def check_math_capabilities(config, moredefs, mathlibs): +def check_math_capabilities(config, ext, moredefs, mathlibs): def check_func(func_name): return config.check_func(func_name, libraries=mathlibs, decl=True, call=True) @@ -167,6 +167,14 @@ def check_funcs(funcs_name): for dec, fn in OPTIONAL_FUNCTION_ATTRIBUTES: if config.check_gcc_function_attribute(dec, fn): moredefs.append((fname2def(fn), 1)) + if fn == 'attribute_target_avx512f': + # GH-14787: Work around GCC<8.4 bug when compiling with AVX512 + # support on Windows-based platforms + if (sys.platform in ('win32', 'cygwin') and + config.check_compiler_gcc() and + not config.check_gcc_version_at_least(8, 4)): + ext.extra_compile_args.extend( + ['-ffixed-xmm%s' % n for n in range(16, 32)]) for dec, fn, code, header in OPTIONAL_FUNCTION_ATTRIBUTES_WITH_INTRINSICS: if config.check_gcc_function_attribute_with_intrinsics(dec, fn, code, @@ -434,7 +442,7 @@ def generate_config_h(ext, build_dir): mathlibs = check_mathlib(config_cmd) moredefs.append(('MATHLIB', ','.join(mathlibs))) - check_math_capabilities(config_cmd, moredefs, mathlibs) + check_math_capabilities(config_cmd, ext, moredefs, mathlibs) moredefs.extend(cocache.check_ieee_macros(config_cmd)[0]) moredefs.extend(cocache.check_complex(config_cmd, mathlibs)[0]) From 18da0ef0480920c6256627ed91469a40997ba4f5 Mon Sep 17 00:00:00 2001 From: Takanori H Date: Wed, 14 Oct 2020 00:09:49 +0900 Subject: [PATCH 263/409] DOC: Add missing types to C function docs (#17549) `.. c:function::` expects its contents to be a valid C prototype, meaning the return types and arguments are all need types. --- doc/source/reference/c-api/array.rst | 165 +++++++++--------- doc/source/reference/c-api/config.rst | 2 +- .../reference/c-api/types-and-structures.rst | 4 +- doc/source/reference/c-api/ufunc.rst | 2 +- 4 files changed, 87 insertions(+), 86 deletions(-) diff --git a/doc/source/reference/c-api/array.rst b/doc/source/reference/c-api/array.rst index 7360c0985ec9..637b6aaeb9bc 100644 --- a/doc/source/reference/c-api/array.rst +++ b/doc/source/reference/c-api/array.rst @@ -326,7 +326,7 @@ From scratch Create a new array with the provided data-type descriptor, *descr*, of the shape determined by *nd* and *dims*. -.. c:function:: PyArray_FILLWBYTE(PyObject* obj, int val) +.. c:function:: void PyArray_FILLWBYTE(PyObject* obj, int val) Fill the array pointed to by *obj* ---which must be a (subclass of) ndarray---with the contents of *val* (evaluated as a byte). @@ -790,17 +790,17 @@ Dealing with types General check of Python Type ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. c:function:: PyArray_Check(PyObject *op) +.. c:function:: int PyArray_Check(PyObject *op) Evaluates true if *op* is a Python object whose type is a sub-type of :c:data:`PyArray_Type`. -.. c:function:: PyArray_CheckExact(PyObject *op) +.. c:function:: int PyArray_CheckExact(PyObject *op) Evaluates true if *op* is a Python object with type :c:data:`PyArray_Type`. -.. c:function:: PyArray_HasArrayInterface(PyObject *op, PyObject *out) +.. c:function:: int PyArray_HasArrayInterface(PyObject *op, PyObject *out) If ``op`` implements any part of the array interface, then ``out`` will contain a new reference to the newly created ndarray using @@ -808,7 +808,8 @@ General check of Python Type conversion occurs. Otherwise, out will contain a borrowed reference to :c:data:`Py_NotImplemented` and no error condition is set. -.. c:function:: PyArray_HasArrayInterfaceType(op, dtype, context, out) +.. c:function:: int PyArray_HasArrayInterfaceType(\ + PyObject *op, PyArray_Descr *dtype, PyObject *context, PyObject *out) If ``op`` implements any part of the array interface, then ``out`` will contain a new reference to the newly created ndarray using @@ -819,7 +820,7 @@ General check of Python Type that looks for the :obj:`~numpy.class.__array__` attribute. `context` is unused. -.. c:function:: PyArray_IsZeroDim(op) +.. c:function:: int PyArray_IsZeroDim(PyObject *op) Evaluates true if *op* is an instance of (a subclass of) :c:data:`PyArray_Type` and has 0 dimensions. @@ -828,29 +829,29 @@ General check of Python Type Evaluates true if *op* is an instance of ``Py{cls}ArrType_Type``. -.. c:function:: PyArray_CheckScalar(op) +.. c:function:: int PyArray_CheckScalar(PyObject *op) Evaluates true if *op* is either an array scalar (an instance of a sub-type of :c:data:`PyGenericArr_Type` ), or an instance of (a sub-class of) :c:data:`PyArray_Type` whose dimensionality is 0. -.. c:function:: PyArray_IsPythonNumber(op) +.. c:function:: int PyArray_IsPythonNumber(PyObject *op) Evaluates true if *op* is an instance of a builtin numeric type (int, float, complex, long, bool) -.. c:function:: PyArray_IsPythonScalar(op) +.. c:function:: int PyArray_IsPythonScalar(PyObject *op) Evaluates true if *op* is a builtin Python scalar object (int, float, complex, bytes, str, long, bool). -.. c:function:: PyArray_IsAnyScalar(op) +.. c:function:: int PyArray_IsAnyScalar(PyObject *op) Evaluates true if *op* is either a Python scalar object (see :c:func:`PyArray_IsPythonScalar`) or an array scalar (an instance of a sub- type of :c:data:`PyGenericArr_Type` ). -.. c:function:: PyArray_CheckAnyScalar(op) +.. c:function:: int PyArray_CheckAnyScalar(PyObject *op) Evaluates true if *op* is a Python scalar object (see :c:func:`PyArray_IsPythonScalar`), an array scalar (an instance of a @@ -866,82 +867,82 @@ enumerated array data type. For the array type checking macros the argument must be a :c:type:`PyObject *` that can be directly interpreted as a :c:type:`PyArrayObject *`. -.. c:function:: PyTypeNum_ISUNSIGNED(int num) +.. c:function:: int PyTypeNum_ISUNSIGNED(int num) -.. c:function:: PyDataType_ISUNSIGNED(PyArray_Descr *descr) +.. c:function:: int PyDataType_ISUNSIGNED(PyArray_Descr *descr) -.. c:function:: PyArray_ISUNSIGNED(PyArrayObject *obj) +.. c:function:: int PyArray_ISUNSIGNED(PyArrayObject *obj) Type represents an unsigned integer. -.. c:function:: PyTypeNum_ISSIGNED(int num) +.. c:function:: int PyTypeNum_ISSIGNED(int num) -.. c:function:: PyDataType_ISSIGNED(PyArray_Descr *descr) +.. c:function:: int PyDataType_ISSIGNED(PyArray_Descr *descr) -.. c:function:: PyArray_ISSIGNED(PyArrayObject *obj) +.. c:function:: int PyArray_ISSIGNED(PyArrayObject *obj) Type represents a signed integer. -.. c:function:: PyTypeNum_ISINTEGER(int num) +.. c:function:: int PyTypeNum_ISINTEGER(int num) -.. c:function:: PyDataType_ISINTEGER(PyArray_Descr* descr) +.. c:function:: int PyDataType_ISINTEGER(PyArray_Descr* descr) -.. c:function:: PyArray_ISINTEGER(PyArrayObject *obj) +.. c:function:: int PyArray_ISINTEGER(PyArrayObject *obj) Type represents any integer. -.. c:function:: PyTypeNum_ISFLOAT(int num) +.. c:function:: int PyTypeNum_ISFLOAT(int num) -.. c:function:: PyDataType_ISFLOAT(PyArray_Descr* descr) +.. c:function:: int PyDataType_ISFLOAT(PyArray_Descr* descr) -.. c:function:: PyArray_ISFLOAT(PyArrayObject *obj) +.. c:function:: int PyArray_ISFLOAT(PyArrayObject *obj) Type represents any floating point number. -.. c:function:: PyTypeNum_ISCOMPLEX(int num) +.. c:function:: int PyTypeNum_ISCOMPLEX(int num) -.. c:function:: PyDataType_ISCOMPLEX(PyArray_Descr* descr) +.. c:function:: int PyDataType_ISCOMPLEX(PyArray_Descr* descr) -.. c:function:: PyArray_ISCOMPLEX(PyArrayObject *obj) +.. c:function:: int PyArray_ISCOMPLEX(PyArrayObject *obj) Type represents any complex floating point number. -.. c:function:: PyTypeNum_ISNUMBER(int num) +.. c:function:: int PyTypeNum_ISNUMBER(int num) -.. c:function:: PyDataType_ISNUMBER(PyArray_Descr* descr) +.. c:function:: int PyDataType_ISNUMBER(PyArray_Descr* descr) -.. c:function:: PyArray_ISNUMBER(PyArrayObject *obj) +.. c:function:: int PyArray_ISNUMBER(PyArrayObject *obj) Type represents any integer, floating point, or complex floating point number. -.. c:function:: PyTypeNum_ISSTRING(int num) +.. c:function:: int PyTypeNum_ISSTRING(int num) -.. c:function:: PyDataType_ISSTRING(PyArray_Descr* descr) +.. c:function:: int PyDataType_ISSTRING(PyArray_Descr* descr) -.. c:function:: PyArray_ISSTRING(PyArrayObject *obj) +.. c:function:: int PyArray_ISSTRING(PyArrayObject *obj) Type represents a string data type. -.. c:function:: PyTypeNum_ISPYTHON(int num) +.. c:function:: int PyTypeNum_ISPYTHON(int num) -.. c:function:: PyDataType_ISPYTHON(PyArray_Descr* descr) +.. c:function:: int PyDataType_ISPYTHON(PyArray_Descr* descr) -.. c:function:: PyArray_ISPYTHON(PyArrayObject *obj) +.. c:function:: int PyArray_ISPYTHON(PyArrayObject *obj) Type represents an enumerated type corresponding to one of the standard Python scalar (bool, int, float, or complex). -.. c:function:: PyTypeNum_ISFLEXIBLE(int num) +.. c:function:: int PyTypeNum_ISFLEXIBLE(int num) -.. c:function:: PyDataType_ISFLEXIBLE(PyArray_Descr* descr) +.. c:function:: int PyDataType_ISFLEXIBLE(PyArray_Descr* descr) -.. c:function:: PyArray_ISFLEXIBLE(PyArrayObject *obj) +.. c:function:: int PyArray_ISFLEXIBLE(PyArrayObject *obj) Type represents one of the flexible array types ( :c:data:`NPY_STRING`, :c:data:`NPY_UNICODE`, or :c:data:`NPY_VOID` ). -.. c:function:: PyDataType_ISUNSIZED(PyArray_Descr* descr): +.. c:function:: int PyDataType_ISUNSIZED(PyArray_Descr* descr): Type has no size information attached, and can be resized. Should only be called on flexible dtypes. Types that are attached to an array will always @@ -951,50 +952,50 @@ argument must be a :c:type:`PyObject *` that can be directly interpret For structured datatypes with no fields this function now returns False. -.. c:function:: PyTypeNum_ISUSERDEF(int num) +.. c:function:: int PyTypeNum_ISUSERDEF(int num) -.. c:function:: PyDataType_ISUSERDEF(PyArray_Descr* descr) +.. c:function:: int PyDataType_ISUSERDEF(PyArray_Descr* descr) -.. c:function:: PyArray_ISUSERDEF(PyArrayObject *obj) +.. c:function:: int PyArray_ISUSERDEF(PyArrayObject *obj) Type represents a user-defined type. -.. c:function:: PyTypeNum_ISEXTENDED(int num) +.. c:function:: int PyTypeNum_ISEXTENDED(int num) -.. c:function:: PyDataType_ISEXTENDED(PyArray_Descr* descr) +.. c:function:: int PyDataType_ISEXTENDED(PyArray_Descr* descr) -.. c:function:: PyArray_ISEXTENDED(PyArrayObject *obj) +.. c:function:: int PyArray_ISEXTENDED(PyArrayObject *obj) Type is either flexible or user-defined. -.. c:function:: PyTypeNum_ISOBJECT(int num) +.. c:function:: int PyTypeNum_ISOBJECT(int num) -.. c:function:: PyDataType_ISOBJECT(PyArray_Descr* descr) +.. c:function:: int PyDataType_ISOBJECT(PyArray_Descr* descr) -.. c:function:: PyArray_ISOBJECT(PyArrayObject *obj) +.. c:function:: int PyArray_ISOBJECT(PyArrayObject *obj) Type represents object data type. -.. c:function:: PyTypeNum_ISBOOL(int num) +.. c:function:: int PyTypeNum_ISBOOL(int num) -.. c:function:: PyDataType_ISBOOL(PyArray_Descr* descr) +.. c:function:: int PyDataType_ISBOOL(PyArray_Descr* descr) -.. c:function:: PyArray_ISBOOL(PyArrayObject *obj) +.. c:function:: int PyArray_ISBOOL(PyArrayObject *obj) Type represents Boolean data type. -.. c:function:: PyDataType_HASFIELDS(PyArray_Descr* descr) +.. c:function:: int PyDataType_HASFIELDS(PyArray_Descr* descr) -.. c:function:: PyArray_HASFIELDS(PyArrayObject *obj) +.. c:function:: int PyArray_HASFIELDS(PyArrayObject *obj) Type has fields associated with it. -.. c:function:: PyArray_ISNOTSWAPPED(m) +.. c:function:: int PyArray_ISNOTSWAPPED(PyArrayObject *m) Evaluates true if the data area of the ndarray *m* is in machine byte-order according to the array's data-type descriptor. -.. c:function:: PyArray_ISBYTESWAPPED(m) +.. c:function:: int PyArray_ISBYTESWAPPED(PyArrayObject *m) Evaluates true if the data area of the ndarray *m* is **not** in machine byte-order according to the array's data-type descriptor. @@ -1019,7 +1020,7 @@ argument must be a :c:type:`PyObject *` that can be directly interpret Special case of :c:func:`PyArray_EquivTypes` (...) that does not accept flexible data types but may be easier to call. -.. c:function:: int PyArray_EquivByteorders({byteorder} b1, {byteorder} b2) +.. c:function:: int PyArray_EquivByteorders(int b1, int b2) True if byteorder characters ( :c:data:`NPY_LITTLE`, :c:data:`NPY_BIG`, :c:data:`NPY_NATIVE`, :c:data:`NPY_IGNORE` ) are @@ -1529,7 +1530,7 @@ Flag checking For all of these macros *arr* must be an instance of a (subclass of) :c:data:`PyArray_Type`. -.. c:function:: PyArray_CHKFLAGS(PyObject *arr, flags) +.. c:function:: int PyArray_CHKFLAGS(PyObject *arr, int flags) The first parameter, arr, must be an ndarray or subclass. The parameter, *flags*, should be an integer consisting of bitwise @@ -1539,60 +1540,60 @@ For all of these macros *arr* must be an instance of a (subclass of) :c:data:`NPY_ARRAY_WRITEABLE`, :c:data:`NPY_ARRAY_WRITEBACKIFCOPY`, :c:data:`NPY_ARRAY_UPDATEIFCOPY`. -.. c:function:: PyArray_IS_C_CONTIGUOUS(PyObject *arr) +.. c:function:: int PyArray_IS_C_CONTIGUOUS(PyObject *arr) Evaluates true if *arr* is C-style contiguous. -.. c:function:: PyArray_IS_F_CONTIGUOUS(PyObject *arr) +.. c:function:: int PyArray_IS_F_CONTIGUOUS(PyObject *arr) Evaluates true if *arr* is Fortran-style contiguous. -.. c:function:: PyArray_ISFORTRAN(PyObject *arr) +.. c:function:: int PyArray_ISFORTRAN(PyObject *arr) Evaluates true if *arr* is Fortran-style contiguous and *not* C-style contiguous. :c:func:`PyArray_IS_F_CONTIGUOUS` is the correct way to test for Fortran-style contiguity. -.. c:function:: PyArray_ISWRITEABLE(PyObject *arr) +.. c:function:: int PyArray_ISWRITEABLE(PyObject *arr) Evaluates true if the data area of *arr* can be written to -.. c:function:: PyArray_ISALIGNED(PyObject *arr) +.. c:function:: int PyArray_ISALIGNED(PyObject *arr) Evaluates true if the data area of *arr* is properly aligned on the machine. -.. c:function:: PyArray_ISBEHAVED(PyObject *arr) +.. c:function:: int PyArray_ISBEHAVED(PyObject *arr) Evaluates true if the data area of *arr* is aligned and writeable and in machine byte-order according to its descriptor. -.. c:function:: PyArray_ISBEHAVED_RO(PyObject *arr) +.. c:function:: int PyArray_ISBEHAVED_RO(PyObject *arr) Evaluates true if the data area of *arr* is aligned and in machine byte-order. -.. c:function:: PyArray_ISCARRAY(PyObject *arr) +.. c:function:: int PyArray_ISCARRAY(PyObject *arr) Evaluates true if the data area of *arr* is C-style contiguous, and :c:func:`PyArray_ISBEHAVED` (*arr*) is true. -.. c:function:: PyArray_ISFARRAY(PyObject *arr) +.. c:function:: int PyArray_ISFARRAY(PyObject *arr) Evaluates true if the data area of *arr* is Fortran-style contiguous and :c:func:`PyArray_ISBEHAVED` (*arr*) is true. -.. c:function:: PyArray_ISCARRAY_RO(PyObject *arr) +.. c:function:: int PyArray_ISCARRAY_RO(PyObject *arr) Evaluates true if the data area of *arr* is C-style contiguous, aligned, and in machine byte-order. -.. c:function:: PyArray_ISFARRAY_RO(PyObject *arr) +.. c:function:: int PyArray_ISFARRAY_RO(PyObject *arr) Evaluates true if the data area of *arr* is Fortran-style contiguous, aligned, and in machine byte-order **.** -.. c:function:: PyArray_ISONESEGMENT(PyObject *arr) +.. c:function:: int PyArray_ISONESEGMENT(PyObject *arr) Evaluates true if the data area of *arr* consists of a single (C-style or Fortran-style) contiguous segment. @@ -2416,12 +2417,12 @@ an element copier function as a primitive.:: functions should never set the Python exception on error, because they may be called from a multi-threaded context. -.. c:function:: NPY_AUXDATA_FREE(auxdata) +.. c:function:: void NPY_AUXDATA_FREE(NpyAuxData *auxdata) A macro which calls the auxdata's free function appropriately, does nothing if auxdata is NULL. -.. c:function:: NPY_AUXDATA_CLONE(auxdata) +.. c:function:: NpyAuxData *NPY_AUXDATA_CLONE(NpyAuxData *auxdata) A macro which calls the auxdata's clone function appropriately, returning a deep copy of the auxiliary data. @@ -2488,7 +2489,7 @@ this useful approach to looping over an array. *destination*, which must have size at least *iterator* ->nd_m1+1. -.. c:function:: PyArray_ITER_GOTO1D(PyObject* iterator, npy_intp index) +.. c:function:: void PyArray_ITER_GOTO1D(PyObject* iterator, npy_intp index) Set the *iterator* index and dataptr to the location in the array indicated by the integer *index* which points to an element in the @@ -3233,7 +3234,7 @@ Memory management .. c:function:: char* PyDataMem_NEW(size_t nbytes) -.. c:function:: PyDataMem_FREE(char* ptr) +.. c:function:: void PyDataMem_FREE(char* ptr) .. c:function:: char* PyDataMem_RENEW(void * ptr, size_t newbytes) @@ -3242,7 +3243,7 @@ Memory management .. c:function:: npy_intp* PyDimMem_NEW(int nd) -.. c:function:: PyDimMem_FREE(char* ptr) +.. c:function:: void PyDimMem_FREE(char* ptr) .. c:function:: npy_intp* PyDimMem_RENEW(void* ptr, size_t newnd) @@ -3250,7 +3251,7 @@ Memory management .. c:function:: void* PyArray_malloc(size_t nbytes) -.. c:function:: PyArray_free(void* ptr) +.. c:function:: void PyArray_free(void* ptr) .. c:function:: void* PyArray_realloc(npy_intp* ptr, size_t nbytes) @@ -3334,18 +3335,18 @@ Group 1 interpreter. This macro acquires the GIL and restores the Python state from the saved variable. - .. c:function:: NPY_BEGIN_THREADS_DESCR(PyArray_Descr *dtype) + .. c:function:: void NPY_BEGIN_THREADS_DESCR(PyArray_Descr *dtype) Useful to release the GIL only if *dtype* does not contain arbitrary Python objects which may need the Python interpreter during execution of the loop. - .. c:function:: NPY_END_THREADS_DESCR(PyArray_Descr *dtype) + .. c:function:: void NPY_END_THREADS_DESCR(PyArray_Descr *dtype) Useful to regain the GIL in situations where it was released using the BEGIN form of this macro. - .. c:function:: NPY_BEGIN_THREADS_THRESHOLDED(int loop_size) + .. c:function:: void NPY_BEGIN_THREADS_THRESHOLDED(int loop_size) Useful to release the GIL only if *loop_size* exceeds a minimum threshold, currently set to 500. Should be matched @@ -3464,7 +3465,7 @@ Other constants Miscellaneous Macros ^^^^^^^^^^^^^^^^^^^^ -.. c:function:: PyArray_SAMESHAPE(PyArrayObject *a1, PyArrayObject *a2) +.. c:function:: int PyArray_SAMESHAPE(PyArrayObject *a1, PyArrayObject *a2) Evaluates as True if arrays *a1* and *a2* have the same shape. @@ -3499,11 +3500,11 @@ Miscellaneous Macros of the ordering which is lexicographic: comparing the real parts first and then the complex parts if the real parts are equal. -.. c:function:: PyArray_REFCOUNT(PyObject* op) +.. c:function:: npy_intp PyArray_REFCOUNT(PyObject* op) Returns the reference count of any Python object. -.. c:function:: PyArray_DiscardWritebackIfCopy(PyObject* obj) +.. c:function:: void PyArray_DiscardWritebackIfCopy(PyObject* obj) If ``obj.flags`` has :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` or (deprecated) :c:data:`NPY_ARRAY_UPDATEIFCOPY`, this function clears the flags, `DECREF` s @@ -3514,7 +3515,7 @@ Miscellaneous Macros error when you are finished with ``obj``, just before ``Py_DECREF(obj)``. It may be called multiple times, or with ``NULL`` input. -.. c:function:: PyArray_XDECREF_ERR(PyObject* obj) +.. c:function:: void PyArray_XDECREF_ERR(PyObject* obj) Deprecated in 1.14, use :c:func:`PyArray_DiscardWritebackIfCopy` followed by ``Py_XDECREF`` diff --git a/doc/source/reference/c-api/config.rst b/doc/source/reference/c-api/config.rst index cec5b973a202..87130699bbf8 100644 --- a/doc/source/reference/c-api/config.rst +++ b/doc/source/reference/c-api/config.rst @@ -97,7 +97,7 @@ Platform information Defined in ``numpy/npy_endian.h``. -.. c:function:: PyArray_GetEndianness() +.. c:function:: int PyArray_GetEndianness() .. versionadded:: 1.3.0 diff --git a/doc/source/reference/c-api/types-and-structures.rst b/doc/source/reference/c-api/types-and-structures.rst index d22a3f03faa8..fb5753c02df5 100644 --- a/doc/source/reference/c-api/types-and-structures.rst +++ b/doc/source/reference/c-api/types-and-structures.rst @@ -318,12 +318,12 @@ PyArrayDescr_Type and PyArray_Descr :c:data:`NPY_ITEM_REFCOUNT` \| :c:data:`NPY_NEEDS_INIT` \| :c:data:`NPY_NEEDS_PYAPI`). - .. c:function:: PyDataType_FLAGCHK(PyArray_Descr *dtype, int flags) + .. c:function:: int PyDataType_FLAGCHK(PyArray_Descr *dtype, int flags) Return true if all the given flags are set for the data-type object. - .. c:function:: PyDataType_REFCHK(PyArray_Descr *dtype) + .. c:function:: int PyDataType_REFCHK(PyArray_Descr *dtype) Equivalent to :c:func:`PyDataType_FLAGCHK` (*dtype*, :c:data:`NPY_ITEM_REFCOUNT`). diff --git a/doc/source/reference/c-api/ufunc.rst b/doc/source/reference/c-api/ufunc.rst index 1b9b6864224d..05ddf0dba119 100644 --- a/doc/source/reference/c-api/ufunc.rst +++ b/doc/source/reference/c-api/ufunc.rst @@ -86,7 +86,7 @@ Types (element-by-element) function :math:`N` times with the following signature: - .. c:function:: void loopfunc( + .. c:function:: void loopfunc(\ char** args, npy_intp const *dimensions, npy_intp const *steps, void* data) *args* From dbc4f5d3fff330a842576427674ee4fc5de0164d Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 13 Oct 2020 18:36:47 -0500 Subject: [PATCH 264/409] NEP: Initial draft for NEP 43 for extensible ufuncs (gh-16723) * NEP: Initial draft for NEP 43 * NEP: Small updates to NEP 43 * Partial iteration on some content, mostly updates the calling convention Its not quite settled, I don't quite like the `npy_intp *` as a default "user-data" yet, but I think I may warm up to it, and I think some kind of default like that is really very convenient. Split out `user_data *`, since I think Marten has a point that separating "constant" and volatile data is conceptionally easier. * Add a bit more overview-intro at the start and refactor "bound" method section * Small fixups based on Marten and also Hameers comments * Revise the "promotion and dispatching" section a bit. * Fix back-compat section and add a tiny bit of implementation * WIP: Further updates nep 43 * Revise NEP 43 and expand on arraymethod (previously the dispatching/promotion was more fleshed out) * Update image and make sure it is included at the right place. * small fixups * Update doc/neps/nep-0043-extensible-ufuncs.rst Co-authored-by: Dave Hirschfeld * Fix comment which was missing the final part of sentence * Pass through, up to "steps involved in a ufunc call" * Review up to ArrayMethod * Review up to ArrayMethod Specs * Review up to loop reuse * Review most of the remaining document * Fix title typo * Wording updates and minor fixups. * Apply suggestions from code review Co-authored-by: Ross Barnowski * Address Ross's additional comments Co-authored-by: Dave Hirschfeld Co-authored-by: Ross Barnowski --- doc/neps/_static/nep43-sketch-with-text.svg | 1304 ++++++++ doc/neps/_static/nep43-sketch.svg | 3009 +++++++++++++++++++ doc/neps/nep-0043-extensible-ufuncs.rst | 1301 ++++++++ 3 files changed, 5614 insertions(+) create mode 100644 doc/neps/_static/nep43-sketch-with-text.svg create mode 100644 doc/neps/_static/nep43-sketch.svg create mode 100644 doc/neps/nep-0043-extensible-ufuncs.rst diff --git a/doc/neps/_static/nep43-sketch-with-text.svg b/doc/neps/_static/nep43-sketch-with-text.svg new file mode 100644 index 000000000000..212cfe89cc5b --- /dev/null +++ b/doc/neps/_static/nep43-sketch-with-text.svg @@ -0,0 +1,1304 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + Loopdescriptors + + ResolverInput + + + + + → + + + + + + → + + >U5 + <U8 + + set descriptorsfor inner-loop… + <U5 + <U8 + <U13 + + User Input + + + + + + Input/OutputOperands + + + + + + → + + DType classesof the ArrayMethod + + + + + → + Promotion (if necessary) + + If provided + + + + + + → + + >U5 + S8 + >U5 + <S8 + Unicode + Unicode + Unicode + If not provided + Cast descriptorsto Loop DTypes + <U13 + + + + + + + + Registered + + resolve_descriptors + + + Perform operation with these descriptors(setup, inner-loop function, teardown) + NumPy + Registered or default + ArrayMethod + + Casting, Result Allocation and Outer Iterationdone by UFunc Machinery (within ArrayMethod) + Promoter + + Inputs + Output + … including correctoutput descriptor + + + ArrayMethod lookup + + diff --git a/doc/neps/_static/nep43-sketch.svg b/doc/neps/_static/nep43-sketch.svg new file mode 100644 index 000000000000..372c0ee46fc0 --- /dev/null +++ b/doc/neps/_static/nep43-sketch.svg @@ -0,0 +1,3009 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/neps/nep-0043-extensible-ufuncs.rst b/doc/neps/nep-0043-extensible-ufuncs.rst new file mode 100644 index 000000000000..008cfad5f1b6 --- /dev/null +++ b/doc/neps/nep-0043-extensible-ufuncs.rst @@ -0,0 +1,1301 @@ +.. _NEP43: + +============================================================================== +NEP 43 — Enhancing the Extensibility of UFuncs +============================================================================== + +:title: Enhancing the Extensibility of UFuncs +:Author: Sebastian Berg +:Status: Draft +:Type: Standard +:Created: 2020-06-20 + + +.. note:: + + This NEP is fourth in a series: + + - :ref:`NEP 40 ` explains the shortcomings of NumPy's dtype implementation. + + - :ref:`NEP 41 ` gives an overview of our proposed replacement. + + - :ref:`NEP 42 ` describes the new design's datatype-related APIs. + + - NEP 43 (this document) describes the new design's API for universal functions. + + +****************************************************************************** +Abstract +****************************************************************************** + +The previous NEP 42 proposes the creation of new DTypes which can +be defined by users outside of NumPy itself. +The implementation of NEP 42 will enable users to create arrays with a custom dtype +and stored values. +This NEP outlines how NumPy will operate on arrays with custom dtypes in the future. +The most important functions operating on NumPy arrays are the so called +"universal functions" (ufunc) which include all math functions, such as +``np.add``, ``np.multiply``, and even ``np.matmul``. +These ufuncs must operate efficiently on multiple arrays with +different datatypes. + +This NEP proposes to expand the design of ufuncs. +It makes a new distinction between the ufunc which can operate +on many different dtypes such as floats or integers, +and a new ``ArrayMethod`` which defines the efficient operation for +specific dtypes. + +.. note:: + + Details of the private and external APIs may change to reflect user + comments and implementation constraints. The underlying principles and + choices should not change significantly. + + +****************************************************************************** +Motivation and scope +****************************************************************************** + +The goal of this NEP is to extend universal +functions support the new DType system detailed in NEPs 41 and 42. +While the main motivation is enabling new user-defined DTypes, this will +also significantly simplify defining universal functions for NumPy strings or +structured DTypes. +Until now, these DTypes are not supported by any of NumPy's functions +(such as ``np.add`` or ``np.equal``), due to difficulties arising from +their parametric nature (compare NEP 41 and 42), e.g. the string length. + +Functions on arrays must handle a number of distinct steps which are +described in more detail in section "`Steps involved in a UFunc call`_". +The most important ones are: + +- Organizing all functionality required to define a ufunc call for specific + DTypes. This is often called the "inner-loop". +- Deal with input for which no exact matching implementation is found. + For example when ``int32`` and ``float64`` are added, the ``int32`` + is cast to ``float64``. This requires a distinct "promotion" step. + +After organizing and defining these, we need to: + +- Define the user API for customizing both of the above points. +- Allow convenient reuse of existing functionality. + For example a DType representing physical units, such as meters, + should be able to fall back to NumPy's existing math implementations. + +This NEP details how these requirements will be achieved in NumPy: + +- All DTyper-specific functionality currently part of the ufunc + definition will be defined as part of a new `ArrayMethod`_ object. + This ``ArrayMethod`` object will be the new, preferred, way to describe any + function operating on arrays. + +- Ufuncs will dispatch to the ``ArrayMethod`` and potentially use promotion + to find the correct ``ArrayMethod`` to use. + This will be described in the `Promotion and dispatching`_ section. + +A new C-API will be outlined in each section. A future Python API is +expected to be very similar and the C-API is presented in terms of Python +code for readability. + +The NEP proposes a large, but necessary, refactor of the NumPy ufunc internals. +This modernization will not affect end users directly and is not only a necessary +step for new DTypes, but in itself a maintenance effort which is expected to +help with future improvements to the ufunc machinery. + +While the most important restructure proposed is the new ``ArrayMethod`` +object, the largest long-term consideration is the API choice for +promotion and dispatching. + + +*********************** +Backwards Compatibility +*********************** + +The general backwards compatibility issues have also been listed +previously in NEP 41. + +The vast majority of users should not see any changes beyond those typical +for NumPy releases. +There are three main users or use-cases impacted by the proposed changes: + +1. The Numba package uses direct access to the NumPy C-loops and modifies + the NumPy ufunc struct directly for its own purposes. +2. Astropy uses its own "type resolver", meaning that a default switch over + from the existing type resolution to a new default Promoter requires care. +3. It is currently possible to register loops for dtype *instances*. + This is theoretically useful for structured dtypes and is a resolution + step happening *after* the DType resolution step proposed here. + +This NEP will try hard to maintain backward compatibility as much as +possible. However, both of these projects have signaled willingness to adapt +to breaking changes. + +The main reason why NumPy will be able to provide backward compatibility +is that: + +* Existing inner-loops can be wrapped, adding an indirection to the call but + maintaining full backwards compatibility. + The ``get_loop`` function can, in this case, search the existing + inner-loop functions (which are stored on the ufunc directly) in order + to maintain full compatibility even with potential direct structure access. +* Legacy type resolvers can be called as a fallback (potentially caching + the result). The resolver may need to be called twice (once for the DType + resolution and once for the ``resolve_descriptor`` implementation). +* The fallback to the legacy type resolver should in most cases handle loops + defined for such structured dtype instances. This is because if there is no + other ``np.Void`` implementation, the legacy fallback will retain the old + behaviour at least initially. + +The masked type resolvers specifically will *not* remain supported, but +has no known users (including NumPy itself, which only uses the default +version). + +While the above changes potentially break some workflows, +we believe that the long-term improvements vastly outweigh this. +Further, packages such as astropy and Numba are capable of adapting so that +end-users may need to update their libraries but not their code. + + +****************************************************************************** +Usage and impact +****************************************************************************** + +This NEP restructures how operations on NumPy arrays are defined both +within NumPy and for external implementers. +The NEP mainly concerns those who either extend ufuncs for custom DTypes +or create custom ufuncs. It does not aim to finalize all +potential use-cases, but rather restructure NumPy to be extensible and allow +addressing new issues or feature requests as they arise. + + +Overview and end user API +========================= + +To give an overview of how this NEP proposes to structure ufuncs, +the following describes the potential exposure of the proposed restructure +to the end user. + +Universal functions are much like a Python method defined on the DType of +the array when considering a ufunc with only a single input:: + + res = np.positive(arr) + +could be implemented (conceptually) as:: + + positive_impl = arr.dtype.positive + res = positive_impl(arr) + +However, unlike methods, ``positive_impl`` is not stored on the dtype itself. +It is rather the implementation of ``np.positive`` for a specific DType. +Current NumPy partially exposes this "choice of implementation" using +the ``dtype`` (or more exact ``signature``) attribute in universal functions, +although these are rarely used:: + + np.positive(arr, dtype=np.float64) + +forces NumPy to use the ``positive_impl`` written specifically for the Float64 +DType. + +This NEP makes the distinction more explicit, by creating a new object to +represent ``positive_impl``:: + + positive_impl = np.positive.resolve_impl((type(arr.dtype), None)) + # The `None` represents the output DType which is automatically chosen. + +While the creation of a ``positive_impl`` object and the ``resolve_impl`` +method is part of this NEP, the following code:: + + res = positive_impl(arr) + +may not be implemented initially and is not central to the redesign. + +In general NumPy universal functions can take many inputs. +This requires looking up the implementation by considering all of them +and makes ufuncs "multi-methods" with respect to the input DTypes:: + + add_impl = np.add.resolve_impl((type(arr1.dtype), type(arr2.dtype), None)) + +This NEP defines how ``positive_impl`` and ``add_impl`` will be represented +as a new ``ArrayMethod`` which can be implemented outside of NumPy. +Further, it defines how ``resolve_impl`` will implement and solve dispatching +and promotion. + +The reasons for this split may be more clear after reviewing the +`Steps involved in a UFunc call`_ section. + + +Defining a new ufunc implementation +=================================== + +The following is a mock-up of how a new implementation, in this case +to define string equality, will be added to a ufunc. + +.. code-block:: python + + class StringEquality(BoundArrayMethod): + nin = 1 + nout = 1 + DTypes = (String, String, Bool) + + def resolve_descriptors(context, given_descrs): + """The strided loop supports all input string dtype instances + and always returns a boolean. (String is always native byte order.) + + Defining this function is not necessary, since NumPy can provide + it by default. + """ + assert isinstance(given_descrs[0], context.DTypes[0]) + assert isinstance(given_descrs[1], context.DTypes[1]) + + # The operation is always "safe" casting (most ufuncs are) + return (given_descrs[0], given_descrs[1], context.DTypes[2]()), "safe" + + def strided_loop(context, n, data, strides): + """The 1-D strided loop, similar to those used in current ufuncs""" + # n: Number of elements in the one dimensional loop + # data: Pointers to the array data. + # strides: strides to iterate all elements + num_chars1 = context.descriptors[0].itemsize + num_chars2 = context.descriptors[0].itemsize + + # C code using the above information to compare the strings in + # both arrays. In particular, this loop requires the `num_chars1` + # and `num_chars2`. Information which is currently not easily + # available. + + np.equal.register_impl(StringEquality) + del StringEquality # may be deleted. + + +This definition will be sufficient to create a new loop, and the +structure allows for expansion in the future; something that is already +required to implement casting within NumPy itself. +We use ``BoundArrayMethod`` and a ``context`` structure here. These +are described and motivated in details later. Briefly: + +* ``context`` is a generalization of the ``self`` that Python passes to its + methods. +* ``BoundArrayMethod`` is equivalent to the Python distinction that + ``class.method`` is a method, while ``class().method`` returns a "bound" method. + + +Customizing Dispatching and Promotion +===================================== + +Finding the correct implementation when ``np.positive.resolve_impl()`` is +called is largely an implementation detail. +But, in some cases it may be necessary to influence this process when no +implementation matches the requested DTypes exactly: + +.. code-block:: python + + np.multiple.resolve_impl((Timedelta64, Int8, None)) + +will not have an exact match, because NumPy only has an implementation for +multiplying ``Timedelta64`` with ``Int64``. +In simple cases, NumPy will use a default promotion step to attempt to find +the correct implementation, but to implement the above step, we will allow +the following: + +.. code-block:: python + + def promote_timedelta_integer(ufunc, dtypes): + new_dtypes = (Timdelta64, Int64, dtypes[-1]) + # Resolve again, using Int64: + return ufunc.resolve_impl(new_dtypes) + + np.multiple.register_promoter( + (Timedelta64, Integer, None), promote_timedelta_integer) + +Where ``Integer`` is an abstract DType (compare NEP 42). + + +.. _steps_of_a_ufunc_call: + +**************************************************************************** +Steps involved in a UFunc call +**************************************************************************** + +Before going into more detailed API choices, it is helpful to review the +steps involved in a call to a universal function in NumPy. + +A UFunc call is split into the following steps: + +1. *Handle ``__array_ufunc__`` protocol:* + + * For array-likes such as a Dask arrays, NumPy can defer the operation. + This step is performed first, and unaffected by this NEP (compare :ref:`NEP18`). + +2. *Promotion and dispatching* + + * Given the DTypes of all inputs, find the correct implementation. + E.g. an implementation for ``float64``, ``int64`` or a user-defined DType. + + * When no exact implementation exists, *promotion* has to be performed. + For example, adding a ``float32`` and a ``float64`` is implemented by + first casting the ``float32`` to ``float64``. + +3. *Parametric ``dtype`` resolution:* + + * In general, whenever an output DType is parametric the parameters have + to be found (resolved). + * For example, if a loop adds two strings, it is necessary to define the + correct output (and possibly input) dtypes. ``S5 + S4 -> S9``, while + an ``upper`` function has the signature ``S5 -> S5``. + * When they are not parametric, a default implementation is provided + which fills in the default dtype instances (ensuring for example native + byte order). + +4. *Preparing the iteration:* + + * This step is largely handled by ``NpyIter`` internally (the iterator). + * Allocate all outputs and temporary buffers necessary to perform casts. + * Find the best iteration order, which includes information to efficiently + implement broadcasting. For example, adding a single value to an array + repeats the same value. + +5. *Setup and fetch the C-level function:* + + * If necessary, allocate temporary working space. + * Find the C-implemented, light weight, inner-loop function. + Finding the inner-loop function can allow specialized implementations + in the future. + For example casting currently optimizes contiguous casts and + reductions have optimizations that are currently handled + inside the inner-loop function itself. + * Signal whether the inner-loop requires the Python API or whether + the GIL may be released (to allow threading). + * Clear floating point exception flags. + +6. *Perform the actual calculation:* + + * Run the DType specific inner-loop function. + * The inner-loop may require access to additional data, such as dtypes or + additional data set in the previous step. + * The inner-loop function may be called an undefined number of times. + +7. *Finalize:* + + * Free any temporary working space allocated in 5. + * Check for floating point exception flags. + * Return the result. + +The ``ArrayMethod`` provides a concept to group steps 3 to 6 and partially 7. +However, implementers of a new ufunc or ``ArrayMethod`` do not need to +customize the behaviour in steps 4 or 6, aside from the inner-loop function. +For the ``ArrayMethod`` implementer, the central steps to have control over +are step 3 and step 5 to provide the custom inner-loop function. +Further customization is a potential future extension. + +Step 2. is promotion and dispatching which will also be restructured +with new API which allows influencing the process where necessary. + +Step 1 is listed for completeness and is unaffected by this NEP. + +The following sketch provides an overview of step 2 to 6 with an emphasize +of how dtypes are handled: + +.. figure:: _static/nep43-sketch.svg + :figclass: align-center + + +***************************************************************************** +ArrayMethod +***************************************************************************** + +A central proposal of this NEP is the creation of the ``ArrayMethod`` as an object +describing each implementation specific to a given set of DTypes. +We use the ``class`` syntax to describe the information required to create +a new ``ArrayMethod`` object: + +.. code-block:: python + :dedent: 0 + + class ArrayMethod: + name: str # Name, mainly useful for debugging + + # Casting safety information (almost always "safe", necessary to + # unify casting and universal functions) + casting: Casting = "safe" + + # More general flags: + flags: int + + @staticmethod + def resolve_descriptors( + Context: context, Tuple[DType]: given_descrs)-> Casting, Tuple[DType]: + """Returns the safety of the operation (casting safety) and the + """ + # A default implementation can be provided for non-parametric + # output dtypes. + raise NotImplementedError + + @staticmethod + def get_loop(Context : context, strides, ...) -> strided_loop_function, flags: + """Returns the low-level C (strided inner-loop) function which + performs the actual operation. + + This method may initially private, users will be able to provide + a set of optimized inner-loop functions instead: + + * `strided_inner_loop` + * `contiguous_inner_loop` + * `unaligned_strided_loop` + * ... + """ + raise NotImplementedError + + @staticmethod + def strided_inner_loop(Context : context, data, strides,...): + """The inner-loop (equivalent to the current ufunc loop) + which is returned by the default `get_loop()` implementation.""" + raise NotImplementedError + +With ``Context`` providing mostly static information about the function call: + +.. code-block:: python + :dedent: 0 + + class Context: + # The ArrayMethod object itself: + ArrayMethod : method + + # Information about the caller, e.g. the ufunc, such as `np.add`: + callable : caller = None + # The number of input arguments: + int : nin = 1 + # The number of output arguments: + int : nout = 1 + # The DTypes this Method operates on/is defined for: + Tuple[DTypeMeta] : dtypes + # The actual dtypes instances the inner-loop operates on: + Tuple[DType] : descriptors + + # Any additional information required. In the future, this will + # generalize or duplicate things currently stored on the ufunc: + # - The ufunc signature of generalized ufuncs + # - The identity used for reductions + +And ``flags`` stored properties, for whether: + +* the ``ArrayMethod`` supports unaligned input and output arrays +* the inner-loop function requires the Python API (GIL) +* NumPy has to check the floating point error CPU flags. + +*Note: More information is expected to be added as necessary.* + + +The call ``Context`` +==================== + +The "context" object is analogous to Python's ``self`` that is +passed to all methods. +To understand why the "context" object is necessary and its +internal structure, it is helpful to remember +that a Python method can be written in the following way +(see also the `documentation of __get__ +`_): + +.. code-block:: python + + class BoundMethod: + def __init__(self, instance, method): + self.instance = instance + self.method = method + + def __call__(self, *args, **kwargs): + return self.method.function(self.instance, *args, **kwargs) + + + class Method: + def __init__(self, function): + self.function = function + + def __get__(self, instance, owner=None): + assert instance is not None # unsupported here + return BoundMethod(instance, self) + + +With which the following ``method1`` and ``method2`` below, behave identically: + +.. code-block:: python + + def function(self): + print(self) + + class MyClass: + def method1(self): + print(self) + + method2 = Method(function) + +And both will print the same result: + +.. code-block:: python + + >>> myinstance = MyClass() + >>> myinstance.method1() + <__main__.MyClass object at 0x7eff65436d00> + >>> myinstance.method2() + <__main__.MyClass object at 0x7eff65436d00> + +Here ``self.instance`` would be all information passed on by ``Context``. +The ``Context`` is a generalization and has to pass additional information: + +* Unlike a method which operates on a single class instance, the ``ArrayMethod`` + operates on many input arrays and thus multiple dtypes. +* The ``__call__`` of the ``BoundMethod`` above contains only a single call + to a function. But an ``ArrayMethod`` has to call ``resolve_descriptors`` + and later pass on that information to the inner-loop function. +* A Python function has no state except that defined by its outer scope. + Within C, ``Context`` is able to provide additional state if necessary. + +Just as Python requires the distinction of a method and a bound method, +NumPy will have a ``BoundArrayMethod``. +This stores all of the constant information that is part of the ``Context``, +such as: + +* the ``DTypes`` +* the number of input and ouput arguments +* the ufunc signature (specific to generalized ufuncs, compare :ref:`NEP20`). + +Fortunately, most users and even ufunc implementers will not have to worry +about these internal details; just like few Python users need to know +about the ``__get__`` dunder method. +The ``Context`` object or C-structure provides all necessary data to the +fast C-functions and NumPy API creates the new ``ArrayMethod`` or +``BoundArrayMethod`` as required. + + +.. _ArrayMethod_specs: + +ArrayMethod Specifications +========================== + +These specifications provide a minimal initial C-API, which shall be expanded +in the future, for example to allow specialized inner-loops. + +Briefly, NumPy currently relies on strided inner-loops and this +will be the only allowed method of defining a ufunc initially. +We expect the addition of a ``setup`` function or exposure of ``get_loop`` +in the future. + +UFuncs require the same information as casting, giving the following +definitions (see also :ref:`NEP 42 ` ``CastingImpl``): + +* A new structure to be passed to the resolve function and inner-loop:: + + typedef struct { + PyObject *caller; /* The ufunc object */ + PyArrayMethodObject *method; + + int nin, nout; + + PyArray_DTypeMeta **dtypes; + /* Operand descriptors, filled in by resolve_desciptors */ + PyArray_Descr **descriptors; + + void *reserved; // For Potential in threading (Interpreter state) + } PyArrayMethod_Context + + This structure may be appended to include additional information in future + versions of NumPy and includes all constant loop metadata. + + We could version this structure, although it may be simpler to version + the ``ArrayMethod`` itself. + +* Similar to casting, ufuncs may need to find the correct loop dtype + or indicate that a loop is only capable of handling certain instances of + the involved DTypes (e.g. only native byteorder). This is handled by + a ``resolve_descriptors`` function (identical to the ``resolve_descriptors`` + of ``CastingImpl``):: + + NPY_CASTING + resolve_descriptors( + PyArrayMethod_Context *context, + PyArray_Descr *given_dtypes[nin+nout], + PyArray_Descr *loop_dtypes[nin+nout]); + + The function fills ``loop_dtypes`` based on the given ``given_dtypes``. + This requires filling in the descriptor of the output(s). + Often also the input descriptor(s) have to be found, e.g. to ensure native + byteorder when needed by the inner-loop. + + In most cases an ``ArrayMethod`` will have non-parametric output DTypes + so that a default implementation can be provided. + +* An additional ``void *user_data`` will usually be typed to extend + the existing ``NpyAuxData *`` struct:: + + struct { + NpyAuxData_FreeFunc *free; + NpyAuxData_CloneFunc *clone; + /* To allow for a bit of expansion without breaking the ABI */ + void *reserved[2]; + } NpyAuxData; + + This struct is currently mainly used for the NumPy internal casting + machinery and as of now both ``free`` and ``clone`` must be provided, + although this could be relaxed. + + Unlike NumPy casts, the vast majority of ufuncs currently do not require + this additional scratch-space, but may need simple flagging capability + for example for implementing warnings (see Error and Warning Handling below). + To simplify this NumPy will pass a single zero initialized ``npy_intp *`` + when ``user_data`` is not set. + *NOTE that it would be possible to pass this as part of ``Context``.* + +* The optional ``get_loop`` function will not be public initially, to avoid + finalizing the API which requires design choices also with casting: + + .. code-block:: + + innerloop * + get_loop( + PyArrayMethod_Context *context, + /* (move_references is currently used internally for casting) */ + int aligned, int move_references, + npy_intp *strides, + PyArray_StridedUnaryOp **out_loop, + NpyAuxData **userdata, + NPY_ARRAYMETHOD_FLAGS *flags); + + The ``NPY_ARRAYMETHOD_FLAGS`` can indicate whether the Python API is required + and floating point errors must be checked. + +* The inner-loop function:: + + int inner_loop(PyArrayMethod_Context *context, ..., void *userdata); + + Will have the identical signature to current inner-loops with the following + changes: + + * A return value to indicate an error when returning ``-1`` instead of ``0``. + When returning ``-1`` a Python error must be set. + * The new first argument ``PyArrayMethod_Context *`` is used to pass in + potentially required information about the ufunc or descriptors in a + convenient way. + * The ``void *userdata`` will be the ``NpyAuxData **userdata`` as set by + ``get_loop``. If ``get_loop`` does not set ``userdata`` an ``npy_intp *`` + is passed instead (see `Error Handling`_ below for the motivation). + + *Note:* Since ``get_loop`` is expected to be private, the exact implementation + of ``userdata`` can be modified until final exposure. + +Creation of a new ``BoundArrayMethod`` will use a ``PyArrayMethod_FromSpec()`` +function. A shorthand will allow direct registration to a ufunc using +``PyUFunc_AddImplementationFromSpec()``. The specification is expected +to contain the following (this may extend in the future):: + + typedef struct { + const char *name; /* Generic name, mainly for debugging */ + int nin, nout; + NPY_CASTING casting; + NPY_ARRAYMETHOD_FLAGS flags; + PyArray_DTypeMeta **dtypes; + PyType_Slot *slots; + } PyArrayMethod_Spec; + + +Discussion and alternatives +=========================== + +The above split into an ``ArrayMethod`` and ``Context`` and the additional +requirement of a ``BoundArrayMethod`` is a necessary split mirroring the +implementation of methods and bound methods in Python. + +One reason for this requirement is that it allows storing the ``ArrayMethod`` +object in many cases without holding references to the ``DTypes`` which may +be important if DTypes are created (and deleted) dynamically. +(This is a complex topic, which does not have a complete solution in current +Python, but the approach solves the issue with respect to casting.) + +There seem to be no alternatives to this structure. Separating the +DType-specific steps from the general ufunc dispatching and promotion is +absolutely necessary to allow future extension and flexibility. +Furthermore, it allows unifying casting and ufuncs. + +Since the structure of ``ArrayMethod`` and ``BoundArrayMethod`` will be +opaque and can be extended, there are few long-term design implications aside +from the choice of making them Python objects. + + +``resolve_descriptors`` +----------------------- + +The ``resolve_descriptors`` method is possibly the main innovation of this +NEP and it is central also in the implementation of casting in NEP 42. + +By ensuring that every ``ArrayMethod`` provides ``resolve_descriptors`` we +define a unified, clear API for step 3 in `Steps involved in a UFunc call`_. +This step is required to allocate output arrays and has to happen before +casting can be prepared. + +While the returned casting-safety (``NPY_CASTING``) will almost always be +"safe" for universal functions, including it has two big advantages: + +* Returning the casting safety is central to NEP 42 for casting and + allows the unmodified use of ``ArrayMethod`` there. +* There may be a future desire to implement fast but unsafe implementations. + For example for ``int64 + int64 -> int32`` which is unsafe from a casting + perspective. Currently, this would use ``int64 + int64 -> int64`` and then + cast to ``int32``. An implementation that skips the cast would + have to signal that it effectively includes the "same-kind" cast and is + thus not considered "safe". + + +``get_loop`` method +------------------- + +Currently, NumPy ufuncs typically only provide a single strided loop, so that +the ``get_loop`` method may seem unnecessary. +For this reason we plan for ``get_loop`` to be a private function initially. + +However, ``get_loop`` is required for casting where specialized loops are +used even beyond strided and contiguous loops. +Thus, the ``get_loop`` function must be a full replacement for +the internal ``PyArray_GetDTypeTransferFunction``. + +In the future, ``get_loop`` may be made public or a new ``setup`` function +be exposed to allow more control, for example to allow allocating +working memory. +Further, we could expand ``get_loop`` and allow the ``ArrayMethod`` implementer +to also control the outer iteration and not only the 1-D inner-loop. + + +Extending the inner-loop signature +---------------------------------- + +Extending the inner-loop signature is another central and necessary part of +the NEP. + +**Passing in the ``Context``:** + +Passing in the ``Context`` potentially allows for the future extension of +the signature by adding new fields to the context struct. +Furthermore it provides direct access to the dtype instances which +the inner-loop operates on. +This is necessary information for parametric dtypes since for example comparing +two strings requires knowing the length of both strings. +The ``Context`` can also hold potentially useful information such as the +the original ``ufunc``, which can be helpful when reporting errors. + +In principle passing in Context is not necessary, as all information could be +included in ``userdata`` and set up in the ``get_loop`` function. +In this NEP we propose passing the struct to simplify creation of loops for +parametric DTypes. + +**Passing in user data:** + +The current casting implementation uses the existing ``NpyAuxData *`` to pass +in additional data as defined by ``get_loop``. +There are certainly alternatives to the use of this structure, but it +provides a simple solution, which is already used in NumPy and public API. + +``NpyAyxData *`` is a light weight, allocated structure and since it already +exists in NumPy for this purpose, it seems a natural choice. +To simplify some use-cases (see "Error Handling" below), we will pass a +``npy_intp *userdata = 0`` instead when ``userdata`` is not provided. + +*Note: Since ``get_loop`` is expected to be private initially we can gain +experience with ``userdata`` before exposing it as public API.* + +**Return value:** + +The return value to indicate an error is an important, but currently missing +feature in NumPy. The error handling is further complicated by the way +CPUs signal floating point errors. +Both are discussed in the next section. + +Error Handling +"""""""""""""" + +We expect that future inner-loops will generally set Python errors as soon +as an error is found. This is complicated when the inner-loop is run without +locking the GIL. In this case the function will have to lock the GIL, +set the Python error and return ``-1`` to indicate an error occurred:: + + int + inner_loop(PyArrayMethod_Context *context, ..., void *userdata) + { + NPY_ALLOW_C_API_DEF + + for (npy_intp i = 0; i < N; i++) { + /* calculation */ + + if (error_occurred) { + NPY_ALLOW_C_API; + PyErr_SetString(PyExc_ValueError, + "Error occurred inside inner_loop."); + NPY_DISABLE_C_API + return -1; + } + } + return 0; + } + +Floating point errors are special, since they require checking the hardware +state which is too expensive if done within the inner-loop function itself. +Thus, NumPy will handle these if flagged by the ``ArrayMethod``. +An ``ArrayMethod`` should never cause floating point error flags to be set +if it flags that these should not be checked. This could interfere when +calling multiple functions; in particular when casting is necessary. + +An alternative solution would be to allow setting the error only at the later +finalization step when NumPy will also check the floating point error flags. + +We decided against this pattern at this time. It seems more complex and +generally unnecessary. +While safely grabbing the GIL in the loop may require passing in an additional +``PyThreadState`` or ``PyInterpreterState`` in the future (for subinterpreter +support), this is acceptable and can be anticipated. +Setting the error at a later point would add complexity: for instance +if an operation is paused (which can currently happen for casting in particular), +the error check needs to run explicitly ever time this happens. + +We expect that setting errors immediately is the easiest and most convenient +solution and more complex solution may be possible future extensions. + +Handling *warnings* is slightly more complex: A warning should be +given exactly once for each function call (i.e. for the whole array) even +if naively it would be given many times. +To simplify such a use case, we will pass in ``npy_intp *userdata = 0`` +by default which can be used to store flags (or other simple persistent data). +For instance, we could imagine an integer multiplication loop which warns +when an overflow occurred:: + + int + integer_multiply(PyArrayMethod_Context *context, ..., npy_intp *userdata) + { + int overflow; + NPY_ALLOW_C_API_DEF + + for (npy_intp i = 0; i < N; i++) { + *out = multiply_integers(*in1, *in2, &overflow); + + if (overflow && !*userdata) { + NPY_ALLOW_C_API; + if (PyErr_Warn(PyExc_UserWarning, + "Integer overflow detected.") < 0) { + NPY_DISABLE_C_API + return -1; + } + *userdata = 1; + NPY_DISABLE_C_API + } + return 0; + } + +*TODO:* The idea of passing an ``npy_intp`` scratch space when ``userdata`` +is not set seems convenient, but I am uncertain about it, since I am not +aware of any similar prior art. This "scratch space" could also be part of +the ``context`` in principle. + + +Reusing existing Loops/Implementations +====================================== + +For many DTypes the above definition for adding additional C-level loops will be +sufficient and require no more than a single strided loop implementation +and if the loop works with parametric DTypes, the +``resolve_descriptors`` function *must* additionally be provided. + +However, in some use-cases it is desirable to call back to an existing implementation. +In Python, this could be achieved by simply calling into the original ufunc. + +For better performance in C, and for large arrays, it is desirable to reuse +an existing ``ArrayMethod`` as directly as possible, so that its inner-loop function +can be used directly without additional overhead. +We will thus allow to create a new, wrapping, ``ArrayMethod`` from an existing +``ArrayMethod``. + +This wrapped ``ArrayMethod`` will have two additional methods: + +* ``view_inputs(Tuple[DType]: input_descr) -> Tuple[DType]`` replacing the + user input descriptors with descriptors matching the wrapped loop. + It must be possible to *view* the inputs as the output. + For example for ``Unit[Float64]("m") + Unit[Float32]("km")`` this will + return ``float64 + int32``. The original ``resolve_descriptors`` will + convert this to ``float64 + float64``. + +* ``wrap_outputs(Tuple[DType]: input_descr) -> Tuple[DType]`` replacing the + resolved descriptors with with the desired actual loop descriptors. + The original ``resolve_descriptors`` function will be called between these + two calls, so that the output descriptors may not be set in the first call. + In the above example it will use the ``float64`` as returned (which might + have changed the byte-order), and further resolve the physical unit making + the final signature:: + + ``Unit[Float64]("m") + Unit[Float64]("m") -> Unit[Float64]("m")`` + + the UFunc machinery will take care of casting the "km" input to "m". + + +The ``view_inputs`` method allows passing the correct inputs into the +original ``resolve_descriptors`` function, while ``wrap_outputs`` ensures +the correct descriptors are used for output allocation and input buffering casts. + +An important use-case for this is that of an abstract Unit DType +with subclasses for each numeric dtype (which could be dynamically created):: + + Unit[Float64]("m") + # with Unit[Float64] being the concrete DType: + isinstance(Unit[Float64], Unit) # is True + +Such a ``Unit[Float64]("m")`` instance has a well-defined signature with +respect to type promotion. +The author of the ``Unit`` DType can implement most necessary logic by +wrapping the existing math functions and using the two additional methods +above. +Using the *promotion* step, this will allow to create a register a single +promoter for the abstract ``Unit`` DType with the ``ufunc``. +The promoter can then add the wrapped concrete ``ArrayMethod`` dynamically +at promotion time, and NumPy can cache (or store it) after the first call. + +**Alternative use-case:** + +A different use-case is that of a ``Unit(float64, "m")`` DType, where +the numerical type is part of the DType parameter. +This approach is possible, but will require a custom ``ArrayMethod`` +which wraps existing loops. +It must also always require require two steps of dispatching +(one to the ``Unit`` DType and a second one for the numerical type). + +Furthermore, the efficient implementation will require the ability to +fetch and reuse the inner-loop function from another ``ArrayMethod``. +(Which is probably necessary for users like Numba, but it is uncertain +whether it should be a common pattern and it cannot be accessible from +Python itself.) + + +.. _promotion_and_dispatching: + +************************* +Promotion and dispatching +************************* + +NumPy ufuncs are multi-methods in the sense that they operate on (or with) +multiple DTypes at once. +While the input (and output) dtypes are attached to NumPy arrays, +the ``ndarray`` type itself does not carry the information of which +function to apply to the data. + +For example, given the input:: + + int_arr = np.array([1, 2, 3], dtype=np.int64) + float_arr = np.array([1, 2, 3], dtype=np.float64) + np.add(int_arr, float_arr) + +has to find the correct ``ArrayMethod`` to perform the operation. +Ideally, there is an exact match defined, e.g. for ``np.add(int_arr, int_arr)`` +the ``ArrayMethod[Int64, Int64, out=Int64]`` matches exactly and can be used. +However, for ``np.add(int_arr, float_arr)`` there is no direct match, +requiring a promotion step. + +Promotion and dispatching process +================================= + +In general the ``ArrayMethod`` is found by searching for an exact match of +all input DTypes. +The output dtypes should *not* affect calculation, but if multiple registered +``ArrayMethod``\ s match exactly, the output DType will be used to find the +better match. +This will allow the current distinction for ``np.equal`` loops which define +both ``Object, Object -> Bool`` (default) and ``Object, Object -> Object``. + +Initially, an ``ArrayMethod`` will be defined for *concrete* DTypes only +and since these cannot be subclassed an exact match is guaranteed. +In the future we expect that ``ArrayMethod``\ s can also be defined for +*abstract* DTypes. In which case the best match is found as detailed below. + +**Promotion:** + +If a matching ``ArrayMethod`` exists, dispatching is straight forward. +However, when it does not, require additional definitions to implement +promotion: + +* By default any UFunc has a promotion which uses the common DType of all + inputs and dispatches a second time. This is well-defined for most + mathematical functions, but can be disabled or customized if necessary. + For instances ``int32 + float64`` tries again using ``float64 + float64`` + which is the common DType. + +* Users can *register* new Promoters just as they can register a + new ``ArrayMethod``. These will use abstract DTypes to allow matching + a large variety of signatures. + The return value of a promotion function shall be a new ``ArrayMethod`` + or ``NotImplemented``. It must be consistent over multiple calls with + the same input to allow caching of the result. + +The signature of a promotion function would be:: + + promoter(np.ufunc: ufunc, Tuple[DTypeMeta]: DTypes): -> Union[ArrayMethod, NotImplemented] + +Note that DTypes may include the output's DType, however, normally the +output DType will *not* affect which ``ArrayMethod`` is chosen. + +In most cases, it should not be necessary to add a custom promotion function. +An example which requires this is multiplication with a unit: +in NumPy ``timedelta64`` can be multiplied with most integers, +but NumPy only defines a loop (``ArrayMethod``) for ``timedelta64 * int64`` +so that multiplying with ``int32`` would fail. + +To allow this, the following promoter can be registered for +``(Timedelta64, Integral, None)``:: + + def promote(ufunc, DTypes): + res = list(DTypes) + try: + res[1] = np.common_dtype(DTypes[1], Int64) + except TypeError: + return NotImplemented + + # Could check that res[1] is actually Int64 + return ufunc.resolve_impl(tuple(res)) + +In this case, just as a ``Timedelta64 * int64`` and ``int64 * timedelta64`` +``ArrayMethod`` is necessary, a second promoter will have to be registered to +handle the case where the integer is passed first. + +**Dispatching rules for ``ArrayMethod`` and Promoters:** + +Promoter and ``ArrayMethod`` are discovered by finding the best match as +defined by the DType class hierarchy. +The best match is defined if: + +* The signature matches for all input DTypes, so that + ``issubclass(input_DType, registered_DType)`` returns true. +* No other promoter or ``ArrayMethod`` is more precise in any input: + ``issubclass(other_DType, this_DType)`` is true (this may include if both + are identical). +* This promoter or ``ArrayMethod`` is more precise in at least one input or + output DType. + +It will be an error if ``NotImplemented`` is returned or if two +promoters match the input equally well. +When an existing promoter is not precise enough for new functionality, a +new promoter has to be added. +To ensure that this promoter takes precedence it may be necessary to define +new abstract DTypes as more precise subclasses of existing ones. + +The above rules enable specialization if an output is supplied +or the full loop is specified. This should not typically be necessary, +but allows resolving ``np.logic_or``, etc. which have both +``Object, Object -> Bool`` and ``Object, Object -> Object`` loops (using the +first by default). + + +Discussion and alternatives +=========================== + +Instead of resolving and returning a new implementation, we could also +return a new set of DTypes to use for dispatching. This works, however, +it has the disadvantage that it is impossible to dispatch to a loop +defined on a different ufunc or to dynamically create a new ``ArrayMethod``. + + +**Rejected Alternatives:** + +In the above the promoters use a multiple dispatching style type resolution +while the current UFunc machinery uses the first +"safe" loop (see also :ref:`NEP 40 `) in an ordered hierarchy. + +While the "safe" casting rule is not restrictive enough, we could imagine +using a new "promote" casting rule, or the common-DType logic to find the +best matching loop by upcasting the inputs as necessary. + +One downside to this approach is that upcasting alone allows upcasting the +result beyond what is expected by users: +Currently (which will remain supported as a fallback) any ufunc which defines +only a float64 loop will also work for float16 and float32 by *upcasting*:: + + >>> from scipy.special import erf + >>> erf(np.array([4.], dtype=np.float16)) # float16 + array([1.], dtype=float32) + +with a float32 result. It is impossible to change the ``erf`` function to +return a float16 result without changing the result of following code. +In general, we argue that automatic upcasting should not occur in cases +where a less precise loop can be defined, *unless* the ufunc +author does this intentionally using a promotion. + +This consideration means that upcasting has to be limited by some additional +method. + +*Alternative 1:* + +Assuming general upcasting is not intended, a rule must be defined to +limit upcasting the input from ``float16 -> float32`` either using generic +logic on the DTypes or the UFunc itself (or a combination of both). +The UFunc cannot do this easily on its own, since it cannot know all possible +DTypes which register loops. +Consider the two examples: + +First (should be rejected): + +* Input: ``float16 * float16`` +* Existing loop: ``float32 * float32`` + +Second (should be accepted): + +* Input: ``timedelta64 * int32`` +* Existing loop: ``timedelta64 * int16`` + + +This requires either: + +1. The ``timedelta64`` to somehow signal that the ``int64`` upcast is + always supported if it is involved in the operation. +2. The ``float32 * float32`` loop to reject upcasting. + +Implementing the first approach requires signaling that upcasts are +acceptable in the specific context. This would require additional hooks +and may not be simple for complex DTypes. + +For the second approach in most cases a simple ``np.common_dtype`` rule will +work for initial dispatching, however, even this is only clearly the case +for homogeneous loops. +This option will require adding a function to check whether the input +is a valid upcast to each loop individually, which seems problematic. +In many cases a default could be provided (homogeneous signature). + +*Alternative 2:* + +An alternative "promotion" step is to ensure that the *output* DType matches +with the loop after first finding the correct output DType. +If the output DTypes are known, finding a safe loop becomes easy. +In the majority of cases this works, the correct output dtype is just:: + + np.common_dtype(*input_DTypes) + +or some fixed DType (e.g. Bool for logical functions). + +However, it fails for example in the ``timedelta64 * int32`` case above since +there is a-priori no way to know that the "expected" result type of this +output is indeed ``timedelta64`` (``np.common_dtype(Datetime64, Int32)`` fails). +This requires some additional knowledge of the timedelta64 precision being +int64. Since a ufunc can have an arbitrary number of (relevant) inputs +it would thus at least require an additional ``__promoted_dtypes__`` method +on ``Datetime64`` (and all DTypes). + +A further limitation is shown by masked DTypes. Logical functions do not +have a boolean result when masked are involved, which would thus require the +original ufunc author to anticipate masked DTypes in this scheme. +Similarly, some functions defined for complex values will return real numbers +while others return complex numbers. If the original author did not anticipate +complex numbers, the promotion may be incorrect for a later added complex loop. + + +We believe that promoters, while allowing for an huge theoretical complexity, +are the best solution: + +1. Promotion allows for dynamically adding new loops. E.g. it is possible + to define an abstract Unit DType, which dynamically creates classes to + wrap other existing DTypes. Using a single promoter, this DType can + dynamically wrap existing ``ArrayMethod`` enabling it to find the correct + loop in a single lookup instead of two. +2. The promotion logic will usually err on the safe side: A newly-added + loop cannot be misused unless a promoter is added as well. +3. They put the burden of carefully thinking of whether the logic is correct + on the programmer adding new loops to a UFunc. (Compared to Alternative 2) +4. In case of incorrect existing promotion, writing a promoter to restrict + or refine a generic rule is possible. In general a promotion rule should + never return an *incorrect* promotion, but if it the existing promotion + logic fails or is incorrect for a newly-added loop, the loop can add a + new promoter to refine the logic. + +The option of having each loop verify that no upcast occured is probably +the best alternative, but does not include the ability to dynamically +adding new loops. + +The main downsides of general promoters is that they allow a possible +very large complexity. +A third-party library *could* add incorrect promotions to NumPy, however, +this is already possible by adding new incorrect loops. +In general we believe we can rely on downstream projects to use this +power and complexity carefully and responsibly. + + +*************** +User Guidelines +*************** + +In general adding a promoter to a UFunc must be done very carefully. +A promoter should never affect loops which can be reasonably defined +by other datatypes. Defining a hypothetical ``erf(UnitFloat16)`` loop +must not lead to ``erf(float16)``. +In general a promoter should fulfill the following requirements: + +* Be conservative when defining a new promotion rule. An incorrect result + is a much more dangerous error than an unexpected error. +* One of the (abstract) DTypes added should typically match specifically with a + DType (or family of DTypes) defined by your project. + Never add promotion rules which go beyond normal common DType rules! + It is *not* reasonable to add a loop for ``int16 + uint16 -> int24`` if + you write an ``int24`` dtype. The result of this operation was already + defined previously as ``int32`` and will be used with this assumption. +* A promoter (or loop) should never affect existing loop results. + This includes adding faster but less precise loops/promoters to replace + existing ones. +* Try to stay within a clear, linear hierarchy for all promotion (and casting) + related logic. NumPy itself breaks this logic for integers and floats + (they are not strictly linear, since int64 cannot promote to float32). +* Loops and promoters can be added by any project, which could be: + + * The project defining the ufunc + * The project defining the DType + * A third-party project + + Try to find out which is the best project to add the loop. If neither + the project defining the ufunc nor the project defining the DType add the + loop, issues with multiple definitions (which are rejected) may arise + and care should be taken that the loop behaviour is always more desirable + than an error. + +In some cases exceptions to these rules may make sense, however, in general +we ask you to use extreme caution and when in doubt create a new UFunc +instead. This clearly notifies the users of differing rules. +When in doubt, ask on the NumPy mailing list or issue tracker! + + +************** +Implementation +************** + +Implementation of this NEP will entail a large refactor and restructuring +of the current ufunc machinery (as well as casting). + +The implementation unfortunately will require large maintenance of the +UFunc machinery, since both the actual UFunc loop calls, as well as the +the initial dispatching steps have to be modified. + +In general, the correct ``ArrayMethod``, also those returned by a promoter, +will be cached (or stored) inside a hashtable for efficient lookup. + + +********** +Discussion +********** + +There is a large space of possible implementations with many discussions +in various places, as well as initial thoughts and design documents. +These are listed in the discussion of :ref:`NEP 40 ` and not repeated here for +brevity. + +A long discussion which touches many of these points and points towards +similar solutions can be found in +`the github issue 12518 "What should be the calling convention for ufunc inner loop signatures?" `_ + + +********** +References +********** + +Please see NEP 40 and 41 for more discussion and references. + + +********* +Copyright +********* + +This document has been placed in the public domain. From d28af3d06160f2db43d7fbf45e9f2d32dc135cb2 Mon Sep 17 00:00:00 2001 From: takanori-pskq Date: Wed, 14 Oct 2020 17:15:02 +0900 Subject: [PATCH 265/409] DOC: Fix wrong blockquotes --- doc/source/reference/arrays.interface.rst | 8 -------- doc/source/user/c-info.how-to-extend.rst | 12 ------------ 2 files changed, 20 deletions(-) diff --git a/doc/source/reference/arrays.interface.rst b/doc/source/reference/arrays.interface.rst index 49772a298772..aeac66f829e5 100644 --- a/doc/source/reference/arrays.interface.rst +++ b/doc/source/reference/arrays.interface.rst @@ -60,7 +60,6 @@ This approach to the interface consists of the object having an The keys are: **shape** (required) - Tuple whose elements are the array size in each dimension. Each entry is an integer (a Python int or long). Note that these integers could be larger than the platform "int" or "long" @@ -70,7 +69,6 @@ This approach to the interface consists of the object having an :c:data:`Py_LONG_LONG` as the C type for the shapes. **typestr** (required) - A string providing the basic type of the homogeneous array The basic string format consists of 3 parts: a character describing the byteorder of the data (``<``: little-endian, ``>``: @@ -97,7 +95,6 @@ This approach to the interface consists of the object having an ===== ================================================================ **descr** (optional) - A list of tuples providing a more detailed description of the memory layout for each item in the homogeneous array. Each tuple in the list has two or three elements. Normally, this @@ -127,7 +124,6 @@ This approach to the interface consists of the object having an **Default**: ``[('', typestr)]`` **data** (optional) - A 2-tuple whose first argument is an integer (a long integer if necessary) that points to the data-area storing the array contents. This pointer must point to the first element of @@ -148,7 +144,6 @@ This approach to the interface consists of the object having an **Default**: None **strides** (optional) - Either None to indicate a C-style contiguous array or a Tuple of strides which provides the number of bytes needed to jump to the next array element in the corresponding @@ -166,7 +161,6 @@ This approach to the interface consists of the object having an **Default**: None (C-style contiguous) **mask** (optional) - None or an object exposing the array interface. All elements of the mask array should be interpreted only as true or not true indicating which elements of this array are valid. @@ -177,7 +171,6 @@ This approach to the interface consists of the object having an **Default**: None (All array values are valid) **offset** (optional) - An integer offset into the array data region. This can only be used when data is None or returns a :class:`buffer` object. @@ -185,7 +178,6 @@ This approach to the interface consists of the object having an **Default**: 0. **version** (required) - An integer showing the version of the interface (i.e. 3 for this version). Be careful not to use this to invalidate objects exposing future versions of the interface. diff --git a/doc/source/user/c-info.how-to-extend.rst b/doc/source/user/c-info.how-to-extend.rst index d7524209202a..ed878be1e92e 100644 --- a/doc/source/user/c-info.how-to-extend.rst +++ b/doc/source/user/c-info.how-to-extend.rst @@ -363,7 +363,6 @@ particular set of requirements ( *e.g.* contiguous, aligned, and writeable). The syntax is :c:func:`PyArray_FROM_OTF` - Return an ndarray from any Python object, *obj*, that can be converted to an array. The number of dimensions in the returned array is determined by the object. The desired data-type of the @@ -375,7 +374,6 @@ writeable). The syntax is exception is set. *obj* - The object can be any Python object convertible to an ndarray. If the object is already (a subclass of) the ndarray that satisfies the requirements then a new reference is returned. @@ -394,11 +392,9 @@ writeable). The syntax is to the requirements flag. *typenum* - One of the enumerated types or :c:data:`NPY_NOTYPE` if the data-type should be determined from the object itself. The C-based names can be used: - :c:data:`NPY_BOOL`, :c:data:`NPY_BYTE`, :c:data:`NPY_UBYTE`, :c:data:`NPY_SHORT`, :c:data:`NPY_USHORT`, :c:data:`NPY_INT`, :c:data:`NPY_UINT`, :c:data:`NPY_LONG`, :c:data:`NPY_ULONG`, @@ -408,7 +404,6 @@ writeable). The syntax is Alternatively, the bit-width names can be used as supported on the platform. For example: - :c:data:`NPY_INT8`, :c:data:`NPY_INT16`, :c:data:`NPY_INT32`, :c:data:`NPY_INT64`, :c:data:`NPY_UINT8`, :c:data:`NPY_UINT16`, :c:data:`NPY_UINT32`, @@ -422,7 +417,6 @@ writeable). The syntax is requirements flag to override this behavior. *requirements* - The memory model for an ndarray admits arbitrary strides in each dimension to advance to the next element of the array. Often, however, you need to interface with code that expects a @@ -446,13 +440,11 @@ writeable). The syntax is :c:data:`NPY_OUT_ARRAY`, and :c:data:`NPY_ARRAY_INOUT_ARRAY`: :c:data:`NPY_ARRAY_IN_ARRAY` - This flag is useful for arrays that must be in C-contiguous order and aligned. These kinds of arrays are usually input arrays for some algorithm. :c:data:`NPY_ARRAY_OUT_ARRAY` - This flag is useful to specify an array that is in C-contiguous order, is aligned, and can be written to as well. Such an array is usually returned as output @@ -460,7 +452,6 @@ writeable). The syntax is scratch). :c:data:`NPY_ARRAY_INOUT_ARRAY` - This flag is useful to specify an array that will be used for both input and output. :c:func:`PyArray_ResolveWritebackIfCopy` must be called before :c:func:`Py_DECREF` at @@ -479,16 +470,13 @@ writeable). The syntax is Other useful flags that can be OR'd as additional requirements are: :c:data:`NPY_ARRAY_FORCECAST` - Cast to the desired type, even if it can't be done without losing information. :c:data:`NPY_ARRAY_ENSURECOPY` - Make sure the resulting array is a copy of the original. :c:data:`NPY_ARRAY_ENSUREARRAY` - Make sure the resulting object is an actual ndarray and not a sub- class. From 82c03d34c4a36f44b951a00efa901cf41457e1f3 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Wed, 14 Oct 2020 11:37:27 +0100 Subject: [PATCH 266/409] Apply suggestions from code review --- doc/source/user/c-info.how-to-extend.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/user/c-info.how-to-extend.rst b/doc/source/user/c-info.how-to-extend.rst index ed878be1e92e..845ce0a74c31 100644 --- a/doc/source/user/c-info.how-to-extend.rst +++ b/doc/source/user/c-info.how-to-extend.rst @@ -395,6 +395,7 @@ writeable). The syntax is One of the enumerated types or :c:data:`NPY_NOTYPE` if the data-type should be determined from the object itself. The C-based names can be used: + :c:data:`NPY_BOOL`, :c:data:`NPY_BYTE`, :c:data:`NPY_UBYTE`, :c:data:`NPY_SHORT`, :c:data:`NPY_USHORT`, :c:data:`NPY_INT`, :c:data:`NPY_UINT`, :c:data:`NPY_LONG`, :c:data:`NPY_ULONG`, @@ -404,6 +405,7 @@ writeable). The syntax is Alternatively, the bit-width names can be used as supported on the platform. For example: + :c:data:`NPY_INT8`, :c:data:`NPY_INT16`, :c:data:`NPY_INT32`, :c:data:`NPY_INT64`, :c:data:`NPY_UINT8`, :c:data:`NPY_UINT16`, :c:data:`NPY_UINT32`, From 81ac8ad9e428affcc9bbe694cab9fd0ac8188157 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Wed, 14 Oct 2020 06:52:35 -0400 Subject: [PATCH 267/409] DOC: MAINT: Add NEP 43 links to NEP 42 --- doc/neps/nep-0042-new-dtypes.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/neps/nep-0042-new-dtypes.rst b/doc/neps/nep-0042-new-dtypes.rst index 6f457a9d4984..ded0fd1aec3a 100644 --- a/doc/neps/nep-0042-new-dtypes.rst +++ b/doc/neps/nep-0042-new-dtypes.rst @@ -23,7 +23,7 @@ NEP 42 — New and extensible DTypes - NEP 42 (this document) describes the new design's datatype-related APIs. - - NEP 43 describes the new design's API for universal functions. + - :ref:`NEP 43 ` describes the new design's API for universal functions. ****************************************************************************** @@ -41,7 +41,7 @@ will return an instance of a ``Float64`` class, a subclass of root class ``np.dtype``. This NEP is one of two that lay out the design and API of this new -architecture. This NEP addresses dtype implementation; NEP 43 addresses +architecture. This NEP addresses dtype implementation; :ref:`NEP 43 ` addresses universal functions. .. note:: @@ -88,7 +88,7 @@ All these are the subjects of this NEP. - The functionality for users to define their own DTypes is described in `Public C-API`_. -The API here and in NEP 43 is entirely on the C side. A Python-side version +The API here and in :ref:`NEP 43 ` is entirely on the C side. A Python-side version will be proposed in a future NEP. A future Python API is expected to be similar, but provide a more convenient API to reuse the functionality of existing DTypes. It could also provide shorthands to create structured DTypes @@ -211,7 +211,7 @@ Other elements of the casting implementation is the ``CastingImpl``: raise NotImplementedError which describes the casting from one DType to another. In -NEP 43 this ``CastingImpl`` object is used unchanged to +:ref:`NEP 43 ` this ``CastingImpl`` object is used unchanged to support universal functions. @@ -337,7 +337,7 @@ hierarchy. ``Float64Unit``. Calling ``Unit(np.float64, "m")`` (``m`` for meters) would be equivalent to ``Float64Unit("m")``. -4. The implementation of universal functions in NEP 43 may require +4. The implementation of universal functions in :ref:`NEP 43 ` may require a class hierarchy. **Example:** A NumPy ``Categorical`` class would be a match for pandas @@ -744,7 +744,7 @@ only if byte order matches. This functionality can be replaced with the combination of "equivalent" casting and the "view" flag. (For more information on the ``resolve_descriptors`` signature see the -:ref:`nep42_C-API` section below and NEP 43.) +:ref:`nep42_C-API` section below and :ref:`NEP 43 `.) **Casting between instances of the same DType** @@ -1312,7 +1312,7 @@ The external API for ``CastingImpl`` will be limited initially to defining: preferable). * ``strided_loop(char **args, npy_intp *dimensions, npy_intp *strides, - ...) -> int {0, -1}`` (signature will be fully defined in NEP 43) + ...) -> int {0, -1}`` (signature will be fully defined in :ref:`NEP 43 `) This is identical to the proposed API for ufuncs. The additional ``...`` part of the signature will include information such as the two ``dtype``\s. @@ -1350,7 +1350,7 @@ incrementally in one of two ways: information, such as the fixed strides (similar to our internal API). The example does not yet include setup and error handling. Since these are -similar to the UFunc machinery, they will be defined in NEP 43 and then +similar to the UFunc machinery, they will be defined in :ref:`NEP 43 ` and then incorporated identically into casting. The slots/methods used will be prefixed with ``NPY_meth_``. From faaf29998d421529113d0bf40dc432c1017544b7 Mon Sep 17 00:00:00 2001 From: takanori-pskq Date: Wed, 14 Oct 2020 20:22:08 +0900 Subject: [PATCH 268/409] DOC: Remove directives for some constants --- doc/source/reference/c-api/dtype.rst | 13 +++---------- doc/source/reference/c-api/ufunc.rst | 9 +++------ 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/doc/source/reference/c-api/dtype.rst b/doc/source/reference/c-api/dtype.rst index 47b998302e86..a1a53cdb600b 100644 --- a/doc/source/reference/c-api/dtype.rst +++ b/doc/source/reference/c-api/dtype.rst @@ -221,24 +221,17 @@ Defines Max and min values for integers ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. c:var:: NPY_MAX_INT{bits} - -.. c:var:: NPY_MAX_UINT{bits} - -.. c:var:: NPY_MIN_INT{bits} - +``NPY_MAX_INT{bits}``, ``NPY_MAX_UINT{bits}``, ``NPY_MIN_INT{bits}`` These are defined for ``{bits}`` = 8, 16, 32, 64, 128, and 256 and provide the maximum (minimum) value of the corresponding (unsigned) integer type. Note: the actual integer type may not be available on all platforms (i.e. 128-bit and 256-bit integers are rare). -.. c:var:: NPY_MIN_{type} - +``NPY_MIN_{type}`` This is defined for ``{type}`` = **BYTE**, **SHORT**, **INT**, **LONG**, **LONGLONG**, **INTP** -.. c:var:: NPY_MAX_{type} - +``NPY_MAX_{type}`` This is defined for all defined for ``{type}`` = **BYTE**, **UBYTE**, **SHORT**, **USHORT**, **INT**, **UINT**, **LONG**, **ULONG**, **LONGLONG**, **ULONGLONG**, **INTP**, **UINTP** diff --git a/doc/source/reference/c-api/ufunc.rst b/doc/source/reference/c-api/ufunc.rst index 05ddf0dba119..9eb70c3fbe34 100644 --- a/doc/source/reference/c-api/ufunc.rst +++ b/doc/source/reference/c-api/ufunc.rst @@ -10,8 +10,7 @@ UFunc API Constants --------- -.. c:var:: UFUNC_ERR_{HANDLER} - +``UFUNC_ERR_{HANDLER}`` .. c:macro:: UFUNC_ERR_IGNORE .. c:macro:: UFUNC_ERR_WARN @@ -20,8 +19,7 @@ Constants .. c:macro:: UFUNC_ERR_CALL -.. c:var:: UFUNC_{THING}_{ERR} - +``UFUNC_{THING}_{ERR}`` .. c:macro:: UFUNC_MASK_DIVIDEBYZERO .. c:macro:: UFUNC_MASK_OVERFLOW @@ -46,8 +44,7 @@ Constants .. c:macro:: UFUNC_FPE_INVALID -.. c:var:: PyUFunc_{VALUE} - +``PyUFunc_{VALUE}`` .. c:macro:: PyUFunc_One .. c:macro:: PyUFunc_Zero From fa2acebfef936df8cc00fa3b4e6c951db1866501 Mon Sep 17 00:00:00 2001 From: Ben Nathanson Date: Wed, 14 Oct 2020 09:09:02 -0400 Subject: [PATCH 269/409] DOC: Include NEP titles for first reference in NEP 42 Rather than saying 'NEP 4x', the links in the note at the top now include the NEP titles as well. --- doc/neps/nep-0042-new-dtypes.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/neps/nep-0042-new-dtypes.rst b/doc/neps/nep-0042-new-dtypes.rst index ded0fd1aec3a..ff92e1612b09 100644 --- a/doc/neps/nep-0042-new-dtypes.rst +++ b/doc/neps/nep-0042-new-dtypes.rst @@ -17,13 +17,13 @@ NEP 42 — New and extensible DTypes This NEP is third in a series: - - :ref:`NEP 40 ` explains the shortcomings of NumPy's dtype implementation. + - :ref:`NEP40` explains the shortcomings of NumPy's dtype implementation. - - :ref:`NEP 41 ` gives an overview of our proposed replacement. + - :ref:`NEP41` gives an overview of our proposed replacement. - NEP 42 (this document) describes the new design's datatype-related APIs. - - :ref:`NEP 43 ` describes the new design's API for universal functions. + - :ref:`NEP43` describes the new design's API for universal functions. ****************************************************************************** From 5b028ae4ec0d23e3fe2e993bd0d1639c48c7b492 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Wed, 14 Oct 2020 15:56:38 +0200 Subject: [PATCH 270/409] MAINT: Updated the annotations in `np.core.numeric` * Added missing overloads * Use `DtypeLike` and `_Shapelike` where applicable * Moved a few annotation-aliases to the top of the module --- numpy/core/numeric.pyi | 120 ++++++++++++++++++++++++++++++++--------- 1 file changed, 96 insertions(+), 24 deletions(-) diff --git a/numpy/core/numeric.pyi b/numpy/core/numeric.pyi index 19720fbdcc9b..b341d136a10f 100644 --- a/numpy/core/numeric.pyi +++ b/numpy/core/numeric.pyi @@ -1,14 +1,45 @@ -from typing import Any, Optional, Union, Sequence, Tuple +import sys +from typing import ( + Any, + Optional, + Union, + Sequence, + Tuple, + Callable, + List, + overload, + TypeVar, + Iterable, +) -from numpy import ndarray, dtype, bool_, _OrderKACF, _OrderCF +from numpy import ndarray, generic, dtype, bool_, int32, int64, _OrderKACF, _OrderCF from numpy.typing import ArrayLike, DtypeLike, _ShapeLike +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + +_T = TypeVar("_T") +_ArrayType = TypeVar("_ArrayType", bound=ndarray) + +_CorrelateMode = Literal["valid", "same", "full"] + +@overload +def zeros_like( + a: _ArrayType, + dtype: None = ..., + order: _OrderKACF = ..., + subok: Literal[True] = ..., + shape: None = ..., +) -> _ArrayType: ... +@overload def zeros_like( a: ArrayLike, dtype: DtypeLike = ..., order: _OrderKACF = ..., subok: bool = ..., - shape: Optional[Union[int, Sequence[int]]] = ..., + shape: Optional[_ShapeLike] = ..., ) -> ndarray: ... def ones( shape: _ShapeLike, @@ -17,6 +48,15 @@ def ones( *, like: ArrayLike = ..., ) -> ndarray: ... +@overload +def ones_like( + a: _ArrayType, + dtype: None = ..., + order: _OrderKACF = ..., + subok: Literal[True] = ..., + shape: None = ..., +) -> _ArrayType: ... +@overload def ones_like( a: ArrayLike, dtype: DtypeLike = ..., @@ -24,6 +64,15 @@ def ones_like( subok: bool = ..., shape: Optional[_ShapeLike] = ..., ) -> ndarray: ... +@overload +def empty_like( + a: _ArrayType, + dtype: None = ..., + order: _OrderKACF = ..., + subok: Literal[True] = ..., + shape: None = ..., +) -> _ArrayType: ... +@overload def empty_like( a: ArrayLike, dtype: DtypeLike = ..., @@ -39,6 +88,16 @@ def full( *, like: ArrayLike = ..., ) -> ndarray: ... +@overload +def full_like( + a: _ArrayType, + fill_value: Any, + dtype: None = ..., + order: _OrderKACF = ..., + subok: Literal[True] = ..., + shape: None = ..., +) -> _ArrayType: ... +@overload def full_like( a: ArrayLike, fill_value: Any, @@ -47,35 +106,38 @@ def full_like( subok: bool = ..., shape: Optional[_ShapeLike] = ..., ) -> ndarray: ... +@overload +def count_nonzero( + a: ArrayLike, axis: None = ..., *, keepdims: Literal[False] = ... +) -> int: ... +@overload def count_nonzero( - a: ArrayLike, axis: Optional[Union[int, Tuple[int], Tuple[int, int]]] = ... -) -> Union[int, ndarray]: ... -def isfortran(a: ndarray) -> bool: ... + a: ArrayLike, axis: _ShapeLike = ..., *, keepdims: bool = ... +) -> Union[int64, int32, ndarray]: ... +def isfortran(a: Union[ndarray, generic]) -> bool: ... def argwhere(a: ArrayLike) -> ndarray: ... def flatnonzero(a: ArrayLike) -> ndarray: ... - -_CorrelateMode = Literal["valid", "same", "full"] - def correlate(a: ArrayLike, v: ArrayLike, mode: _CorrelateMode = ...) -> ndarray: ... def convolve(a: ArrayLike, v: ArrayLike, mode: _CorrelateMode = ...) -> ndarray: ... -def outer(a: ArrayLike, b: ArrayLike, out: ndarray = ...) -> ndarray: ... +@overload +def outer(a: ArrayLike, b: ArrayLike, out: None = ...) -> ndarray: ... +@overload +def outer(a: ArrayLike, b: ArrayLike, out: _ArrayType = ...) -> _ArrayType: ... def tensordot( a: ArrayLike, b: ArrayLike, - axes: Union[ - int, Tuple[int, int], Tuple[Tuple[int, int], ...], Tuple[List[int, int], ...] - ] = ..., + axes: _ShapeLike = ..., ) -> ndarray: ... def roll( a: ArrayLike, - shift: Union[int, Tuple[int, ...]], - axis: Optional[Union[int, Tuple[int, ...]]] = ..., + shift: _ShapeLike, + axis: Optional[_ShapeLike] = ..., ) -> ndarray: ... -def rollaxis(a: ArrayLike, axis: int, start: int = ...) -> ndarray: ... +def rollaxis(a: ndarray, axis: int, start: int = ...) -> ndarray: ... def moveaxis( a: ndarray, - source: Union[int, Sequence[int]], - destination: Union[int, Sequence[int]], + source: _ShapeLike, + destination: _ShapeLike, ) -> ndarray: ... def cross( a: ArrayLike, @@ -85,16 +147,26 @@ def cross( axisc: int = ..., axis: Optional[int] = ..., ) -> ndarray: ... +@overload def indices( - dimensions: Sequence[int], dtype: dtype = ..., sparse: bool = ... -) -> Union[ndarray, Tuple[ndarray, ...]]: ... + dimensions: Sequence[int], + dtype: DtypeLike = ..., + sparse: Literal[False] = ..., +) -> ndarray: ... +@overload +def indices( + dimensions: Sequence[int], + dtype: DtypeLike = ..., + sparse: Literal[True] = ..., +) -> Tuple[ndarray, ...]: ... def fromfunction( - function: Callable, - shape: Tuple[int, int], + function: Callable[..., _T], + shape: Sequence[int], *, + dtype: DtypeLike = ..., like: ArrayLike = ..., - **kwargs, -) -> Any: ... + **kwargs: Any, +) -> _T: ... def isscalar(element: Any) -> bool: ... def binary_repr(num: int, width: Optional[int] = ...) -> str: ... def base_repr(number: int, base: int = ..., padding: int = ...) -> str: ... From 4e3c6bd71b170667c8c73f0d28fe6f67f5a4252b Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Wed, 14 Oct 2020 15:57:03 +0200 Subject: [PATCH 271/409] TST,MAINT: Moved a few tests to `array_constructors` --- numpy/typing/tests/data/pass/simple.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/numpy/typing/tests/data/pass/simple.py b/numpy/typing/tests/data/pass/simple.py index 4d397bd29f29..243caf229f13 100644 --- a/numpy/typing/tests/data/pass/simple.py +++ b/numpy/typing/tests/data/pass/simple.py @@ -17,14 +17,6 @@ def ndarray_func(x): array == 1 array.dtype == float -ndarray_func(np.zeros([1, 2])) -ndarray_func(np.ones([1, 2])) -ndarray_func(np.empty([1, 2])) - -ndarray_func(np.zeros_like(array)) -ndarray_func(np.ones_like(array)) -ndarray_func(np.empty_like(array)) - # Dtype construction np.dtype(float) np.dtype(np.float64) From 71292e07e0329d0b256804146d06b99d83fd36c9 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Wed, 14 Oct 2020 16:57:08 +0200 Subject: [PATCH 272/409] TST: Added tests for `np.core.numeric` --- .../tests/data/pass/array_constructors.py | 40 ++++++++- numpy/typing/tests/data/pass/numeric.py | 89 +++++++++++++++++++ .../tests/data/reveal/array_constructors.py | 39 +++++++- numpy/typing/tests/data/reveal/numeric.py | 89 +++++++++++++++++++ 4 files changed, 255 insertions(+), 2 deletions(-) create mode 100644 numpy/typing/tests/data/pass/numeric.py create mode 100644 numpy/typing/tests/data/reveal/numeric.py diff --git a/numpy/typing/tests/data/pass/array_constructors.py b/numpy/typing/tests/data/pass/array_constructors.py index bf29e52b91b9..08f4b6211dc9 100644 --- a/numpy/typing/tests/data/pass/array_constructors.py +++ b/numpy/typing/tests/data/pass/array_constructors.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Any import numpy as np class Index: @@ -7,10 +7,15 @@ def __index__(self) -> int: class SubClass(np.ndarray): ... +i8 = np.int64(1) + A = np.array([1]) B = A.view(SubClass).copy() C = [1] +def func(i: int, j: int, **kwargs: Any) -> SubClass: + return B + np.array(1, dtype=float) np.array(1, copy=False) np.array(1, order='F') @@ -64,3 +69,36 @@ class SubClass(np.ndarray): ... np.logspace(0, 2, base=[1j, 2j], num=2) np.geomspace(1, 2) + +np.zeros_like(A) +np.zeros_like(C) +np.zeros_like(B) +np.zeros_like(B, dtype=np.int64) + +np.ones_like(A) +np.ones_like(C) +np.ones_like(B) +np.ones_like(B, dtype=np.int64) + +np.empty_like(A) +np.empty_like(C) +np.empty_like(B) +np.empty_like(B, dtype=np.int64) + +np.full_like(A, i8) +np.full_like(C, i8) +np.full_like(B, i8) +np.full_like(B, i8, dtype=np.int64) + +np.ones(1) +np.ones([1, 1, 1]) + +np.full(1, i8) +np.full([1, 1, 1], i8) + +np.indices([1, 2, 3]) +np.indices([1, 2, 3], sparse=True) + +np.fromfunction(func, (3, 5)) + +np.identity(10) diff --git a/numpy/typing/tests/data/pass/numeric.py b/numpy/typing/tests/data/pass/numeric.py new file mode 100644 index 000000000000..34fef7270443 --- /dev/null +++ b/numpy/typing/tests/data/pass/numeric.py @@ -0,0 +1,89 @@ +""" +Tests for :mod:`numpy.core.numeric`. + +Does not include tests which fall under ``array_constructors``. + +""" + +from typing import List +import numpy as np + +class SubClass(np.ndarray): + ... + +i8 = np.int64(1) + +A = np.arange(27).reshape(3, 3, 3) +B: List[List[List[int]]] = A.tolist() +C = np.empty((27, 27)).view(SubClass) + +np.count_nonzero(i8) +np.count_nonzero(A) +np.count_nonzero(B) +np.count_nonzero(A, keepdims=True) +np.count_nonzero(A, axis=0) + +np.isfortran(i8) +np.isfortran(A) + +np.argwhere(i8) +np.argwhere(A) + +np.flatnonzero(i8) +np.flatnonzero(A) + +np.correlate(B[0][0], A.ravel(), mode="valid") +np.correlate(A.ravel(), A.ravel(), mode="same") + +np.convolve(B[0][0], A.ravel(), mode="valid") +np.convolve(A.ravel(), A.ravel(), mode="same") + +np.outer(i8, A) +np.outer(B, A) +np.outer(A, A) +np.outer(A, A, out=C) + +np.tensordot(B, A) +np.tensordot(A, A) +np.tensordot(A, A, axes=0) +np.tensordot(A, A, axes=(0, 1)) + +np.isscalar(i8) +np.isscalar(A) +np.isscalar(B) + +np.roll(A, 1) +np.roll(A, (1, 2)) +np.roll(B, 1) + +np.rollaxis(A, 0, 1) + +np.moveaxis(A, 0, 1) +np.moveaxis(A, (0, 1), (1, 2)) + +np.cross(B, A) +np.cross(A, A) + +np.indices([0, 1, 2]) +np.indices([0, 1, 2], sparse=False) +np.indices([0, 1, 2], sparse=True) + +np.binary_repr(1) + +np.base_repr(1) + +np.allclose(i8, A) +np.allclose(B, A) +np.allclose(A, A) + +np.isclose(i8, A) +np.isclose(B, A) +np.isclose(A, A) + +np.array_equal(i8, A) +np.array_equal(B, A) +np.array_equal(A, A) + +np.array_equiv(i8, A) +np.array_equiv(B, A) +np.array_equiv(A, A) diff --git a/numpy/typing/tests/data/reveal/array_constructors.py b/numpy/typing/tests/data/reveal/array_constructors.py index ba8a8eda1e94..9475b3884b9a 100644 --- a/numpy/typing/tests/data/reveal/array_constructors.py +++ b/numpy/typing/tests/data/reveal/array_constructors.py @@ -1,12 +1,16 @@ -from typing import List +from typing import List, Any import numpy as np class SubClass(np.ndarray): ... +i8: np.int64 + A: np.ndarray B: SubClass C: List[int] +def func(i: int, j: int, **kwargs: Any) -> SubClass: ... + reveal_type(np.asarray(A)) # E: ndarray reveal_type(np.asarray(B)) # E: ndarray reveal_type(np.asarray(C)) # E: ndarray @@ -40,3 +44,36 @@ class SubClass(np.ndarray): ... reveal_type(np.linspace(0, 10, retstep=True)) # E: Tuple[numpy.ndarray, numpy.inexact] reveal_type(np.logspace(0, 10)) # E: numpy.ndarray reveal_type(np.geomspace(1, 10)) # E: numpy.ndarray + +reveal_type(np.zeros_like(A)) # E: numpy.ndarray +reveal_type(np.zeros_like(C)) # E: numpy.ndarray +reveal_type(np.zeros_like(B)) # E: SubClass +reveal_type(np.zeros_like(B, dtype=np.int64)) # E: numpy.ndarray + +reveal_type(np.ones_like(A)) # E: numpy.ndarray +reveal_type(np.ones_like(C)) # E: numpy.ndarray +reveal_type(np.ones_like(B)) # E: SubClass +reveal_type(np.ones_like(B, dtype=np.int64)) # E: numpy.ndarray + +reveal_type(np.empty_like(A)) # E: numpy.ndarray +reveal_type(np.empty_like(C)) # E: numpy.ndarray +reveal_type(np.empty_like(B)) # E: SubClass +reveal_type(np.empty_like(B, dtype=np.int64)) # E: numpy.ndarray + +reveal_type(np.full_like(A, i8)) # E: numpy.ndarray +reveal_type(np.full_like(C, i8)) # E: numpy.ndarray +reveal_type(np.full_like(B, i8)) # E: SubClass +reveal_type(np.full_like(B, i8, dtype=np.int64)) # E: numpy.ndarray + +reveal_type(np.ones(1)) # E: numpy.ndarray +reveal_type(np.ones([1, 1, 1])) # E: numpy.ndarray + +reveal_type(np.full(1, i8)) # E: numpy.ndarray +reveal_type(np.full([1, 1, 1], i8)) # E: numpy.ndarray + +reveal_type(np.indices([1, 2, 3])) # E: numpy.ndarray +reveal_type(np.indices([1, 2, 3], sparse=True)) # E: tuple[numpy.ndarray] + +reveal_type(np.fromfunction(func, (3, 5))) # E: SubClass + +reveal_type(np.identity(10)) # E: numpy.ndarray diff --git a/numpy/typing/tests/data/reveal/numeric.py b/numpy/typing/tests/data/reveal/numeric.py new file mode 100644 index 000000000000..6bd4ed9fdd1e --- /dev/null +++ b/numpy/typing/tests/data/reveal/numeric.py @@ -0,0 +1,89 @@ +""" +Tests for :mod:`numpy.core.numeric`. + +Does not include tests which fall under ``array_constructors``. + +""" + +from typing import List +import numpy as np + +class SubClass(np.ndarray): + ... + +i8: np.int64 + +A: np.ndarray +B: List[int] +C: SubClass + +reveal_type(np.count_nonzero(i8)) # E: int +reveal_type(np.count_nonzero(A)) # E: int +reveal_type(np.count_nonzero(B)) # E: int +reveal_type(np.count_nonzero(A, keepdims=True)) # E: Union[numpy.int64, numpy.int32, numpy.ndarray] +reveal_type(np.count_nonzero(A, axis=0)) # E: Union[numpy.int64, numpy.int32, numpy.ndarray] + +reveal_type(np.isfortran(i8)) # E: bool +reveal_type(np.isfortran(A)) # E: bool + +reveal_type(np.argwhere(i8)) # E: numpy.ndarray +reveal_type(np.argwhere(A)) # E: numpy.ndarray + +reveal_type(np.flatnonzero(i8)) # E: numpy.ndarray +reveal_type(np.flatnonzero(A)) # E: numpy.ndarray + +reveal_type(np.correlate(B, A, mode="valid")) # E: numpy.ndarray +reveal_type(np.correlate(A, A, mode="same")) # E: numpy.ndarray + +reveal_type(np.convolve(B, A, mode="valid")) # E: numpy.ndarray +reveal_type(np.convolve(A, A, mode="same")) # E: numpy.ndarray + +reveal_type(np.outer(i8, A)) # E: numpy.ndarray +reveal_type(np.outer(B, A)) # E: numpy.ndarray +reveal_type(np.outer(A, A)) # E: numpy.ndarray +reveal_type(np.outer(A, A, out=C)) # E: SubClass + +reveal_type(np.tensordot(B, A)) # E: numpy.ndarray +reveal_type(np.tensordot(A, A)) # E: numpy.ndarray +reveal_type(np.tensordot(A, A, axes=0)) # E: numpy.ndarray +reveal_type(np.tensordot(A, A, axes=(0, 1))) # E: numpy.ndarray + +reveal_type(np.isscalar(i8)) # E: bool +reveal_type(np.isscalar(A)) # E: bool +reveal_type(np.isscalar(B)) # E: bool + +reveal_type(np.roll(A, 1)) # E: numpy.ndarray +reveal_type(np.roll(A, (1, 2))) # E: numpy.ndarray +reveal_type(np.roll(B, 1)) # E: numpy.ndarray + +reveal_type(np.rollaxis(A, 0, 1)) # E: numpy.ndarray + +reveal_type(np.moveaxis(A, 0, 1)) # E: numpy.ndarray +reveal_type(np.moveaxis(A, (0, 1), (1, 2))) # E: numpy.ndarray + +reveal_type(np.cross(B, A)) # E: numpy.ndarray +reveal_type(np.cross(A, A)) # E: numpy.ndarray + +reveal_type(np.indices([0, 1, 2])) # E: numpy.ndarray +reveal_type(np.indices([0, 1, 2], sparse=False)) # E: numpy.ndarray +reveal_type(np.indices([0, 1, 2], sparse=True)) # E: tuple[numpy.ndarray] + +reveal_type(np.binary_repr(1)) # E: str + +reveal_type(np.base_repr(1)) # E: str + +reveal_type(np.allclose(i8, A)) # E: bool +reveal_type(np.allclose(B, A)) # E: bool +reveal_type(np.allclose(A, A)) # E: bool + +reveal_type(np.isclose(i8, A)) # E: Union[numpy.bool_, numpy.ndarray] +reveal_type(np.isclose(B, A)) # E: Union[numpy.bool_, numpy.ndarray] +reveal_type(np.isclose(A, A)) # E: Union[numpy.bool_, numpy.ndarray] + +reveal_type(np.array_equal(i8, A)) # E: bool +reveal_type(np.array_equal(B, A)) # E: bool +reveal_type(np.array_equal(A, A)) # E: bool + +reveal_type(np.array_equiv(i8, A)) # E: bool +reveal_type(np.array_equiv(B, A)) # E: bool +reveal_type(np.array_equiv(A, A)) # E: bool From 33ce3d7388e8202497a9e2ac163c26c26c1d2115 Mon Sep 17 00:00:00 2001 From: takanori-pskq Date: Wed, 14 Oct 2020 00:29:13 +0900 Subject: [PATCH 273/409] DOC: Fix typos --- doc/source/reference/c-api/array.rst | 4 ++-- doc/source/reference/routines.io.rst | 2 +- doc/source/user/c-info.beyond-basics.rst | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/source/reference/c-api/array.rst b/doc/source/reference/c-api/array.rst index 637b6aaeb9bc..3a024e203fe0 100644 --- a/doc/source/reference/c-api/array.rst +++ b/doc/source/reference/c-api/array.rst @@ -24,7 +24,7 @@ These macros access the :c:type:`PyArrayObject` structure members and are defined in ``ndarraytypes.h``. The input argument, *arr*, can be any :c:type:`PyObject *` that is directly interpretable as a :c:type:`PyArrayObject *` (any instance of the :c:data:`PyArray_Type` -and itssub-types). +and its sub-types). .. c:function:: int PyArray_NDIM(PyArrayObject *arr) @@ -942,7 +942,7 @@ argument must be a :c:type:`PyObject *` that can be directly interpret Type represents one of the flexible array types ( :c:data:`NPY_STRING`, :c:data:`NPY_UNICODE`, or :c:data:`NPY_VOID` ). -.. c:function:: int PyDataType_ISUNSIZED(PyArray_Descr* descr): +.. c:function:: int PyDataType_ISUNSIZED(PyArray_Descr* descr) Type has no size information attached, and can be resized. Should only be called on flexible dtypes. Types that are attached to an array will always diff --git a/doc/source/reference/routines.io.rst b/doc/source/reference/routines.io.rst index 2e119af9a111..3052ee1fbee8 100644 --- a/doc/source/reference/routines.io.rst +++ b/doc/source/reference/routines.io.rst @@ -88,4 +88,4 @@ Binary Format Description .. autosummary:: :toctree: generated/ - lib.format + lib.format diff --git a/doc/source/user/c-info.beyond-basics.rst b/doc/source/user/c-info.beyond-basics.rst index 9e9cd30679a0..124162d6ce17 100644 --- a/doc/source/user/c-info.beyond-basics.rst +++ b/doc/source/user/c-info.beyond-basics.rst @@ -115,7 +115,7 @@ processors that use pipelining to enhance fundamental operations. The :c:func:`PyArray_IterAllButAxis` ( ``array``, ``&dim`` ) constructs an iterator object that is modified so that it will not iterate over the dimension indicated by dim. The only restriction on this iterator -object, is that the :c:func:`PyArray_Iter_GOTO1D` ( ``it``, ``ind`` ) macro +object, is that the :c:func:`PyArray_ITER_GOTO1D` ( ``it``, ``ind`` ) macro cannot be used (thus flat indexing won't work either if you pass this object back to Python --- so you shouldn't do this). Note that the returned object from this routine is still usually cast to From d87800a3c619a9cd051b5fac63a8486b48fb1c1c Mon Sep 17 00:00:00 2001 From: takanori-pskq Date: Thu, 15 Oct 2020 18:49:16 +0900 Subject: [PATCH 274/409] DOC: Add the entry for ``NPY_FEATURE_VERSION`` --- doc/source/reference/c-api/array.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/source/reference/c-api/array.rst b/doc/source/reference/c-api/array.rst index 637b6aaeb9bc..ff8e911bfc70 100644 --- a/doc/source/reference/c-api/array.rst +++ b/doc/source/reference/c-api/array.rst @@ -3443,6 +3443,11 @@ Other constants variable is defined to guarantee the numpy/arrayobject.h header is being used). +.. c:macro:: NPY_FEATURE_VERSION + + The current version of the API. :c:data:`NPY_FEATURE_VERSION` changes + whenever the API changes (e.g. a function is added). + .. c:macro:: NPY_FALSE Defined as 0 for use with Bool. From c7fa76b307deba855804b6d01e5c2e991daf53b2 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Thu, 15 Oct 2020 13:04:18 +0200 Subject: [PATCH 275/409] ENH: Added annotations for 3 new constants `False_`, `True_` and `UFUNC_PYVALS_NAME` --- numpy/__init__.pyi | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index e41c3cd78a51..99d0e6db243f 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -218,11 +218,8 @@ __all__ = [ ] DataSource: Any -False_: Any MachAr: Any ScalarType: Any -True_: Any -UFUNC_PYVALS_NAME: Any angle: Any append: Any apply_along_axis: Any @@ -1667,6 +1664,11 @@ WRAP: Final[int] little_endian: Final[int] tracemalloc_domain: Final[int] +True_: Final[bool_] +False_: Final[bool_] + +UFUNC_PYVALS_NAME: Final[str] + class ufunc: @property def __name__(self) -> str: ... From 7decfcf481abfc823b2bb6df966088383d6695a5 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Thu, 15 Oct 2020 13:04:41 +0200 Subject: [PATCH 276/409] MAINT: Fix the type of `little_endian` It should be a boolean, not an integeer --- numpy/__init__.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 99d0e6db243f..d76121865d70 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -1661,9 +1661,9 @@ SHIFT_OVERFLOW: Final[int] SHIFT_UNDERFLOW: Final[int] UFUNC_BUFSIZE_DEFAULT: Final[int] WRAP: Final[int] -little_endian: Final[int] tracemalloc_domain: Final[int] +little_endian: Final[bool] True_: Final[bool_] False_: Final[bool_] From 50e09dbffba0d8f0b1682c3ed570885cae68fdbb Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Thu, 15 Oct 2020 13:13:05 +0200 Subject: [PATCH 277/409] TST: Updated the typing tests --- numpy/typing/tests/data/fail/constants.py | 6 ++++++ numpy/typing/tests/data/reveal/constants.py | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 numpy/typing/tests/data/fail/constants.py diff --git a/numpy/typing/tests/data/fail/constants.py b/numpy/typing/tests/data/fail/constants.py new file mode 100644 index 000000000000..67ee0e0bc0bf --- /dev/null +++ b/numpy/typing/tests/data/fail/constants.py @@ -0,0 +1,6 @@ +import numpy as np + +np.Inf = np.Inf # E: Cannot assign to final +np.ALLOW_THREADS = np.ALLOW_THREADS # E: Cannot assign to final +np.little_endian = np.little_endian # E: Cannot assign to final +np.UFUNC_PYVALS_NAME = np.UFUNC_PYVALS_NAME # E: Cannot assign to final diff --git a/numpy/typing/tests/data/reveal/constants.py b/numpy/typing/tests/data/reveal/constants.py index 8e00810bd7c4..dc5803974332 100644 --- a/numpy/typing/tests/data/reveal/constants.py +++ b/numpy/typing/tests/data/reveal/constants.py @@ -40,5 +40,10 @@ reveal_type(np.SHIFT_UNDERFLOW) # E: int reveal_type(np.UFUNC_BUFSIZE_DEFAULT) # E: int reveal_type(np.WRAP) # E: int -reveal_type(np.little_endian) # E: int reveal_type(np.tracemalloc_domain) # E: int + +reveal_type(np.little_endian) # E: bool +reveal_type(np.True_) # E: numpy.bool_ +reveal_type(np.False_) # E: numpy.bool_ + +reveal_type(np.UFUNC_PYVALS_NAME) # E: str From 44be9665f7730a80d04c4e114b6c99c54d297b46 Mon Sep 17 00:00:00 2001 From: Ben Derrett Date: Fri, 16 Oct 2020 13:09:47 +0100 Subject: [PATCH 278/409] DOC: Fix Boolean array indexing typo --- doc/source/reference/arrays.indexing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/reference/arrays.indexing.rst b/doc/source/reference/arrays.indexing.rst index b2a9f1d219f3..9f82875ea65a 100644 --- a/doc/source/reference/arrays.indexing.rst +++ b/doc/source/reference/arrays.indexing.rst @@ -453,7 +453,7 @@ also supports boolean arrays and will work without any surprises. array([[ 3, 5], [ 9, 11]]) - Without the ``np.ix_`` call or only the diagonal elements would be + Without the ``np.ix_`` call, only the diagonal elements would be selected. Or without ``np.ix_`` (compare the integer array examples): From 2ce4ab4063457f558e5aab1051aedc68ea17eb25 Mon Sep 17 00:00:00 2001 From: Stefan van der Walt Date: Sat, 17 Oct 2020 06:49:57 -0700 Subject: [PATCH 279/409] NEP 36 (fair play) (#14779) --- doc/neps/nep-0036-fair-play.rst | 173 ++++++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 doc/neps/nep-0036-fair-play.rst diff --git a/doc/neps/nep-0036-fair-play.rst b/doc/neps/nep-0036-fair-play.rst new file mode 100644 index 000000000000..b8cf88788383 --- /dev/null +++ b/doc/neps/nep-0036-fair-play.rst @@ -0,0 +1,173 @@ +================== +NEP 36 — Fair play +================== + +:Author: Stéfan van der Walt +:Status: Draft +:Type: Informational +:Created: 2019-10-24 +:Resolution: Draft + + +Abstract +-------- + +This document sets out Rules of Play for companies and outside +developers that engage with the NumPy project. It covers: + +- Restrictions on use of the NumPy name +- How and whether to publish a modified distribution +- How to make us aware of patched versions + +Companies and developers will know after reading this NEP what kinds +of behavior the community would like to see, and which we consider +troublesome, bothersome, and unacceptable. + +Motivation +---------- + +We sometimes learn of NumPy versions modified and circulated by outsiders. +These patched versions can cause problems for the NumPy community. + +- In December 2018, a `bug report + `__ was filed against + `np.erf` -- a function that didn't exist in the NumPy distribution. + It came to light that a company had published a NumPy version with + an extended API footprint. After several months of discussion, the + company agreed to make its patches public, and we added a label to + the NumPy issue tracker to identify issues pertaining to that + distribution. + +- In another case, after a security issue (CVE-2019-6446) was filed + against NumPy, distributions put in their own fixes, most often by + changing a default keyword value. As a result the NumPy API was + inconsistent across distributions. + +When issues arise in cases like these, our developers waste time +identifying the problematic release, locating alterations, +and determining an appropriate course of action. + +During a community call on `October 16th, 2019 +`__ +the community resolved to draft guidelines on the distribution of +modified NumPy versions. + +Scope +----- + +This document aims to define a minimal set of rules that, when +followed, will be considered good faith efforts in line with the +expectations of the NumPy developers. + +Our hope is that developers who feel they need to modify NumPy will +first consider contributing to the project, or use one of several existing +mechanisms for extending our APIs and for operating on +externally defined array objects. + +When in doubt, please `talk to us first +`__. We may suggest an +alternative; at minimum, we'll be prepared. + +Fair play rules +--------------- + +1. Do not reuse the NumPy name for projects not developed by the NumPy + community. + + At time of writing, there are only a handful of `numpy`-named + packages developed by the community, including `numpy`, + `numpy-financial`, and `unumpy`. We ask that external packages not + include the phrase `numpy`, i.e., avoid names such as + `mycompany_numpy`. + + NumPy is a trademark owned by NumFOCUS. + +2. Do not republish modified versions of NumPy. + + Modified versions of NumPy make it very difficult for the + developers to address bug reports, since we typically do not know + which parts of NumPy have been modified. + + If you have to break this rule (and we implore you not + to!), then make it clear in the `__version__` tag that + you have modified NumPy, e.g.:: + + >>> print(np.__version__) + '1.17.2+mycompany.15` + + We understand that minor patches are often required to make a + library work under a certain distribution. E.g., Debian may patch + NumPy so that it searches for optimized BLAS libraries in the + correct locations. But we ask that no substantive changes are + made. + +3. Do not extend NumPy's API footprint. + + If you absolutely have to break rule two, please do not add + additional functions to the namespace. NumPy's API is already + quite large, and we are working hard to reduce it where feasible. + Having additional functions exposed in distributed versions is + confusing for users and developers alike. + +4. *DO* use official mechanism to engage with the API. + + Protocols such as `__array_ufunc__ + `__ and + `__array_function__ + `__ + were designed to help external packages interact more easily with + NumPy. E.g., the latter allows objects from foreign libraries to + pass through NumPy unharmed. We actively encourage using any of + these "officialy sanctioned" mechanisms for overriding or + interacting with NumPy. + + If these mechanisms are deemed insufficient, please start a + discussion on the mailing list before monkeypatching NumPy. + +Questions and answers +------------------- + +**Q:** We would like to distribute an optimized version of NumPy that +utilizes special instructions for our company's CPU. You recommend +against that, so what are we to do? + +**A:** Please consider including the patches required in the official +NumPy repository. Not only do we encourage such contributions, but we +already have optimized loops for some platforms available. + +**Q:** We would like to ship a much faster version of FFT than NumPy +provides, but NumPy has no mechanism for overriding its FFT routines. +How do we proceed? + +**A:** There are two solutions that we approve of: let the users +install your optimizations using a piece of code, such as:: + + from my_company_accel import patch_numpy_fft + patch_numpy_fft() + +or have your distribution automatically perform the above, but print a +message to the terminal clearly stating what is happening:: + + We are now patching NumPy for optimal performance under MyComp + Special Platform. Please direct all bug reports to + https://mycomp.com/numpy-bugs + +If you require additional mechanisms for overriding code, please +discuss this with the development team on the mailing list. + +**Q:** We would like to distribute NumPy with faster linear algebra +routines. Are we allowed to do this? + +**A:** Yes, this is explicitly supported by linking to a different +version of BLAS. + +Discussion +---------- + +References and footnotes +------------------------ + +Copyright +--------- + +This document has been placed in the public domain. From 7b0a764fee6e1614f3249e9082d8c4acf1dc62d5 Mon Sep 17 00:00:00 2001 From: madhulikajc <53166646+madhulikajc@users.noreply.github.com> Date: Sat, 17 Oct 2020 06:58:33 -0700 Subject: [PATCH 280/409] ENH: add function to get broadcast shape from a given set of shapes. (#17535) * ENH: add function to get broadcast shape from a given set of shapes. Add new function numpy.broadcast_shape which takes tuples for the shapes to be broadcast against each other. Return the broadcasted shape as a tuple. See #17217 * Perform array allocations of size 0 for provided shape tuples Co-authored-by: Eric Wieser * Test for int as input shape Also update docstring to include both ints and tuples of ints as input * Remove unnecessary array_function_dispatch * Add missing set_module * Add release notes. Add versionadded to docstring. Also fix up docstring details. * follow convention for trailing comma Co-authored-by: Sebastian Berg * Change name to broadcast_shapes. Also add test case, and type hint. * follow convention Co-authored-by: Eric Wieser * Update docstring Co-authored-by: Eric Wieser * Add reference to numpy docs on broadcasting to docstring Also move versionadded * Fix spelling Co-authored-by: Warren Weckesser * Add broadcast_shapes to reference docs and add See Also sections Co-authored-by: Eric Wieser Co-authored-by: Sebastian Berg Co-authored-by: Warren Weckesser --- .../upcoming_changes/17535.new_function.rst | 15 +++++ doc/source/reference/routines.other.rst | 1 + numpy/__init__.pyi | 2 + numpy/core/_add_newdocs.py | 1 + numpy/lib/stride_tricks.py | 59 ++++++++++++++++- numpy/lib/tests/test_stride_tricks.py | 65 ++++++++++++++++++- 6 files changed, 139 insertions(+), 4 deletions(-) create mode 100644 doc/release/upcoming_changes/17535.new_function.rst diff --git a/doc/release/upcoming_changes/17535.new_function.rst b/doc/release/upcoming_changes/17535.new_function.rst new file mode 100644 index 000000000000..4c3c11de4e26 --- /dev/null +++ b/doc/release/upcoming_changes/17535.new_function.rst @@ -0,0 +1,15 @@ +`numpy.broadcast_shapes` is a new user-facing function +------------------------------------------------------ +`broadcast_shapes` gets the resulting shape from +broadcasting the given shape tuples against each other. + +.. code:: python + + >>> np.broadcast_shapes((1, 2), (3, 1)) + (3, 2) + + >>> np.broadcast_shapes(2, (3, 1)) + (3, 2) + + >>> np.broadcast_shapes((6, 7), (5, 6, 1), (7,), (5, 1, 7)) + (5, 6, 7) diff --git a/doc/source/reference/routines.other.rst b/doc/source/reference/routines.other.rst index def5b3e3c9f5..aefd680bbcd1 100644 --- a/doc/source/reference/routines.other.rst +++ b/doc/source/reference/routines.other.rst @@ -47,6 +47,7 @@ Utility show_config deprecate deprecate_with_doc + broadcast_shapes Matlab-like Functions --------------------- diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 80fb213fdadd..2fff82d59365 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -1926,6 +1926,8 @@ def empty( like: ArrayLike = ..., ) -> ndarray: ... +def broadcast_shapes(*args: _ShapeLike) -> _Shape: ... + # # Constants # diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py index aa858761da78..c9968f122942 100644 --- a/numpy/core/_add_newdocs.py +++ b/numpy/core/_add_newdocs.py @@ -605,6 +605,7 @@ -------- broadcast_arrays broadcast_to + broadcast_shapes Examples -------- diff --git a/numpy/lib/stride_tricks.py b/numpy/lib/stride_tricks.py index 502235bdfbcd..d8a8b325e3a4 100644 --- a/numpy/lib/stride_tricks.py +++ b/numpy/lib/stride_tricks.py @@ -6,9 +6,9 @@ """ import numpy as np -from numpy.core.overrides import array_function_dispatch +from numpy.core.overrides import array_function_dispatch, set_module -__all__ = ['broadcast_to', 'broadcast_arrays'] +__all__ = ['broadcast_to', 'broadcast_arrays', 'broadcast_shapes'] class DummyArray: @@ -165,6 +165,12 @@ def broadcast_to(array, shape, subok=False): If the array is not compatible with the new shape according to NumPy's broadcasting rules. + See Also + -------- + broadcast + broadcast_arrays + broadcast_shapes + Notes ----- .. versionadded:: 1.10.0 @@ -197,6 +203,49 @@ def _broadcast_shape(*args): return b.shape +@set_module('numpy') +def broadcast_shapes(*args): + """ + Broadcast the input shapes into a single shape. + + :ref:`Learn more about broadcasting here `. + + .. versionadded:: 1.20.0 + + Parameters + ---------- + `*args` : tuples of ints, or ints + The shapes to be broadcast against each other. + + Returns + ------- + tuple + Broadcasted shape. + + Raises + ------ + ValueError + If the shapes are not compatible and cannot be broadcast according + to NumPy's broadcasting rules. + + See Also + -------- + broadcast + broadcast_arrays + broadcast_to + + Examples + -------- + >>> np.broadcast_shapes((1, 2), (3, 1), (3, 2)) + (3, 2) + + >>> np.broadcast_shapes((6, 7), (5, 6, 1), (7,), (5, 1, 7)) + (5, 6, 7) + """ + arrays = [np.empty(x, dtype=[]) for x in args] + return _broadcast_shape(*arrays) + + def _broadcast_arrays_dispatcher(*args, subok=None): return args @@ -230,6 +279,12 @@ def broadcast_arrays(*args, subok=False): warning will be emitted. A future version will set the ``writable`` flag False so writing to it will raise an error. + See Also + -------- + broadcast + broadcast_to + broadcast_shapes + Examples -------- >>> x = np.array([[1,2,3]]) diff --git a/numpy/lib/tests/test_stride_tricks.py b/numpy/lib/tests/test_stride_tricks.py index 9d95eb9d0321..10d7a19abec0 100644 --- a/numpy/lib/tests/test_stride_tricks.py +++ b/numpy/lib/tests/test_stride_tricks.py @@ -5,7 +5,8 @@ assert_raises_regex, assert_warns, ) from numpy.lib.stride_tricks import ( - as_strided, broadcast_arrays, _broadcast_shape, broadcast_to + as_strided, broadcast_arrays, _broadcast_shape, broadcast_to, + broadcast_shapes, ) def assert_shapes_correct(input_shapes, expected_shape): @@ -274,7 +275,9 @@ def test_broadcast_to_raises(): def test_broadcast_shape(): - # broadcast_shape is already exercized indirectly by broadcast_arrays + # tests internal _broadcast_shape + # _broadcast_shape is already exercised indirectly by broadcast_arrays + # _broadcast_shape is also exercised by the public broadcast_shapes function assert_equal(_broadcast_shape(), ()) assert_equal(_broadcast_shape([1, 2]), (2,)) assert_equal(_broadcast_shape(np.ones((1, 1))), (1, 1)) @@ -288,6 +291,64 @@ def test_broadcast_shape(): assert_raises(ValueError, lambda: _broadcast_shape(*bad_args)) +def test_broadcast_shapes_succeeds(): + # tests public broadcast_shapes + data = [ + [[], ()], + [[()], ()], + [[(7,)], (7,)], + [[(1, 2), (2,)], (1, 2)], + [[(1, 1)], (1, 1)], + [[(1, 1), (3, 4)], (3, 4)], + [[(6, 7), (5, 6, 1), (7,), (5, 1, 7)], (5, 6, 7)], + [[(5, 6, 1)], (5, 6, 1)], + [[(1, 3), (3, 1)], (3, 3)], + [[(1, 0), (0, 0)], (0, 0)], + [[(0, 1), (0, 0)], (0, 0)], + [[(1, 0), (0, 1)], (0, 0)], + [[(1, 1), (0, 0)], (0, 0)], + [[(1, 1), (1, 0)], (1, 0)], + [[(1, 1), (0, 1)], (0, 1)], + [[(), (0,)], (0,)], + [[(0,), (0, 0)], (0, 0)], + [[(0,), (0, 1)], (0, 0)], + [[(1,), (0, 0)], (0, 0)], + [[(), (0, 0)], (0, 0)], + [[(1, 1), (0,)], (1, 0)], + [[(1,), (0, 1)], (0, 1)], + [[(1,), (1, 0)], (1, 0)], + [[(), (1, 0)], (1, 0)], + [[(), (0, 1)], (0, 1)], + [[(1,), (3,)], (3,)], + [[2, (3, 2)], (3, 2)], + ] + for input_shapes, target_shape in data: + assert_equal(broadcast_shapes(*input_shapes), target_shape) + + assert_equal(broadcast_shapes(*([(1, 2)] * 32)), (1, 2)) + assert_equal(broadcast_shapes(*([(1, 2)] * 100)), (1, 2)) + + # regression tests for gh-5862 + assert_equal(broadcast_shapes(*([(2,)] * 32)), (2,)) + + +def test_broadcast_shapes_raises(): + # tests public broadcast_shapes + data = [ + [(3,), (4,)], + [(2, 3), (2,)], + [(3,), (3,), (4,)], + [(1, 3, 4), (2, 3, 3)], + [(1, 2), (3,1), (3,2), (10, 5)], + [2, (2, 3)], + ] + for input_shapes in data: + assert_raises(ValueError, lambda: broadcast_shapes(*input_shapes)) + + bad_args = [(2,)] * 32 + [(3,)] * 32 + assert_raises(ValueError, lambda: broadcast_shapes(*bad_args)) + + def test_as_strided(): a = np.array([None]) a_view = as_strided(a) From c53797e838f002b14e0d33c9651bffecd9934404 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Sat, 3 Oct 2020 01:06:56 +0200 Subject: [PATCH 281/409] ENH: Added support for `number` precision --- numpy/__init__.pyi | 224 ++++++++++++++++++-------------------- numpy/typing/__init__.py | 80 ++++++++++++++ numpy/typing/_callable.py | 107 ++++++++++++++---- 3 files changed, 269 insertions(+), 142 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 2fff82d59365..854d9e1ce418 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -17,6 +17,11 @@ from numpy.typing import ( _NumberLike, _SupportsDtype, _VoidDtypeLike, + NBitBase, + _64Bit, + _32Bit, + _16Bit, + _8Bit, ) from numpy.typing._callable import ( _BoolOp, @@ -1606,13 +1611,15 @@ class ndarray(_ArrayOrScalarCommon, Iterable, Sized, Container): # See https://github.com/numpy/numpy-stubs/pull/80 for more details. +_NBit_co = TypeVar("_NBit_co", covariant=True, bound=NBitBase) + class generic(_ArrayOrScalarCommon): @abstractmethod def __init__(self, *args: Any, **kwargs: Any) -> None: ... @property def base(self) -> None: ... -class number(generic): # type: ignore +class number(generic, Generic[_NBit_co]): # type: ignore @property def real(self: _ArraySelf) -> _ArraySelf: ... @property @@ -1699,12 +1706,12 @@ else: _FloatValue = Union[None, _CharLike, SupportsFloat] _ComplexValue = Union[None, _CharLike, SupportsFloat, SupportsComplex] -class integer(number): # type: ignore +class integer(number[_NBit_co]): # type: ignore # NOTE: `__index__` is technically defined in the bottom-most # sub-classes (`int64`, `uint32`, etc) def __index__(self) -> int: ... - __truediv__: _IntTrueDiv - __rtruediv__: _IntTrueDiv + __truediv__: _IntTrueDiv[_NBit_co] + __rtruediv__: _IntTrueDiv[_NBit_co] def __invert__(self: _IntType) -> _IntType: ... # Ensure that objects annotated as `integer` support bit-wise operations def __lshift__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... @@ -1718,39 +1725,33 @@ class integer(number): # type: ignore def __xor__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... def __rxor__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... -class signedinteger(integer): # type: ignore - __add__: _SignedIntOp - __radd__: _SignedIntOp - __sub__: _SignedIntOp - __rsub__: _SignedIntOp - __mul__: _SignedIntOp - __rmul__: _SignedIntOp - __floordiv__: _SignedIntOp - __rfloordiv__: _SignedIntOp - __pow__: _SignedIntOp - __rpow__: _SignedIntOp - __lshift__: _SignedIntBitOp - __rlshift__: _SignedIntBitOp - __rshift__: _SignedIntBitOp - __rrshift__: _SignedIntBitOp - __and__: _SignedIntBitOp - __rand__: _SignedIntBitOp - __xor__: _SignedIntBitOp - __rxor__: _SignedIntBitOp - __or__: _SignedIntBitOp - __ror__: _SignedIntBitOp - -class int8(signedinteger): - def __init__(self, __value: _IntValue = ...) -> None: ... - -class int16(signedinteger): - def __init__(self, __value: _IntValue = ...) -> None: ... - -class int32(signedinteger): - def __init__(self, __value: _IntValue = ...) -> None: ... - -class int64(signedinteger): +class signedinteger(integer[_NBit_co]): def __init__(self, __value: _IntValue = ...) -> None: ... + __add__: _SignedIntOp[_NBit_co] + __radd__: _SignedIntOp[_NBit_co] + __sub__: _SignedIntOp[_NBit_co] + __rsub__: _SignedIntOp[_NBit_co] + __mul__: _SignedIntOp[_NBit_co] + __rmul__: _SignedIntOp[_NBit_co] + __floordiv__: _SignedIntOp[_NBit_co] + __rfloordiv__: _SignedIntOp[_NBit_co] + __pow__: _SignedIntOp[_NBit_co] + __rpow__: _SignedIntOp[_NBit_co] + __lshift__: _SignedIntBitOp[_NBit_co] + __rlshift__: _SignedIntBitOp[_NBit_co] + __rshift__: _SignedIntBitOp[_NBit_co] + __rrshift__: _SignedIntBitOp[_NBit_co] + __and__: _SignedIntBitOp[_NBit_co] + __rand__: _SignedIntBitOp[_NBit_co] + __xor__: _SignedIntBitOp[_NBit_co] + __rxor__: _SignedIntBitOp[_NBit_co] + __or__: _SignedIntBitOp[_NBit_co] + __ror__: _SignedIntBitOp[_NBit_co] + +int8 = signedinteger[_8Bit] +int16 = signedinteger[_16Bit] +int32 = signedinteger[_32Bit] +int64 = signedinteger[_64Bit] class timedelta64(generic): def __init__( @@ -1765,98 +1766,85 @@ class timedelta64(generic): def __mul__(self, other: Union[_FloatLike, _BoolLike]) -> timedelta64: ... def __rmul__(self, other: Union[_FloatLike, _BoolLike]) -> timedelta64: ... __truediv__: _TD64Div[float64] - __floordiv__: _TD64Div[signedinteger] + __floordiv__: _TD64Div[int64] def __rtruediv__(self, other: timedelta64) -> float64: ... - def __rfloordiv__(self, other: timedelta64) -> signedinteger: ... + def __rfloordiv__(self, other: timedelta64) -> int64: ... def __mod__(self, other: timedelta64) -> timedelta64: ... -class unsignedinteger(integer): # type: ignore +class unsignedinteger(integer[_NBit_co]): # type: ignore # NOTE: `uint64 + signedinteger -> float64` - __add__: _UnsignedIntOp - __radd__: _UnsignedIntOp - __sub__: _UnsignedIntOp - __rsub__: _UnsignedIntOp - __mul__: _UnsignedIntOp - __rmul__: _UnsignedIntOp - __floordiv__: _UnsignedIntOp - __rfloordiv__: _UnsignedIntOp - __pow__: _UnsignedIntOp - __rpow__: _UnsignedIntOp - __lshift__: _UnsignedIntBitOp - __rlshift__: _UnsignedIntBitOp - __rshift__: _UnsignedIntBitOp - __rrshift__: _UnsignedIntBitOp - __and__: _UnsignedIntBitOp - __rand__: _UnsignedIntBitOp - __xor__: _UnsignedIntBitOp - __rxor__: _UnsignedIntBitOp - __or__: _UnsignedIntBitOp - __ror__: _UnsignedIntBitOp - -class uint8(unsignedinteger): - def __init__(self, __value: _IntValue = ...) -> None: ... - -class uint16(unsignedinteger): - def __init__(self, __value: _IntValue = ...) -> None: ... - -class uint32(unsignedinteger): - def __init__(self, __value: _IntValue = ...) -> None: ... - -class uint64(unsignedinteger): - def __init__(self, __value: _IntValue = ...) -> None: ... - -class inexact(number): ... # type: ignore - -class floating(inexact): # type: ignore - __add__: _FloatOp - __radd__: _FloatOp - __sub__: _FloatOp - __rsub__: _FloatOp - __mul__: _FloatOp - __rmul__: _FloatOp - __truediv__: _FloatOp - __rtruediv__: _FloatOp - __floordiv__: _FloatOp - __rfloordiv__: _FloatOp - __pow__: _FloatOp - __rpow__: _FloatOp + __add__: _UnsignedIntOp[_NBit_co] + __radd__: _UnsignedIntOp[_NBit_co] + __sub__: _UnsignedIntOp[_NBit_co] + __rsub__: _UnsignedIntOp[_NBit_co] + __mul__: _UnsignedIntOp[_NBit_co] + __rmul__: _UnsignedIntOp[_NBit_co] + __floordiv__: _UnsignedIntOp[_NBit_co] + __rfloordiv__: _UnsignedIntOp[_NBit_co] + __pow__: _UnsignedIntOp[_NBit_co] + __rpow__: _UnsignedIntOp[_NBit_co] + __lshift__: _UnsignedIntBitOp[_NBit_co] + __rlshift__: _UnsignedIntBitOp[_NBit_co] + __rshift__: _UnsignedIntBitOp[_NBit_co] + __rrshift__: _UnsignedIntBitOp[_NBit_co] + __and__: _UnsignedIntBitOp[_NBit_co] + __rand__: _UnsignedIntBitOp[_NBit_co] + __xor__: _UnsignedIntBitOp[_NBit_co] + __rxor__: _UnsignedIntBitOp[_NBit_co] + __or__: _UnsignedIntBitOp[_NBit_co] + __ror__: _UnsignedIntBitOp[_NBit_co] + +uint8 = unsignedinteger[_8Bit] +uint16 = unsignedinteger[_16Bit] +uint32 = unsignedinteger[_32Bit] +uint64 = unsignedinteger[_64Bit] + +class inexact(number[_NBit_co]): ... # type: ignore _IntType = TypeVar("_IntType", bound=integer) _FloatType = TypeVar('_FloatType', bound=floating) -class float16(floating): +class floating(inexact[_NBit_co]): def __init__(self, __value: _FloatValue = ...) -> None: ... - -class float32(floating): - def __init__(self, __value: _FloatValue = ...) -> None: ... - -class float64(floating, float): - def __init__(self, __value: _FloatValue = ...) -> None: ... - -class complexfloating(inexact, Generic[_FloatType]): # type: ignore - @property - def real(self) -> _FloatType: ... # type: ignore[override] - @property - def imag(self) -> _FloatType: ... # type: ignore[override] - def __abs__(self) -> _FloatType: ... # type: ignore[override] - __add__: _ComplexOp - __radd__: _ComplexOp - __sub__: _ComplexOp - __rsub__: _ComplexOp - __mul__: _ComplexOp - __rmul__: _ComplexOp - __truediv__: _ComplexOp - __rtruediv__: _ComplexOp - __floordiv__: _ComplexOp - __rfloordiv__: _ComplexOp - __pow__: _ComplexOp - __rpow__: _ComplexOp - -class complex64(complexfloating[float32]): - def __init__(self, __value: _ComplexValue = ...) -> None: ... - -class complex128(complexfloating[float64], complex): + __add__: _FloatOp[_NBit_co] + __radd__: _FloatOp[_NBit_co] + __sub__: _FloatOp[_NBit_co] + __rsub__: _FloatOp[_NBit_co] + __mul__: _FloatOp[_NBit_co] + __rmul__: _FloatOp[_NBit_co] + __truediv__: _FloatOp[_NBit_co] + __rtruediv__: _FloatOp[_NBit_co] + __floordiv__: _FloatOp[_NBit_co] + __rfloordiv__: _FloatOp[_NBit_co] + __pow__: _FloatOp[_NBit_co] + __rpow__: _FloatOp[_NBit_co] + +float16 = floating[_16Bit] +float32 = floating[_32Bit] +float64 = floating[_64Bit] + +class complexfloating(inexact[_NBit_co]): def __init__(self, __value: _ComplexValue = ...) -> None: ... + @property + def real(self) -> floating[_NBit_co]: ... # type: ignore[override] + @property + def imag(self) -> floating[_NBit_co]: ... # type: ignore[override] + def __abs__(self) -> floating[_NBit_co]: ... # type: ignore[override] + __add__: _ComplexOp[_NBit_co] + __radd__: _ComplexOp[_NBit_co] + __sub__: _ComplexOp[_NBit_co] + __rsub__: _ComplexOp[_NBit_co] + __mul__: _ComplexOp[_NBit_co] + __rmul__: _ComplexOp[_NBit_co] + __truediv__: _ComplexOp[_NBit_co] + __rtruediv__: _ComplexOp[_NBit_co] + __floordiv__: _ComplexOp[_NBit_co] + __rfloordiv__: _ComplexOp[_NBit_co] + __pow__: _ComplexOp[_NBit_co] + __rpow__: _ComplexOp[_NBit_co] + +complex64 = complexfloating[_32Bit] +complex128 = complexfloating[_64Bit] class flexible(generic): ... # type: ignore diff --git a/numpy/typing/__init__.py b/numpy/typing/__init__.py index dafabd95aefd..5c6e5df99830 100644 --- a/numpy/typing/__init__.py +++ b/numpy/typing/__init__.py @@ -89,7 +89,87 @@ since its usage is discouraged. Please see : https://numpy.org/devdocs/reference/arrays.dtypes.html +NBitBase +~~~~~~~~ + +.. autoclass:: numpy.typing.NBitBase + """ + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import sys + if sys.version_info >= (3, 8): + from typing import final + else: + from typing_extensions import final +else: + def final(f): return f + + +@final # Dissallow the creation of arbitrary `NBitBase` subclasses +class NBitBase: + """ + An object representing `number` precision during static type checking. + + Used exclusively for the purpose static type checking, `NBitBase` + represents the base of a hierachieral set of subclasses. + Each subsequent subclass is herein used for representing a lower level + of precision, _e.g._ `64Bit > 32Bit > 16Bit`. + + Examples + -------- + Below is a typical usage example: `NBitBase` is herein used for annotating a + function that takes a float and integer of arbitrary precision as arguments + and returns a new float of whichever precision is largest + (_e.g._ `np.float16 + np.int64 -> np.float64`). + + >>> from typing import TypeVar, TYPE_CHECKING + >>> import numpy as np + >>> import numpy.typing as npt + + >>> T = TypeVar("T", bound=npt.NBitBase) + + >>> def add(a: "np.floating[T]", b: "np.integer[T]") -> "np.floating[T]": + ... return a + b + + >>> a = np.float16() + >>> b = np.int64() + >>> out = add(a, b) + + >>> if TYPE_CHECKING: + ... reveal_locals() + ... # note: Revealed local types are: + ... # note: a: numpy.floating[numpy.typing._16Bit*] + ... # note: b: numpy.signedinteger[numpy.typing._64Bit*] + ... # note: out: numpy.floating[numpy.typing._64Bit*] + + """ + + def __init_subclass__(cls) -> None: + allowed_names = { + "NBitBase", "_256Bit", "_128Bit", "_96Bit", "_80Bit", + "_64Bit", "_32Bit", "_16Bit", "_8Bit", + } + if cls.__name__ not in allowed_names: + raise TypeError('cannot inherit from final class "NBitBase"') + super().__init_subclass__() + + +# Silence errors about subclassing a `@final`-decorated class +class _256Bit(NBitBase): ... # type: ignore[misc] +class _128Bit(_256Bit): ... # type: ignore[misc] +class _96Bit(_128Bit): ... # type: ignore[misc] +class _80Bit(_96Bit): ... # type: ignore[misc] +class _64Bit(_80Bit): ... # type: ignore[misc] +class _32Bit(_64Bit): ... # type: ignore[misc] +class _16Bit(_32Bit): ... # type: ignore[misc] +class _8Bit(_16Bit): ... # type: ignore[misc] + +# Clean up the namespace +del TYPE_CHECKING, final + from ._scalars import ( _CharLike, _BoolLike, diff --git a/numpy/typing/_callable.py b/numpy/typing/_callable.py index 7c2ee86cbf2d..bc58f27740d0 100644 --- a/numpy/typing/_callable.py +++ b/numpy/typing/_callable.py @@ -9,7 +9,7 @@ """ import sys -from typing import Union, TypeVar, overload, Any +from typing import Union, TypeVar, overload, Any, TYPE_CHECKING, NoReturn from numpy import ( generic, @@ -25,6 +25,7 @@ float32, float64, complexfloating, + complex64, complex128, ) from ._scalars import ( @@ -34,6 +35,7 @@ _ComplexLike, _NumberLike, ) +from . import NBitBase, _64Bit if sys.version_info >= (3, 8): from typing import Protocol @@ -46,7 +48,8 @@ else: HAVE_PROTOCOL = True -if HAVE_PROTOCOL: +if TYPE_CHECKING or HAVE_PROTOCOL: + _NBit_co = TypeVar("_NBit_co", covariant=True, bound=NBitBase) _IntType = TypeVar("_IntType", bound=integer) _NumberType = TypeVar("_NumberType", bound=number) _NumberType_co = TypeVar("_NumberType_co", covariant=True, bound=number) @@ -74,6 +77,8 @@ def __call__(self, __other: _IntType) -> _IntType: ... class _BoolSub(Protocol): # Note that `__other: bool_` is absent here + @overload + def __call__(self, __other: bool) -> NoReturn: ... @overload # platform dependent def __call__(self, __other: int) -> Union[int32, int64]: ... @overload @@ -97,51 +102,105 @@ def __call__(self, __other: timedelta64) -> _NumberType_co: ... @overload def __call__(self, __other: _FloatLike) -> timedelta64: ... - class _IntTrueDiv(Protocol): + class _IntTrueDiv(Protocol[_NBit_co]): # type: ignore[misc] + @overload + def __call__(self, __other: bool) -> floating[_NBit_co]: ... + @overload + def __call__(self, __other: int) -> Union[float32, float64]: ... + @overload + def __call__(self, __other: float) -> float64: ... @overload - def __call__(self, __other: Union[_IntLike, float]) -> floating: ... + def __call__(self, __other: complex) -> complex128: ... @overload - def __call__(self, __other: complex) -> complexfloating[floating]: ... + def __call__(self, __other: integer[_NBit_co]) -> floating[_NBit_co]: ... - class _UnsignedIntOp(Protocol): + class _UnsignedIntOp(Protocol[_NBit_co]): # type: ignore[misc] # NOTE: `uint64 + signedinteger -> float64` @overload - def __call__(self, __other: Union[bool, unsignedinteger]) -> unsignedinteger: ... + def __call__(self, __other: bool) -> unsignedinteger[_NBit_co]: ... @overload - def __call__(self, __other: Union[int, signedinteger]) -> Union[signedinteger, float64]: ... + def __call__( + self, __other: Union[int, signedinteger[Any]] + ) -> Union[signedinteger[Any], float64]: ... + @overload + def __call__(self, __other: float) -> float64: ... @overload - def __call__(self, __other: float) -> floating: ... + def __call__(self, __other: complex) -> complex128: ... @overload - def __call__(self, __other: complex) -> complexfloating[floating]: ... + def __call__( + self, __other: unsignedinteger[_NBit_co] + ) -> unsignedinteger[_NBit_co]: ... - class _UnsignedIntBitOp(Protocol): + class _UnsignedIntBitOp(Protocol[_NBit_co]): # type: ignore[misc] # TODO: The likes of `uint64 | np.signedinteger` will fail as there # is no signed integer type large enough to hold a `uint64` # See https://github.com/numpy/numpy/issues/2524 @overload - def __call__(self, __other: Union[bool, unsignedinteger]) -> unsignedinteger: ... + def __call__(self, __other: bool) -> unsignedinteger[_NBit_co]: ... + @overload + def __call__( + self, __other: unsignedinteger[_NBit_co] + ) -> unsignedinteger[_NBit_co]: ... + @overload + def __call__( + self: _UnsignedIntBitOp[_64Bit], + __other: Union[int, signedinteger[Any]], + ) -> NoReturn: ... + @overload + def __call__(self, __other: int) -> Union[int32, int64]: ... @overload - def __call__(self, __other: Union[int, signedinteger]) -> signedinteger: ... + def __call__(self, __other: signedinteger[Any]) -> signedinteger[Any]: ... - class _SignedIntOp(Protocol): + class _SignedIntOp(Protocol[_NBit_co]): # type: ignore[misc] + @overload + def __call__(self, __other: bool) -> signedinteger[_NBit_co]: ... @overload - def __call__(self, __other: Union[int, signedinteger]) -> signedinteger: ... + def __call__(self, __other: int) -> Union[int32, int64]: ... @overload - def __call__(self, __other: float) -> floating: ... + def __call__(self, __other: float) -> float64: ... @overload - def __call__(self, __other: complex) -> complexfloating[floating]: ... + def __call__(self, __other: complex) -> complex128: ... + @overload + def __call__( + self, __other: signedinteger[_NBit_co] + ) -> signedinteger[_NBit_co]: ... - class _SignedIntBitOp(Protocol): - def __call__(self, __other: Union[int, signedinteger]) -> signedinteger: ... + class _SignedIntBitOp(Protocol[_NBit_co]): # type: ignore[misc] + @overload + def __call__(self, __other: bool) -> signedinteger[_NBit_co]: ... + @overload + def __call__(self, __other: int) -> Union[int32, int64]: ... + @overload + def __call__( + self, __other: signedinteger[_NBit_co] + ) -> signedinteger[_NBit_co]: ... - class _FloatOp(Protocol): + class _FloatOp(Protocol[_NBit_co]): # type: ignore[misc] @overload - def __call__(self, __other: _FloatLike) -> floating: ... + def __call__(self, __other: bool) -> floating[_NBit_co]: ... @overload - def __call__(self, __other: complex) -> complexfloating[floating]: ... + def __call__(self, __other: int) -> Union[float32, float64]: ... + @overload + def __call__(self, __other: float) -> float64: ... + @overload + def __call__(self, __other: complex) -> complex128: ... + @overload + def __call__( + self, __other: Union[integer[_NBit_co], floating[_NBit_co]] + ) -> floating[_NBit_co]: ... - class _ComplexOp(Protocol): - def __call__(self, __other: _ComplexLike) -> complexfloating[floating]: ... + class _ComplexOp(Protocol[_NBit_co]): # type: ignore[misc] + @overload + def __call__(self, __other: bool) -> complexfloating[_NBit_co, _NBit_co]: ... + @overload + def __call__(self, __other: int) -> Union[complex64, complex128]: ... + @overload + def __call__(self, __other: Union[float, complex]) -> complex128: ... + @overload + def __call__( + self, + __other: Union[integer[_NBit_co], floating[_NBit_co], complexfloating[_NBit_co]] + ) -> complexfloating[_NBit_co]: ... class _NumberOp(Protocol): def __call__(self, __other: _NumberLike) -> number: ... From 1816e44ec688dfb8f9683ecb41f069520a306a89 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Mon, 12 Oct 2020 14:50:07 +0200 Subject: [PATCH 282/409] ENH: Make `complexfloating` generic w.r.t 2 typevars --- numpy/__init__.pyi | 13 +++++++++---- numpy/typing/_callable.py | 8 ++++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 854d9e1ce418..32b6017ce4f7 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -1612,6 +1612,7 @@ class ndarray(_ArrayOrScalarCommon, Iterable, Sized, Container): # See https://github.com/numpy/numpy-stubs/pull/80 for more details. _NBit_co = TypeVar("_NBit_co", covariant=True, bound=NBitBase) +_NBit_co2 = TypeVar("_NBit_co2", covariant=True, bound=NBitBase) class generic(_ArrayOrScalarCommon): @abstractmethod @@ -1823,12 +1824,16 @@ float16 = floating[_16Bit] float32 = floating[_32Bit] float64 = floating[_64Bit] -class complexfloating(inexact[_NBit_co]): +# The main reason for `complexfloating` having two typevars is cosmetic. +# It is used to clarify why `complex128`s precision is `_64Bit`, the latter +# describing the two 64 bit floats representing its real and imaginary component + +class complexfloating(inexact[_NBit_co], Generic[_NBit_co, _NBit_co2]): def __init__(self, __value: _ComplexValue = ...) -> None: ... @property def real(self) -> floating[_NBit_co]: ... # type: ignore[override] @property - def imag(self) -> floating[_NBit_co]: ... # type: ignore[override] + def imag(self) -> floating[_NBit_co2]: ... # type: ignore[override] def __abs__(self) -> floating[_NBit_co]: ... # type: ignore[override] __add__: _ComplexOp[_NBit_co] __radd__: _ComplexOp[_NBit_co] @@ -1843,8 +1848,8 @@ class complexfloating(inexact[_NBit_co]): __pow__: _ComplexOp[_NBit_co] __rpow__: _ComplexOp[_NBit_co] -complex64 = complexfloating[_32Bit] -complex128 = complexfloating[_64Bit] +complex64 = complexfloating[_32Bit, _32Bit] +complex128 = complexfloating[_64Bit, _64Bit] class flexible(generic): ... # type: ignore diff --git a/numpy/typing/_callable.py b/numpy/typing/_callable.py index bc58f27740d0..f75828a7a125 100644 --- a/numpy/typing/_callable.py +++ b/numpy/typing/_callable.py @@ -199,8 +199,12 @@ def __call__(self, __other: Union[float, complex]) -> complex128: ... @overload def __call__( self, - __other: Union[integer[_NBit_co], floating[_NBit_co], complexfloating[_NBit_co]] - ) -> complexfloating[_NBit_co]: ... + __other: Union[ + integer[_NBit_co], + floating[_NBit_co], + complexfloating[_NBit_co, _NBit_co], + ] + ) -> complexfloating[_NBit_co, _NBit_co]: ... class _NumberOp(Protocol): def __call__(self, __other: _NumberLike) -> number: ... From 9e732c85a259c500088df39e5492b014b56ef5da Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Mon, 12 Oct 2020 16:13:01 +0200 Subject: [PATCH 283/409] MAINT: Removed a now redundant `TODO` comment --- numpy/typing/_callable.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/numpy/typing/_callable.py b/numpy/typing/_callable.py index f75828a7a125..372a39928c5a 100644 --- a/numpy/typing/_callable.py +++ b/numpy/typing/_callable.py @@ -132,9 +132,6 @@ def __call__( ) -> unsignedinteger[_NBit_co]: ... class _UnsignedIntBitOp(Protocol[_NBit_co]): # type: ignore[misc] - # TODO: The likes of `uint64 | np.signedinteger` will fail as there - # is no signed integer type large enough to hold a `uint64` - # See https://github.com/numpy/numpy/issues/2524 @overload def __call__(self, __other: bool) -> unsignedinteger[_NBit_co]: ... @overload From fdf30fb70821462fc636e11c361688e46cfeaccf Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Tue, 13 Oct 2020 17:26:20 +0200 Subject: [PATCH 284/409] DOC: Fixed some docstring formatting --- numpy/typing/__init__.py | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/numpy/typing/__init__.py b/numpy/typing/__init__.py index 5c6e5df99830..2738da071198 100644 --- a/numpy/typing/__init__.py +++ b/numpy/typing/__init__.py @@ -111,39 +111,41 @@ def final(f): return f @final # Dissallow the creation of arbitrary `NBitBase` subclasses class NBitBase: """ - An object representing `number` precision during static type checking. + An object representing `numpy.number` precision during static type checking. Used exclusively for the purpose static type checking, `NBitBase` represents the base of a hierachieral set of subclasses. Each subsequent subclass is herein used for representing a lower level - of precision, _e.g._ `64Bit > 32Bit > 16Bit`. + of precision, *e.g.* ``64Bit > 32Bit > 16Bit``. Examples -------- Below is a typical usage example: `NBitBase` is herein used for annotating a function that takes a float and integer of arbitrary precision as arguments and returns a new float of whichever precision is largest - (_e.g._ `np.float16 + np.int64 -> np.float64`). + (*e.g.* ``np.float16 + np.int64 -> np.float64``). - >>> from typing import TypeVar, TYPE_CHECKING - >>> import numpy as np - >>> import numpy.typing as npt + .. code-block:: python - >>> T = TypeVar("T", bound=npt.NBitBase) + >>> from typing import TypeVar, TYPE_CHECKING + >>> import numpy as np + >>> import numpy.typing as npt - >>> def add(a: "np.floating[T]", b: "np.integer[T]") -> "np.floating[T]": - ... return a + b + >>> T = TypeVar("T", bound=npt.NBitBase) - >>> a = np.float16() - >>> b = np.int64() - >>> out = add(a, b) + >>> def add(a: "np.floating[T]", b: "np.integer[T]") -> "np.floating[T]": + ... return a + b - >>> if TYPE_CHECKING: - ... reveal_locals() - ... # note: Revealed local types are: - ... # note: a: numpy.floating[numpy.typing._16Bit*] - ... # note: b: numpy.signedinteger[numpy.typing._64Bit*] - ... # note: out: numpy.floating[numpy.typing._64Bit*] + >>> a = np.float16() + >>> b = np.int64() + >>> out = add(a, b) + + >>> if TYPE_CHECKING: + ... reveal_locals() + ... # note: Revealed local types are: + ... # note: a: numpy.floating[numpy.typing._16Bit*] + ... # note: b: numpy.signedinteger[numpy.typing._64Bit*] + ... # note: out: numpy.floating[numpy.typing._64Bit*] """ From 3ebabec4a68f980bc680d8fa8445fb08aa350e79 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Fri, 16 Oct 2020 16:25:28 +0200 Subject: [PATCH 285/409] MAINT: Added a missing `__init__` method --- numpy/__init__.pyi | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 32b6017ce4f7..f670a5e3b4da 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -1772,8 +1772,9 @@ class timedelta64(generic): def __rfloordiv__(self, other: timedelta64) -> int64: ... def __mod__(self, other: timedelta64) -> timedelta64: ... -class unsignedinteger(integer[_NBit_co]): # type: ignore +class unsignedinteger(integer[_NBit_co]): # NOTE: `uint64 + signedinteger -> float64` + def __init__(self, __value: _IntValue = ...) -> None: ... __add__: _UnsignedIntOp[_NBit_co] __radd__: _UnsignedIntOp[_NBit_co] __sub__: _UnsignedIntOp[_NBit_co] From a5f17578bc31caccce5c5113dae5d1994bab22e4 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Fri, 16 Oct 2020 16:27:55 +0200 Subject: [PATCH 286/409] MAINT: Fixed a few issues with the `__call__`-based protocols * Removed redundant `type: ignore` messages * Set the return precision as `Union[_NBit_co, _NBit]` * Type the precision of `builtins.int` operations as `Any` --- numpy/typing/_callable.py | 72 ++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 38 deletions(-) diff --git a/numpy/typing/_callable.py b/numpy/typing/_callable.py index 372a39928c5a..e8c8615b5a7f 100644 --- a/numpy/typing/_callable.py +++ b/numpy/typing/_callable.py @@ -50,6 +50,7 @@ if TYPE_CHECKING or HAVE_PROTOCOL: _NBit_co = TypeVar("_NBit_co", covariant=True, bound=NBitBase) + _NBit = TypeVar("_NBit", bound=NBitBase) _IntType = TypeVar("_IntType", bound=integer) _NumberType = TypeVar("_NumberType", bound=number) _NumberType_co = TypeVar("_NumberType_co", covariant=True, bound=number) @@ -59,7 +60,7 @@ class _BoolOp(Protocol[_GenericType_co]): @overload def __call__(self, __other: _BoolLike) -> _GenericType_co: ... @overload # platform dependent - def __call__(self, __other: int) -> Union[int32, int64]: ... + def __call__(self, __other: int) -> signedinteger[Any]: ... @overload def __call__(self, __other: float) -> float64: ... @overload @@ -71,7 +72,7 @@ class _BoolBitOp(Protocol[_GenericType_co]): @overload def __call__(self, __other: _BoolLike) -> _GenericType_co: ... @overload # platform dependent - def __call__(self, __other: int) -> Union[int32, int64]: ... + def __call__(self, __other: int) -> signedinteger[Any]: ... @overload def __call__(self, __other: _IntType) -> _IntType: ... @@ -80,7 +81,7 @@ class _BoolSub(Protocol): @overload def __call__(self, __other: bool) -> NoReturn: ... @overload # platform dependent - def __call__(self, __other: int) -> Union[int32, int64]: ... + def __call__(self, __other: int) -> signedinteger[Any]: ... @overload def __call__(self, __other: float) -> float64: ... @overload @@ -102,19 +103,19 @@ def __call__(self, __other: timedelta64) -> _NumberType_co: ... @overload def __call__(self, __other: _FloatLike) -> timedelta64: ... - class _IntTrueDiv(Protocol[_NBit_co]): # type: ignore[misc] + class _IntTrueDiv(Protocol[_NBit_co]): @overload def __call__(self, __other: bool) -> floating[_NBit_co]: ... @overload - def __call__(self, __other: int) -> Union[float32, float64]: ... + def __call__(self, __other: int) -> floating[Any]: ... @overload def __call__(self, __other: float) -> float64: ... @overload def __call__(self, __other: complex) -> complex128: ... @overload - def __call__(self, __other: integer[_NBit_co]) -> floating[_NBit_co]: ... + def __call__(self, __other: integer[_NBit]) -> floating[Union[_NBit_co, _NBit]]: ... - class _UnsignedIntOp(Protocol[_NBit_co]): # type: ignore[misc] + class _UnsignedIntOp(Protocol[_NBit_co]): # NOTE: `uint64 + signedinteger -> float64` @overload def __call__(self, __other: bool) -> unsignedinteger[_NBit_co]: ... @@ -128,80 +129,75 @@ def __call__(self, __other: float) -> float64: ... def __call__(self, __other: complex) -> complex128: ... @overload def __call__( - self, __other: unsignedinteger[_NBit_co] - ) -> unsignedinteger[_NBit_co]: ... + self, __other: unsignedinteger[_NBit] + ) -> unsignedinteger[Union[_NBit_co, _NBit]]: ... - class _UnsignedIntBitOp(Protocol[_NBit_co]): # type: ignore[misc] + class _UnsignedIntBitOp(Protocol[_NBit_co]): @overload def __call__(self, __other: bool) -> unsignedinteger[_NBit_co]: ... @overload - def __call__( - self, __other: unsignedinteger[_NBit_co] - ) -> unsignedinteger[_NBit_co]: ... - @overload - def __call__( - self: _UnsignedIntBitOp[_64Bit], - __other: Union[int, signedinteger[Any]], - ) -> NoReturn: ... - @overload - def __call__(self, __other: int) -> Union[int32, int64]: ... + def __call__(self, __other: int) -> signedinteger[Any]: ... @overload def __call__(self, __other: signedinteger[Any]) -> signedinteger[Any]: ... + @overload + def __call__( + self, __other: unsignedinteger[_NBit] + ) -> unsignedinteger[Union[_NBit_co, _NBit]]: ... - class _SignedIntOp(Protocol[_NBit_co]): # type: ignore[misc] + class _SignedIntOp(Protocol[_NBit_co]): @overload def __call__(self, __other: bool) -> signedinteger[_NBit_co]: ... @overload - def __call__(self, __other: int) -> Union[int32, int64]: ... + def __call__(self, __other: int) -> signedinteger[Any]: ... @overload def __call__(self, __other: float) -> float64: ... @overload def __call__(self, __other: complex) -> complex128: ... @overload def __call__( - self, __other: signedinteger[_NBit_co] - ) -> signedinteger[_NBit_co]: ... + self, __other: signedinteger[_NBit] + ) -> signedinteger[Union[_NBit_co, _NBit]]: ... - class _SignedIntBitOp(Protocol[_NBit_co]): # type: ignore[misc] + class _SignedIntBitOp(Protocol[_NBit_co]): @overload def __call__(self, __other: bool) -> signedinteger[_NBit_co]: ... @overload - def __call__(self, __other: int) -> Union[int32, int64]: ... + def __call__(self, __other: int) -> signedinteger[Any]: ... @overload def __call__( - self, __other: signedinteger[_NBit_co] - ) -> signedinteger[_NBit_co]: ... + self, __other: signedinteger[_NBit] + ) -> signedinteger[Union[_NBit_co, _NBit]]: ... - class _FloatOp(Protocol[_NBit_co]): # type: ignore[misc] + class _FloatOp(Protocol[_NBit_co]): @overload def __call__(self, __other: bool) -> floating[_NBit_co]: ... @overload - def __call__(self, __other: int) -> Union[float32, float64]: ... + def __call__(self, __other: int) -> floating[Any]: ... @overload def __call__(self, __other: float) -> float64: ... @overload def __call__(self, __other: complex) -> complex128: ... @overload def __call__( - self, __other: Union[integer[_NBit_co], floating[_NBit_co]] - ) -> floating[_NBit_co]: ... + self, __other: Union[integer[_NBit], floating[_NBit]] + ) -> floating[Union[_NBit_co, _NBit]]: ... - class _ComplexOp(Protocol[_NBit_co]): # type: ignore[misc] + class _ComplexOp(Protocol[_NBit_co]): @overload def __call__(self, __other: bool) -> complexfloating[_NBit_co, _NBit_co]: ... @overload - def __call__(self, __other: int) -> Union[complex64, complex128]: ... + def __call__(self, __other: int) -> complexfloating[Any, Any]: ... @overload def __call__(self, __other: Union[float, complex]) -> complex128: ... @overload def __call__( self, __other: Union[ - integer[_NBit_co], - floating[_NBit_co], - complexfloating[_NBit_co, _NBit_co], + integer[_NBit], + floating[_NBit], + complexfloating[_NBit, _NBit], ] - ) -> complexfloating[_NBit_co, _NBit_co]: ... + ) -> complexfloating[Union[_NBit_co, _NBit], Union[_NBit_co, _NBit]]: ... class _NumberOp(Protocol): def __call__(self, __other: _NumberLike) -> number: ... From b84946d15e5b63f44cd4bbbc04e5230a18720afc Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Fri, 16 Oct 2020 16:36:16 +0200 Subject: [PATCH 287/409] TST: Ignore all `*` characters in the reveal tests Mypy uses a `*` whenever an annotation or one of its parameters is based on a TypeVar. Its added value is neglible and it unnecessarily complicates the `reveal` tests so lets just ignore them. Note that this is done after running mypy, so it won't affect cases where `*` is used as multiplication operator. --- numpy/typing/tests/test_typing.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/numpy/typing/tests/test_typing.py b/numpy/typing/tests/test_typing.py index cba1dc1be7eb..709eda7a614b 100644 --- a/numpy/typing/tests/test_typing.py +++ b/numpy/typing/tests/test_typing.py @@ -114,9 +114,10 @@ def test_reveal(path): ]) with open(path) as fin: - lines = fin.readlines() + lines = fin.read().replace('*', '').split("\n") - for error_line in stdout.split("\n"): + stdout_list = stdout.replace('*', '').split("\n") + for error_line in stdout_list: error_line = error_line.strip() if not error_line: continue From 27d586465760a47cd1754ad07fff813f3517e775 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Fri, 16 Oct 2020 16:37:08 +0200 Subject: [PATCH 288/409] TST: Update the pre-existing tests --- numpy/typing/tests/data/fail/bitwise_ops.py | 4 +- numpy/typing/tests/data/fail/numerictypes.py | 2 +- numpy/typing/tests/data/fail/scalars.py | 4 - .../tests/data/reveal/array_constructors.py | 2 +- numpy/typing/tests/data/reveal/fromnumeric.py | 194 +++++++++--------- .../typing/tests/data/reveal/ndarray_misc.py | 94 ++++----- numpy/typing/tests/data/reveal/scalars.py | 12 +- 7 files changed, 154 insertions(+), 158 deletions(-) diff --git a/numpy/typing/tests/data/fail/bitwise_ops.py b/numpy/typing/tests/data/fail/bitwise_ops.py index 3b5be95c0d96..8a8f89755a36 100644 --- a/numpy/typing/tests/data/fail/bitwise_ops.py +++ b/numpy/typing/tests/data/fail/bitwise_ops.py @@ -9,9 +9,9 @@ f8 = np.float64() b_ >> f8 # E: No overload variant -i8 << f8 # E: incompatible type +i8 << f8 # E: No overload variant i | f8 # E: Unsupported operand types -i8 ^ f8 # E: incompatible type +i8 ^ f8 # E: No overload variant u8 & f8 # E: No overload variant ~f8 # E: Unsupported operand type diff --git a/numpy/typing/tests/data/fail/numerictypes.py b/numpy/typing/tests/data/fail/numerictypes.py index dd03eacc1c97..94537a23b682 100644 --- a/numpy/typing/tests/data/fail/numerictypes.py +++ b/numpy/typing/tests/data/fail/numerictypes.py @@ -10,4 +10,4 @@ np.issubdtype(1, np.int64) # E: incompatible type "int" -np.find_common_type(np.int64, np.int64) # E: incompatible type "Type[int64]" +np.find_common_type(np.int64, np.int64) # E: incompatible type "Type[signedinteger[Any]]" diff --git a/numpy/typing/tests/data/fail/scalars.py b/numpy/typing/tests/data/fail/scalars.py index 13bb45483353..e7c4c7485603 100644 --- a/numpy/typing/tests/data/fail/scalars.py +++ b/numpy/typing/tests/data/fail/scalars.py @@ -47,11 +47,7 @@ def __float__(self): np.generic(1) # E: Cannot instantiate abstract class np.number(1) # E: Cannot instantiate abstract class np.integer(1) # E: Cannot instantiate abstract class -np.signedinteger(1) # E: Cannot instantiate abstract class -np.unsignedinteger(1) # E: Cannot instantiate abstract class np.inexact(1) # E: Cannot instantiate abstract class -np.floating(1) # E: Cannot instantiate abstract class -np.complexfloating(1) # E: Cannot instantiate abstract class np.character("test") # E: Cannot instantiate abstract class np.flexible(b"test") # E: Cannot instantiate abstract class diff --git a/numpy/typing/tests/data/reveal/array_constructors.py b/numpy/typing/tests/data/reveal/array_constructors.py index ba8a8eda1e94..13a04777f20e 100644 --- a/numpy/typing/tests/data/reveal/array_constructors.py +++ b/numpy/typing/tests/data/reveal/array_constructors.py @@ -37,6 +37,6 @@ class SubClass(np.ndarray): ... reveal_type(np.require(C)) # E: ndarray reveal_type(np.linspace(0, 10)) # E: numpy.ndarray -reveal_type(np.linspace(0, 10, retstep=True)) # E: Tuple[numpy.ndarray, numpy.inexact] +reveal_type(np.linspace(0, 10, retstep=True)) # E: Tuple[numpy.ndarray, numpy.inexact[Any]] reveal_type(np.logspace(0, 10)) # E: numpy.ndarray reveal_type(np.geomspace(1, 10)) # E: numpy.ndarray diff --git a/numpy/typing/tests/data/reveal/fromnumeric.py b/numpy/typing/tests/data/reveal/fromnumeric.py index 06501f6e2f87..75865c285f36 100644 --- a/numpy/typing/tests/data/reveal/fromnumeric.py +++ b/numpy/typing/tests/data/reveal/fromnumeric.py @@ -13,7 +13,7 @@ d = np.array(1.0, dtype=np.float32) # writeable reveal_type(np.take(a, 0)) # E: numpy.bool_ -reveal_type(np.take(b, 0)) # E: numpy.float32 +reveal_type(np.take(b, 0)) # E: numpy.floating[numpy.typing._32Bit] reveal_type( np.take(c, 0) # E: Union[numpy.generic, datetime.datetime, datetime.timedelta] ) @@ -66,8 +66,8 @@ reveal_type(np.partition(A, 0)) # E: numpy.ndarray reveal_type(np.partition(B, 0)) # E: numpy.ndarray -reveal_type(np.argpartition(a, 0)) # E: numpy.integer -reveal_type(np.argpartition(b, 0)) # E: numpy.integer +reveal_type(np.argpartition(a, 0)) # E: numpy.integer[Any] +reveal_type(np.argpartition(b, 0)) # E: numpy.integer[Any] reveal_type(np.argpartition(c, 0)) # E: numpy.ndarray reveal_type(np.argpartition(A, 0)) # E: numpy.ndarray reveal_type(np.argpartition(B, 0)) # E: numpy.ndarray @@ -78,18 +78,18 @@ reveal_type(np.argsort(A, 0)) # E: numpy.ndarray reveal_type(np.argsort(B, 0)) # E: numpy.ndarray -reveal_type(np.argmax(A)) # E: numpy.integer -reveal_type(np.argmax(B)) # E: numpy.integer -reveal_type(np.argmax(A, axis=0)) # E: Union[numpy.integer, numpy.ndarray] -reveal_type(np.argmax(B, axis=0)) # E: Union[numpy.integer, numpy.ndarray] +reveal_type(np.argmax(A)) # E: numpy.integer[Any] +reveal_type(np.argmax(B)) # E: numpy.integer[Any] +reveal_type(np.argmax(A, axis=0)) # E: Union[numpy.integer[Any], numpy.ndarray] +reveal_type(np.argmax(B, axis=0)) # E: Union[numpy.integer[Any], numpy.ndarray] -reveal_type(np.argmin(A)) # E: numpy.integer -reveal_type(np.argmin(B)) # E: numpy.integer -reveal_type(np.argmin(A, axis=0)) # E: Union[numpy.integer, numpy.ndarray] -reveal_type(np.argmin(B, axis=0)) # E: Union[numpy.integer, numpy.ndarray] +reveal_type(np.argmin(A)) # E: numpy.integer[Any] +reveal_type(np.argmin(B)) # E: numpy.integer[Any] +reveal_type(np.argmin(A, axis=0)) # E: Union[numpy.integer[Any], numpy.ndarray] +reveal_type(np.argmin(B, axis=0)) # E: Union[numpy.integer[Any], numpy.ndarray] -reveal_type(np.searchsorted(A[0], 0)) # E: numpy.integer -reveal_type(np.searchsorted(B[0], 0)) # E: numpy.integer +reveal_type(np.searchsorted(A[0], 0)) # E: numpy.integer[Any] +reveal_type(np.searchsorted(B[0], 0)) # E: numpy.integer[Any] reveal_type(np.searchsorted(A[0], [0])) # E: numpy.ndarray reveal_type(np.searchsorted(B[0], [0])) # E: numpy.ndarray @@ -100,7 +100,7 @@ reveal_type(np.resize(B, (5, 5))) # E: numpy.ndarray reveal_type(np.squeeze(a)) # E: numpy.bool_ -reveal_type(np.squeeze(b)) # E: numpy.float32 +reveal_type(np.squeeze(b)) # E: numpy.floating[numpy.typing._32Bit] reveal_type(np.squeeze(c)) # E: numpy.ndarray reveal_type(np.squeeze(A)) # E: numpy.ndarray reveal_type(np.squeeze(B)) # E: numpy.ndarray @@ -108,8 +108,8 @@ reveal_type(np.diagonal(A)) # E: numpy.ndarray reveal_type(np.diagonal(B)) # E: numpy.ndarray -reveal_type(np.trace(A)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.trace(B)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.trace(A)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.trace(B)) # E: Union[numpy.number[Any], numpy.ndarray] reveal_type(np.ravel(a)) # E: numpy.ndarray reveal_type(np.ravel(b)) # E: numpy.ndarray @@ -135,19 +135,19 @@ reveal_type(np.compress([True], A)) # E: numpy.ndarray reveal_type(np.compress([True], B)) # E: numpy.ndarray -reveal_type(np.clip(a, 0, 1.0)) # E: numpy.number -reveal_type(np.clip(b, -1, 1)) # E: numpy.float32 -reveal_type(np.clip(c, 0, 1)) # E: numpy.number -reveal_type(np.clip(A, 0, 1)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.clip(B, 0, 1)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.clip(a, 0, 1.0)) # E: numpy.number[Any] +reveal_type(np.clip(b, -1, 1)) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(np.clip(c, 0, 1)) # E: numpy.number[Any] +reveal_type(np.clip(A, 0, 1)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.clip(B, 0, 1)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.sum(a)) # E: numpy.number -reveal_type(np.sum(b)) # E: numpy.float32 -reveal_type(np.sum(c)) # E: numpy.number -reveal_type(np.sum(A)) # E: numpy.number -reveal_type(np.sum(B)) # E: numpy.number -reveal_type(np.sum(A, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.sum(B, axis=0)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.sum(a)) # E: numpy.number[Any] +reveal_type(np.sum(b)) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(np.sum(c)) # E: numpy.number[Any] +reveal_type(np.sum(A)) # E: numpy.number[Any] +reveal_type(np.sum(B)) # E: numpy.number[Any] +reveal_type(np.sum(A, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.sum(B, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] reveal_type(np.all(a)) # E: numpy.bool_ reveal_type(np.all(b)) # E: numpy.bool_ @@ -175,45 +175,45 @@ reveal_type(np.cumsum(A)) # E: numpy.ndarray reveal_type(np.cumsum(B)) # E: numpy.ndarray -reveal_type(np.ptp(a)) # E: numpy.number -reveal_type(np.ptp(b)) # E: numpy.float32 -reveal_type(np.ptp(c)) # E: numpy.number -reveal_type(np.ptp(A)) # E: numpy.number -reveal_type(np.ptp(B)) # E: numpy.number -reveal_type(np.ptp(A, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.ptp(B, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.ptp(A, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.ptp(B, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] - -reveal_type(np.amax(a)) # E: numpy.number -reveal_type(np.amax(b)) # E: numpy.float32 -reveal_type(np.amax(c)) # E: numpy.number -reveal_type(np.amax(A)) # E: numpy.number -reveal_type(np.amax(B)) # E: numpy.number -reveal_type(np.amax(A, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.amax(B, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.amax(A, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.amax(B, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] - -reveal_type(np.amin(a)) # E: numpy.number -reveal_type(np.amin(b)) # E: numpy.float32 -reveal_type(np.amin(c)) # E: numpy.number -reveal_type(np.amin(A)) # E: numpy.number -reveal_type(np.amin(B)) # E: numpy.number -reveal_type(np.amin(A, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.amin(B, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.amin(A, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.amin(B, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] - -reveal_type(np.prod(a)) # E: numpy.number -reveal_type(np.prod(b)) # E: numpy.float32 -reveal_type(np.prod(c)) # E: numpy.number -reveal_type(np.prod(A)) # E: numpy.number -reveal_type(np.prod(B)) # E: numpy.number -reveal_type(np.prod(A, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.prod(B, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.prod(A, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.prod(B, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.ptp(a)) # E: numpy.number[Any] +reveal_type(np.ptp(b)) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(np.ptp(c)) # E: numpy.number[Any] +reveal_type(np.ptp(A)) # E: numpy.number[Any] +reveal_type(np.ptp(B)) # E: numpy.number[Any] +reveal_type(np.ptp(A, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.ptp(B, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.ptp(A, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.ptp(B, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] + +reveal_type(np.amax(a)) # E: numpy.number[Any] +reveal_type(np.amax(b)) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(np.amax(c)) # E: numpy.number[Any] +reveal_type(np.amax(A)) # E: numpy.number[Any] +reveal_type(np.amax(B)) # E: numpy.number[Any] +reveal_type(np.amax(A, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.amax(B, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.amax(A, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.amax(B, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] + +reveal_type(np.amin(a)) # E: numpy.number[Any] +reveal_type(np.amin(b)) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(np.amin(c)) # E: numpy.number[Any] +reveal_type(np.amin(A)) # E: numpy.number[Any] +reveal_type(np.amin(B)) # E: numpy.number[Any] +reveal_type(np.amin(A, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.amin(B, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.amin(A, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.amin(B, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] + +reveal_type(np.prod(a)) # E: numpy.number[Any] +reveal_type(np.prod(b)) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(np.prod(c)) # E: numpy.number[Any] +reveal_type(np.prod(A)) # E: numpy.number[Any] +reveal_type(np.prod(B)) # E: numpy.number[Any] +reveal_type(np.prod(A, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.prod(B, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.prod(A, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.prod(B, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] reveal_type(np.prod(b, out=d)) # E: numpy.ndarray reveal_type(np.prod(B, out=d)) # E: numpy.ndarray @@ -235,44 +235,44 @@ reveal_type(np.size(A)) # E: int reveal_type(np.size(B)) # E: int -reveal_type(np.around(a)) # E: numpy.number -reveal_type(np.around(b)) # E: numpy.float32 -reveal_type(np.around(c)) # E: numpy.number +reveal_type(np.around(a)) # E: numpy.number[Any] +reveal_type(np.around(b)) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(np.around(c)) # E: numpy.number[Any] reveal_type(np.around(A)) # E: numpy.ndarray reveal_type(np.around(B)) # E: numpy.ndarray -reveal_type(np.mean(a)) # E: numpy.number -reveal_type(np.mean(b)) # E: numpy.number -reveal_type(np.mean(c)) # E: numpy.number -reveal_type(np.mean(A)) # E: numpy.number -reveal_type(np.mean(B)) # E: numpy.number -reveal_type(np.mean(A, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.mean(B, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.mean(A, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.mean(B, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.mean(a)) # E: numpy.number[Any] +reveal_type(np.mean(b)) # E: numpy.number[Any] +reveal_type(np.mean(c)) # E: numpy.number[Any] +reveal_type(np.mean(A)) # E: numpy.number[Any] +reveal_type(np.mean(B)) # E: numpy.number[Any] +reveal_type(np.mean(A, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.mean(B, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.mean(A, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.mean(B, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] reveal_type(np.mean(b, out=d)) # E: numpy.ndarray reveal_type(np.mean(B, out=d)) # E: numpy.ndarray -reveal_type(np.std(a)) # E: numpy.number -reveal_type(np.std(b)) # E: numpy.number -reveal_type(np.std(c)) # E: numpy.number -reveal_type(np.std(A)) # E: numpy.number -reveal_type(np.std(B)) # E: numpy.number -reveal_type(np.std(A, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.std(B, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.std(A, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.std(B, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.std(a)) # E: numpy.number[Any] +reveal_type(np.std(b)) # E: numpy.number[Any] +reveal_type(np.std(c)) # E: numpy.number[Any] +reveal_type(np.std(A)) # E: numpy.number[Any] +reveal_type(np.std(B)) # E: numpy.number[Any] +reveal_type(np.std(A, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.std(B, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.std(A, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.std(B, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] reveal_type(np.std(b, out=d)) # E: numpy.ndarray reveal_type(np.std(B, out=d)) # E: numpy.ndarray -reveal_type(np.var(a)) # E: numpy.number -reveal_type(np.var(b)) # E: numpy.number -reveal_type(np.var(c)) # E: numpy.number -reveal_type(np.var(A)) # E: numpy.number -reveal_type(np.var(B)) # E: numpy.number -reveal_type(np.var(A, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.var(B, axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.var(A, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(np.var(B, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.var(a)) # E: numpy.number[Any] +reveal_type(np.var(b)) # E: numpy.number[Any] +reveal_type(np.var(c)) # E: numpy.number[Any] +reveal_type(np.var(A)) # E: numpy.number[Any] +reveal_type(np.var(B)) # E: numpy.number[Any] +reveal_type(np.var(A, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.var(B, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.var(A, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.var(B, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] reveal_type(np.var(b, out=d)) # E: numpy.ndarray reveal_type(np.var(B, out=d)) # E: numpy.ndarray diff --git a/numpy/typing/tests/data/reveal/ndarray_misc.py b/numpy/typing/tests/data/reveal/ndarray_misc.py index 22a4564df1b7..826c8aaa6d12 100644 --- a/numpy/typing/tests/data/reveal/ndarray_misc.py +++ b/numpy/typing/tests/data/reveal/ndarray_misc.py @@ -26,14 +26,14 @@ class SubClass(np.ndarray): ... reveal_type(A.any(keepdims=True)) # E: Union[numpy.bool_, numpy.ndarray] reveal_type(A.any(out=B)) # E: SubClass -reveal_type(f8.argmax()) # E: numpy.signedinteger -reveal_type(A.argmax()) # E: numpy.signedinteger -reveal_type(A.argmax(axis=0)) # E: Union[numpy.signedinteger, numpy.ndarray] +reveal_type(f8.argmax()) # E: numpy.signedinteger[Any] +reveal_type(A.argmax()) # E: numpy.signedinteger[Any] +reveal_type(A.argmax(axis=0)) # E: Union[numpy.signedinteger[Any], numpy.ndarray] reveal_type(A.argmax(out=B)) # E: SubClass -reveal_type(f8.argmin()) # E: numpy.signedinteger -reveal_type(A.argmin()) # E: numpy.signedinteger -reveal_type(A.argmin(axis=0)) # E: Union[numpy.signedinteger, numpy.ndarray] +reveal_type(f8.argmin()) # E: numpy.signedinteger[Any] +reveal_type(A.argmin()) # E: numpy.signedinteger[Any] +reveal_type(A.argmin(axis=0)) # E: Union[numpy.signedinteger[Any], numpy.ndarray] reveal_type(A.argmin(out=B)) # E: SubClass reveal_type(f8.argsort()) # E: numpy.ndarray @@ -43,9 +43,9 @@ class SubClass(np.ndarray): ... reveal_type(A.choose([0])) # E: numpy.ndarray reveal_type(A.choose([0], out=B)) # E: SubClass -reveal_type(f8.clip(1)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(A.clip(1)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(A.clip(None, 1)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(f8.clip(1)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.clip(1)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.clip(None, 1)) # E: Union[numpy.number[Any], numpy.ndarray] reveal_type(A.clip(1, out=B)) # E: SubClass reveal_type(A.clip(None, 1, out=B)) # E: SubClass @@ -53,11 +53,11 @@ class SubClass(np.ndarray): ... reveal_type(A.compress([0])) # E: numpy.ndarray reveal_type(A.compress([0], out=B)) # E: SubClass -reveal_type(f8.conj()) # E: numpy.float64 +reveal_type(f8.conj()) # E: numpy.floating[numpy.typing._64Bit] reveal_type(A.conj()) # E: numpy.ndarray reveal_type(B.conj()) # E: SubClass -reveal_type(f8.conjugate()) # E: numpy.float64 +reveal_type(f8.conjugate()) # E: numpy.floating[numpy.typing._64Bit] reveal_type(A.conjugate()) # E: numpy.ndarray reveal_type(B.conjugate()) # E: SubClass @@ -69,41 +69,41 @@ class SubClass(np.ndarray): ... reveal_type(A.cumsum()) # E: numpy.ndarray reveal_type(A.cumsum(out=B)) # E: SubClass -reveal_type(f8.max()) # E: numpy.number -reveal_type(A.max()) # E: numpy.number -reveal_type(A.max(axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(A.max(keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(f8.max()) # E: numpy.number[Any] +reveal_type(A.max()) # E: numpy.number[Any] +reveal_type(A.max(axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.max(keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] reveal_type(A.max(out=B)) # E: SubClass -reveal_type(f8.mean()) # E: numpy.number -reveal_type(A.mean()) # E: numpy.number -reveal_type(A.mean(axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(A.mean(keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(f8.mean()) # E: numpy.number[Any] +reveal_type(A.mean()) # E: numpy.number[Any] +reveal_type(A.mean(axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.mean(keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] reveal_type(A.mean(out=B)) # E: SubClass -reveal_type(f8.min()) # E: numpy.number -reveal_type(A.min()) # E: numpy.number -reveal_type(A.min(axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(A.min(keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(f8.min()) # E: numpy.number[Any] +reveal_type(A.min()) # E: numpy.number[Any] +reveal_type(A.min(axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.min(keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] reveal_type(A.min(out=B)) # E: SubClass -reveal_type(f8.newbyteorder()) # E: numpy.float64 +reveal_type(f8.newbyteorder()) # E: numpy.floating[numpy.typing._64Bit] reveal_type(A.newbyteorder()) # E: numpy.ndarray reveal_type(B.newbyteorder('|')) # E: SubClass -reveal_type(f8.prod()) # E: numpy.number -reveal_type(A.prod()) # E: numpy.number -reveal_type(A.prod(axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(A.prod(keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(f8.prod()) # E: numpy.number[Any] +reveal_type(A.prod()) # E: numpy.number[Any] +reveal_type(A.prod(axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.prod(keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] reveal_type(A.prod(out=B)) # E: SubClass -reveal_type(f8.ptp()) # E: numpy.number -reveal_type(A.ptp()) # E: numpy.number -reveal_type(A.ptp(axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(A.ptp(keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(f8.ptp()) # E: numpy.number[Any] +reveal_type(A.ptp()) # E: numpy.number[Any] +reveal_type(A.ptp(axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.ptp(keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] reveal_type(A.ptp(out=B)) # E: SubClass -reveal_type(f8.round()) # E: numpy.float64 +reveal_type(f8.round()) # E: numpy.floating[numpy.typing._64Bit] reveal_type(A.round()) # E: numpy.ndarray reveal_type(A.round(out=B)) # E: SubClass @@ -111,16 +111,16 @@ class SubClass(np.ndarray): ... reveal_type(A.repeat(1)) # E: numpy.ndarray reveal_type(B.repeat(1)) # E: numpy.ndarray -reveal_type(f8.std()) # E: numpy.number -reveal_type(A.std()) # E: numpy.number -reveal_type(A.std(axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(A.std(keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(f8.std()) # E: numpy.number[Any] +reveal_type(A.std()) # E: numpy.number[Any] +reveal_type(A.std(axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.std(keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] reveal_type(A.std(out=B)) # E: SubClass -reveal_type(f8.sum()) # E: numpy.number -reveal_type(A.sum()) # E: numpy.number -reveal_type(A.sum(axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(A.sum(keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(f8.sum()) # E: numpy.number[Any] +reveal_type(A.sum()) # E: numpy.number[Any] +reveal_type(A.sum(axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.sum(keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] reveal_type(A.sum(out=B)) # E: SubClass reveal_type(f8.take(0)) # E: numpy.generic @@ -129,22 +129,22 @@ class SubClass(np.ndarray): ... reveal_type(A.take(0, out=B)) # E: SubClass reveal_type(A.take([0], out=B)) # E: SubClass -reveal_type(f8.var()) # E: numpy.number -reveal_type(A.var()) # E: numpy.number -reveal_type(A.var(axis=0)) # E: Union[numpy.number, numpy.ndarray] -reveal_type(A.var(keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(f8.var()) # E: numpy.number[Any] +reveal_type(A.var()) # E: numpy.number[Any] +reveal_type(A.var(axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(A.var(keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] reveal_type(A.var(out=B)) # E: SubClass reveal_type(A.argpartition([0])) # E: numpy.ndarray reveal_type(A.diagonal()) # E: numpy.ndarray -reveal_type(A.dot(1)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(A.dot(1)) # E: Union[numpy.number[Any], numpy.ndarray] reveal_type(A.dot(1, out=B)) # E: SubClass reveal_type(A.nonzero()) # E: tuple[numpy.ndarray] reveal_type(A.searchsorted([1])) # E: numpy.ndarray -reveal_type(A.trace()) # E: Union[numpy.number, numpy.ndarray] +reveal_type(A.trace()) # E: Union[numpy.number[Any], numpy.ndarray] reveal_type(A.trace(out=B)) # E: SubClass diff --git a/numpy/typing/tests/data/reveal/scalars.py b/numpy/typing/tests/data/reveal/scalars.py index 0168ebed8c6a..72a3e4d2e628 100644 --- a/numpy/typing/tests/data/reveal/scalars.py +++ b/numpy/typing/tests/data/reveal/scalars.py @@ -2,18 +2,18 @@ x = np.complex64(3 + 2j) -reveal_type(x.real) # E: numpy.float32 -reveal_type(x.imag) # E: numpy.float32 +reveal_type(x.real) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(x.imag) # E: numpy.floating[numpy.typing._32Bit] -reveal_type(x.real.real) # E: numpy.float32 -reveal_type(x.real.imag) # E: numpy.float32 +reveal_type(x.real.real) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(x.real.imag) # E: numpy.floating[numpy.typing._32Bit] reveal_type(x.itemsize) # E: int reveal_type(x.shape) # E: tuple[builtins.int] reveal_type(x.strides) # E: tuple[builtins.int] -reveal_type(np.complex64().real) # E: numpy.float32 -reveal_type(np.complex128().imag) # E: numpy.float64 +reveal_type(np.complex64().real) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(np.complex128().imag) # E: numpy.floating[numpy.typing._64Bit] reveal_type(np.unicode_('foo')) # E: numpy.str_ reveal_type(np.str0('foo')) # E: numpy.str_ From 95a391c53cd227fe48c33e03ff1d95f342ae31f5 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Fri, 16 Oct 2020 16:37:42 +0200 Subject: [PATCH 289/409] TST: Add support for precision to the arithmetic and bitwise ops tests --- numpy/typing/tests/data/reveal/arithmetic.py | 332 +++++++++--------- numpy/typing/tests/data/reveal/bitwise_ops.py | 206 +++++------ 2 files changed, 269 insertions(+), 269 deletions(-) diff --git a/numpy/typing/tests/data/reveal/arithmetic.py b/numpy/typing/tests/data/reveal/arithmetic.py index b8c457aafb53..a1290304a015 100644 --- a/numpy/typing/tests/data/reveal/arithmetic.py +++ b/numpy/typing/tests/data/reveal/arithmetic.py @@ -45,212 +45,212 @@ reveal_type(td / f) # E: numpy.timedelta64 reveal_type(td / f4) # E: numpy.timedelta64 reveal_type(td / f8) # E: numpy.timedelta64 -reveal_type(td / td) # E: float64 -reveal_type(td // td) # E: signedinteger +reveal_type(td / td) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(td // td) # E: numpy.signedinteger[numpy.typing._64Bit] reveal_type(td % td) # E: numpy.timedelta64 # boolean -reveal_type(b_ / b) # E: float64 -reveal_type(b_ / b_) # E: float64 -reveal_type(b_ / i) # E: float64 -reveal_type(b_ / i8) # E: float64 -reveal_type(b_ / i4) # E: float64 -reveal_type(b_ / u8) # E: float64 -reveal_type(b_ / u4) # E: float64 -reveal_type(b_ / f) # E: float64 -reveal_type(b_ / f8) # E: float64 -reveal_type(b_ / f4) # E: float32 -reveal_type(b_ / c) # E: complex128 -reveal_type(b_ / c16) # E: complex128 -reveal_type(b_ / c8) # E: complex64 +reveal_type(b_ / b) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ / b_) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ / i) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ / i8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ / i4) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ / u8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ / u4) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ / f) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ / f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ / f4) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(b_ / c) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(b_ / c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(b_ / c8) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] -reveal_type(b / b_) # E: float64 -reveal_type(b_ / b_) # E: float64 -reveal_type(i / b_) # E: float64 -reveal_type(i8 / b_) # E: float64 -reveal_type(i4 / b_) # E: float64 -reveal_type(u8 / b_) # E: float64 -reveal_type(u4 / b_) # E: float64 -reveal_type(f / b_) # E: float64 -reveal_type(f8 / b_) # E: float64 -reveal_type(f4 / b_) # E: float32 -reveal_type(c / b_) # E: complex128 -reveal_type(c16 / b_) # E: complex128 -reveal_type(c8 / b_) # E: complex64 +reveal_type(b / b_) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ / b_) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i / b_) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i8 / b_) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i4 / b_) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(u8 / b_) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(u4 / b_) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f / b_) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f8 / b_) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f4 / b_) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(c / b_) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c16 / b_) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c8 / b_) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] # Complex -reveal_type(c16 + c16) # E: complexfloating -reveal_type(c16 + f8) # E: complexfloating -reveal_type(c16 + i8) # E: complexfloating -reveal_type(c16 + c8) # E: complexfloating -reveal_type(c16 + f4) # E: complexfloating -reveal_type(c16 + i4) # E: complexfloating -reveal_type(c16 + b_) # E: complex128 -reveal_type(c16 + b) # E: complexfloating -reveal_type(c16 + c) # E: complexfloating -reveal_type(c16 + f) # E: complexfloating -reveal_type(c16 + i) # E: complexfloating +reveal_type(c16 + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c16 + f8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c16 + i8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c16 + c8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c16 + f4) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c16 + i4) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c16 + b_) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c16 + b) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c16 + c) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c16 + f) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c16 + i) # E: numpy.complexfloating[Any, Any] reveal_type(c16 + AR) # E: Union[numpy.ndarray, numpy.generic] -reveal_type(c16 + c16) # E: complexfloating -reveal_type(f8 + c16) # E: complexfloating -reveal_type(i8 + c16) # E: complexfloating -reveal_type(c8 + c16) # E: complexfloating -reveal_type(f4 + c16) # E: complexfloating -reveal_type(i4 + c16) # E: complexfloating -reveal_type(b_ + c16) # E: complex128 -reveal_type(b + c16) # E: complexfloating -reveal_type(c + c16) # E: complexfloating -reveal_type(f + c16) # E: complexfloating -reveal_type(i + c16) # E: complexfloating +reveal_type(c16 + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(f8 + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(i8 + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c8 + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(f4 + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(i4 + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(b_ + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(b + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(f + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(i + c16) # E: numpy.complexfloating[Any, Any] reveal_type(AR + c16) # E: Union[numpy.ndarray, numpy.generic] -reveal_type(c8 + c16) # E: complexfloating -reveal_type(c8 + f8) # E: complexfloating -reveal_type(c8 + i8) # E: complexfloating -reveal_type(c8 + c8) # E: complexfloating -reveal_type(c8 + f4) # E: complexfloating -reveal_type(c8 + i4) # E: complexfloating -reveal_type(c8 + b_) # E: complex64 -reveal_type(c8 + b) # E: complexfloating -reveal_type(c8 + c) # E: complexfloating -reveal_type(c8 + f) # E: complexfloating -reveal_type(c8 + i) # E: complexfloating +reveal_type(c8 + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c8 + f8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c8 + i8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c8 + c8) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] +reveal_type(c8 + f4) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] +reveal_type(c8 + i4) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] +reveal_type(c8 + b_) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] +reveal_type(c8 + b) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] +reveal_type(c8 + c) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c8 + f) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c8 + i) # E: numpy.complexfloating[Any, Any] reveal_type(c8 + AR) # E: Union[numpy.ndarray, numpy.generic] -reveal_type(c16 + c8) # E: complexfloating -reveal_type(f8 + c8) # E: complexfloating -reveal_type(i8 + c8) # E: complexfloating -reveal_type(c8 + c8) # E: complexfloating -reveal_type(f4 + c8) # E: complexfloating -reveal_type(i4 + c8) # E: complexfloating -reveal_type(b_ + c8) # E: complex64 -reveal_type(b + c8) # E: complexfloating -reveal_type(c + c8) # E: complexfloating -reveal_type(f + c8) # E: complexfloating -reveal_type(i + c8) # E: complexfloating +reveal_type(c16 + c8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(f8 + c8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(i8 + c8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(c8 + c8) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] +reveal_type(f4 + c8) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] +reveal_type(i4 + c8) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] +reveal_type(b_ + c8) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] +reveal_type(b + c8) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] +reveal_type(c + c8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(f + c8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(i + c8) # E: numpy.complexfloating[Any, Any] reveal_type(AR + c8) # E: Union[numpy.ndarray, numpy.generic] # Float -reveal_type(f8 + f8) # E: floating -reveal_type(f8 + i8) # E: floating -reveal_type(f8 + f4) # E: floating -reveal_type(f8 + i4) # E: floating -reveal_type(f8 + b_) # E: float64 -reveal_type(f8 + b) # E: floating -reveal_type(f8 + c) # E: complexfloating -reveal_type(f8 + f) # E: floating -reveal_type(f8 + i) # E: floating +reveal_type(f8 + f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f8 + i8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f8 + f4) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f8 + i4) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f8 + b_) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f8 + b) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f8 + c) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(f8 + f) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f8 + i) # E: numpy.floating[Any] reveal_type(f8 + AR) # E: Union[numpy.ndarray, numpy.generic] -reveal_type(f8 + f8) # E: floating -reveal_type(i8 + f8) # E: floating -reveal_type(f4 + f8) # E: floating -reveal_type(i4 + f8) # E: floating -reveal_type(b_ + f8) # E: float64 -reveal_type(b + f8) # E: floating -reveal_type(c + f8) # E: complexfloating -reveal_type(f + f8) # E: floating -reveal_type(i + f8) # E: floating +reveal_type(f8 + f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i8 + f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f4 + f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i4 + f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ + f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b + f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(c + f8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(f + f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i + f8) # E: numpy.floating[Any] reveal_type(AR + f8) # E: Union[numpy.ndarray, numpy.generic] -reveal_type(f4 + f8) # E: floating -reveal_type(f4 + i8) # E: floating -reveal_type(f4 + f4) # E: floating -reveal_type(f4 + i4) # E: floating -reveal_type(f4 + b_) # E: float32 -reveal_type(f4 + b) # E: floating -reveal_type(f4 + c) # E: complexfloating -reveal_type(f4 + f) # E: floating -reveal_type(f4 + i) # E: floating +reveal_type(f4 + f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f4 + i8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f4 + f4) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(f4 + i4) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(f4 + b_) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(f4 + b) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(f4 + c) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(f4 + f) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f4 + i) # E: numpy.floating[Any] reveal_type(f4 + AR) # E: Union[numpy.ndarray, numpy.generic] -reveal_type(f8 + f4) # E: floating -reveal_type(i8 + f4) # E: floating -reveal_type(f4 + f4) # E: floating -reveal_type(i4 + f4) # E: floating -reveal_type(b_ + f4) # E: float32 -reveal_type(b + f4) # E: floating -reveal_type(c + f4) # E: complexfloating -reveal_type(f + f4) # E: floating -reveal_type(i + f4) # E: floating +reveal_type(f8 + f4) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i8 + f4) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f4 + f4) # E: umpy.floating[numpy.typing._32Bit] +reveal_type(i4 + f4) # E: umpy.floating[numpy.typing._32Bit] +reveal_type(b_ + f4) # E: umpy.floating[numpy.typing._32Bit] +reveal_type(b + f4) # E: umpy.floating[numpy.typing._32Bit] +reveal_type(c + f4) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(f + f4) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i + f4) # E: numpy.floating[Any] reveal_type(AR + f4) # E: Union[numpy.ndarray, numpy.generic] # Int -reveal_type(i8 + i8) # E: signedinteger -reveal_type(i8 + u8) # E: Union[numpy.signedinteger, numpy.float64] -reveal_type(i8 + i4) # E: signedinteger -reveal_type(i8 + u4) # E: Union[numpy.signedinteger, numpy.float64] -reveal_type(i8 + b_) # E: int64 -reveal_type(i8 + b) # E: signedinteger -reveal_type(i8 + c) # E: complexfloating -reveal_type(i8 + f) # E: floating -reveal_type(i8 + i) # E: signedinteger +reveal_type(i8 + i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 + u8) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] +reveal_type(i8 + i4) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 + u4) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] +reveal_type(i8 + b_) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 + b) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 + c) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(i8 + f) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i8 + i) # E: numpy.signedinteger[Any] reveal_type(i8 + AR) # E: Union[numpy.ndarray, numpy.generic] -reveal_type(u8 + u8) # E: unsignedinteger -reveal_type(u8 + i4) # E: Union[numpy.signedinteger, numpy.float64] -reveal_type(u8 + u4) # E: unsignedinteger -reveal_type(u8 + b_) # E: uint64 -reveal_type(u8 + b) # E: unsignedinteger -reveal_type(u8 + c) # E: complexfloating -reveal_type(u8 + f) # E: floating -reveal_type(u8 + i) # E: Union[numpy.signedinteger, numpy.float64] +reveal_type(u8 + u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 + i4) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] +reveal_type(u8 + u4) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 + b_) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 + b) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 + c) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(u8 + f) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(u8 + i) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] reveal_type(u8 + AR) # E: Union[numpy.ndarray, numpy.generic] -reveal_type(i8 + i8) # E: signedinteger -reveal_type(u8 + i8) # E: Union[numpy.signedinteger, numpy.float64] -reveal_type(i4 + i8) # E: signedinteger -reveal_type(u4 + i8) # E: Union[numpy.signedinteger, numpy.float64] -reveal_type(b_ + i8) # E: int64 -reveal_type(b + i8) # E: signedinteger -reveal_type(c + i8) # E: complexfloating -reveal_type(f + i8) # E: floating -reveal_type(i + i8) # E: signedinteger +reveal_type(i8 + i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(u8 + i8) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] +reveal_type(i4 + i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(u4 + i8) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] +reveal_type(b_ + i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(b + i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(c + i8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(f + i8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i + i8) # E: numpy.signedinteger[Any] reveal_type(AR + i8) # E: Union[numpy.ndarray, numpy.generic] -reveal_type(u8 + u8) # E: unsignedinteger -reveal_type(i4 + u8) # E: Union[numpy.signedinteger, numpy.float64] -reveal_type(u4 + u8) # E: unsignedinteger -reveal_type(b_ + u8) # E: uint64 -reveal_type(b + u8) # E: unsignedinteger -reveal_type(c + u8) # E: complexfloating -reveal_type(f + u8) # E: floating -reveal_type(i + u8) # E: Union[numpy.signedinteger, numpy.float64] +reveal_type(u8 + u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(i4 + u8) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] +reveal_type(u4 + u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(b_ + u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(b + u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(c + u8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(f + u8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i + u8) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] reveal_type(AR + u8) # E: Union[numpy.ndarray, numpy.generic] -reveal_type(i4 + i8) # E: signedinteger -reveal_type(i4 + i4) # E: signedinteger -reveal_type(i4 + i) # E: signedinteger -reveal_type(i4 + b_) # E: int32 -reveal_type(i4 + b) # E: signedinteger +reveal_type(i4 + i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i4 + i4) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(i4 + i) # E: numpy.signedinteger[Any] +reveal_type(i4 + b_) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(i4 + b) # E: numpy.signedinteger[numpy.typing._32Bit] reveal_type(i4 + AR) # E: Union[numpy.ndarray, numpy.generic] -reveal_type(u4 + i8) # E: Union[numpy.signedinteger, numpy.float64] -reveal_type(u4 + i4) # E: Union[numpy.signedinteger, numpy.float64] -reveal_type(u4 + u8) # E: unsignedinteger -reveal_type(u4 + u4) # E: unsignedinteger -reveal_type(u4 + i) # E: Union[numpy.signedinteger, numpy.float64] -reveal_type(u4 + b_) # E: uint32 -reveal_type(u4 + b) # E: unsignedinteger +reveal_type(u4 + i8) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] +reveal_type(u4 + i4) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] +reveal_type(u4 + u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u4 + u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(u4 + i) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] +reveal_type(u4 + b_) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(u4 + b) # E: numpy.unsignedinteger[numpy.typing._32Bit] reveal_type(u4 + AR) # E: Union[numpy.ndarray, numpy.generic] -reveal_type(i8 + i4) # E: signedinteger -reveal_type(i4 + i4) # E: signedinteger -reveal_type(i + i4) # E: signedinteger -reveal_type(b_ + i4) # E: int32 -reveal_type(b + i4) # E: signedinteger +reveal_type(i8 + i4) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i4 + i4) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(i + i4) # E: numpy.signedinteger[Any] +reveal_type(b_ + i4) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(b + i4) # E: numpy.signedinteger[numpy.typing._32Bit] reveal_type(AR + i4) # E: Union[numpy.ndarray, numpy.generic] -reveal_type(i8 + u4) # E: Union[numpy.signedinteger, numpy.float64] -reveal_type(i4 + u4) # E: Union[numpy.signedinteger, numpy.float64] -reveal_type(u8 + u4) # E: unsignedinteger -reveal_type(u4 + u4) # E: unsignedinteger -reveal_type(b_ + u4) # E: uint32 -reveal_type(b + u4) # E: unsignedinteger -reveal_type(i + u4) # E: Union[numpy.signedinteger, numpy.float64] +reveal_type(i8 + u4) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] +reveal_type(i4 + u4) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] +reveal_type(u8 + u4) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u4 + u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(b_ + u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(b + u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(i + u4) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] reveal_type(AR + u4) # E: Union[numpy.ndarray, numpy.generic] diff --git a/numpy/typing/tests/data/reveal/bitwise_ops.py b/numpy/typing/tests/data/reveal/bitwise_ops.py index ae899f47b690..6883532f2502 100644 --- a/numpy/typing/tests/data/reveal/bitwise_ops.py +++ b/numpy/typing/tests/data/reveal/bitwise_ops.py @@ -15,117 +15,117 @@ AR.setflags(write=False) -reveal_type(i8 << i8) # E: numpy.signedinteger -reveal_type(i8 >> i8) # E: numpy.signedinteger -reveal_type(i8 | i8) # E: numpy.signedinteger -reveal_type(i8 ^ i8) # E: numpy.signedinteger -reveal_type(i8 & i8) # E: numpy.signedinteger - -reveal_type(i8 << AR) # E: Union[numpy.ndarray, numpy.integer] -reveal_type(i8 >> AR) # E: Union[numpy.ndarray, numpy.integer] -reveal_type(i8 | AR) # E: Union[numpy.ndarray, numpy.integer, numpy.bool_] -reveal_type(i8 ^ AR) # E: Union[numpy.ndarray, numpy.integer, numpy.bool_] -reveal_type(i8 & AR) # E: Union[numpy.ndarray, numpy.integer, numpy.bool_] - -reveal_type(i4 << i4) # E: numpy.signedinteger -reveal_type(i4 >> i4) # E: numpy.signedinteger -reveal_type(i4 | i4) # E: numpy.signedinteger -reveal_type(i4 ^ i4) # E: numpy.signedinteger -reveal_type(i4 & i4) # E: numpy.signedinteger - -reveal_type(i8 << i4) # E: numpy.signedinteger -reveal_type(i8 >> i4) # E: numpy.signedinteger -reveal_type(i8 | i4) # E: numpy.signedinteger -reveal_type(i8 ^ i4) # E: numpy.signedinteger -reveal_type(i8 & i4) # E: numpy.signedinteger - -reveal_type(i8 << i) # E: numpy.signedinteger -reveal_type(i8 >> i) # E: numpy.signedinteger -reveal_type(i8 | i) # E: numpy.signedinteger -reveal_type(i8 ^ i) # E: numpy.signedinteger -reveal_type(i8 & i) # E: numpy.signedinteger - -reveal_type(i8 << b_) # E: numpy.int64 -reveal_type(i8 >> b_) # E: numpy.int64 -reveal_type(i8 | b_) # E: numpy.int64 -reveal_type(i8 ^ b_) # E: numpy.int64 -reveal_type(i8 & b_) # E: numpy.int64 - -reveal_type(i8 << b) # E: numpy.signedinteger -reveal_type(i8 >> b) # E: numpy.signedinteger -reveal_type(i8 | b) # E: numpy.signedinteger -reveal_type(i8 ^ b) # E: numpy.signedinteger -reveal_type(i8 & b) # E: numpy.signedinteger - -reveal_type(u8 << u8) # E: numpy.unsignedinteger -reveal_type(u8 >> u8) # E: numpy.unsignedinteger -reveal_type(u8 | u8) # E: numpy.unsignedinteger -reveal_type(u8 ^ u8) # E: numpy.unsignedinteger -reveal_type(u8 & u8) # E: numpy.unsignedinteger - -reveal_type(u8 << AR) # E: Union[numpy.ndarray, numpy.integer] -reveal_type(u8 >> AR) # E: Union[numpy.ndarray, numpy.integer] -reveal_type(u8 | AR) # E: Union[numpy.ndarray, numpy.integer, numpy.bool_] -reveal_type(u8 ^ AR) # E: Union[numpy.ndarray, numpy.integer, numpy.bool_] -reveal_type(u8 & AR) # E: Union[numpy.ndarray, numpy.integer, numpy.bool_] - -reveal_type(u4 << u4) # E: numpy.unsignedinteger -reveal_type(u4 >> u4) # E: numpy.unsignedinteger -reveal_type(u4 | u4) # E: numpy.unsignedinteger -reveal_type(u4 ^ u4) # E: numpy.unsignedinteger -reveal_type(u4 & u4) # E: numpy.unsignedinteger - -reveal_type(u4 << i4) # E: numpy.signedinteger -reveal_type(u4 >> i4) # E: numpy.signedinteger -reveal_type(u4 | i4) # E: numpy.signedinteger -reveal_type(u4 ^ i4) # E: numpy.signedinteger -reveal_type(u4 & i4) # E: numpy.signedinteger - -reveal_type(u4 << i) # E: numpy.signedinteger -reveal_type(u4 >> i) # E: numpy.signedinteger -reveal_type(u4 | i) # E: numpy.signedinteger -reveal_type(u4 ^ i) # E: numpy.signedinteger -reveal_type(u4 & i) # E: numpy.signedinteger - -reveal_type(u8 << b_) # E: numpy.uint64 -reveal_type(u8 >> b_) # E: numpy.uint64 -reveal_type(u8 | b_) # E: numpy.uint64 -reveal_type(u8 ^ b_) # E: numpy.uint64 -reveal_type(u8 & b_) # E: numpy.uint64 - -reveal_type(u8 << b) # E: numpy.unsignedinteger -reveal_type(u8 >> b) # E: numpy.unsignedinteger -reveal_type(u8 | b) # E: numpy.unsignedinteger -reveal_type(u8 ^ b) # E: numpy.unsignedinteger -reveal_type(u8 & b) # E: numpy.unsignedinteger - -reveal_type(b_ << b_) # E: numpy.int8 -reveal_type(b_ >> b_) # E: numpy.int8 +reveal_type(i8 << i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 >> i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 | i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 ^ i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 & i8) # E: numpy.signedinteger[numpy.typing._64Bit] + +reveal_type(i8 << AR) # E: Union[numpy.ndarray, numpy.integer[Any]] +reveal_type(i8 >> AR) # E: Union[numpy.ndarray, numpy.integer[Any]] +reveal_type(i8 | AR) # E: Union[numpy.ndarray, numpy.integer[Any], numpy.bool_] +reveal_type(i8 ^ AR) # E: Union[numpy.ndarray, numpy.integer[Any], numpy.bool_] +reveal_type(i8 & AR) # E: Union[numpy.ndarray, numpy.integer[Any], numpy.bool_] + +reveal_type(i4 << i4) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(i4 >> i4) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(i4 | i4) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(i4 ^ i4) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(i4 & i4) # E: numpy.signedinteger[numpy.typing._32Bit] + +reveal_type(i8 << i4) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 >> i4) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 | i4) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 ^ i4) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 & i4) # E: numpy.signedinteger[numpy.typing._64Bit] + +reveal_type(i8 << i) # E: numpy.signedinteger[Any] +reveal_type(i8 >> i) # E: numpy.signedinteger[Any] +reveal_type(i8 | i) # E: numpy.signedinteger[Any] +reveal_type(i8 ^ i) # E: numpy.signedinteger[Any] +reveal_type(i8 & i) # E: numpy.signedinteger[Any] + +reveal_type(i8 << b_) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 >> b_) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 | b_) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 ^ b_) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 & b_) # E: numpy.signedinteger[numpy.typing._64Bit] + +reveal_type(i8 << b) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 >> b) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 | b) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 ^ b) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 & b) # E: numpy.signedinteger[numpy.typing._64Bit] + +reveal_type(u8 << u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 >> u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 | u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 ^ u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 & u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] + +reveal_type(u8 << AR) # E: Union[numpy.ndarray, numpy.integer[Any]] +reveal_type(u8 >> AR) # E: Union[numpy.ndarray, numpy.integer[Any]] +reveal_type(u8 | AR) # E: Union[numpy.ndarray, numpy.integer[Any], numpy.bool_] +reveal_type(u8 ^ AR) # E: Union[numpy.ndarray, numpy.integer[Any], numpy.bool_] +reveal_type(u8 & AR) # E: Union[numpy.ndarray, numpy.integer[Any], numpy.bool_] + +reveal_type(u4 << u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(u4 >> u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(u4 | u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(u4 ^ u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(u4 & u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] + +reveal_type(u4 << i4) # E: numpy.signedinteger[Any] +reveal_type(u4 >> i4) # E: numpy.signedinteger[Any] +reveal_type(u4 | i4) # E: numpy.signedinteger[Any] +reveal_type(u4 ^ i4) # E: numpy.signedinteger[Any] +reveal_type(u4 & i4) # E: numpy.signedinteger[Any] + +reveal_type(u4 << i) # E: numpy.signedinteger[Any] +reveal_type(u4 >> i) # E: numpy.signedinteger[Any] +reveal_type(u4 | i) # E: numpy.signedinteger[Any] +reveal_type(u4 ^ i) # E: numpy.signedinteger[Any] +reveal_type(u4 & i) # E: numpy.signedinteger[Any] + +reveal_type(u8 << b_) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 >> b_) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 | b_) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 ^ b_) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 & b_) # E: numpy.unsignedinteger[numpy.typing._64Bit] + +reveal_type(u8 << b) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 >> b) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 | b) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 ^ b) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 & b) # E: numpy.unsignedinteger[numpy.typing._64Bit] + +reveal_type(b_ << b_) # E: numpy.signedinteger[numpy.typing._8Bit] +reveal_type(b_ >> b_) # E: numpy.signedinteger[numpy.typing._8Bit] reveal_type(b_ | b_) # E: numpy.bool_ reveal_type(b_ ^ b_) # E: numpy.bool_ reveal_type(b_ & b_) # E: numpy.bool_ -reveal_type(b_ << AR) # E: Union[numpy.ndarray, numpy.integer] -reveal_type(b_ >> AR) # E: Union[numpy.ndarray, numpy.integer] -reveal_type(b_ | AR) # E: Union[numpy.ndarray, numpy.integer, numpy.bool_] -reveal_type(b_ ^ AR) # E: Union[numpy.ndarray, numpy.integer, numpy.bool_] -reveal_type(b_ & AR) # E: Union[numpy.ndarray, numpy.integer, numpy.bool_] +reveal_type(b_ << AR) # E: Union[numpy.ndarray, numpy.integer[Any]] +reveal_type(b_ >> AR) # E: Union[numpy.ndarray, numpy.integer[Any]] +reveal_type(b_ | AR) # E: Union[numpy.ndarray, numpy.integer[Any], numpy.bool_] +reveal_type(b_ ^ AR) # E: Union[numpy.ndarray, numpy.integer[Any], numpy.bool_] +reveal_type(b_ & AR) # E: Union[numpy.ndarray, numpy.integer[Any], numpy.bool_] -reveal_type(b_ << b) # E: numpy.int8 -reveal_type(b_ >> b) # E: numpy.int8 +reveal_type(b_ << b) # E: numpy.signedinteger[numpy.typing._8Bit] +reveal_type(b_ >> b) # E: numpy.signedinteger[numpy.typing._8Bit] reveal_type(b_ | b) # E: numpy.bool_ reveal_type(b_ ^ b) # E: numpy.bool_ reveal_type(b_ & b) # E: numpy.bool_ -reveal_type(b_ << i) # E: Union[numpy.int32, numpy.int64] -reveal_type(b_ >> i) # E: Union[numpy.int32, numpy.int64] -reveal_type(b_ | i) # E: Union[numpy.int32, numpy.int64] -reveal_type(b_ ^ i) # E: Union[numpy.int32, numpy.int64] -reveal_type(b_ & i) # E: Union[numpy.int32, numpy.int64] +reveal_type(b_ << i) # E: numpy.signedinteger[Any] +reveal_type(b_ >> i) # E: numpy.signedinteger[Any] +reveal_type(b_ | i) # E: numpy.signedinteger[Any] +reveal_type(b_ ^ i) # E: numpy.signedinteger[Any] +reveal_type(b_ & i) # E: numpy.signedinteger[Any] -reveal_type(~i8) # E: numpy.int64 -reveal_type(~i4) # E: numpy.int32 -reveal_type(~u8) # E: numpy.uint64 -reveal_type(~u4) # E: numpy.uint32 +reveal_type(~i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(~i4) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(~u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(~u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] reveal_type(~b_) # E: numpy.bool_ -reveal_type(~AR) # E: Union[numpy.ndarray*, numpy.integer, numpy.bool_] +reveal_type(~AR) # E: Union[numpy.ndarray*, numpy.integer[Any], numpy.bool_] From 3082bd7e6b23d6076cc6da8b526862019610202c Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Fri, 16 Oct 2020 16:38:52 +0200 Subject: [PATCH 290/409] TST: Added a test for the example in the `NBitBase` docstring --- .../tests/data/reveal/nbit_base_example.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 numpy/typing/tests/data/reveal/nbit_base_example.py diff --git a/numpy/typing/tests/data/reveal/nbit_base_example.py b/numpy/typing/tests/data/reveal/nbit_base_example.py new file mode 100644 index 000000000000..0c4c53f9b1f1 --- /dev/null +++ b/numpy/typing/tests/data/reveal/nbit_base_example.py @@ -0,0 +1,18 @@ +from typing import TypeVar, Union +import numpy as np +import numpy.typing as npt + +T = TypeVar("T", bound=npt.NBitBase) + +def add(a: np.floating[T], b: np.integer[T]) -> np.floating[T]: + return a + b + +i8: np.int64 +i4: np.int32 +f8: np.float64 +f4: np.float32 + +reveal_type(add(f8, i8)) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(add(f4, i8)) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(add(f8, i4)) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(add(f4, i4)) # E: numpy.floating[numpy.typing._32Bit] From 226b46bb422953b08e38756d3b9384988d3d35fc Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Sat, 17 Oct 2020 18:04:31 +0200 Subject: [PATCH 291/409] MAINT: Removed an unused import --- numpy/typing/_callable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/typing/_callable.py b/numpy/typing/_callable.py index e8c8615b5a7f..b16891b0e24f 100644 --- a/numpy/typing/_callable.py +++ b/numpy/typing/_callable.py @@ -35,7 +35,7 @@ _ComplexLike, _NumberLike, ) -from . import NBitBase, _64Bit +from . import NBitBase if sys.version_info >= (3, 8): from typing import Protocol From cd15084bf9f21f4e255f9c788f903ffcbf703a58 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Sat, 17 Oct 2020 19:50:25 +0200 Subject: [PATCH 292/409] TST: Update the tests introduced in the latest rebase --- numpy/typing/tests/data/reveal/dtype.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/numpy/typing/tests/data/reveal/dtype.py b/numpy/typing/tests/data/reveal/dtype.py index e0802299e973..d414f2c4934f 100644 --- a/numpy/typing/tests/data/reveal/dtype.py +++ b/numpy/typing/tests/data/reveal/dtype.py @@ -1,30 +1,30 @@ import numpy as np -reveal_type(np.dtype(np.float64)) # E: numpy.dtype[numpy.float64*] -reveal_type(np.dtype(np.int64)) # E: numpy.dtype[numpy.int64*] +reveal_type(np.dtype(np.float64)) # E: numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(np.dtype(np.int64)) # E: numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]] # String aliases -reveal_type(np.dtype("float64")) # E: numpy.dtype[numpy.float64] -reveal_type(np.dtype("float32")) # E: numpy.dtype[numpy.float32] -reveal_type(np.dtype("int64")) # E: numpy.dtype[numpy.int64] -reveal_type(np.dtype("int32")) # E: numpy.dtype[numpy.int32] +reveal_type(np.dtype("float64")) # E: numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(np.dtype("float32")) # E: numpy.dtype[numpy.floating[numpy.typing._32Bit]] +reveal_type(np.dtype("int64")) # E: numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]] +reveal_type(np.dtype("int32")) # E: numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]] reveal_type(np.dtype("bool")) # E: numpy.dtype[numpy.bool_] reveal_type(np.dtype("bytes")) # E: numpy.dtype[numpy.bytes_] reveal_type(np.dtype("str")) # E: numpy.dtype[numpy.str_] # Python types -reveal_type(np.dtype(complex)) # E: numpy.dtype[numpy.complex128] -reveal_type(np.dtype(float)) # E: numpy.dtype[numpy.float64] +reveal_type(np.dtype(complex)) # E: numpy.dtype[numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit]] +reveal_type(np.dtype(float)) # E: numpy.dtype[numpy.floating[numpy.typing._64Bit]] reveal_type(np.dtype(int)) # E: numpy.dtype reveal_type(np.dtype(bool)) # E: numpy.dtype[numpy.bool_] reveal_type(np.dtype(str)) # E: numpy.dtype[numpy.str_] reveal_type(np.dtype(bytes)) # E: numpy.dtype[numpy.bytes_] # Special case for None -reveal_type(np.dtype(None)) # E: numpy.dtype[numpy.float64] +reveal_type(np.dtype(None)) # E: numpy.dtype[numpy.floating[numpy.typing._64Bit]] # Dtypes of dtypes -reveal_type(np.dtype(np.dtype(np.float64))) # E: numpy.dtype[numpy.float64*] +reveal_type(np.dtype(np.dtype(np.float64))) # E: numpy.dtype[numpy.floating[numpy.typing._64Bit]] # Parameterized dtypes reveal_type(np.dtype("S8")) # E: numpy.dtype From 56dedfc23f8f4ee0ce17c710b505254528566960 Mon Sep 17 00:00:00 2001 From: Cameron Blocker Date: Sat, 17 Oct 2020 19:00:00 -0400 Subject: [PATCH 293/409] BUG: Respect dtype of all-zero argument to poly1d Fixes gh-16354. Previously np.poly1d(z).coeffs.dtype would always be np.float64 for zero array z, regardless of z's dtype. --- numpy/lib/polynomial.py | 2 +- numpy/lib/tests/test_polynomial.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/numpy/lib/polynomial.py b/numpy/lib/polynomial.py index 7b89eeb70367..28c3f5e99cd8 100644 --- a/numpy/lib/polynomial.py +++ b/numpy/lib/polynomial.py @@ -1236,7 +1236,7 @@ def __init__(self, c_or_r, r=False, variable=None): raise ValueError("Polynomial must be 1d only.") c_or_r = trim_zeros(c_or_r, trim='f') if len(c_or_r) == 0: - c_or_r = NX.array([0.]) + c_or_r = NX.array([0.], dtype=c_or_r.dtype) self._coeffs = c_or_r if variable is None: variable = 'x' diff --git a/numpy/lib/tests/test_polynomial.py b/numpy/lib/tests/test_polynomial.py index ab6691b4384f..6c3e4fa02212 100644 --- a/numpy/lib/tests/test_polynomial.py +++ b/numpy/lib/tests/test_polynomial.py @@ -227,6 +227,20 @@ def test_poly_int_overflow(self): v = np.arange(1, 21) assert_almost_equal(np.poly(v), np.poly(np.diag(v))) + def test_zero_poly_dtype(self): + """ + Regression test for gh-16354. + """ + z = np.array([0, 0, 0]) + p = np.poly1d(z.astype(np.int64)) + assert_equal(p.coeffs.dtype, np.int64) + + p = np.poly1d(z.astype(np.float32)) + assert_equal(p.coeffs.dtype, np.float32) + + p = np.poly1d(z.astype(np.complex64)) + assert_equal(p.coeffs.dtype, np.complex64) + def test_poly_eq(self): p = np.poly1d([1, 2, 3]) p2 = np.poly1d([1, 2, 4]) From bb76bff17f86e468ca9fbc5071d95782dc932c10 Mon Sep 17 00:00:00 2001 From: Ganesh Kathiresan Date: Sun, 18 Oct 2020 14:33:30 +0530 Subject: [PATCH 294/409] BUG: Fixed crash on self-referential dtypes (#17536) Instead of causing a C stack overflow, these now raise RecursionError. It shouldn't be possible to recurse through the string, type, or byte conversion paths; only the tuple, dict, and list paths are at risk. --- numpy/core/src/multiarray/descriptor.c | 27 +++++++++++++++++++++++--- numpy/core/tests/test_dtype.py | 20 +++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c index 24a3507bc408..a8d575248bb7 100644 --- a/numpy/core/src/multiarray/descriptor.c +++ b/numpy/core/src/multiarray/descriptor.c @@ -1497,15 +1497,36 @@ _convert_from_any(PyObject *obj, int align) } else if (PyTuple_Check(obj)) { /* or a tuple */ - return _convert_from_tuple(obj, align); + if (Py_EnterRecursiveCall( + " while trying to convert the given data type from" + " a tuple object" ) != 0) { + return NULL; + } + PyArray_Descr *ret = _convert_from_tuple(obj, align); + Py_LeaveRecursiveCall(); + return ret; } else if (PyList_Check(obj)) { /* or a list */ - return _convert_from_array_descr(obj, align); + if (Py_EnterRecursiveCall( + " while trying to convert the given data type from" + " a list object" ) != 0) { + return NULL; + } + PyArray_Descr *ret = _convert_from_array_descr(obj, align); + Py_LeaveRecursiveCall(); + return ret; } else if (PyDict_Check(obj) || PyDictProxy_Check(obj)) { /* or a dictionary */ - return _convert_from_dict(obj, align); + if (Py_EnterRecursiveCall( + " while trying to convert the given data type from" + " a dict object" ) != 0) { + return NULL; + } + PyArray_Descr *ret = _convert_from_dict(obj, align); + Py_LeaveRecursiveCall(); + return ret; } else if (PyArray_Check(obj)) { PyErr_SetString(PyExc_TypeError, "Cannot construct a dtype from an array"); diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index 45cc0b8b30cd..ba50690245c1 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -767,6 +767,26 @@ def test1(self): ('yi', np.dtype((a, (3, 2))))]) assert_dtype_equal(c, d) + def test_list_recursion(self): + l = list() + l.append(('f', l)) + with pytest.raises(RecursionError): + np.dtype(l) + + def test_tuple_recursion(self): + d = np.int32 + for i in range(100000): + d = (d, (1,)) + with pytest.raises(RecursionError): + np.dtype(d) + + def test_dict_recursion(self): + d = dict(names=['self'], formats=[None], offsets=[0]) + d['formats'][0] = d + with pytest.raises(RecursionError): + np.dtype(d) + + class TestMetadata: def test_no_metadata(self): d = np.dtype(int) From 10d28b6dc9cda2be1ca1236515a5936c3571207b Mon Sep 17 00:00:00 2001 From: Cameron Blocker Date: Sun, 18 Oct 2020 14:11:47 -0400 Subject: [PATCH 295/409] use integer literal instead of float Co-authored-by: Eric Wieser --- numpy/lib/polynomial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/lib/polynomial.py b/numpy/lib/polynomial.py index 28c3f5e99cd8..351f2b5730f4 100644 --- a/numpy/lib/polynomial.py +++ b/numpy/lib/polynomial.py @@ -1236,7 +1236,7 @@ def __init__(self, c_or_r, r=False, variable=None): raise ValueError("Polynomial must be 1d only.") c_or_r = trim_zeros(c_or_r, trim='f') if len(c_or_r) == 0: - c_or_r = NX.array([0.], dtype=c_or_r.dtype) + c_or_r = NX.array([0], dtype=c_or_r.dtype) self._coeffs = c_or_r if variable is None: variable = 'x' From cf06cd41ba8bd49b3ada8aa333df8e562829247b Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Sun, 18 Oct 2020 14:18:10 -0600 Subject: [PATCH 296/409] MAINT: Cleanup swig for Python 3. Replaces ``PyInt_Check`` and ``PyInt_AsLong`` in a few places. --- doc/release/upcoming_changes/17580.compatibility.rst | 4 ++++ tools/swig/numpy.i | 12 +++++++----- tools/swig/pyfragments.swg | 11 ++++------- 3 files changed, 15 insertions(+), 12 deletions(-) create mode 100644 doc/release/upcoming_changes/17580.compatibility.rst diff --git a/doc/release/upcoming_changes/17580.compatibility.rst b/doc/release/upcoming_changes/17580.compatibility.rst new file mode 100644 index 000000000000..b8e1849afcbe --- /dev/null +++ b/doc/release/upcoming_changes/17580.compatibility.rst @@ -0,0 +1,4 @@ +The numpy.i file for swig is Python 3 only. +------------------------------------------- +Uses of Python 2.7 C-API functions have been updated to Python 3 only. Users +who need the old version should take it from an older version of NumPy. diff --git a/tools/swig/numpy.i b/tools/swig/numpy.i index a2e7a335fdb0..60930c098330 100644 --- a/tools/swig/numpy.i +++ b/tools/swig/numpy.i @@ -115,7 +115,7 @@ if (py_obj == Py_None ) return "Python None" ; if (PyCallable_Check(py_obj)) return "callable" ; if (PyBytes_Check( py_obj)) return "string" ; - if (PyInt_Check( py_obj)) return "int" ; + if (PyLong_Check( py_obj)) return "int" ; if (PyFloat_Check( py_obj)) return "float" ; if (PyDict_Check( py_obj)) return "dict" ; if (PyList_Check( py_obj)) return "list" ; @@ -2002,7 +2002,7 @@ (PyObject* array = NULL) { npy_intp dims[1]; - if (!PyInt_Check($input)) + if (!PyLong_Check($input)) { const char* typestring = pytype_string($input); PyErr_Format(PyExc_TypeError, @@ -2010,7 +2010,8 @@ typestring); SWIG_fail; } - $2 = (DIM_TYPE) PyInt_AsLong($input); + $2 = (DIM_TYPE) PyLong_AsSsize_t($input); + if ($2 == -1 && PyErr_Occurred()) SWIG_fail; dims[0] = (npy_intp) $2; array = PyArray_SimpleNew(1, dims, DATA_TYPECODE); if (!array) SWIG_fail; @@ -2030,7 +2031,7 @@ (PyObject* array = NULL) { npy_intp dims[1]; - if (!PyInt_Check($input)) + if (!PyLong_Check($input)) { const char* typestring = pytype_string($input); PyErr_Format(PyExc_TypeError, @@ -2038,7 +2039,8 @@ typestring); SWIG_fail; } - $1 = (DIM_TYPE) PyInt_AsLong($input); + $1 = (DIM_TYPE) PyLong_AsSsize_t($input); + if ($1 == -1 && PyErr_Occurred()) SWIG_fail; dims[0] = (npy_intp) $1; array = PyArray_SimpleNew(1, dims, DATA_TYPECODE); if (!array) SWIG_fail; diff --git a/tools/swig/pyfragments.swg b/tools/swig/pyfragments.swg index ce2452f8020d..558633733da3 100644 --- a/tools/swig/pyfragments.swg +++ b/tools/swig/pyfragments.swg @@ -22,12 +22,9 @@ SWIGINTERN int SWIG_AsVal_dec(long)(PyObject * obj, long * val) { - if (PyInt_Check(obj)) { - if (val) *val = PyInt_AsLong(obj); - return SWIG_OK; - } else if (PyLong_Check(obj)) { + if (PyLong_Check(obj)) { long v = PyLong_AsLong(obj); - if (!PyErr_Occurred()) { + if (v != -1 || !PyErr_Occurred()) { if (val) *val = v; return SWIG_OK; } else { @@ -37,8 +34,8 @@ %#ifdef SWIG_PYTHON_CAST_MODE { int dispatch = 0; - long v = PyInt_AsLong(obj); - if (!PyErr_Occurred()) { + long v = PyLong_AsLong(obj); + if (v != -1 || !PyErr_Occurred()) { if (val) *val = v; return SWIG_AddCast(SWIG_OK); } else { From d3e40dc59e638f2210a20e40f7edca5c5ee7c124 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Mon, 19 Oct 2020 00:40:52 +0200 Subject: [PATCH 297/409] MAINT: Move the `np.core.numerictypes` annotations to their own stub file --- numpy/__init__.pyi | 36 ++++++++++++------------------------ numpy/core/numerictypes.pyi | 26 ++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 24 deletions(-) create mode 100644 numpy/core/numerictypes.pyi diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 2fff82d59365..f70965c419f9 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -141,7 +141,7 @@ from numpy.core._asarray import ( require as require, ) -from numpy.core.numeric import( +from numpy.core.numeric import ( zeros_like as zeros_like, ones as ones, ones_like as ones_like, @@ -172,6 +172,17 @@ from numpy.core.numeric import( array_equiv as array_equiv, ) +from numpy.core.numerictypes import ( + maximum_sctype as maximum_sctype, + issctype as issctype, + obj2sctype as obj2sctype, + issubclass_ as issubclass_, + issubsctype as issubsctype, + issubdtype as issubdtype, + sctype2char as sctype2char, + find_common_type as find_common_type, +) + # Add an object to `__all__` if their stubs are defined in an external file; # their stubs will not be recognized otherwise. # NOTE: This is redundant for objects defined within this file. @@ -2149,26 +2160,3 @@ class AxisError(ValueError, IndexError): def __init__( self, axis: int, ndim: Optional[int] = ..., msg_prefix: Optional[str] = ... ) -> None: ... - -# Functions from np.core.numerictypes -_DefaultType = TypeVar("_DefaultType") - -def maximum_sctype(t: DtypeLike) -> dtype: ... -def issctype(rep: object) -> bool: ... -@overload -def obj2sctype(rep: object) -> Optional[generic]: ... -@overload -def obj2sctype(rep: object, default: None) -> Optional[generic]: ... -@overload -def obj2sctype( - rep: object, default: Type[_DefaultType] -) -> Union[generic, Type[_DefaultType]]: ... -def issubclass_(arg1: object, arg2: Union[object, Tuple[object, ...]]) -> bool: ... -def issubsctype( - arg1: Union[ndarray, DtypeLike], arg2: Union[ndarray, DtypeLike] -) -> bool: ... -def issubdtype(arg1: DtypeLike, arg2: DtypeLike) -> bool: ... -def sctype2char(sctype: object) -> str: ... -def find_common_type( - array_types: Sequence[DtypeLike], scalar_types: Sequence[DtypeLike] -) -> dtype: ... diff --git a/numpy/core/numerictypes.pyi b/numpy/core/numerictypes.pyi new file mode 100644 index 000000000000..86688a66e76f --- /dev/null +++ b/numpy/core/numerictypes.pyi @@ -0,0 +1,26 @@ +from typing import TypeVar, Optional, Type, Union, Tuple, Sequence, overload, Any + +from numpy import generic, ndarray, dtype +from numpy.typing import DtypeLike + +_DefaultType = TypeVar("_DefaultType") + +def maximum_sctype(t: DtypeLike) -> dtype: ... +def issctype(rep: object) -> bool: ... +@overload +def obj2sctype(rep: object) -> Optional[generic]: ... +@overload +def obj2sctype(rep: object, default: None) -> Optional[generic]: ... +@overload +def obj2sctype( + rep: object, default: Type[_DefaultType] +) -> Union[generic, Type[_DefaultType]]: ... +def issubclass_(arg1: object, arg2: Union[object, Tuple[object, ...]]) -> bool: ... +def issubsctype( + arg1: Union[ndarray, DtypeLike], arg2: Union[ndarray, DtypeLike] +) -> bool: ... +def issubdtype(arg1: DtypeLike, arg2: DtypeLike) -> bool: ... +def sctype2char(sctype: object) -> str: ... +def find_common_type( + array_types: Sequence[DtypeLike], scalar_types: Sequence[DtypeLike] +) -> dtype: ... From 7d92ceb111b4cb3ca625f67470fea860f61e8bfd Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Mon, 19 Oct 2020 00:41:24 +0200 Subject: [PATCH 298/409] MAINT: Add a note about to-be annotated objects --- numpy/core/numerictypes.pyi | 3 +++ 1 file changed, 3 insertions(+) diff --git a/numpy/core/numerictypes.pyi b/numpy/core/numerictypes.pyi index 86688a66e76f..0aba5f1d1519 100644 --- a/numpy/core/numerictypes.pyi +++ b/numpy/core/numerictypes.pyi @@ -24,3 +24,6 @@ def sctype2char(sctype: object) -> str: ... def find_common_type( array_types: Sequence[DtypeLike], scalar_types: Sequence[DtypeLike] ) -> dtype: ... + +# TODO: Add annotations for the following objects: +# typeDict, nbytes, cast, ScalarType & typecodes From 44b6e2426a1f74d2bf1c596d6d9df91a1341510d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2020 07:21:32 +0000 Subject: [PATCH 299/409] MAINT: Bump hypothesis from 5.37.1 to 5.37.3 Bumps [hypothesis](https://github.com/HypothesisWorks/hypothesis) from 5.37.1 to 5.37.3. - [Release notes](https://github.com/HypothesisWorks/hypothesis/releases) - [Commits](https://github.com/HypothesisWorks/hypothesis/compare/hypothesis-python-5.37.1...hypothesis-python-5.37.3) Signed-off-by: dependabot-preview[bot] --- test_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index c189631ed1ab..6d8b94cae630 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,7 +1,7 @@ cython==0.29.21 wheel setuptools<49.2.0 -hypothesis==5.37.1 +hypothesis==5.37.3 pytest==6.0.2 pytz==2020.1 pytest-cov==2.10.1 From 953117443a4919a20ae0759209f76d5f5dba256f Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Mon, 19 Oct 2020 00:25:08 +0200 Subject: [PATCH 300/409] ENH: Add annotations for `np.core._type_aliases` --- numpy/__init__.pyi | 7 +++++-- numpy/core/_type_aliases.pyi | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 numpy/core/_type_aliases.pyi diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index f70965c419f9..c62227e5e673 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -141,6 +141,11 @@ from numpy.core._asarray import ( require as require, ) +from numpy.core._type_aliases import ( + sctypes as sctypes, + sctypeDict as sctypeDict, +) + from numpy.core.numeric import ( zeros_like as zeros_like, ones as ones, @@ -460,8 +465,6 @@ save: Any savetxt: Any savez: Any savez_compressed: Any -sctypeDict: Any -sctypes: Any select: Any set_printoptions: Any set_string_function: Any diff --git a/numpy/core/_type_aliases.pyi b/numpy/core/_type_aliases.pyi new file mode 100644 index 000000000000..6a1099cd3fad --- /dev/null +++ b/numpy/core/_type_aliases.pyi @@ -0,0 +1,19 @@ +import sys +from typing import Dict, Union, Type, List + +from numpy import generic, signedinteger, unsignedinteger, floating, complexfloating + +if sys.version_info >= (3, 8): + from typing import TypedDict +else: + from typing_extensions import TypedDict + +class _SCTypes(TypedDict): + int: List[Type[signedinteger]] + uint: List[Type[unsignedinteger]] + float: List[Type[floating]] + complex: List[Type[complexfloating]] + others: List[type] + +sctypeDict: Dict[Union[int, str], Type[generic]] +sctypes: _SCTypes From f1b9adb1c794ea36a31bdc304d3f8dc80833c008 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Mon, 19 Oct 2020 13:31:16 +0200 Subject: [PATCH 301/409] TST: Added new typing tests --- numpy/typing/tests/data/reveal/constants.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/numpy/typing/tests/data/reveal/constants.py b/numpy/typing/tests/data/reveal/constants.py index dc5803974332..b2382e8611a3 100644 --- a/numpy/typing/tests/data/reveal/constants.py +++ b/numpy/typing/tests/data/reveal/constants.py @@ -47,3 +47,6 @@ reveal_type(np.False_) # E: numpy.bool_ reveal_type(np.UFUNC_PYVALS_NAME) # E: str + +reveal_type(np.sctypeDict) # E: dict +reveal_type(np.sctypes) # E: TypedDict From 136185cee387e0e3638abf38b4faf9b0f0cfadd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Melissa=20Weber=20Mendon=C3=A7a?= Date: Mon, 19 Oct 2020 14:51:50 -0300 Subject: [PATCH 302/409] DOC: Added a PR "Reviewer guidelines" document. (#17252) --- doc/source/dev/development_environment.rst | 2 +- doc/source/dev/index.rst | 12 ++- doc/source/dev/reviewer_guidelines.rst | 119 +++++++++++++++++++++ 3 files changed, 130 insertions(+), 3 deletions(-) create mode 100644 doc/source/dev/reviewer_guidelines.rst diff --git a/doc/source/dev/development_environment.rst b/doc/source/dev/development_environment.rst index ff78cecc5315..cb027c6620f8 100644 --- a/doc/source/dev/development_environment.rst +++ b/doc/source/dev/development_environment.rst @@ -273,7 +273,7 @@ pull requests aren't perfect, the community is always happy to help. As a volunteer project, things do sometimes get dropped and it's totally fine to ping us if something has sat without a response for about two to four weeks. -So go ahead and pick something that annoys or confuses you about numpy, +So go ahead and pick something that annoys or confuses you about NumPy, experiment with the code, hang around for discussions or go through the reference documents to try to fix it. Things will fall in place and soon you'll have a pretty good understanding of the project as a whole. Good Luck! diff --git a/doc/source/dev/index.rst b/doc/source/dev/index.rst index d2c162c9a4ec..4641a7e2fff6 100644 --- a/doc/source/dev/index.rst +++ b/doc/source/dev/index.rst @@ -122,7 +122,8 @@ Here's the short summary, complete TOC links are below: overall code quality benefits. Therefore, please don't let the review discourage you from contributing: its only aim is to improve the quality of project, not to criticize (we are, after all, very grateful for the - time you're donating!). + time you're donating!). See our :ref:`Reviewer Guidelines + ` for more information. * To update your PR, make your changes on your local repository, commit, **run tests, and only if they succeed** push to your fork. As soon as @@ -179,6 +180,8 @@ be merged automatically, you have to incorporate changes that have been made since you started into your branch. Our recommended way to do this is to :ref:`rebase on master`. +.. _guidelines: + Guidelines ---------- @@ -186,9 +189,11 @@ Guidelines * All code should be `documented `_. * No changes are ever committed without review and approval by a core - team member.Please ask politely on the PR or on the `mailing list`_ if you + team member. Please ask politely on the PR or on the `mailing list`_ if you get no response to your pull request within a week. +.. _stylistic-guidelines: + Stylistic Guidelines -------------------- @@ -233,6 +238,8 @@ This will create a report in ``build/coverage``, which can be viewed with:: $ firefox build/coverage/index.html +.. _building-docs: + Building docs ------------- @@ -295,6 +302,7 @@ The rest of the story Git Basics development_environment development_workflow + reviewer_guidelines ../benchmarking NumPy C style guide releasing diff --git a/doc/source/dev/reviewer_guidelines.rst b/doc/source/dev/reviewer_guidelines.rst new file mode 100644 index 000000000000..0b225b9b6b14 --- /dev/null +++ b/doc/source/dev/reviewer_guidelines.rst @@ -0,0 +1,119 @@ +.. _reviewer-guidelines: + +=================== +Reviewer Guidelines +=================== + +Reviewing open pull requests (PRs) helps move the project forward. We encourage +people outside the project to get involved as well; it's a great way to get +familiar with the codebase. + +Who can be a reviewer? +====================== + +Reviews can come from outside the NumPy team -- we welcome contributions from +domain experts (for instance, `linalg` or `fft`) or maintainers of other +projects. You do not need to be a NumPy maintainer (a NumPy team member with +permission to merge a PR) to review. + +If we do not know you yet, consider introducing yourself in `the mailing list or +Slack `_ before you start reviewing pull requests. + +Communication Guidelines +======================== + +- Every PR, good or bad, is an act of generosity. Opening with a positive + comment will help the author feel rewarded, and your subsequent remarks may be + heard more clearly. You may feel good also. +- Begin if possible with the large issues, so the author knows they've been + understood. Resist the temptation to immediately go line by line, or to open + with small pervasive issues. +- You are the face of the project, and NumPy some time ago decided `the kind of + project it will be `_: open, empathetic, + welcoming, friendly and patient. Be `kind + `_ to contributors. +- Do not let perfect be the enemy of the good, particularly for documentation. + If you find yourself making many small suggestions, or being too nitpicky on + style or grammar, consider merging the current PR when all important concerns + are addressed. Then, either push a commit directly (if you are a maintainer) + or open a follow-up PR yourself. +- If you need help writing replies in reviews, check out some `Standard replies + for reviewing + `_. + +Reviewer Checklist +================== + +- Is the intended behavior clear under all conditions? Some things to watch: + - What happens with unexpected inputs like empty arrays or nan/inf values? + - Are axis or shape arguments tested to be `int` or `tuples`? + - Are unusual `dtypes` tested if a function supports those? +- Should variable names be improved for clarity or consistency? +- Should comments be added, or rather removed as unhelpful or extraneous? +- Does the documentation follow the :ref:`NumPy guidelines`? Are + the docstrings properly formatted? +- Does the code follow NumPy's :ref:`Stylistic Guidelines`? +- If you are a maintainer, and it is not obvious from the PR description, add a + short explanation of what a branch did to the merge message and, if closing an + issue, also add "Closes gh-123" where 123 is the issue number. +- For code changes, at least one maintainer (i.e. someone with commit rights) + should review and approve a pull request. If you are the first to review a + PR and approve of the changes use the GitHub `approve review + `_ tool + to mark it as such. If a PR is straightforward, for example it's a clearly + correct bug fix, it can be merged straight away. If it's more complex or + changes public API, please leave it open for at least a couple of days so + other maintainers get a chance to review. +- If you are a subsequent reviewer on an already approved PR, please use the + same review method as for a new PR (focus on the larger issues, resist the + temptation to add only a few nitpicks). If you have commit rights and think + no more review is needed, merge the PR. + +For maintainers +--------------- + +- Make sure all automated CI tests pass before merging a PR, and that the + :ref:`documentation builds ` without any errors. +- In case of merge conflicts, ask the PR submitter to :ref:`rebase on master + `. +- For PRs that add new features or are in some way complex, wait at least a day + or two before merging it. That way, others get a chance to comment before the + code goes in. Consider adding it to the release notes. +- When merging contributions, a committer is responsible for ensuring that those + meet the requirements outlined in the :ref:`Development process guidelines + ` for NumPy. Also, check that new features and backwards + compatibility breaks were discussed on the `numpy-discussion mailing list + `_. +- Squashing commits or cleaning up commit messages of a PR that you consider too + messy is OK. Remember to retain the original author's name when doing this. + Make sure commit messages follow the :ref:`rules for NumPy + `. +- When you want to reject a PR: if it's very obvious, you can just close it and + explain why. If it's not, then it's a good idea to first explain why you + think the PR is not suitable for inclusion in NumPy and then let a second + committer comment or close. + +GitHub Workflow +--------------- + +When reviewing pull requests, please use workflow tracking features on GitHub as +appropriate: + +- After you have finished reviewing, if you want to ask for the submitter to + make changes, change your review status to "Changes requested." This can be + done on GitHub, PR page, Files changed tab, Review changes (button on the top + right). +- If you're happy about the current status, mark the pull request as Approved + (same way as Changes requested). Alternatively (for maintainers): merge + the pull request, if you think it is ready to be merged. + +It may be helpful to have a copy of the pull request code checked out on your +own machine so that you can play with it locally. You can use the `GitHub CLI +`_ to +do this by clicking the ``Open with`` button in the upper right-hand corner of +the PR page. + +Assuming you have your :ref:`development environment` +set up, you can now build the code and test it. + +.. include:: gitwash/git_links.inc From 76e80b35bcf279f7837b999cc8c7d6dd2d0a3402 Mon Sep 17 00:00:00 2001 From: Rashmi K A <39820442+Rashmi-K-A@users.noreply.github.com> Date: Tue, 20 Oct 2020 03:09:38 +0530 Subject: [PATCH 303/409] DOC: adding examples to ufunc documentation for array ops (#17418) Add wording and examples to ufuncs highlighting corresponding operators. Co-authored-by: Ross Barnowski --- .../core/code_generators/ufunc_docstrings.py | 228 +++++++++++++++++- 1 file changed, 222 insertions(+), 6 deletions(-) diff --git a/numpy/core/code_generators/ufunc_docstrings.py b/numpy/core/code_generators/ufunc_docstrings.py index 1f8e4a6fb314..b7edd2834b43 100644 --- a/numpy/core/code_generators/ufunc_docstrings.py +++ b/numpy/core/code_generators/ufunc_docstrings.py @@ -107,6 +107,13 @@ def add_newdoc(place, name, doc): >>> plt.imshow(np.abs(xx), extent=[-10, 10, -10, 10], cmap='gray') >>> plt.show() + The `abs` function can be used as a shorthand for ``np.absolute`` on + ndarrays. + + >>> x = np.array([-1.2, 1.2]) + >>> abs(x) + array([1.2, 1.2]) + """) add_newdoc('numpy.core.umath', 'add', @@ -141,6 +148,14 @@ def add_newdoc(place, name, doc): [ 3., 5., 7.], [ 6., 8., 10.]]) + The ``+`` operator can be used as a shorthand for ``np.add`` on ndarrays. + + >>> x1 = np.arange(9.0).reshape((3, 3)) + >>> x2 = np.arange(3.0) + >>> x1 + x2 + array([[ 0., 2., 4.], + [ 3., 5., 7.], + [ 6., 8., 10.]]) """) add_newdoc('numpy.core.umath', 'arccos', @@ -608,6 +623,14 @@ def add_newdoc(place, name, doc): >>> np.bitwise_and([True, True], [False, True]) array([False, True]) + The ``&`` operator can be used as a shorthand for ``np.bitwise_and`` on + ndarrays. + + >>> x1 = np.array([2, 5, 255]) + >>> x2 = np.array([3, 14, 16]) + >>> x1 & x2 + array([ 2, 4, 16]) + """) add_newdoc('numpy.core.umath', 'bitwise_or', @@ -667,6 +690,14 @@ def add_newdoc(place, name, doc): >>> np.bitwise_or([True, True], [False, True]) array([ True, True]) + The ``|`` operator can be used as a shorthand for ``np.bitwise_or`` on + ndarrays. + + >>> x1 = np.array([2, 5, 255]) + >>> x2 = np.array([4, 4, 4]) + >>> x1 | x2 + array([ 6, 5, 255]) + """) add_newdoc('numpy.core.umath', 'bitwise_xor', @@ -719,6 +750,14 @@ def add_newdoc(place, name, doc): >>> np.bitwise_xor([True, True], [False, True]) array([ True, False]) + The ``^`` operator can be used as a shorthand for ``np.bitwise_xor`` on + ndarrays. + + >>> x1 = np.array([True, True]) + >>> x2 = np.array([False, True]) + >>> x1 ^ x2 + array([ True, False]) + """) add_newdoc('numpy.core.umath', 'ceil', @@ -1088,6 +1127,16 @@ def add_newdoc(place, name, doc): >>> np.divide(1, 0) 0 + The ``/`` operator can be used as a shorthand for ``np.divide`` on + ndarrays. + + >>> x1 = np.arange(9.0).reshape((3, 3)) + >>> x2 = 2 * np.ones(3) + >>> x1 / x2 + array([[0. , 0.5, 1. ], + [1.5, 2. , 2.5], + [3. , 3.5, 4. ]]) + """) add_newdoc('numpy.core.umath', 'equal', @@ -1123,6 +1172,14 @@ def add_newdoc(place, name, doc): >>> np.equal(1, np.ones(1)) array([ True]) + The ``==`` operator can be used as a shorthand for ``np.equal`` on + ndarrays. + + >>> a = np.array([2, 4, 6]) + >>> b = np.array([2, 4, 2]) + >>> a == b + array([ True, True, False]) + """) add_newdoc('numpy.core.umath', 'exp', @@ -1370,6 +1427,13 @@ def add_newdoc(place, name, doc): >>> np.floor_divide([1., 2., 3., 4.], 2.5) array([ 0., 0., 1., 1.]) + The ``//`` operator can be used as a shorthand for ``np.floor_divide`` + on ndarrays. + + >>> x1 = np.array([1., 2., 3., 4.]) + >>> x1 // 2.5 + array([0., 0., 1., 1.]) + """) add_newdoc('numpy.core.umath', 'fmod', @@ -1458,10 +1522,11 @@ def add_newdoc(place, name, doc): >>> np.greater([4,2],[2,2]) array([ True, False]) - If the inputs are ndarrays, then np.greater is equivalent to '>'. + The ``>`` operator can be used as a shorthand for ``np.greater`` on + ndarrays. - >>> a = np.array([4,2]) - >>> b = np.array([2,2]) + >>> a = np.array([4, 2]) + >>> b = np.array([2, 2]) >>> a > b array([ True, False]) @@ -1494,6 +1559,14 @@ def add_newdoc(place, name, doc): >>> np.greater_equal([4, 2, 1], [2, 2, 2]) array([ True, True, False]) + The ``>=`` operator can be used as a shorthand for ``np.greater_equal`` + on ndarrays. + + >>> a = np.array([4, 2, 1]) + >>> b = np.array([2, 2, 2]) + >>> a >= b + array([ True, True, False]) + """) add_newdoc('numpy.core.umath', 'hypot', @@ -1612,6 +1685,13 @@ def add_newdoc(place, name, doc): >>> np.invert(np.array([True, False])) array([False, True]) + The ``~`` operator can be used as a shorthand for ``np.invert`` on + ndarrays. + + >>> x1 = np.array([True, False]) + >>> ~x1 + array([False, True]) + """) add_newdoc('numpy.core.umath', 'isfinite', @@ -1846,6 +1926,14 @@ def add_newdoc(place, name, doc): >>> print(b, type(b)) 254 + The ``<<`` operator can be used as a shorthand for ``np.left_shift`` on + ndarrays. + + >>> x1 = 5 + >>> x2 = np.array([1, 2, 3]) + >>> x1 << x2 + array([10, 20, 40]) + """) add_newdoc('numpy.core.umath', 'less', @@ -1875,11 +1963,18 @@ def add_newdoc(place, name, doc): >>> np.less([1, 2], [2, 2]) array([ True, False]) + The ``<`` operator can be used as a shorthand for ``np.less`` on ndarrays. + + >>> a = np.array([1, 2]) + >>> b = np.array([2, 2]) + >>> a < b + array([ True, False]) + """) add_newdoc('numpy.core.umath', 'less_equal', """ - Return the truth value of (x1 =< x2) element-wise. + Return the truth value of (x1 <= x2) element-wise. Parameters ---------- @@ -1904,6 +1999,14 @@ def add_newdoc(place, name, doc): >>> np.less_equal([4, 2, 1], [2, 2, 2]) array([False, True, True]) + The ``<=`` operator can be used as a shorthand for ``np.less_equal`` on + ndarrays. + + >>> a = np.array([4, 2, 1]) + >>> b = np.array([2, 2, 2]) + >>> a <= b + array([False, True, True]) + """) add_newdoc('numpy.core.umath', 'log', @@ -2231,6 +2334,15 @@ def add_newdoc(place, name, doc): >>> np.logical_and(x>1, x<4) array([False, False, True, True, False]) + + The ``&`` operator can be used as a shorthand for ``np.logical_and`` on + boolean ndarrays. + + >>> a = np.array([True, False]) + >>> b = np.array([False, False]) + >>> a & b + array([False, False]) + """) add_newdoc('numpy.core.umath', 'logical_not', @@ -2301,6 +2413,14 @@ def add_newdoc(place, name, doc): >>> np.logical_or(x < 1, x > 3) array([ True, False, False, False, True]) + The ``|`` operator can be used as a shorthand for ``np.logical_or`` on + boolean ndarrays. + + >>> a = np.array([True, False]) + >>> b = np.array([False, False]) + >>> a | b + array([ True, False]) + """) add_newdoc('numpy.core.umath', 'logical_xor', @@ -2738,6 +2858,14 @@ def add_newdoc(place, name, doc): ... ValueError: matmul: Input operand 1 does not have enough dimensions ... + The ``@`` operator can be used as a shorthand for ``np.matmul`` on + ndarrays. + + >>> x1 = np.array([2j, 3j]) + >>> x2 = np.array([2j, 3j]) + >>> x1 @ x2 + (-13+0j) + .. versionadded:: 1.10.0 """) @@ -2814,6 +2942,16 @@ def add_newdoc(place, name, doc): [ 0., 4., 10.], [ 0., 7., 16.]]) + The ``*`` operator can be used as a shorthand for ``np.multiply`` on + ndarrays. + + >>> x1 = np.arange(9.0).reshape((3, 3)) + >>> x2 = np.arange(3.0) + >>> x1 * x2 + array([[ 0., 1., 4.], + [ 0., 4., 10.], + [ 0., 7., 16.]]) + """) add_newdoc('numpy.core.umath', 'negative', @@ -2837,6 +2975,13 @@ def add_newdoc(place, name, doc): >>> np.negative([1.,-1.]) array([-1., 1.]) + The unary ``-`` operator can be used as a shorthand for ``np.negative`` on + ndarrays. + + >>> x1 = np.array(([1., -1.])) + >>> -x1 + array([-1., 1.]) + """) add_newdoc('numpy.core.umath', 'positive', @@ -2861,6 +3006,20 @@ def add_newdoc(place, name, doc): Equivalent to `x.copy()`, but only defined for types that support arithmetic. + Examples + -------- + + >>> x1 = np.array(([1., -1.])) + >>> np.positive(x1) + array([ 1., -1.]) + + The unary ``+`` operator can be used as a shorthand for ``np.positive`` on + ndarrays. + + >>> x1 = np.array(([1., -1.])) + >>> +x1 + array([ 1., -1.]) + """) add_newdoc('numpy.core.umath', 'not_equal', @@ -2893,6 +3052,15 @@ def add_newdoc(place, name, doc): array([[False, True], [False, True]]) + The ``!=`` operator can be used as a shorthand for ``np.not_equal`` on + ndarrays. + + >>> a = np.array([1., 2.]) + >>> b = np.array([1., 3.]) + >>> a != b + array([False, True]) + + """) add_newdoc('numpy.core.umath', '_ones_like', @@ -2936,9 +3104,9 @@ def add_newdoc(place, name, doc): Examples -------- - Cube each element in a list. + Cube each element in an array. - >>> x1 = range(6) + >>> x1 = np.arange(6) >>> x1 [0, 1, 2, 3, 4, 5] >>> np.power(x1, 3) @@ -2960,6 +3128,14 @@ def add_newdoc(place, name, doc): array([[ 0, 1, 8, 27, 16, 5], [ 0, 1, 8, 27, 16, 5]]) + The ``**`` operator can be used as a shorthand for ``np.power`` on + ndarrays. + + >>> x2 = np.array([1, 2, 3, 3, 2, 1]) + >>> x1 = np.arange(6) + >>> x1 ** x2 + array([ 0, 1, 8, 27, 16, 5]) + """) add_newdoc('numpy.core.umath', 'float_power', @@ -3183,6 +3359,13 @@ def add_newdoc(place, name, doc): >>> np.remainder(np.arange(7), 5) array([0, 1, 2, 3, 4, 0, 1]) + The ``%`` operator can be used as a shorthand for ``np.remainder`` on + ndarrays. + + >>> x1 = np.arange(7) + >>> x1 % 5 + array([0, 1, 2, 3, 4, 0, 1]) + """) add_newdoc('numpy.core.umath', 'divmod', @@ -3225,6 +3408,13 @@ def add_newdoc(place, name, doc): >>> np.divmod(np.arange(5), 3) (array([0, 0, 0, 1, 1]), array([0, 1, 2, 0, 1])) + The `divmod` function can be used as a shorthand for ``np.divmod`` on + ndarrays. + + >>> x = np.arange(5) + >>> divmod(x, 3) + (array([0, 0, 0, 1, 1]), array([0, 1, 2, 0, 1])) + """) add_newdoc('numpy.core.umath', 'right_shift', @@ -3268,6 +3458,14 @@ def add_newdoc(place, name, doc): >>> np.right_shift(10, [1,2,3]) array([5, 2, 1]) + The ``>>`` operator can be used as a shorthand for ``np.right_shift`` on + ndarrays. + + >>> x1 = 10 + >>> x2 = np.array([1,2,3]) + >>> x1 >> x2 + array([5, 2, 1]) + """) add_newdoc('numpy.core.umath', 'rint', @@ -3709,6 +3907,16 @@ def add_newdoc(place, name, doc): [ 3., 3., 3.], [ 6., 6., 6.]]) + The ``-`` operator can be used as a shorthand for ``np.subtract`` on + ndarrays. + + >>> x1 = np.arange(9.0).reshape((3, 3)) + >>> x2 = np.arange(3.0) + >>> x1 - x2 + array([[0., 0., 0.], + [3., 3., 3.], + [6., 6., 6.]]) + """) add_newdoc('numpy.core.umath', 'tan', @@ -3851,6 +4059,14 @@ def add_newdoc(place, name, doc): >>> x//4 array([0, 0, 0, 0, 1]) + + The ``/`` operator can be used as a shorthand for ``np.true_divide`` on + ndarrays. + + >>> x = np.arange(5) + >>> x / 4 + array([0. , 0.25, 0.5 , 0.75, 1. ]) + """) add_newdoc('numpy.core.umath', 'frexp', From 028a61af17b2b3d960323fc0a9595bdfd91b9a7e Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 23 Jul 2020 11:14:38 -0500 Subject: [PATCH 304/409] BUG: Fix memory leak of buffer-info cache due to relaxed strides When relaxed strides is active (and has an effect), we recalculate the strides to export "clean" strides in the buffer interface. (Python and probably some other exporters expect this, i.e. NumPy has fully switched to and embraced relaxed strides, but the buffer interface at large probably not.) The place where "fixing" the strides occured however meant that when the strides are fixed, the old, cached buffer-info was not reused when it should have been reused. This moves the "fixing" logic so that reuse will occur. It leaves one issue open in that an array shaped e.g. `(1, 10)` is both C- and F-contiguous. Thus, if it is exported as C-contiguous and then as F-contiguous, and then *again* as C-contiguous, this will work, but the last export will compare to the export as an F-contig buffer and thus still leak memory. Address gh-16934 (but does leave a small hole) --- numpy/core/src/multiarray/buffer.c | 86 ++++++++++++++++------------- numpy/core/tests/test_multiarray.py | 7 ++- 2 files changed, 53 insertions(+), 40 deletions(-) diff --git a/numpy/core/src/multiarray/buffer.c b/numpy/core/src/multiarray/buffer.c index af40cdc2c911..770813e7012e 100644 --- a/numpy/core/src/multiarray/buffer.c +++ b/numpy/core/src/multiarray/buffer.c @@ -456,7 +456,7 @@ static PyObject *_buffer_info_cache = NULL; /* Fill in the info structure */ static _buffer_info_t* -_buffer_info_new(PyObject *obj) +_buffer_info_new(PyObject *obj, npy_bool f_contiguous) { /* * Note that the buffer info is cached as PyLongObjects making them appear @@ -504,9 +504,43 @@ _buffer_info_new(PyObject *obj) info->shape = (npy_intp *)((char *)info + sizeof(_buffer_info_t)); assert((size_t)info->shape % sizeof(npy_intp) == 0); info->strides = info->shape + PyArray_NDIM(arr); - for (k = 0; k < PyArray_NDIM(arr); ++k) { - info->shape[k] = PyArray_DIMS(arr)[k]; - info->strides[k] = PyArray_STRIDES(arr)[k]; + +#if NPY_RELAXED_STRIDES_CHECKING + /* + * When NPY_RELAXED_STRIDES_CHECKING is used, some buffer users + * may expect a contiguous buffer to have well formatted strides + * also when a dimension is 1, but we do not guarantee this + * internally. Thus, recalculate strides for contiguous arrays. + * (This is unnecessary, but has no effect in the case where + * NPY_RELAXED_STRIDES CHECKING is disabled.) + */ + if (PyArray_IS_C_CONTIGUOUS(arr) && !( + f_contiguous && PyArray_IS_F_CONTIGUOUS(arr))) { + Py_ssize_t sd = PyArray_ITEMSIZE(arr); + for (k = info->ndim-1; k >= 0; --k) { + info->shape[k] = PyArray_DIMS(arr)[k]; + info->strides[k] = sd; + sd *= info->shape[k]; + } + } + else if (PyArray_IS_F_CONTIGUOUS(arr)) { + Py_ssize_t sd = PyArray_ITEMSIZE(arr); + for (k = 0; k < info->ndim; ++k) { + info->shape[k] = PyArray_DIMS(arr)[k]; + info->strides[k] = sd; + sd *= info->shape[k]; + } + } + else { +#else /* NPY_RELAXED_STRIDES_CHECKING */ + /* We can always use the arrays strides directly */ + { +#endif + + for (k = 0; k < PyArray_NDIM(arr); ++k) { + info->shape[k] = PyArray_DIMS(arr)[k]; + info->strides[k] = PyArray_STRIDES(arr)[k]; + } } } Py_INCREF(descr); @@ -565,7 +599,7 @@ _buffer_info_free(_buffer_info_t *info) /* Get buffer info from the global dictionary */ static _buffer_info_t* -_buffer_get_info(PyObject *obj) +_buffer_get_info(PyObject *obj, npy_bool f_contiguous) { PyObject *key = NULL, *item_list = NULL, *item = NULL; _buffer_info_t *info = NULL, *old_info = NULL; @@ -578,7 +612,7 @@ _buffer_get_info(PyObject *obj) } /* Compute information */ - info = _buffer_info_new(obj); + info = _buffer_info_new(obj, f_contiguous); if (info == NULL) { return NULL; } @@ -596,6 +630,13 @@ _buffer_get_info(PyObject *obj) item = PyList_GetItem(item_list, PyList_GET_SIZE(item_list) - 1); old_info = (_buffer_info_t*)PyLong_AsVoidPtr(item); + /* + * TODO: In rare cases when an array is both C- and + * F-contiguous, and buffers are exported alternating + * we may always have a cache miss even though the correct + * info had already been exported before (causing a memory + * leak). + */ if (_buffer_info_cmp(info, old_info) == 0) { _buffer_info_free(info); info = old_info; @@ -706,7 +747,7 @@ array_getbuffer(PyObject *obj, Py_buffer *view, int flags) } /* Fill in information */ - info = _buffer_get_info(obj); + info = _buffer_get_info(obj, (flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS); if (info == NULL) { goto fail; } @@ -742,35 +783,6 @@ array_getbuffer(PyObject *obj, Py_buffer *view, int flags) } if ((flags & PyBUF_STRIDES) == PyBUF_STRIDES) { view->strides = info->strides; - -#ifdef NPY_RELAXED_STRIDES_CHECKING - /* - * If NPY_RELAXED_STRIDES_CHECKING is on, the array may be - * contiguous, but it won't look that way to Python when it - * tries to determine contiguity by looking at the strides - * (since one of the elements may be -1). In that case, just - * regenerate strides from shape. - */ - if (PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS) && - !((flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS)) { - Py_ssize_t sd = view->itemsize; - int i; - - for (i = view->ndim-1; i >= 0; --i) { - view->strides[i] = sd; - sd *= view->shape[i]; - } - } - else if (PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)) { - Py_ssize_t sd = view->itemsize; - int i; - - for (i = 0; i < view->ndim; ++i) { - view->strides[i] = sd; - sd *= view->shape[i]; - } - } -#endif } else { view->strides = NULL; @@ -800,7 +812,7 @@ void_getbuffer(PyObject *self, Py_buffer *view, int flags) } /* Fill in information */ - info = _buffer_get_info(self); + info = _buffer_get_info(self, 0); if (info == NULL) { goto fail; } diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 6f8af175703c..d376fe6b7a8a 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -7178,9 +7178,10 @@ def test_padded_struct_array(self): x3 = np.arange(dt3.itemsize, dtype=np.int8).view(dt3) self._check_roundtrip(x3) - def test_relaxed_strides(self): - # Test that relaxed strides are converted to non-relaxed - c = np.ones((1, 10, 10), dtype='i8') + @pytest.mark.valgrind_error(reason="leaks buffer info cache temporarily.") + def test_relaxed_strides(self, c=np.ones((1, 10, 10), dtype='i8')): + # Note: c defined as parameter so that it is persistent and leak + # checks will notice gh-16934 (buffer info cache leak). # Check for NPY_RELAXED_STRIDES_CHECKING: if np.ones((10, 1), order="C").flags.f_contiguous: From 78080362911a36a2f4eff99a41b42dfb4c18c39c Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 13 Aug 2020 15:22:52 -0500 Subject: [PATCH 305/409] BUG: Fix leak for relaxed strides when exporting both C- and F-order Exporting these multiple times alternating would previously cause a new buffer-info to be created each time. --- numpy/core/src/multiarray/buffer.c | 30 ++++++++++++++++++++--------- numpy/core/tests/test_multiarray.py | 15 +++++++++++++++ 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/numpy/core/src/multiarray/buffer.c b/numpy/core/src/multiarray/buffer.c index 770813e7012e..6415dc276a7c 100644 --- a/numpy/core/src/multiarray/buffer.c +++ b/numpy/core/src/multiarray/buffer.c @@ -625,22 +625,34 @@ _buffer_get_info(PyObject *obj, npy_bool f_contiguous) item_list = PyDict_GetItem(_buffer_info_cache, key); if (item_list != NULL) { + Py_ssize_t item_list_length = PyList_GET_SIZE(item_list); Py_INCREF(item_list); - if (PyList_GET_SIZE(item_list) > 0) { - item = PyList_GetItem(item_list, PyList_GET_SIZE(item_list) - 1); + if (item_list_length > 0) { + item = PyList_GetItem(item_list, item_list_length - 1); old_info = (_buffer_info_t*)PyLong_AsVoidPtr(item); - /* - * TODO: In rare cases when an array is both C- and - * F-contiguous, and buffers are exported alternating - * we may always have a cache miss even though the correct - * info had already been exported before (causing a memory - * leak). - */ if (_buffer_info_cmp(info, old_info) == 0) { _buffer_info_free(info); info = old_info; } + else if (item_list_length > 1 && info->ndim > 1) { + /* + * Some arrays are C- and F-contiguous and if they have more + * than one dimension, the buffer-info may differ between the + * two due to RELAXED_STRIDES_CHECKING. + * If we export both buffers, the first stored one may be the + * one for the other contiguity, so check both in this case. + * This is generally very unlikely in all other cases, since + * in all other cases the first one will match unless array + * metadata was modified in-place (which is discouraged). + */ + item = PyList_GetItem(item_list, item_list_length - 2); + old_info = (_buffer_info_t*)PyLong_AsVoidPtr(item); + if (_buffer_info_cmp(info, old_info) == 0) { + _buffer_info_free(info); + info = old_info; + } + } } } else { diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index d376fe6b7a8a..c80d0c073691 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -7206,6 +7206,21 @@ def test_relaxed_strides(self, c=np.ones((1, 10, 10), dtype='i8')): arr, ['C_CONTIGUOUS']) assert_(strides[-1] == 8) + @pytest.mark.valgrind_error(reason="leaks buffer info cache temporarily.") + @pytest.mark.skipif(not np.ones((10, 1), order="C").flags.f_contiguous, + reason="Test is unnecessary (but fails) without relaxed strides.") + def test_relaxed_strides_buffer_info_leak(self, arr=np.ones((1, 10))): + """Test that alternating export of C- and F-order buffers from + an array which is both C- and F-order when relaxed strides is + active works. + This test defines array in the signature to ensure leaking more + references every time the test is run (catching the leak with + pytest-leaks). + """ + for i in range(10): + _multiarray_tests.get_buffer_info(arr, ['F_CONTIGUOUS']) + _multiarray_tests.get_buffer_info(arr, ['C_CONTIGUOUS']) + def test_out_of_order_fields(self): dt = np.dtype(dict( formats=[' Date: Fri, 11 Sep 2020 09:59:19 -0500 Subject: [PATCH 306/409] MAINT: simplify buffer info reuse check --- numpy/core/src/multiarray/buffer.c | 39 +++++++++++++++--------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/numpy/core/src/multiarray/buffer.c b/numpy/core/src/multiarray/buffer.c index 6415dc276a7c..ca88e070f77c 100644 --- a/numpy/core/src/multiarray/buffer.c +++ b/numpy/core/src/multiarray/buffer.c @@ -631,28 +631,29 @@ _buffer_get_info(PyObject *obj, npy_bool f_contiguous) item = PyList_GetItem(item_list, item_list_length - 1); old_info = (_buffer_info_t*)PyLong_AsVoidPtr(item); - if (_buffer_info_cmp(info, old_info) == 0) { + if (_buffer_info_cmp(info, old_info) != 0) { + if (item_list_length > 1 && info->ndim > 1) { + /* + * Some arrays are C- and F-contiguous and if they have more + * than one dimension, the buffer-info may differ between + * the two due to RELAXED_STRIDES_CHECKING. + * If we export both buffers, the first stored one may be + * the one for the other contiguity, so check both. + * This is generally very unlikely in all other cases, since + * in all other cases the first one will match unless array + * metadata was modified in-place (which is discouraged). + */ + item = PyList_GetItem(item_list, item_list_length - 2); + old_info = (_buffer_info_t*)PyLong_AsVoidPtr(item); + if (_buffer_info_cmp(info, old_info) != 0) { + old_info = NULL; + } + } + } + if (old_info != NULL) { _buffer_info_free(info); info = old_info; } - else if (item_list_length > 1 && info->ndim > 1) { - /* - * Some arrays are C- and F-contiguous and if they have more - * than one dimension, the buffer-info may differ between the - * two due to RELAXED_STRIDES_CHECKING. - * If we export both buffers, the first stored one may be the - * one for the other contiguity, so check both in this case. - * This is generally very unlikely in all other cases, since - * in all other cases the first one will match unless array - * metadata was modified in-place (which is discouraged). - */ - item = PyList_GetItem(item_list, item_list_length - 2); - old_info = (_buffer_info_t*)PyLong_AsVoidPtr(item); - if (_buffer_info_cmp(info, old_info) == 0) { - _buffer_info_free(info); - info = old_info; - } - } } } else { From c42e6904b723c096cdebd9abbb877ed5b4f00b20 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Mon, 19 Oct 2020 23:11:55 -0500 Subject: [PATCH 307/409] Fixup, an earlier cleanup got things wrong and I forgot to test the strides :( --- numpy/core/src/multiarray/buffer.c | 16 ++++++++-------- numpy/core/tests/test_multiarray.py | 6 ++++-- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/numpy/core/src/multiarray/buffer.c b/numpy/core/src/multiarray/buffer.c index ca88e070f77c..e676682deec2 100644 --- a/numpy/core/src/multiarray/buffer.c +++ b/numpy/core/src/multiarray/buffer.c @@ -630,8 +630,11 @@ _buffer_get_info(PyObject *obj, npy_bool f_contiguous) if (item_list_length > 0) { item = PyList_GetItem(item_list, item_list_length - 1); old_info = (_buffer_info_t*)PyLong_AsVoidPtr(item); - - if (_buffer_info_cmp(info, old_info) != 0) { + if (_buffer_info_cmp(info, old_info) == 0) { + _buffer_info_free(info); + info = old_info; + } + else { if (item_list_length > 1 && info->ndim > 1) { /* * Some arrays are C- and F-contiguous and if they have more @@ -645,15 +648,12 @@ _buffer_get_info(PyObject *obj, npy_bool f_contiguous) */ item = PyList_GetItem(item_list, item_list_length - 2); old_info = (_buffer_info_t*)PyLong_AsVoidPtr(item); - if (_buffer_info_cmp(info, old_info) != 0) { - old_info = NULL; + if (_buffer_info_cmp(info, old_info) == 0) { + _buffer_info_free(info); + info = old_info; } } } - if (old_info != NULL) { - _buffer_info_free(info); - info = old_info; - } } } else { diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index c80d0c073691..d46e4ce9b840 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -7218,8 +7218,10 @@ def test_relaxed_strides_buffer_info_leak(self, arr=np.ones((1, 10))): pytest-leaks). """ for i in range(10): - _multiarray_tests.get_buffer_info(arr, ['F_CONTIGUOUS']) - _multiarray_tests.get_buffer_info(arr, ['C_CONTIGUOUS']) + _, s = _multiarray_tests.get_buffer_info(arr, ['F_CONTIGUOUS']) + assert s == (8, 8) + _, s = _multiarray_tests.get_buffer_info(arr, ['C_CONTIGUOUS']) + assert s == (80, 8) def test_out_of_order_fields(self): dt = np.dtype(dict( From 75927fcd2fe2613daa372638a8539dea4c1b29be Mon Sep 17 00:00:00 2001 From: andryandrew Date: Tue, 20 Oct 2020 16:02:17 +0200 Subject: [PATCH 308/409] DOC: Typo in lexsort docstring ...it's rows... --> ...its rows... --- numpy/core/multiarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/multiarray.py b/numpy/core/multiarray.py index 6b335f1a64bb..d293700f2ee9 100644 --- a/numpy/core/multiarray.py +++ b/numpy/core/multiarray.py @@ -406,7 +406,7 @@ def lexsort(keys, axis=None): for the primary sort order, the second-to-last key for the secondary sort order, and so on. The keys argument must be a sequence of objects that can be converted to arrays of the same shape. If a 2D array is provided - for the keys argument, it's rows are interpreted as the sorting keys and + for the keys argument, its rows are interpreted as the sorting keys and sorting is according to the last row, second last row etc. Parameters From 30adfe2db06060fdbe60d0cd73b892aa8fa29977 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Wed, 21 Oct 2020 01:06:02 +0200 Subject: [PATCH 309/409] TST: Clean up the errors of the typing tests --- numpy/typing/tests/test_typing.py | 47 +++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/numpy/typing/tests/test_typing.py b/numpy/typing/tests/test_typing.py index cba1dc1be7eb..f5cc29c2a7ba 100644 --- a/numpy/typing/tests/test_typing.py +++ b/numpy/typing/tests/test_typing.py @@ -3,6 +3,7 @@ import os import re from collections import defaultdict +from typing import Optional import pytest try: @@ -55,6 +56,8 @@ def test_success(path): @pytest.mark.skipif(NO_MYPY, reason="Mypy is not installed") @pytest.mark.parametrize("path", get_test_cases(FAIL_DIR)) def test_fail(path): + __tracebackhide__ = True + stdout, stderr, exitcode = api.run([ "--config-file", MYPY_INI, @@ -95,16 +98,37 @@ def test_fail(path): target_line = lines[lineno - 1] if "# E:" in target_line: marker = target_line.split("# E:")[-1].strip() - assert lineno in errors, f'Extra error "{marker}"' - assert marker in errors[lineno] + expected_error = errors.get(lineno) + _test_fail(path, marker, expected_error, lineno) else: pytest.fail(f"Error {repr(errors[lineno])} not found") +_FAIL_MSG1 = """Extra error at line {} + +Extra error: {!r} +""" + +_FAIL_MSG2 = """Error mismatch at line {} + +Expected error: {!r} +Observed error: {!r} +""" + + +def _test_fail(path: str, error: str, expected_error: Optional[str], lineno: int) -> None: + if expected_error is None: + raise AssertionError(_FAIL_MSG1.format(lineno, error)) + elif error not in expected_error: + raise AssertionError(_FAIL_MSG2.format(lineno, expected_error, error)) + + @pytest.mark.slow @pytest.mark.skipif(NO_MYPY, reason="Mypy is not installed") @pytest.mark.parametrize("path", get_test_cases(REVEAL_DIR)) def test_reveal(path): + __tracebackhide__ = True + stdout, stderr, exitcode = api.run([ "--config-file", MYPY_INI, @@ -127,10 +151,23 @@ def test_reveal(path): ) if match is None: raise ValueError(f"Unexpected reveal line format: {error_line}") - lineno = int(match.group('lineno')) + lineno = int(match.group('lineno')) - 1 assert "Revealed type is" in error_line - marker = lines[lineno - 1].split("# E:")[-1].strip() - assert marker in error_line + + marker = lines[lineno].split("# E:")[-1].strip() + _test_reveal(path, marker, error_line, lineno) + + +_REVEAL_MSG = """Reveal mismatch at line {} + +Expected reveal: {!r} +Observed reveal: {!r} +""" + + +def _test_reveal(path: str, reveal: str, expected_reveal: str, lineno: int) -> None: + if reveal not in expected_reveal: + raise AssertionError(_REVEAL_MSG.format(lineno, expected_reveal, reveal)) @pytest.mark.slow From 3a6857c99d9012650b8b58726783eee30308d944 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Wed, 21 Oct 2020 13:01:11 +0200 Subject: [PATCH 310/409] MAINT: Update the `axes` parameter of `tensordot` `axes` can take either an integer or a 2-tuple of shape-like objects --- numpy/core/numeric.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/numeric.pyi b/numpy/core/numeric.pyi index b341d136a10f..f917f74dec41 100644 --- a/numpy/core/numeric.pyi +++ b/numpy/core/numeric.pyi @@ -126,7 +126,7 @@ def outer(a: ArrayLike, b: ArrayLike, out: _ArrayType = ...) -> _ArrayType: ... def tensordot( a: ArrayLike, b: ArrayLike, - axes: _ShapeLike = ..., + axes: Union[int, Tuple[_ShapeLike, _ShapeLike]] = ..., ) -> ndarray: ... def roll( a: ArrayLike, From 6161fd97eed9c6244a7adf343b8df2291df56a76 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Wed, 21 Oct 2020 16:03:23 +0200 Subject: [PATCH 311/409] TST: Fixed a broken `np.core.numeric` test --- numpy/core/numeric.pyi | 4 ++-- numpy/typing/tests/data/reveal/numeric.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/numpy/core/numeric.pyi b/numpy/core/numeric.pyi index f917f74dec41..de80cfe08c31 100644 --- a/numpy/core/numeric.pyi +++ b/numpy/core/numeric.pyi @@ -12,7 +12,7 @@ from typing import ( Iterable, ) -from numpy import ndarray, generic, dtype, bool_, int32, int64, _OrderKACF, _OrderCF +from numpy import ndarray, generic, dtype, bool_, signedinteger, _OrderKACF, _OrderCF from numpy.typing import ArrayLike, DtypeLike, _ShapeLike if sys.version_info >= (3, 8): @@ -113,7 +113,7 @@ def count_nonzero( @overload def count_nonzero( a: ArrayLike, axis: _ShapeLike = ..., *, keepdims: bool = ... -) -> Union[int64, int32, ndarray]: ... +) -> Union[signedinteger[Any], ndarray]: ... # TODO: np.intp def isfortran(a: Union[ndarray, generic]) -> bool: ... def argwhere(a: ArrayLike) -> ndarray: ... def flatnonzero(a: ArrayLike) -> ndarray: ... diff --git a/numpy/typing/tests/data/reveal/numeric.py b/numpy/typing/tests/data/reveal/numeric.py index 6bd4ed9fdd1e..5cbfa4ac765c 100644 --- a/numpy/typing/tests/data/reveal/numeric.py +++ b/numpy/typing/tests/data/reveal/numeric.py @@ -20,8 +20,8 @@ class SubClass(np.ndarray): reveal_type(np.count_nonzero(i8)) # E: int reveal_type(np.count_nonzero(A)) # E: int reveal_type(np.count_nonzero(B)) # E: int -reveal_type(np.count_nonzero(A, keepdims=True)) # E: Union[numpy.int64, numpy.int32, numpy.ndarray] -reveal_type(np.count_nonzero(A, axis=0)) # E: Union[numpy.int64, numpy.int32, numpy.ndarray] +reveal_type(np.count_nonzero(A, keepdims=True)) # E: Union[numpy.signedinteger[Any], numpy.ndarray] +reveal_type(np.count_nonzero(A, axis=0)) # E: Union[numpy.signedinteger[Any], numpy.ndarray] reveal_type(np.isfortran(i8)) # E: bool reveal_type(np.isfortran(A)) # E: bool From 250ab1270ad54f133816956bc64de39b7533968c Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Wed, 21 Oct 2020 10:12:24 -0600 Subject: [PATCH 312/409] MAINT: Mark dead code as intentional for clang. Clang is giving dead code warnings in places where we have intentionally marked code to be ignored by inserting a zero in an if statement. --- numpy/core/src/multiarray/array_coercion.c | 2 +- numpy/core/src/npysort/selection.c.src | 3 ++- numpy/core/src/umath/loops.c.src | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/numpy/core/src/multiarray/array_coercion.c b/numpy/core/src/multiarray/array_coercion.c index 4831dfca6552..ffb9bdbe848e 100644 --- a/numpy/core/src/multiarray/array_coercion.c +++ b/numpy/core/src/multiarray/array_coercion.c @@ -298,7 +298,7 @@ discover_dtype_from_pyobject( Py_INCREF(DType); Py_DECREF(legacy_descr); /* TODO: Enable warning about subclass handling */ - if (0 && !((*flags) & GAVE_SUBCLASS_WARNING)) { + if ((0) && !((*flags) & GAVE_SUBCLASS_WARNING)) { if (DEPRECATE_FUTUREWARNING( "in the future NumPy will not automatically find the " "dtype for subclasses of scalars known to NumPy (i.e. " diff --git a/numpy/core/src/npysort/selection.c.src b/numpy/core/src/npysort/selection.c.src index 4fd955200404..0e285b320b91 100644 --- a/numpy/core/src/npysort/selection.c.src +++ b/numpy/core/src/npysort/selection.c.src @@ -323,7 +323,8 @@ NPY_NO_EXPORT int store_pivot(kth, kth, pivots, npiv); return 0; } - else if (@inexact@ && kth == num - 1) { + // Parenthesis around @inexact@ tells clang dead code as intentional + else if ((@inexact@) && kth == num - 1) { /* useful to check if NaN present via partition(d, (x, -1)) */ npy_intp k; npy_intp maxidx = low; diff --git a/numpy/core/src/umath/loops.c.src b/numpy/core/src/umath/loops.c.src index d9591ab3353b..ef3d5a21a413 100644 --- a/numpy/core/src/umath/loops.c.src +++ b/numpy/core/src/umath/loops.c.src @@ -2647,7 +2647,8 @@ pairwise_sum_@TYPE@(@ftype@ *rr, @ftype@ * ri, char * a, npy_intp n, NPY_NO_EXPORT void @TYPE@_@kind@(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { - if (IS_BINARY_REDUCE && @PW@) { + // Parenthesis around @PW@ tells clang dead code is intentional + if (IS_BINARY_REDUCE && (@PW@)) { npy_intp n = dimensions[0]; @ftype@ * or = ((@ftype@ *)args[0]); @ftype@ * oi = ((@ftype@ *)args[0]) + 1; From fcc394033c5902bbd8104694c5f7d33a8b5eb99f Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 20 Oct 2020 16:35:09 -0500 Subject: [PATCH 313/409] DEP,BUG: Coercion/cast of array to a subarray dtype will be fixed This currently appends the subarray dtype dimensions first and then tries to assign to the result array which uses incorrect broadcasting (broadcasting against the subarray dimensions instead of repeating each element according to the subarray dimensions). This also fixes the python scalar pathway `np.array(2, dtype="(2)f4,")` which previously only filled the first value. I consider that a clear bug fix. Closes gh-17511 --- .../upcoming_changes/17419.deprecation.rst | 24 ---- doc/release/upcoming_changes/17596.future.rst | 30 +++++ numpy/core/src/multiarray/ctors.c | 111 ++++++++++++------ numpy/core/src/multiarray/methods.c | 21 ++-- numpy/core/tests/test_deprecations.py | 56 ++++----- numpy/core/tests/test_dtype.py | 18 +++ 6 files changed, 165 insertions(+), 95 deletions(-) delete mode 100644 doc/release/upcoming_changes/17419.deprecation.rst create mode 100644 doc/release/upcoming_changes/17596.future.rst diff --git a/doc/release/upcoming_changes/17419.deprecation.rst b/doc/release/upcoming_changes/17419.deprecation.rst deleted file mode 100644 index fcab3a8adef9..000000000000 --- a/doc/release/upcoming_changes/17419.deprecation.rst +++ /dev/null @@ -1,24 +0,0 @@ -Arrays cannot be using subarray dtypes --------------------------------------- -Array creation and casting using ``np.array(obj, dtype)`` -and ``arr.astype(dtype)`` will not support ``dtype`` -to be a subarray dtype such as ``np.dtype("(2)i,")``. - -For such a ``dtype`` the following behaviour occurs currently:: - - res = np.array(obj, dtype) - - res.dtype is not dtype - res.dtype is dtype.base - res.shape[-dtype.ndim:] == dtype.shape - -The shape of the dtype is included into the array. -This leads to inconsistencies when ``obj`` is: - -* a scalar, such as ``np.array(1, dtype="(2)i")`` -* an array, such as ``np.array(np.array([1]), dtype="(2)i")`` - -In most cases the work-around is to pass the output dtype directly -and possibly check ``res.shape[-dtype.ndim:] == dtype.shape``. -If this is insufficient, please open an issue on the NumPy issue -tracker. diff --git a/doc/release/upcoming_changes/17596.future.rst b/doc/release/upcoming_changes/17596.future.rst new file mode 100644 index 000000000000..6e697c8d1f76 --- /dev/null +++ b/doc/release/upcoming_changes/17596.future.rst @@ -0,0 +1,30 @@ +Arrays cannot be using subarray dtypes +-------------------------------------- +Array creation and casting using ``np.array(arr, dtype)`` +and ``arr.astype(dtype)`` will use different logic when ``dtype`` +is a subarray dtype such as ``np.dtype("(2)i,")``. + +For such a ``dtype`` the following behaviour is true:: + + res = np.array(arr, dtype) + + res.dtype is not dtype + res.dtype is dtype.base + res.shape == arr.shape + dtype.shape + +But ``res`` is filled using the logic: + + res = np.empty(arr.shape + dtype.shape, dtype=dtype.base) + res[...] = arr + +which uses incorrect broadcasting (and often leads to an error). +In the future, this will instead cast each element individually, +leading to the same result as:: + + res = np.array(arr, dtype=np.dtype(["f", dtype]))["f"] + +Which can normally be used to opt-in to the new behaviour. + +This change does not affect ``np.array(list, dtype="(2)i,")`` unless the +``list`` itself includes at least one array. In particular, the behaviour +is unchanged for a list of tuples. diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 05e45fbf52c5..ff262369bff6 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1565,49 +1565,66 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, if (NPY_UNLIKELY(fixed_descriptor != NULL && PyDataType_HASSUBARRAY(dtype))) { /* - * When a subarray dtype was passed in, its dimensions are absorbed - * into the array dimension (causing a dimension mismatch). - * We can't reasonably handle this because of inconsistencies in - * how it was handled (depending on nested list vs. embed array-likes). - * So we give a deprecation warning and fall back to legacy code. + * When a subarray dtype was passed in, its dimensions are appended + * to the array dimension (causing a dimension mismatch). + * There is a problem with that, because if we coerce from non-arrays + * we do this correctly by element (as defined by tuples), but for + * arrays we first append the dimensions and then assign to the base + * dtype and then assign which causes the problem. + * + * Thus, we check if there is an array included, in that case we + * give a FutureWarning. + * When the warning is removed, PyArray_Pack will have to ensure + * that that it does not append the dimensions when creating the + * subarrays to assign `arr[0] = obj[0]`. */ - ret = (PyArrayObject *)PyArray_NewFromDescr( - &PyArray_Type, dtype, ndim, dims, NULL, NULL, - flags&NPY_ARRAY_F_CONTIGUOUS, NULL); - if (ret == NULL) { - npy_free_coercion_cache(cache); - return NULL; + int includes_array = 0; + if (cache != NULL) { + /* This is not ideal, but it is a pretty special case */ + coercion_cache_obj *next = cache; + while (next != NULL) { + if (!next->sequence) { + includes_array = 1; + break; + } + next = next->next; + } } - assert(PyArray_NDIM(ret) != ndim); + if (includes_array) { + npy_free_coercion_cache(cache); - if (cache == NULL) { - /* This is a single item. Sets only first subarray element. */ - assert(ndim == 0); - if (PyArray_Pack(PyArray_DESCR(ret), PyArray_DATA(ret), op) < 0) { + ret = (PyArrayObject *) PyArray_NewFromDescr( + &PyArray_Type, dtype, ndim, dims, NULL, NULL, + flags & NPY_ARRAY_F_CONTIGUOUS, NULL); + if (ret == NULL) { + return NULL; + } + assert(PyArray_NDIM(ret) != ndim); + + /* NumPy 1.20, 2020-10-01 */ + if (DEPRECATE_FUTUREWARNING( + "creating an array with a subarray dtype will behave " + "differently when the `np.array()` (or `asarray`, etc.) " + "call includes an array or array object.\n" + "If you are converting a single array or a list of arrays," + "you can opt-in to the future behaviour using:\n" + " np.array(arr, dtype=np.dtype(['f', dtype]))['f']\n" + " np.array([arr1, arr2], dtype=np.dtype(['f', dtype]))['f']\n" + "\n" + "By including a new field and indexing it after the " + "conversion.\n" + "This may lead to a different result or to current failures " + "succeeding. (FutureWarning since NumPy 1.20)") < 0) { Py_DECREF(ret); return NULL; } - } - else { - npy_free_coercion_cache(cache); + if (setArrayFromSequence(ret, op, 0, NULL) < 0) { Py_DECREF(ret); return NULL; } + return (PyObject *)ret; } - /* NumPy 1.20, 2020-10-01 */ - if (DEPRECATE( - "using a dtype with a subarray field is deprecated. " - "This can lead to inconsistent behaviour due to the resulting " - "dtype being different from the input dtype. " - "You may try to use `dtype=dtype.base`, which should give the " - "same result for most inputs, but does not guarantee the " - "output dimensions to match the subarray ones. " - "(Deprecated NumPy 1.20)")) { - Py_DECREF(ret); - return NULL; - } - return (PyObject *)ret; } if (dtype == NULL) { @@ -1700,26 +1717,52 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, } /* Create a new array and copy the data */ + Py_INCREF(dtype); /* hold on in case of a subarray that is replaced */ ret = (PyArrayObject *)PyArray_NewFromDescr( &PyArray_Type, dtype, ndim, dims, NULL, NULL, flags&NPY_ARRAY_F_CONTIGUOUS, NULL); if (ret == NULL) { npy_free_coercion_cache(cache); + Py_DECREF(dtype); return NULL; } + if (ndim == PyArray_NDIM(ret)) { + /* + * Appending of dimensions did not occur, so use the actual dtype + * below. This is relevant for S0 or U0 which can be replaced with + * S1 or U1, although that should likely change. + */ + Py_SETREF(dtype, PyArray_DESCR(ret)); + Py_INCREF(dtype); + } + if (cache == NULL) { /* This is a single item. Set it directly. */ assert(ndim == 0); - if (PyArray_Pack(PyArray_DESCR(ret), PyArray_BYTES(ret), op) < 0) { + if (PyArray_Pack(dtype, PyArray_BYTES(ret), op) < 0) { + Py_DECREF(dtype); Py_DECREF(ret); return NULL; } + Py_DECREF(dtype); return (PyObject *)ret; } assert(ndim != 0); assert(op == cache->converted_obj); - if (PyArray_AssignFromCache(ret, cache) < 0) { + + /* Decrease the number of dimensions to the detected ones */ + int out_ndim = PyArray_NDIM(ret); + PyArray_Descr *out_descr = PyArray_DESCR(ret); + ((PyArrayObject_fields *)ret)->nd = ndim; + ((PyArrayObject_fields *)ret)->descr = dtype; + + int success = PyArray_AssignFromCache(ret, cache); + + ((PyArrayObject_fields *)ret)->nd = out_ndim; + ((PyArrayObject_fields *)ret)->descr = out_descr; + Py_DECREF(dtype); + if (success < 0) { Py_DECREF(ret); return NULL; } diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index e4421b41b7df..084f7cee1782 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -845,16 +845,17 @@ array_astype(PyArrayObject *self, PyObject *args, PyObject *kwds) return NULL; } /* NumPy 1.20, 2020-10-01 */ - if ((PyArray_NDIM(self) != PyArray_NDIM(ret)) && DEPRECATE( - "using a dtype with a subarray field is deprecated. " - "This can lead to inconsistent behaviour due to the resulting " - "dtype being different from the input dtype. " - "You may try to use `dtype=dtype.base`, which should give the " - "same result for most inputs, but does not guarantee the " - "output dimensions to match the subarray ones. " - "For `arr.astype()` the old, surprising, behaviour can be " - "retained using `res = np.empty(arr.shape, dtype)` followed" - "by `res[...] = arr`. (Deprecated NumPy 1.20)")) { + if ((PyArray_NDIM(self) != PyArray_NDIM(ret)) && + DEPRECATE_FUTUREWARNING( + "casting an array to a subarray dtype " + "will not using broadcasting in the future, but cast each " + "element to the new dtype and then append the dtype's shape " + "to the new array. You can opt-in to the new behaviour, by " + "additional field to the cast: " + "`arr.astype(np.dtype([('f', dtype)]))['f']`.\n" + "This may lead to a different result or to current failures " + "succeeding. " + "(FutureWarning since NumPy 1.20)") < 0) { Py_DECREF(ret); return NULL; } diff --git a/numpy/core/tests/test_deprecations.py b/numpy/core/tests/test_deprecations.py index 91fd95d4d8f2..380b78f678db 100644 --- a/numpy/core/tests/test_deprecations.py +++ b/numpy/core/tests/test_deprecations.py @@ -732,40 +732,42 @@ def test_not_deprecated(self): class TestDeprecateSubarrayDTypeDuringArrayCoercion(_DeprecationTestCase): - message = "using a dtype with a subarray field is deprecated" - - @pytest.mark.parametrize(["obj", "dtype"], - [([((0, 1), (1, 2)), ((2,),)], '(2,2)f4'), - (["1", "2"], "(2)i,")]) - def test_deprecated_sequence(self, obj, dtype): - dtype = np.dtype(dtype) - self.assert_deprecated(lambda: np.array(obj, dtype=dtype)) - with pytest.warns(DeprecationWarning): - res = np.array(obj, dtype=dtype) - - # Using `arr.astype(subarray_dtype)` is also deprecated, because - # it uses broadcasting instead of casting each element. - self.assert_deprecated(lambda: res.astype(dtype)) - expected = np.empty(len(obj), dtype=dtype) - for i in range(len(expected)): - expected[i] = obj[i] - - assert_array_equal(res, expected) + warning_cls = FutureWarning + message = "(creating|casting) an array (with|to) a subarray dtype" def test_deprecated_array(self): # Arrays are more complex, since they "broadcast" on success: arr = np.array([1, 2]) + + self.assert_deprecated(lambda: arr.astype("(2)i,")) + with pytest.warns(FutureWarning): + res = arr.astype("(2)i,") + + assert_array_equal(res, [[1, 2], [1, 2]]) + self.assert_deprecated(lambda: np.array(arr, dtype="(2)i,")) - with pytest.warns(DeprecationWarning): + with pytest.warns(FutureWarning): res = np.array(arr, dtype="(2)i,") assert_array_equal(res, [[1, 2], [1, 2]]) - def test_not_deprecated(self): - # These error paths are not deprecated, the tests should be retained - # when the deprecation is finalized. + with pytest.warns(FutureWarning): + res = np.array([[(1,), (2,)], arr], dtype="(2)i,") + + assert_array_equal(res, [[[1, 1], [2, 2]], [[1, 2], [1, 2]]]) + + def test_deprecated_and_error(self): + # These error paths do not give a warning, but will succeed in the + # future. arr = np.arange(5 * 2).reshape(5, 2) - with pytest.raises(ValueError): - arr.astype("(2,2)f") - with pytest.raises(ValueError): - np.array(arr, dtype="(2,2)f") + def check(): + with pytest.raises(ValueError): + arr.astype("(2,2)f") + + self.assert_deprecated(check) + + def check(): + with pytest.raises(ValueError): + np.array(arr, dtype="(2,2)f") + + self.assert_deprecated(check) diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index ba50690245c1..1b2b85cc1c29 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -314,6 +314,24 @@ def test_union_struct(self): 'formats':['i1', 'O'], 'offsets':[np.dtype('intp').itemsize, 0]}) + @pytest.mark.parametrize(["obj", "dtype", "expected"], + [([], ("(2)f4,"), np.empty((0, 2), dtype="f4")), + (3, "(3)f4,", [3, 3, 3]), + (np.float64(2), "(2)f4,", [2, 2]), + ([((0, 1), (1, 2)), ((2,),)], '(2,2)f4', None), + (["1", "2"], "(2)i,", None)]) + def test_subarray_list(self, obj, dtype, expected): + dtype = np.dtype(dtype) + res = np.array(obj, dtype=dtype) + + if expected is None: + # iterate the 1-d list to fill the array + expected = np.empty(len(obj), dtype=dtype) + for i in range(len(expected)): + expected[i] = obj[i] + + assert_array_equal(res, expected) + def test_comma_datetime(self): dt = np.dtype('M8[D],datetime64[Y],i8') assert_equal(dt, np.dtype([('f0', 'M8[D]'), From 83b9349ac705241a72437ef58d08f4ff51dd595f Mon Sep 17 00:00:00 2001 From: mitch Date: Wed, 21 Oct 2020 14:06:43 -0700 Subject: [PATCH 314/409] DOC: removed old references to submodule licenses (closes #17606) --- LICENSES_bundled.txt | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/LICENSES_bundled.txt b/LICENSES_bundled.txt index e9c66d1dcfb6..00b7473777ca 100644 --- a/LICENSES_bundled.txt +++ b/LICENSES_bundled.txt @@ -1,16 +1,6 @@ The NumPy repository and source distributions bundle several libraries that are compatibly licensed. We list these here. -Name: Numpydoc -Files: doc/sphinxext/numpydoc/* -License: BSD-2-Clause - For details, see doc/sphinxext/LICENSE.txt - -Name: scipy-sphinx-theme -Files: doc/scipy-sphinx-theme/* -License: BSD-3-Clause AND PSF-2.0 AND Apache-2.0 - For details, see doc/scipy-sphinx-theme/LICENSE.txt - Name: lapack-lite Files: numpy/linalg/lapack_lite/* License: BSD-3-Clause From 8bb56b7325beddb74ca08696bcc2d3c184047279 Mon Sep 17 00:00:00 2001 From: Alexander Kuhn-Regnier Date: Wed, 21 Oct 2020 23:14:53 +0100 Subject: [PATCH 315/409] DOC: Fix typos (general documentation) --- doc/source/dev/development_workflow.rst | 2 +- doc/source/user/basics.subclassing.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/dev/development_workflow.rst b/doc/source/dev/development_workflow.rst index 1665cfddb9b7..34535b2f54b6 100644 --- a/doc/source/dev/development_workflow.rst +++ b/doc/source/dev/development_workflow.rst @@ -309,7 +309,7 @@ Rewriting commit history Do this only for your own feature branches. -There's an embarrassing typo in a commit you made? Or perhaps the you +There's an embarrassing typo in a commit you made? Or perhaps you made several false starts you would like the posterity not to see. This can be done via *interactive rebasing*. diff --git a/doc/source/user/basics.subclassing.rst b/doc/source/user/basics.subclassing.rst index d8d104220e0d..8ffa31688531 100644 --- a/doc/source/user/basics.subclassing.rst +++ b/doc/source/user/basics.subclassing.rst @@ -92,8 +92,8 @@ Implications for subclassing If we subclass ndarray, we need to deal not only with explicit construction of our array type, but also :ref:`view-casting` or -:ref:`new-from-template`. NumPy has the machinery to do this, and this -machinery that makes subclassing slightly non-standard. +:ref:`new-from-template`. NumPy has the machinery to do this, and it is +this machinery that makes subclassing slightly non-standard. There are two aspects to the machinery that ndarray uses to support views and new-from-template in subclasses. From 44b7698b9bba6b9212498e3659cede7d14a8b79a Mon Sep 17 00:00:00 2001 From: takanori-pskq Date: Thu, 22 Oct 2020 17:01:47 +0900 Subject: [PATCH 316/409] DOC: Fixup --- doc/source/reference/c-api/array.rst | 33 ++++++++++++++-------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/doc/source/reference/c-api/array.rst b/doc/source/reference/c-api/array.rst index ff8e911bfc70..c737690d4df5 100644 --- a/doc/source/reference/c-api/array.rst +++ b/doc/source/reference/c-api/array.rst @@ -3139,21 +3139,31 @@ calling the function). That's why several functions are provided to check for numpy versions. The macros :c:data:`NPY_VERSION` and :c:data:`NPY_FEATURE_VERSION` corresponds to the numpy version used to build the extension, whereas the versions returned by the functions -PyArray_GetNDArrayCVersion and PyArray_GetNDArrayCFeatureVersion corresponds to -the runtime numpy's version. +:c:func:`PyArray_GetNDArrayCVersion` and :c:func:`PyArray_GetNDArrayCFeatureVersion` +corresponds to the runtime numpy's version. The rules for ABI and API compatibilities can be summarized as follows: - * Whenever :c:data:`NPY_VERSION` != PyArray_GetNDArrayCVersion, the + * Whenever :c:data:`NPY_VERSION` != ``PyArray_GetNDArrayCVersion()``, the extension has to be recompiled (ABI incompatibility). - * :c:data:`NPY_VERSION` == PyArray_GetNDArrayCVersion and - :c:data:`NPY_FEATURE_VERSION` <= PyArray_GetNDArrayCFeatureVersion means + * :c:data:`NPY_VERSION` == ``PyArray_GetNDArrayCVersion()`` and + :c:data:`NPY_FEATURE_VERSION` <= ``PyArray_GetNDArrayCFeatureVersion()`` means backward compatible changes. ABI incompatibility is automatically detected in every numpy's version. API incompatibility detection was added in numpy 1.4.0. If you want to supported many different numpy versions with one extension binary, you have to build your -extension with the lowest NPY_FEATURE_VERSION as possible. +extension with the lowest :c:data:`NPY_FEATURE_VERSION` as possible. + +.. c:macro:: NPY_VERSION + + The current version of the ndarray object (check to see if this + variable is defined to guarantee the ``numpy/arrayobject.h`` header is + being used). + +.. c:macro:: NPY_FEATURE_VERSION + + The current version of the C-API. .. c:function:: unsigned int PyArray_GetNDArrayCVersion(void) @@ -3437,17 +3447,6 @@ Other constants The maximum number of array arguments that can be used in functions. -.. c:macro:: NPY_VERSION - - The current version of the ndarray object (check to see if this - variable is defined to guarantee the numpy/arrayobject.h header is - being used). - -.. c:macro:: NPY_FEATURE_VERSION - - The current version of the API. :c:data:`NPY_FEATURE_VERSION` changes - whenever the API changes (e.g. a function is added). - .. c:macro:: NPY_FALSE Defined as 0 for use with Bool. From 5cb80d60f2727b7ed734e4617c4d846cf6422f55 Mon Sep 17 00:00:00 2001 From: johnthagen Date: Thu, 22 Oct 2020 08:43:03 -0400 Subject: [PATCH 317/409] Fully qualify license trove classifier --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index acd015d1d92a..c64a5c491617 100755 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ Development Status :: 5 - Production/Stable Intended Audience :: Science/Research Intended Audience :: Developers -License :: OSI Approved +License :: OSI Approved :: BSD License Programming Language :: C Programming Language :: Python Programming Language :: Python :: 3 From 2eefb6d1fdd46193328332b84de0e607f0a40318 Mon Sep 17 00:00:00 2001 From: Bharat Medasani Date: Thu, 22 Oct 2020 10:34:14 -0400 Subject: [PATCH 318/409] BUG: mac dylib treated as extra library by f2py --- numpy/f2py/f2py2e.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index e68b2e01aceb..be2c345d1a99 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -604,7 +604,7 @@ def run_compile(): if modulename: break - extra_objects, sources = filter_files('', '[.](o|a|so)', sources) + extra_objects, sources = filter_files('', '[.](o|a|so|dylib)', sources) include_dirs, sources = filter_files('-I', '', sources, remove_prefix=1) library_dirs, sources = filter_files('-L', '', sources, remove_prefix=1) libraries, sources = filter_files('-l', '', sources, remove_prefix=1) From 8f6f84f22c43e8f273f8a826b490af6d83d8b752 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Thu, 22 Oct 2020 14:54:12 +0200 Subject: [PATCH 319/409] ENH: Add annotations for `ndarray` / `generic` magic methods * `__int__` * `__float__` * `__complex__` * `__neg__` * `__pos__` * `__abs__` --- numpy/__init__.pyi | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index ce4aa1ae6fea..8bcc89bad460 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -1004,9 +1004,6 @@ class _ArrayOrScalarCommon( @property def strides(self) -> _Shape: ... def __array__(self, __dtype: DtypeLike = ...) -> ndarray: ... - def __int__(self) -> int: ... - def __float__(self) -> float: ... - def __complex__(self) -> complex: ... def __bool__(self) -> bool: ... def __bytes__(self) -> bytes: ... def __str__(self) -> str: ... @@ -1023,9 +1020,6 @@ class _ArrayOrScalarCommon( def __rmod__(self, other): ... def __divmod__(self, other): ... def __rdivmod__(self, other): ... - def __neg__(self: _ArraySelf) -> _ArraySelf: ... - def __pos__(self: _ArraySelf) -> _ArraySelf: ... - def __abs__(self: _ArraySelf) -> _ArraySelf: ... def astype( self: _ArraySelf, dtype: DtypeLike, @@ -1572,14 +1566,20 @@ class ndarray(_ArrayOrScalarCommon, Iterable, Sized, Container): # Many of these special methods are irrelevant currently, since protocols # aren't supported yet. That said, I'm adding them for completeness. # https://docs.python.org/3/reference/datamodel.html + def __int__(self) -> int: ... + def __float__(self) -> float: ... + def __complex__(self) -> complex: ... def __len__(self) -> int: ... def __setitem__(self, key, value): ... def __iter__(self) -> Any: ... def __contains__(self, key) -> bool: ... def __index__(self) -> int: ... - def __matmul__(self, other): ... - def __imatmul__(self, other): ... - def __rmatmul__(self, other): ... + def __matmul__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + # NOTE: `ndarray` does not implement `__imatmul__` + def __rmatmul__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __neg__(self: _ArraySelf) -> Union[_ArraySelf, generic]: ... + def __pos__(self: _ArraySelf) -> Union[_ArraySelf, generic]: ... + def __abs__(self: _ArraySelf) -> Union[_ArraySelf, generic]: ... def __add__(self, other: ArrayLike) -> Union[ndarray, generic]: ... def __radd__(self, other: ArrayLike) -> Union[ndarray, generic]: ... def __sub__(self, other: ArrayLike) -> Union[ndarray, generic]: ... @@ -1610,7 +1610,7 @@ class ndarray(_ArrayOrScalarCommon, Iterable, Sized, Container): def __itruediv__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... def __ifloordiv__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... def __ipow__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... - def __imod__(self, other): ... + def __imod__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... def __ilshift__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... def __irshift__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... def __iand__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... @@ -1639,6 +1639,11 @@ class number(generic, Generic[_NBit_co]): # type: ignore def real(self: _ArraySelf) -> _ArraySelf: ... @property def imag(self: _ArraySelf) -> _ArraySelf: ... + def __int__(self) -> int: ... + def __float__(self) -> float: ... + def __neg__(self: _ArraySelf) -> _ArraySelf: ... + def __pos__(self: _ArraySelf) -> _ArraySelf: ... + def __abs__(self: _ArraySelf) -> _ArraySelf: ... # Ensure that objects annotated as `number` support arithmetic operations __add__: _NumberOp __radd__: _NumberOp @@ -1659,6 +1664,9 @@ class bool_(generic): def real(self: _ArraySelf) -> _ArraySelf: ... @property def imag(self: _ArraySelf) -> _ArraySelf: ... + def __int__(self) -> int: ... + def __float__(self) -> float: ... + def __abs__(self: _ArraySelf) -> _ArraySelf: ... __add__: _BoolOp[bool_] __radd__: _BoolOp[bool_] __sub__: _BoolSub @@ -1774,6 +1782,11 @@ class timedelta64(generic): __value: Union[None, int, _CharLike, dt.timedelta, timedelta64] = ..., __format: Union[_CharLike, Tuple[_CharLike, _IntLike]] = ..., ) -> None: ... + def __int__(self) -> int: ... + def __float__(self) -> float: ... + def __neg__(self: _ArraySelf) -> _ArraySelf: ... + def __pos__(self: _ArraySelf) -> _ArraySelf: ... + def __abs__(self: _ArraySelf) -> _ArraySelf: ... def __add__(self, other: Union[timedelta64, _IntLike, _BoolLike]) -> timedelta64: ... def __radd__(self, other: Union[timedelta64, _IntLike, _BoolLike]) -> timedelta64: ... def __sub__(self, other: Union[timedelta64, _IntLike, _BoolLike]) -> timedelta64: ... @@ -1850,6 +1863,7 @@ class complexfloating(inexact[_NBit_co], Generic[_NBit_co, _NBit_co2]): @property def imag(self) -> floating[_NBit_co2]: ... # type: ignore[override] def __abs__(self) -> floating[_NBit_co]: ... # type: ignore[override] + def __complex__(self) -> complex: ... __add__: _ComplexOp[_NBit_co] __radd__: _ComplexOp[_NBit_co] __sub__: _ComplexOp[_NBit_co] @@ -1878,7 +1892,9 @@ class void(flexible): self, val: ArrayLike, dtype: DtypeLike, offset: int = ... ) -> None: ... -class character(flexible): ... # type: ignore +class character(flexible): # type: ignore + def __int__(self) -> int: ... + def __float__(self) -> float: ... # NOTE: Most `np.bytes_` / `np.str_` methods return their # builtin `bytes` / `str` counterpart From 6f08cd3b55ae00b80cf33e172a4df1daf439b50f Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Thu, 22 Oct 2020 14:56:25 +0200 Subject: [PATCH 320/409] ENH: Add annotations for `__mod__` & `__divmod__` --- numpy/__init__.pyi | 41 +++++++++++-- numpy/typing/_callable.py | 124 +++++++++++++++++++++++++++++++++++++- 2 files changed, 160 insertions(+), 5 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 8bcc89bad460..27fdbed3dccc 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -28,13 +28,21 @@ from numpy.typing._callable import ( _BoolBitOp, _BoolSub, _BoolTrueDiv, + _BoolMod, + _BoolDivMod, _TD64Div, _IntTrueDiv, _UnsignedIntOp, _UnsignedIntBitOp, + _UnsignedIntMod, + _UnsignedIntDivMod, _SignedIntOp, _SignedIntBitOp, + _SignedIntMod, + _SignedIntDivMod, _FloatOp, + _FloatMod, + _FloatDivMod, _ComplexOp, _NumberOp, ) @@ -1016,10 +1024,6 @@ class _ArrayOrScalarCommon( def __ne__(self, other): ... def __gt__(self, other): ... def __ge__(self, other): ... - def __mod__(self, other): ... - def __rmod__(self, other): ... - def __divmod__(self, other): ... - def __rdivmod__(self, other): ... def astype( self: _ArraySelf, dtype: DtypeLike, @@ -1580,6 +1584,14 @@ class ndarray(_ArrayOrScalarCommon, Iterable, Sized, Container): def __neg__(self: _ArraySelf) -> Union[_ArraySelf, generic]: ... def __pos__(self: _ArraySelf) -> Union[_ArraySelf, generic]: ... def __abs__(self: _ArraySelf) -> Union[_ArraySelf, generic]: ... + def __mod__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __rmod__(self, other: ArrayLike) -> Union[ndarray, generic]: ... + def __divmod__( + self, other: ArrayLike + ) -> Union[Tuple[ndarray, ndarray], Tuple[generic, generic]]: ... + def __rdivmod__( + self, other: ArrayLike + ) -> Union[Tuple[ndarray, ndarray], Tuple[generic, generic]]: ... def __add__(self, other: ArrayLike) -> Union[ndarray, generic]: ... def __radd__(self, other: ArrayLike) -> Union[ndarray, generic]: ... def __sub__(self, other: ArrayLike) -> Union[ndarray, generic]: ... @@ -1690,6 +1702,10 @@ class bool_(generic): __rxor__: _BoolBitOp[bool_] __or__: _BoolBitOp[bool_] __ror__: _BoolBitOp[bool_] + __mod__: _BoolMod + __rmod__: _BoolMod + __divmod__: _BoolDivMod + __rdivmod__: _BoolDivMod class object_(generic): def __init__(self, __value: object = ...) -> None: ... @@ -1735,6 +1751,8 @@ class integer(number[_NBit_co]): # type: ignore def __index__(self) -> int: ... __truediv__: _IntTrueDiv[_NBit_co] __rtruediv__: _IntTrueDiv[_NBit_co] + def __mod__(self, value: Union[_IntLike, integer]) -> integer: ... + def __rmod__(self, value: Union[_IntLike, integer]) -> integer: ... def __invert__(self: _IntType) -> _IntType: ... # Ensure that objects annotated as `integer` support bit-wise operations def __lshift__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... @@ -1770,6 +1788,10 @@ class signedinteger(integer[_NBit_co]): __rxor__: _SignedIntBitOp[_NBit_co] __or__: _SignedIntBitOp[_NBit_co] __ror__: _SignedIntBitOp[_NBit_co] + __mod__: _SignedIntMod[_NBit_co] + __rmod__: _SignedIntMod[_NBit_co] + __divmod__: _SignedIntDivMod[_NBit_co] + __rdivmod__: _SignedIntDivMod[_NBit_co] int8 = signedinteger[_8Bit] int16 = signedinteger[_16Bit] @@ -1798,6 +1820,9 @@ class timedelta64(generic): def __rtruediv__(self, other: timedelta64) -> float64: ... def __rfloordiv__(self, other: timedelta64) -> int64: ... def __mod__(self, other: timedelta64) -> timedelta64: ... + def __rmod__(self, other: timedelta64) -> timedelta64: ... + def __divmod__(self, other: timedelta64) -> Tuple[int64, timedelta64]: ... + def __rdivmod__(self, other: timedelta64) -> Tuple[int64, timedelta64]: ... class unsignedinteger(integer[_NBit_co]): # NOTE: `uint64 + signedinteger -> float64` @@ -1822,6 +1847,10 @@ class unsignedinteger(integer[_NBit_co]): __rxor__: _UnsignedIntBitOp[_NBit_co] __or__: _UnsignedIntBitOp[_NBit_co] __ror__: _UnsignedIntBitOp[_NBit_co] + __mod__: _UnsignedIntMod[_NBit_co] + __rmod__: _UnsignedIntMod[_NBit_co] + __divmod__: _UnsignedIntDivMod[_NBit_co] + __rdivmod__: _UnsignedIntDivMod[_NBit_co] uint8 = unsignedinteger[_8Bit] uint16 = unsignedinteger[_16Bit] @@ -1847,6 +1876,10 @@ class floating(inexact[_NBit_co]): __rfloordiv__: _FloatOp[_NBit_co] __pow__: _FloatOp[_NBit_co] __rpow__: _FloatOp[_NBit_co] + __mod__: _FloatMod[_NBit_co] + __rmod__: _FloatMod[_NBit_co] + __divmod__: _FloatDivMod[_NBit_co] + __rdivmod__: _FloatDivMod[_NBit_co] float16 = floating[_16Bit] float32 = floating[_32Bit] diff --git a/numpy/typing/_callable.py b/numpy/typing/_callable.py index b16891b0e24f..8b7bfedaca12 100644 --- a/numpy/typing/_callable.py +++ b/numpy/typing/_callable.py @@ -9,7 +9,15 @@ """ import sys -from typing import Union, TypeVar, overload, Any, TYPE_CHECKING, NoReturn +from typing import ( + Union, + TypeVar, + overload, + Any, + Tuple, + NoReturn, + TYPE_CHECKING, +) from numpy import ( generic, @@ -19,6 +27,7 @@ integer, unsignedinteger, signedinteger, + int8, int32, int64, floating, @@ -49,9 +58,14 @@ HAVE_PROTOCOL = True if TYPE_CHECKING or HAVE_PROTOCOL: + _T = TypeVar("_T") + _2Tuple = Tuple[_T, _T] + _NBit_co = TypeVar("_NBit_co", covariant=True, bound=NBitBase) _NBit = TypeVar("_NBit", bound=NBitBase) + _IntType = TypeVar("_IntType", bound=integer) + _FloatType = TypeVar("_FloatType", bound=floating) _NumberType = TypeVar("_NumberType", bound=number) _NumberType_co = TypeVar("_NumberType_co", covariant=True, bound=number) _GenericType_co = TypeVar("_GenericType_co", covariant=True, bound=generic) @@ -97,6 +111,30 @@ def __call__(self, __other: complex) -> complex128: ... @overload def __call__(self, __other: _NumberType) -> _NumberType: ... + class _BoolMod(Protocol): + @overload + def __call__(self, __other: _BoolLike) -> int8: ... + @overload # platform dependent + def __call__(self, __other: int) -> signedinteger[Any]: ... + @overload + def __call__(self, __other: float) -> float64: ... + @overload + def __call__(self, __other: _IntType) -> _IntType: ... + @overload + def __call__(self, __other: _FloatType) -> _FloatType: ... + + class _BoolDivMod(Protocol): + @overload + def __call__(self, __other: _BoolLike) -> _2Tuple[int8]: ... + @overload # platform dependent + def __call__(self, __other: int) -> _2Tuple[signedinteger[Any]]: ... + @overload + def __call__(self, __other: float) -> _2Tuple[float64]: ... + @overload + def __call__(self, __other: _IntType) -> _2Tuple[_IntType]: ... + @overload + def __call__(self, __other: _FloatType) -> _2Tuple[_FloatType]: ... + class _TD64Div(Protocol[_NumberType_co]): @overload def __call__(self, __other: timedelta64) -> _NumberType_co: ... @@ -144,6 +182,34 @@ def __call__( self, __other: unsignedinteger[_NBit] ) -> unsignedinteger[Union[_NBit_co, _NBit]]: ... + class _UnsignedIntMod(Protocol[_NBit_co]): + @overload + def __call__(self, __other: bool) -> unsignedinteger[_NBit_co]: ... + @overload + def __call__( + self, __other: Union[int, signedinteger[Any]] + ) -> Union[signedinteger[Any], float64]: ... + @overload + def __call__(self, __other: float) -> float64: ... + @overload + def __call__( + self, __other: unsignedinteger[_NBit] + ) -> unsignedinteger[Union[_NBit_co, _NBit]]: ... + + class _UnsignedIntDivMod(Protocol[_NBit_co]): + @overload + def __call__(self, __other: bool) -> _2Tuple[signedinteger[_NBit_co]]: ... + @overload + def __call__( + self, __other: Union[int, signedinteger[Any]] + ) -> Union[_2Tuple[signedinteger[Any]], _2Tuple[float64]]: ... + @overload + def __call__(self, __other: float) -> _2Tuple[float64]: ... + @overload + def __call__( + self, __other: unsignedinteger[_NBit] + ) -> _2Tuple[unsignedinteger[Union[_NBit_co, _NBit]]]: ... + class _SignedIntOp(Protocol[_NBit_co]): @overload def __call__(self, __other: bool) -> signedinteger[_NBit_co]: ... @@ -168,6 +234,30 @@ def __call__( self, __other: signedinteger[_NBit] ) -> signedinteger[Union[_NBit_co, _NBit]]: ... + class _SignedIntMod(Protocol[_NBit_co]): + @overload + def __call__(self, __other: bool) -> signedinteger[_NBit_co]: ... + @overload + def __call__(self, __other: int) -> signedinteger[Any]: ... + @overload + def __call__(self, __other: float) -> float64: ... + @overload + def __call__( + self, __other: signedinteger[_NBit] + ) -> signedinteger[Union[_NBit_co, _NBit]]: ... + + class _SignedIntDivMod(Protocol[_NBit_co]): + @overload + def __call__(self, __other: bool) -> _2Tuple[signedinteger[_NBit_co]]: ... + @overload + def __call__(self, __other: int) -> _2Tuple[signedinteger[Any]]: ... + @overload + def __call__(self, __other: float) -> _2Tuple[float64]: ... + @overload + def __call__( + self, __other: signedinteger[_NBit] + ) -> _2Tuple[signedinteger[Union[_NBit_co, _NBit]]]: ... + class _FloatOp(Protocol[_NBit_co]): @overload def __call__(self, __other: bool) -> floating[_NBit_co]: ... @@ -182,6 +272,30 @@ def __call__( self, __other: Union[integer[_NBit], floating[_NBit]] ) -> floating[Union[_NBit_co, _NBit]]: ... + class _FloatMod(Protocol[_NBit_co]): + @overload + def __call__(self, __other: bool) -> floating[_NBit_co]: ... + @overload + def __call__(self, __other: int) -> floating[Any]: ... + @overload + def __call__(self, __other: float) -> float64: ... + @overload + def __call__( + self, __other: Union[integer[_NBit], floating[_NBit]] + ) -> floating[Union[_NBit_co, _NBit]]: ... + + class _FloatDivMod(Protocol[_NBit_co]): + @overload + def __call__(self, __other: bool) -> _2Tuple[floating[_NBit_co]]: ... + @overload + def __call__(self, __other: int) -> _2Tuple[floating[Any]]: ... + @overload + def __call__(self, __other: float) -> _2Tuple[float64]: ... + @overload + def __call__( + self, __other: Union[integer[_NBit], floating[_NBit]] + ) -> _2Tuple[floating[Union[_NBit_co, _NBit]]]: ... + class _ComplexOp(Protocol[_NBit_co]): @overload def __call__(self, __other: bool) -> complexfloating[_NBit_co, _NBit_co]: ... @@ -207,12 +321,20 @@ def __call__(self, __other: _NumberLike) -> number: ... _BoolBitOp = Any _BoolSub = Any _BoolTrueDiv = Any + _BoolMod = Any + _BoolDivMod = Any _TD64Div = Any _IntTrueDiv = Any _UnsignedIntOp = Any _UnsignedIntBitOp = Any + _UnsignedIntMod = Any + _UnsignedIntDivMod = Any _SignedIntOp = Any _SignedIntBitOp = Any + _SignedIntMod = Any + _SignedIntDivMod = Any _FloatOp = Any + _FloatMod = Any + _FloatDivMod = Any _ComplexOp = Any _NumberOp = Any From 5602b48f72de4f2311d8979cecc1c5d82468a115 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Thu, 22 Oct 2020 17:36:03 +0200 Subject: [PATCH 321/409] MAINT: Add `__complex__` to all numeric types In practice only `complexfloating` defines a `__complex__` method, but this is the only make all numeric types compatible with `builtins.complex`. --- numpy/__init__.pyi | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 27fdbed3dccc..e70075ee6e83 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -1653,6 +1653,7 @@ class number(generic, Generic[_NBit_co]): # type: ignore def imag(self: _ArraySelf) -> _ArraySelf: ... def __int__(self) -> int: ... def __float__(self) -> float: ... + def __complex__(self) -> complex: ... def __neg__(self: _ArraySelf) -> _ArraySelf: ... def __pos__(self: _ArraySelf) -> _ArraySelf: ... def __abs__(self: _ArraySelf) -> _ArraySelf: ... @@ -1678,6 +1679,7 @@ class bool_(generic): def imag(self: _ArraySelf) -> _ArraySelf: ... def __int__(self) -> int: ... def __float__(self) -> float: ... + def __complex__(self) -> complex: ... def __abs__(self: _ArraySelf) -> _ArraySelf: ... __add__: _BoolOp[bool_] __radd__: _BoolOp[bool_] @@ -1806,6 +1808,7 @@ class timedelta64(generic): ) -> None: ... def __int__(self) -> int: ... def __float__(self) -> float: ... + def __complex__(self) -> complex: ... def __neg__(self: _ArraySelf) -> _ArraySelf: ... def __pos__(self: _ArraySelf) -> _ArraySelf: ... def __abs__(self: _ArraySelf) -> _ArraySelf: ... @@ -1896,7 +1899,6 @@ class complexfloating(inexact[_NBit_co], Generic[_NBit_co, _NBit_co2]): @property def imag(self) -> floating[_NBit_co2]: ... # type: ignore[override] def __abs__(self) -> floating[_NBit_co]: ... # type: ignore[override] - def __complex__(self) -> complex: ... __add__: _ComplexOp[_NBit_co] __radd__: _ComplexOp[_NBit_co] __sub__: _ComplexOp[_NBit_co] From c1dde64e78ae4cbfcba88727c4d2872f5238999a Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Thu, 22 Oct 2020 14:56:05 +0200 Subject: [PATCH 322/409] MAINT: Removed a redundant protocol inheritance --- numpy/__init__.pyi | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index e70075ee6e83..c8ef60c2d2d4 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -63,8 +63,6 @@ from typing import ( overload, Sequence, Sized, - SupportsAbs, - SupportsBytes, SupportsComplex, SupportsFloat, SupportsInt, @@ -986,9 +984,7 @@ _ArrayLikeIntOrBool = Union[ _ArraySelf = TypeVar("_ArraySelf", bound=_ArrayOrScalarCommon) -class _ArrayOrScalarCommon( - SupportsInt, SupportsFloat, SupportsComplex, SupportsBytes, SupportsAbs[Any] -): +class _ArrayOrScalarCommon: @property def T(self: _ArraySelf) -> _ArraySelf: ... @property From db55397e4c705b7f5a5bf7ee5a45c9ce78aba4c4 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Thu, 22 Oct 2020 15:49:38 +0200 Subject: [PATCH 323/409] MAINT: Removed unused imports --- numpy/typing/_callable.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/numpy/typing/_callable.py b/numpy/typing/_callable.py index 8b7bfedaca12..91b7a4ec21c0 100644 --- a/numpy/typing/_callable.py +++ b/numpy/typing/_callable.py @@ -28,13 +28,9 @@ unsignedinteger, signedinteger, int8, - int32, - int64, floating, - float32, float64, complexfloating, - complex64, complex128, ) from ._scalars import ( From af3765d4adfe12acae55285e6646d7990dc20907 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Thu, 22 Oct 2020 16:53:08 +0200 Subject: [PATCH 324/409] TST: Fixed the line number offset --- numpy/typing/tests/test_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/typing/tests/test_typing.py b/numpy/typing/tests/test_typing.py index 84b0ab3cc98a..90de4fd6dc41 100644 --- a/numpy/typing/tests/test_typing.py +++ b/numpy/typing/tests/test_typing.py @@ -156,7 +156,7 @@ def test_reveal(path): assert "Revealed type is" in error_line marker = lines[lineno].split("# E:")[-1].strip() - _test_reveal(path, marker, error_line, lineno) + _test_reveal(path, marker, error_line, 1 + lineno) _REVEAL_MSG = """Reveal mismatch at line {} From cdcb14017513ad70e7957d59c37459253f0b40ef Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Thu, 22 Oct 2020 17:01:01 +0200 Subject: [PATCH 325/409] TST: Added typing tests for `__mod__` & `__divmod__` --- numpy/typing/tests/data/pass/mod.py | 149 +++++++++++++++++++ numpy/typing/tests/data/reveal/arithmetic.py | 1 - numpy/typing/tests/data/reveal/mod.py | 149 +++++++++++++++++++ 3 files changed, 298 insertions(+), 1 deletion(-) create mode 100644 numpy/typing/tests/data/pass/mod.py create mode 100644 numpy/typing/tests/data/reveal/mod.py diff --git a/numpy/typing/tests/data/pass/mod.py b/numpy/typing/tests/data/pass/mod.py new file mode 100644 index 000000000000..b5b9afb2a544 --- /dev/null +++ b/numpy/typing/tests/data/pass/mod.py @@ -0,0 +1,149 @@ +import numpy as np + +f8 = np.float64(1) +i8 = np.int64(1) +u8 = np.uint64(1) + +f4 = np.float32(1) +i4 = np.int32(1) +u4 = np.uint32(1) + +td = np.timedelta64(1, "D") +b_ = np.bool_(1) + +b = bool(1) +f = float(1) +i = int(1) + +AR = np.array([1], dtype=np.bool_) +AR.setflags(write=False) + +AR2 = np.array([1], dtype=np.timedelta64) +AR2.setflags(write=False) + +# Time structures + +td % td +td % AR2 +AR2 % td + +divmod(td, td) +divmod(td, AR2) +divmod(AR2, td) + +# Bool + +b_ % b +b_ % i +b_ % f +b_ % b_ +b_ % i8 +b_ % u8 +b_ % f8 +b_ % AR + +divmod(b_, b) +divmod(b_, i) +divmod(b_, f) +divmod(b_, b_) +divmod(b_, i8) +divmod(b_, u8) +divmod(b_, f8) +divmod(b_, AR) + +b % b_ +i % b_ +f % b_ +b_ % b_ +i8 % b_ +u8 % b_ +f8 % b_ +AR % b_ + +divmod(b, b_) +divmod(i, b_) +divmod(f, b_) +divmod(b_, b_) +divmod(i8, b_) +divmod(u8, b_) +divmod(f8, b_) +divmod(AR, b_) + +# int + +i8 % b +i8 % i +i8 % f +i8 % i8 +i8 % f8 +i4 % i8 +i4 % f8 +i4 % i4 +i4 % f4 +i8 % AR + +divmod(i8, b) +divmod(i8, i) +divmod(i8, f) +divmod(i8, i8) +divmod(i8, f8) +divmod(i8, i4) +divmod(i8, f4) +divmod(i4, i4) +divmod(i4, f4) +divmod(i8, AR) + +b % i8 +i % i8 +f % i8 +i8 % i8 +f8 % i8 +i8 % i4 +f8 % i4 +i4 % i4 +f4 % i4 +AR % i8 + +divmod(b, i8) +divmod(i, i8) +divmod(f, i8) +divmod(i8, i8) +divmod(f8, i8) +divmod(i4, i8) +divmod(f4, i8) +divmod(i4, i4) +divmod(f4, i4) +divmod(AR, i8) + +# float + +f8 % b +f8 % i +f8 % f +i8 % f4 +f4 % f4 +f8 % AR + +divmod(f8, b) +divmod(f8, i) +divmod(f8, f) +divmod(f8, f8) +divmod(f8, f4) +divmod(f4, f4) +divmod(f8, AR) + +b % f8 +i % f8 +f % f8 +f8 % f8 +f8 % f8 +f4 % f4 +AR % f8 + +divmod(b, f8) +divmod(i, f8) +divmod(f, f8) +divmod(f8, f8) +divmod(f4, f8) +divmod(f4, f4) +divmod(AR, f8) diff --git a/numpy/typing/tests/data/reveal/arithmetic.py b/numpy/typing/tests/data/reveal/arithmetic.py index a1290304a015..24fc672d0533 100644 --- a/numpy/typing/tests/data/reveal/arithmetic.py +++ b/numpy/typing/tests/data/reveal/arithmetic.py @@ -47,7 +47,6 @@ reveal_type(td / f8) # E: numpy.timedelta64 reveal_type(td / td) # E: numpy.floating[numpy.typing._64Bit] reveal_type(td // td) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(td % td) # E: numpy.timedelta64 # boolean diff --git a/numpy/typing/tests/data/reveal/mod.py b/numpy/typing/tests/data/reveal/mod.py new file mode 100644 index 000000000000..9cd2c2fcfd60 --- /dev/null +++ b/numpy/typing/tests/data/reveal/mod.py @@ -0,0 +1,149 @@ +import numpy as np + +f8 = np.float64() +i8 = np.int64() +u8 = np.uint64() + +f4 = np.float32() +i4 = np.int32() +u4 = np.uint32() + +td = np.timedelta64(0, "D") +b_ = np.bool_() + +b = bool() +f = float() +i = int() + +AR = np.array([1], dtype=np.bool_) +AR.setflags(write=False) + +AR2 = np.array([1], dtype=np.timedelta64) +AR2.setflags(write=False) + +# Time structures + +reveal_type(td % td) # E: numpy.timedelta64 +reveal_type(AR2 % td) # E: Union[numpy.ndarray, numpy.generic] +reveal_type(td % AR2) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(divmod(td, td)) # E: Tuple[numpy.signedinteger[numpy.typing._64Bit], numpy.timedelta64] +reveal_type(divmod(AR2, td)) # E: Union[Tuple[numpy.ndarray, numpy.ndarray], Tuple[numpy.generic, numpy.generic]] +reveal_type(divmod(td, AR2)) # E: Union[Tuple[numpy.ndarray, numpy.ndarray], Tuple[numpy.generic, numpy.generic]] + +# Bool + +reveal_type(b_ % b) # E: numpy.signedinteger[numpy.typing._8Bit] +reveal_type(b_ % i) # E: numpy.signedinteger[Any] +reveal_type(b_ % f) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ % b_) # E: numpy.signedinteger[numpy.typing._8Bit] +reveal_type(b_ % i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(b_ % u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(b_ % f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ % AR) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(divmod(b_, b)) # E: Tuple[numpy.signedinteger[numpy.typing._8Bit], numpy.signedinteger[numpy.typing._8Bit]] +reveal_type(divmod(b_, i)) # E: Tuple[numpy.signedinteger[Any], numpy.signedinteger[Any]] +reveal_type(divmod(b_, f)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(b_, b_)) # E: Tuple[numpy.signedinteger[numpy.typing._8Bit], numpy.signedinteger[numpy.typing._8Bit]] +reveal_type(divmod(b_, i8)) # E: Tuple[numpy.signedinteger[numpy.typing._64Bit], numpy.signedinteger[numpy.typing._64Bit]] +reveal_type(divmod(b_, u8)) # E: Tuple[numpy.unsignedinteger[numpy.typing._64Bit], numpy.unsignedinteger[numpy.typing._64Bit]] +reveal_type(divmod(b_, f8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(b_, AR)) # E: Union[Tuple[numpy.ndarray, numpy.ndarray], Tuple[numpy.generic, numpy.generic]] + +reveal_type(b % b_) # E: numpy.signedinteger[numpy.typing._8Bit] +reveal_type(i % b_) # E: numpy.signedinteger[Any] +reveal_type(f % b_) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(b_ % b_) # E: numpy.signedinteger[numpy.typing._8Bit] +reveal_type(i8 % b_) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(u8 % b_) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(f8 % b_) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(AR % b_) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(divmod(b, b_)) # E: Tuple[numpy.signedinteger[numpy.typing._8Bit], numpy.signedinteger[numpy.typing._8Bit]] +reveal_type(divmod(i, b_)) # E: Tuple[numpy.signedinteger[Any], numpy.signedinteger[Any]] +reveal_type(divmod(f, b_)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(b_, b_)) # E: Tuple[numpy.signedinteger[numpy.typing._8Bit], numpy.signedinteger[numpy.typing._8Bit]] +reveal_type(divmod(i8, b_)) # E: Tuple[numpy.signedinteger[numpy.typing._64Bit], numpy.signedinteger[numpy.typing._64Bit]] +reveal_type(divmod(u8, b_)) # E: Tuple[numpy.unsignedinteger[numpy.typing._64Bit], numpy.unsignedinteger[numpy.typing._64Bit]] +reveal_type(divmod(f8, b_)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(AR, b_)) # E: Union[Tuple[numpy.ndarray, numpy.ndarray], Tuple[numpy.generic, numpy.generic]] + +# int + +reveal_type(i8 % b) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 % i) # E: numpy.signedinteger[Any] +reveal_type(i8 % f) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i8 % i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 % f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i4 % i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i4 % f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i4 % i4) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(i4 % f4) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(i8 % AR) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(divmod(i8, b)) # E: Tuple[numpy.signedinteger[numpy.typing._64Bit], numpy.signedinteger[numpy.typing._64Bit]] +reveal_type(divmod(i8, i)) # E: Tuple[numpy.signedinteger[Any], numpy.signedinteger[Any]] +reveal_type(divmod(i8, f)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(i8, i8)) # E: Tuple[numpy.signedinteger[numpy.typing._64Bit], numpy.signedinteger[numpy.typing._64Bit]] +reveal_type(divmod(i8, f8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(i8, i4)) # E: Tuple[numpy.signedinteger[numpy.typing._64Bit], numpy.signedinteger[numpy.typing._64Bit]] +reveal_type(divmod(i8, f4)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(i4, i4)) # E: Tuple[numpy.signedinteger[numpy.typing._32Bit], numpy.signedinteger[numpy.typing._32Bit]] +reveal_type(divmod(i4, f4)) # E: Tuple[numpy.floating[numpy.typing._32Bit], numpy.floating[numpy.typing._32Bit]] +reveal_type(divmod(i8, AR)) # E: Union[Tuple[numpy.ndarray, numpy.ndarray], Tuple[numpy.generic, numpy.generic]] + +reveal_type(b % i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i % i8) # E: numpy.signedinteger[Any] +reveal_type(f % i8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i8 % i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(f8 % i8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i8 % i4) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(f8 % i4) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i4 % i4) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(f4 % i4) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(AR % i8) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(divmod(b, i8)) # E: Tuple[numpy.signedinteger[numpy.typing._64Bit], numpy.signedinteger[numpy.typing._64Bit]] +reveal_type(divmod(i, i8)) # E: Tuple[numpy.signedinteger[Any], numpy.signedinteger[Any]] +reveal_type(divmod(f, i8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(i8, i8)) # E: Tuple[numpy.signedinteger[numpy.typing._64Bit], numpy.signedinteger[numpy.typing._64Bit]] +reveal_type(divmod(f8, i8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(i4, i8)) # E: Tuple[numpy.signedinteger[numpy.typing._64Bit], numpy.signedinteger[numpy.typing._64Bit]] +reveal_type(divmod(f4, i8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(i4, i4)) # E: Tuple[numpy.signedinteger[numpy.typing._32Bit], numpy.signedinteger[numpy.typing._32Bit]] +reveal_type(divmod(f4, i4)) # E: Tuple[numpy.floating[numpy.typing._32Bit], numpy.floating[numpy.typing._32Bit]] +reveal_type(divmod(AR, i8)) # E: Union[Tuple[numpy.ndarray, numpy.ndarray], Tuple[numpy.generic, numpy.generic]] + +# float + +reveal_type(f8 % b) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f8 % i) # E: numpy.floating[Any] +reveal_type(f8 % f) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i8 % f4) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f4 % f4) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(f8 % AR) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(divmod(f8, b)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(f8, i)) # E: Tuple[numpy.floating[Any], numpy.floating[Any]] +reveal_type(divmod(f8, f)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(f8, f8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(f8, f4)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(f4, f4)) # E: Tuple[numpy.floating[numpy.typing._32Bit], numpy.floating[numpy.typing._32Bit]] +reveal_type(divmod(f8, AR)) # E: Union[Tuple[numpy.ndarray, numpy.ndarray], Tuple[numpy.generic, numpy.generic]] + +reveal_type(b % f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(i % f8) # E: numpy.floating[Any] +reveal_type(f % f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f8 % f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f8 % f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(f4 % f4) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(AR % f8) # E: Union[numpy.ndarray, numpy.generic] + +reveal_type(divmod(b, f8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(i, f8)) # E: Tuple[numpy.floating[Any], numpy.floating[Any]] +reveal_type(divmod(f, f8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(f8, f8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(f4, f8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] +reveal_type(divmod(f4, f4)) # E: Tuple[numpy.floating[numpy.typing._32Bit], numpy.floating[numpy.typing._32Bit]] +reveal_type(divmod(AR, f8)) # E: Union[Tuple[numpy.ndarray, numpy.ndarray], Tuple[numpy.generic, numpy.generic]] From 33cc725339191f57ef2f646f9b734f97e673846e Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Thu, 22 Oct 2020 17:18:43 +0200 Subject: [PATCH 326/409] TST: Added typing tests for unary operations --- numpy/typing/tests/data/fail/arithmetic.py | 9 ++--- numpy/typing/tests/data/pass/arithmetic.py | 36 ++++++++++++++++++++ numpy/typing/tests/data/reveal/arithmetic.py | 36 ++++++++++++++++++++ 3 files changed, 75 insertions(+), 6 deletions(-) diff --git a/numpy/typing/tests/data/fail/arithmetic.py b/numpy/typing/tests/data/fail/arithmetic.py index 169e104f9a22..f32eddc4bc8d 100644 --- a/numpy/typing/tests/data/fail/arithmetic.py +++ b/numpy/typing/tests/data/fail/arithmetic.py @@ -10,10 +10,7 @@ td - dt # E: Unsupported operand types td % 1 # E: Unsupported operand types td / dt # E: No overload +td % dt # E: Unsupported operand types -# NOTE: The 1 tests below currently don't work due to the broad -# (i.e. untyped) signature of `.__mod__()`. -# TODO: Revisit this once annotations are added to the -# `_ArrayOrScalarCommon` magic methods. - -# td % dt # E: Unsupported operand types +-b_ # E: Unsupported operand type ++b_ # E: Unsupported operand type diff --git a/numpy/typing/tests/data/pass/arithmetic.py b/numpy/typing/tests/data/pass/arithmetic.py index f26eab879255..ffbaf2975332 100644 --- a/numpy/typing/tests/data/pass/arithmetic.py +++ b/numpy/typing/tests/data/pass/arithmetic.py @@ -23,6 +23,42 @@ AR = np.ones(1, dtype=np.float64) AR.setflags(write=False) +# unary ops + +-c16 +-c8 +-f8 +-f4 +-i8 +-i4 +-u8 +-u4 +-td +-AR + ++c16 ++c8 ++f8 ++f4 ++i8 ++i4 ++u8 ++u4 ++td ++AR + +abs(c16) +abs(c8) +abs(f8) +abs(f4) +abs(i8) +abs(i4) +abs(u8) +abs(u4) +abs(td) +abs(b_) +abs(AR) + # Time structures dt + td diff --git a/numpy/typing/tests/data/reveal/arithmetic.py b/numpy/typing/tests/data/reveal/arithmetic.py index 24fc672d0533..8a133c136339 100644 --- a/numpy/typing/tests/data/reveal/arithmetic.py +++ b/numpy/typing/tests/data/reveal/arithmetic.py @@ -23,6 +23,42 @@ AR = np.array([0], dtype=np.float64) AR.setflags(write=False) +# unary ops + +reveal_type(-c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(-c8) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] +reveal_type(-f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(-f4) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(-i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(-i4) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(-u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(-u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(-td) # E: numpy.timedelta64 +reveal_type(-AR) # E: Union[numpy.ndarray*, numpy.generic] + +reveal_type(+c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] +reveal_type(+c8) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] +reveal_type(+f8) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(+f4) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(+i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(+i4) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(+u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(+u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(+td) # E: numpy.timedelta64 +reveal_type(+AR) # E: Union[numpy.ndarray*, numpy.generic] + +reveal_type(abs(c16)) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(abs(c8)) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(abs(f8)) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(abs(f4)) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(abs(i8)) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(abs(i4)) # E: numpy.signedinteger[numpy.typing._32Bit] +reveal_type(abs(u8)) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(abs(u4)) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(abs(td)) # E: numpy.timedelta64 +reveal_type(abs(b_)) # E: numpy.bool_ +reveal_type(abs(AR)) # E: Union[numpy.ndarray*, numpy.generic] + # Time structures reveal_type(dt + td) # E: numpy.datetime64 From 939985d0859bbce349c8cccf866679eb113aeab3 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Thu, 22 Oct 2020 17:44:59 +0200 Subject: [PATCH 327/409] TST: Add tests for conversions to builtin scalar types --- numpy/typing/tests/data/fail/scalars.py | 2 + numpy/typing/tests/data/pass/scalars.py | 53 ++++++++++++++++++++----- 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/numpy/typing/tests/data/fail/scalars.py b/numpy/typing/tests/data/fail/scalars.py index e7c4c7485603..5fffa89b142e 100644 --- a/numpy/typing/tests/data/fail/scalars.py +++ b/numpy/typing/tests/data/fail/scalars.py @@ -64,3 +64,5 @@ def __float__(self): np.bytes_(b"hello", encoding='utf-8') # E: No overload variant np.str_("hello", encoding='utf-8') # E: No overload variant + +complex(np.bytes_("1")) # E: No overload variant diff --git a/numpy/typing/tests/data/pass/scalars.py b/numpy/typing/tests/data/pass/scalars.py index 49ddb8ed91f5..04e97498304d 100644 --- a/numpy/typing/tests/data/pass/scalars.py +++ b/numpy/typing/tests/data/pass/scalars.py @@ -1,6 +1,7 @@ import sys import datetime as dt +import pytest import numpy as np @@ -62,16 +63,6 @@ def __float__(self) -> float: np.str_(b"hello", 'utf-8') np.str_(b"hello", encoding='utf-8') -# Protocols -float(np.int8(4)) -int(np.int16(5)) -np.int8(np.float32(6)) - -# TODO(alan): test after https://github.com/python/typeshed/pull/2004 -# complex(np.int32(8)) - -abs(np.int8(4)) - # Array-ish semantics np.int8().real np.int16().imag @@ -114,3 +105,45 @@ def __float__(self) -> float: np.void(np.bool_(True)) np.void(b"test") np.void(np.bytes_("test")) + +# Protocols +i8 = np.int64() +u8 = np.uint64() +f8 = np.float64() +c16 = np.complex128() +b_ = np.bool_() +td = np.timedelta64() +U = np.str_("1") +S = np.bytes_("1") +AR = np.array(1, dtype=np.float64) + +int(i8) +int(u8) +int(f8) +int(b_) +int(td) +int(U) +int(S) +int(AR) +with pytest.warns(np.ComplexWarning): + int(c16) + +float(i8) +float(u8) +float(f8) +float(b_) +float(td) +float(U) +float(S) +float(AR) +with pytest.warns(np.ComplexWarning): + float(c16) + +complex(i8) +complex(u8) +complex(f8) +complex(c16) +complex(b_) +complex(td) +complex(U) +complex(AR) From d92f151b961ac0a39915a0e7769d6455031e826f Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 11 Sep 2020 11:31:45 -0500 Subject: [PATCH 328/409] ENH: allow exporting user-dtype as buffers without FORMAT --- numpy/core/src/multiarray/buffer.c | 64 +++++++++++++++++++----------- 1 file changed, 40 insertions(+), 24 deletions(-) diff --git a/numpy/core/src/multiarray/buffer.c b/numpy/core/src/multiarray/buffer.c index e676682deec2..e27dadc9dbe0 100644 --- a/numpy/core/src/multiarray/buffer.c +++ b/numpy/core/src/multiarray/buffer.c @@ -456,7 +456,8 @@ static PyObject *_buffer_info_cache = NULL; /* Fill in the info structure */ static _buffer_info_t* -_buffer_info_new(PyObject *obj, npy_bool f_contiguous) +_buffer_info_new( + PyObject *obj, npy_bool f_contiguous, npy_bool format_requested) { /* * Note that the buffer info is cached as PyLongObjects making them appear @@ -547,16 +548,20 @@ _buffer_info_new(PyObject *obj, npy_bool f_contiguous) } /* Fill in format */ - err = _buffer_format_string(descr, &fmt, obj, NULL, NULL); - Py_DECREF(descr); - if (err != 0) { - goto fail; + if (format_requested) { + err = _buffer_format_string(descr, &fmt, obj, NULL, NULL); + Py_DECREF(descr); + if (err != 0) { + goto fail; + } + if (_append_char(&fmt, '\0') < 0) { + goto fail; + } + info->format = fmt.s; } - if (_append_char(&fmt, '\0') < 0) { - goto fail; + else { + info->format = NULL; } - info->format = fmt.s; - return info; fail: @@ -572,9 +577,10 @@ _buffer_info_cmp(_buffer_info_t *a, _buffer_info_t *b) Py_ssize_t c; int k; - c = strcmp(a->format, b->format); - if (c != 0) return c; - + if (a->format != NULL && b->format != NULL) { + c = strcmp(a->format, b->format); + if (c != 0) return c; + } c = a->ndim - b->ndim; if (c != 0) return c; @@ -599,7 +605,8 @@ _buffer_info_free(_buffer_info_t *info) /* Get buffer info from the global dictionary */ static _buffer_info_t* -_buffer_get_info(PyObject *obj, npy_bool f_contiguous) +_buffer_get_info( + PyObject *obj, npy_bool f_contiguous, npy_bool format_requested) { PyObject *key = NULL, *item_list = NULL, *item = NULL; _buffer_info_t *info = NULL, *old_info = NULL; @@ -612,7 +619,7 @@ _buffer_get_info(PyObject *obj, npy_bool f_contiguous) } /* Compute information */ - info = _buffer_info_new(obj, f_contiguous); + info = _buffer_info_new(obj, f_contiguous, format_requested); if (info == NULL) { return NULL; } @@ -630,11 +637,7 @@ _buffer_get_info(PyObject *obj, npy_bool f_contiguous) if (item_list_length > 0) { item = PyList_GetItem(item_list, item_list_length - 1); old_info = (_buffer_info_t*)PyLong_AsVoidPtr(item); - if (_buffer_info_cmp(info, old_info) == 0) { - _buffer_info_free(info); - info = old_info; - } - else { + if (_buffer_info_cmp(info, old_info) != 0) { if (item_list_length > 1 && info->ndim > 1) { /* * Some arrays are C- and F-contiguous and if they have more @@ -648,11 +651,22 @@ _buffer_get_info(PyObject *obj, npy_bool f_contiguous) */ item = PyList_GetItem(item_list, item_list_length - 2); old_info = (_buffer_info_t*)PyLong_AsVoidPtr(item); - if (_buffer_info_cmp(info, old_info) == 0) { - _buffer_info_free(info); - info = old_info; + if (_buffer_info_cmp(info, old_info) != 0) { + old_info = NULL; } } + else { + old_info = NULL; + } + } + + if (old_info != NULL) { + if (old_info->format == NULL) { + old_info->format = info->format; + info->format = NULL; + } + _buffer_info_free(info); + info = old_info; } } } @@ -760,7 +774,9 @@ array_getbuffer(PyObject *obj, Py_buffer *view, int flags) } /* Fill in information */ - info = _buffer_get_info(obj, (flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS); + info = _buffer_get_info(obj, + (flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS, + (flags & PyBUF_FORMAT) == PyBUF_FORMAT); if (info == NULL) { goto fail; } @@ -825,7 +841,7 @@ void_getbuffer(PyObject *self, Py_buffer *view, int flags) } /* Fill in information */ - info = _buffer_get_info(self, 0); + info = _buffer_get_info(self, 0, (flags & PyBUF_FORMAT) == PyBUF_FORMAT); if (info == NULL) { goto fail; } From 83b517b84d34f1a847702c3d6d12eddf5643bf33 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 11 Sep 2020 11:09:15 -0500 Subject: [PATCH 329/409] ENH: Allow buffer export and pickling of `np.generic` subclasses Previously we had code that would allow exporting the buffer, but then fail for any reasonable subclass, because such a subclass should have its own user-dtype. The change is, that now a subclass without its own user-dtype will inherit the correct behaviour directly. This allows pickling of of such user-defined scalars (with user-defined dtype) if no FORMAT was requested in the buffer export. The latter allows the generic pickling code to succeed. Closes gh-17294 --- numpy/core/src/multiarray/scalartypes.c.src | 45 +++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src index 1a50927a8cd4..d2ae6ce31b7c 100644 --- a/numpy/core/src/multiarray/scalartypes.c.src +++ b/numpy/core/src/multiarray/scalartypes.c.src @@ -2383,6 +2383,50 @@ static PySequenceMethods voidtype_as_sequence = { }; +/* + * This function implements simple buffer export for user defined subclasses + * of `np.generic`. All other scalar types override the buffer export. + */ +static int +gentype_arrtype_getbuffer(PyObject *self, Py_buffer *view, int flags) +{ + if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) { + PyErr_Format(PyExc_TypeError, + "NumPy scalar %R can only exported as a buffer without format.", + self); + return -1; + } + PyArray_Descr *descr = PyArray_DescrFromScalar(self); + if (descr == NULL) { + return -1; + } + if (!PyDataType_ISUSERDEF(descr)) { + /* This path would also reject the (hopefully) impossible "object" */ + PyErr_Format(PyExc_TypeError, + "user-defined scalar %R registered for built-in dtype %S? " + "This should be impossible.", + self, descr); + return -1; + } + view->ndim = 0; + view->len = descr->elsize; + view->itemsize = descr->elsize; + view->shape = NULL; + view->strides = NULL; + view->suboffsets = NULL; + Py_INCREF(self); + view->obj = self; + view->buf = scalar_value(self, descr); + Py_DECREF(descr); + view->format = NULL; + return 0; +} + + +static PyBufferProcs gentype_arrtype_as_buffer = { + .bf_getbuffer = (getbufferproc)gentype_arrtype_getbuffer, +}; + /**begin repeat * #name = bool, byte, short, int, long, longlong, ubyte, ushort, uint, ulong, @@ -3794,6 +3838,7 @@ initialize_numeric_types(void) PyGenericArrType_Type.tp_alloc = gentype_alloc; PyGenericArrType_Type.tp_free = (freefunc)gentype_free; PyGenericArrType_Type.tp_richcompare = gentype_richcompare; + PyGenericArrType_Type.tp_as_buffer = &gentype_arrtype_as_buffer; PyBoolArrType_Type.tp_as_number = &bool_arrtype_as_number; /* From fae94b72f122501c4da98239e8253be34e474012 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 11 Sep 2020 12:00:23 -0500 Subject: [PATCH 330/409] TST: Fix `tp_name` of rational This is necessary to allow pickling of the type object, which is necessary to test pickling of the scalar (and in arrays) --- numpy/core/src/umath/_rational_tests.c.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/src/umath/_rational_tests.c.src b/numpy/core/src/umath/_rational_tests.c.src index 08c259d98f25..7b1e5627ae7f 100644 --- a/numpy/core/src/umath/_rational_tests.c.src +++ b/numpy/core/src/umath/_rational_tests.c.src @@ -663,7 +663,7 @@ static PyGetSetDef pyrational_getset[] = { static PyTypeObject PyRational_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "rational", /* tp_name */ + "numpy.core._rational_tests.rational", /* tp_name */ sizeof(PyRational), /* tp_basicsize */ 0, /* tp_itemsize */ 0, /* tp_dealloc */ From 1725f315bfc4f088c6d1773e20d120c663c9b740 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 11 Sep 2020 11:11:49 -0500 Subject: [PATCH 331/409] TST: Add test for non-FORMAT user dtype array/scalar export This also tests pickling as a regression test, since at least at this time it is directly related to the buffer export. --- numpy/core/tests/test_multiarray.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index d46e4ce9b840..e7243c510f9f 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -22,6 +22,7 @@ import numpy as np import numpy.core._multiarray_tests as _multiarray_tests +from numpy.core._rational_tests import rational from numpy.testing import ( assert_, assert_raises, assert_warns, assert_equal, assert_almost_equal, assert_array_equal, assert_raises_regex, assert_array_almost_equal, @@ -7143,6 +7144,20 @@ def test_export_flags(self): _multiarray_tests.get_buffer_info, np.arange(5)[::2], ('SIMPLE',)) + @pytest.mark.parametrize(["obj", "error"], [ + pytest.param(np.array([1, 2], dtype=rational), ValueError, id="array"), + pytest.param(rational(1, 2), TypeError, id="scalar")]) + def test_export_and_pickle_user_dtype(self, obj, error): + # User dtypes should export successfully when FORMAT was not requested. + with pytest.raises(error): + _multiarray_tests.get_buffer_info(obj, ("STRIDED", "FORMAT")) + + _multiarray_tests.get_buffer_info(obj, ("STRIDED",)) + + # This is currently also necessary to implement pickling: + res = pickle.loads(pickle.dumps(obj)) + assert_array_equal(res, obj) + def test_padding(self): for j in range(8): x = np.array([(1,), (2,)], dtype={'f0': (int, j)}) From 2fc4dbb3601adb38d27e9c4d35b9ea7a49369499 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 22 Oct 2020 11:51:17 -0500 Subject: [PATCH 332/409] MAINT: Pass in flags instead of format and contig explicitly --- numpy/core/src/multiarray/buffer.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/numpy/core/src/multiarray/buffer.c b/numpy/core/src/multiarray/buffer.c index e27dadc9dbe0..bf19770cfa6a 100644 --- a/numpy/core/src/multiarray/buffer.c +++ b/numpy/core/src/multiarray/buffer.c @@ -456,8 +456,7 @@ static PyObject *_buffer_info_cache = NULL; /* Fill in the info structure */ static _buffer_info_t* -_buffer_info_new( - PyObject *obj, npy_bool f_contiguous, npy_bool format_requested) +_buffer_info_new(PyObject *obj, int flags) { /* * Note that the buffer info is cached as PyLongObjects making them appear @@ -515,6 +514,7 @@ _buffer_info_new( * (This is unnecessary, but has no effect in the case where * NPY_RELAXED_STRIDES CHECKING is disabled.) */ + int f_contiguous = (flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS; if (PyArray_IS_C_CONTIGUOUS(arr) && !( f_contiguous && PyArray_IS_F_CONTIGUOUS(arr))) { Py_ssize_t sd = PyArray_ITEMSIZE(arr); @@ -548,7 +548,7 @@ _buffer_info_new( } /* Fill in format */ - if (format_requested) { + if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) { err = _buffer_format_string(descr, &fmt, obj, NULL, NULL); Py_DECREF(descr); if (err != 0) { @@ -605,8 +605,7 @@ _buffer_info_free(_buffer_info_t *info) /* Get buffer info from the global dictionary */ static _buffer_info_t* -_buffer_get_info( - PyObject *obj, npy_bool f_contiguous, npy_bool format_requested) +_buffer_get_info(PyObject *obj, int flags) { PyObject *key = NULL, *item_list = NULL, *item = NULL; _buffer_info_t *info = NULL, *old_info = NULL; @@ -619,7 +618,7 @@ _buffer_get_info( } /* Compute information */ - info = _buffer_info_new(obj, f_contiguous, format_requested); + info = _buffer_info_new(obj, flags); if (info == NULL) { return NULL; } @@ -774,9 +773,7 @@ array_getbuffer(PyObject *obj, Py_buffer *view, int flags) } /* Fill in information */ - info = _buffer_get_info(obj, - (flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS, - (flags & PyBUF_FORMAT) == PyBUF_FORMAT); + info = _buffer_get_info(obj, flags); if (info == NULL) { goto fail; } @@ -841,7 +838,7 @@ void_getbuffer(PyObject *self, Py_buffer *view, int flags) } /* Fill in information */ - info = _buffer_get_info(self, 0, (flags & PyBUF_FORMAT) == PyBUF_FORMAT); + info = _buffer_get_info(self, flags); if (info == NULL) { goto fail; } From e9ce0f485c16f57fcc619de76b691acf8294f09c Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 22 Oct 2020 12:00:45 -0500 Subject: [PATCH 333/409] DOC: Add a comment to explain the format transfer --- numpy/core/src/multiarray/buffer.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/numpy/core/src/multiarray/buffer.c b/numpy/core/src/multiarray/buffer.c index bf19770cfa6a..5ba343d6dac0 100644 --- a/numpy/core/src/multiarray/buffer.c +++ b/numpy/core/src/multiarray/buffer.c @@ -660,6 +660,12 @@ _buffer_get_info(PyObject *obj, int flags) } if (old_info != NULL) { + /* + * The two info->format are considered equal if one of them + * has no format set (meaning the format is arbitrary and can + * be modified). If the new info has a format, but we reuse + * the old one, this transfers the ownership to the old one. + */ if (old_info->format == NULL) { old_info->format = info->format; info->format = NULL; From 63f90a8734ead51150c9b9a62239c42ef6a387df Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 22 Oct 2020 13:41:44 -0500 Subject: [PATCH 334/409] TST: Modify test to see if the error is in pickling or unpickling --- numpy/core/tests/test_multiarray.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index e7243c510f9f..291e8ba8eb15 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -7155,7 +7155,8 @@ def test_export_and_pickle_user_dtype(self, obj, error): _multiarray_tests.get_buffer_info(obj, ("STRIDED",)) # This is currently also necessary to implement pickling: - res = pickle.loads(pickle.dumps(obj)) + pickle_obj = pickle.dumps(obj) + res = pickle.loads(pickle_obj) assert_array_equal(res, obj) def test_padding(self): From 09934b2b5227cdc9b9f28dc5d719dbc515cbe69e Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 22 Oct 2020 14:51:15 -0500 Subject: [PATCH 335/409] BUG: Fix memloc to be unsigned and add debug info to test --- numpy/core/src/multiarray/scalarapi.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/core/src/multiarray/scalarapi.c b/numpy/core/src/multiarray/scalarapi.c index f610ad4680e3..0e93cbbe9f57 100644 --- a/numpy/core/src/multiarray/scalarapi.c +++ b/numpy/core/src/multiarray/scalarapi.c @@ -35,7 +35,7 @@ scalar_value(PyObject *scalar, PyArray_Descr *descr) { int type_num; int align; - npy_intp memloc; + uintptr_t memloc; if (descr == NULL) { descr = PyArray_DescrFromScalar(scalar); type_num = descr->type_num; @@ -168,7 +168,7 @@ scalar_value(PyObject *scalar, PyArray_Descr *descr) * Use the alignment flag to figure out where the data begins * after a PyObject_HEAD */ - memloc = (npy_intp)scalar; + memloc = (uintptr_t)scalar; memloc += sizeof(PyObject); /* now round-up to the nearest alignment value */ align = descr->alignment; From d02ca96090ea2fed97b7789a855668c1ddc98294 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 22 Oct 2020 15:07:38 -0500 Subject: [PATCH 336/409] MAINT: Simplify code path slightly --- numpy/core/src/multiarray/buffer.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/numpy/core/src/multiarray/buffer.c b/numpy/core/src/multiarray/buffer.c index 5ba343d6dac0..3b3bba663aac 100644 --- a/numpy/core/src/multiarray/buffer.c +++ b/numpy/core/src/multiarray/buffer.c @@ -637,6 +637,8 @@ _buffer_get_info(PyObject *obj, int flags) item = PyList_GetItem(item_list, item_list_length - 1); old_info = (_buffer_info_t*)PyLong_AsVoidPtr(item); if (_buffer_info_cmp(info, old_info) != 0) { + old_info = NULL; /* Can't use this one, but possibly next */ + if (item_list_length > 1 && info->ndim > 1) { /* * Some arrays are C- and F-contiguous and if they have more @@ -654,9 +656,6 @@ _buffer_get_info(PyObject *obj, int flags) old_info = NULL; } } - else { - old_info = NULL; - } } if (old_info != NULL) { From b20b06f8ed33eef1afc5640567cffa5203bcef2f Mon Sep 17 00:00:00 2001 From: takanori-pskq Date: Fri, 23 Oct 2020 13:28:07 +0900 Subject: [PATCH 337/409] DOC: Fix the document for arrays interface --- doc/source/reference/arrays.interface.rst | 28 +++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/source/reference/arrays.interface.rst b/doc/source/reference/arrays.interface.rst index aeac66f829e5..aee95ba5bf69 100644 --- a/doc/source/reference/arrays.interface.rst +++ b/doc/source/reference/arrays.interface.rst @@ -61,12 +61,12 @@ This approach to the interface consists of the object having an **shape** (required) Tuple whose elements are the array size in each dimension. Each - entry is an integer (a Python int or long). Note that these - integers could be larger than the platform "int" or "long" - could hold (a Python int is a C long). It is up to the code + entry is an integer (a Python :py:class:`int`). Note that these + integers could be larger than the platform ``int`` or ``long`` + could hold (a Python :py:class`int` is a C ``long``). It is up to the code using this attribute to handle this appropriately; either by raising an error when overflow is possible, or by using - :c:data:`Py_LONG_LONG` as the C type for the shapes. + ``long long`` as the C type for the shapes. **typestr** (required) A string providing the basic type of the homogeneous array The @@ -132,7 +132,7 @@ This approach to the interface consists of the object having an means the data area is read-only). This attribute can also be an object exposing the - :c:func:`buffer interface ` which + :ref:`buffer interface ` which will be used to share the data. If this key is not present (or returns None), then memory sharing will be done through the buffer interface of the object itself. In this @@ -144,21 +144,21 @@ This approach to the interface consists of the object having an **Default**: None **strides** (optional) - Either None to indicate a C-style contiguous array or + Either ``None`` to indicate a C-style contiguous array or a Tuple of strides which provides the number of bytes needed to jump to the next array element in the corresponding dimension. Each entry must be an integer (a Python - :const:`int` or :const:`long`). As with shape, the values may - be larger than can be represented by a C "int" or "long"; the + :py:class:`int`). As with shape, the values may + be larger than can be represented by a C ``int`` or ``long``; the calling code should handle this appropriately, either by - raising an error, or by using :c:type:`Py_LONG_LONG` in C. The - default is None which implies a C-style contiguous - memory buffer. In this model, the last dimension of the array + raising an error, or by using ``long long`` in C. The + default is ``None`` which implies a C-style contiguous + memory buffer. In this model, the last dimension of the array varies the fastest. For example, the default strides tuple for an object whose array entries are 8 bytes long and whose - shape is (10,20,30) would be (4800, 240, 8) + shape is ``(10, 20, 30)`` would be ``(4800, 240, 8)`` - **Default**: None (C-style contiguous) + **Default**: ``None`` (C-style contiguous) **mask** (optional) None or an object exposing the array interface. All @@ -172,7 +172,7 @@ This approach to the interface consists of the object having an **offset** (optional) An integer offset into the array data region. This can only be - used when data is None or returns a :class:`buffer` + used when data is ``None`` or returns a :class:`buffer` object. **Default**: 0. From 54d8c9f50228da85f79755e2891a368e572425ad Mon Sep 17 00:00:00 2001 From: Jakob Date: Fri, 23 Oct 2020 20:56:18 +0200 Subject: [PATCH 338/409] Change string to f-string --- numpy/tests/test_public_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/tests/test_public_api.py b/numpy/tests/test_public_api.py index 6ba279c61706..a7bd0f115d15 100644 --- a/numpy/tests/test_public_api.py +++ b/numpy/tests/test_public_api.py @@ -349,7 +349,7 @@ def test_all_modules_are_expected(): modnames.append(modname) if modnames: - raise AssertionError("Found unexpected modules: {}".format(modnames)) + raise AssertionError(f'Found unexpected modules: {modnames}') # Stuff that clearly shouldn't be in the API and is detected by the next test From 1c2b8da1bbc8458b1d878ae10d957bd6b7546131 Mon Sep 17 00:00:00 2001 From: Jakob Date: Fri, 23 Oct 2020 21:33:30 +0200 Subject: [PATCH 339/409] mostly string conversions --- numpy/testing/_private/decorators.py | 2 +- numpy/testing/_private/noseclasses.py | 4 +- numpy/testing/_private/nosetester.py | 18 +++--- numpy/testing/_private/parameterized.py | 2 +- numpy/testing/_private/utils.py | 84 ++++++++++++------------- 5 files changed, 53 insertions(+), 57 deletions(-) diff --git a/numpy/testing/_private/decorators.py b/numpy/testing/_private/decorators.py index b4b6259a0497..4c87d1a4919c 100644 --- a/numpy/testing/_private/decorators.py +++ b/numpy/testing/_private/decorators.py @@ -136,7 +136,7 @@ def get_msg(func,msg=None): else: out = msg - return "Skipping test: %s: %s" % (func.__name__, out) + return f'Skipping test: {func.__name__}: {out}' # We need to define *two* skippers because Python doesn't allow both # return with value and yield inside the same function. diff --git a/numpy/testing/_private/noseclasses.py b/numpy/testing/_private/noseclasses.py index 69e19e9594d4..48fa4dc1f5af 100644 --- a/numpy/testing/_private/noseclasses.py +++ b/numpy/testing/_private/noseclasses.py @@ -76,7 +76,7 @@ def _find(self, tests, obj, name, module, source_lines, globs, seen): # Look for tests in a module's contained objects. if ismodule(obj) and self._recurse: for valname, val in obj.__dict__.items(): - valname1 = '%s.%s' % (name, valname) + valname1 = f'{name}.{valname}' if ( (isroutine(val) or isclass(val)) and self._from_module(module, val)): @@ -96,7 +96,7 @@ def _find(self, tests, obj, name, module, source_lines, globs, seen): if ((isfunction(val) or isclass(val) or ismethod(val) or isinstance(val, property)) and self._from_module(module, val)): - valname = '%s.%s' % (name, valname) + valname = f'{name}.{valname}' self._find(tests, val, valname, module, source_lines, globs, seen) diff --git a/numpy/testing/_private/nosetester.py b/numpy/testing/_private/nosetester.py index 57691a448f73..bccec8236912 100644 --- a/numpy/testing/_private/nosetester.py +++ b/numpy/testing/_private/nosetester.py @@ -233,20 +233,20 @@ def _show_system_info(self): nose = import_nose() import numpy - print("NumPy version %s" % numpy.__version__) + print(f'NumPy version {numpy.__version__}') relaxed_strides = numpy.ones((10, 1), order="C").flags.f_contiguous print("NumPy relaxed strides checking option:", relaxed_strides) npdir = os.path.dirname(numpy.__file__) - print("NumPy is installed in %s" % npdir) + print(f'NumPy is installed in {npdir}') if 'scipy' in self.package_name: import scipy - print("SciPy version %s" % scipy.__version__) + print(f'SciPy version {scipy.__version__}') spdir = os.path.dirname(scipy.__file__) - print("SciPy is installed in %s" % spdir) + print(f'SciPy is installed in {spdir}') pyversion = sys.version.replace('\n', '') - print("Python version %s" % pyversion) + print(f'Python version {pyversion}') print("nose version %d.%d.%d" % nose.__versioninfo__) def _get_custom_doctester(self): @@ -278,7 +278,7 @@ def prepare_test_args(self, label='fast', verbose=1, extra_argv=None, argv = self._test_argv(label, verbose, extra_argv) # our way of doing coverage if coverage: - argv += ['--cover-package=%s' % self.package_name, '--with-coverage', + argv += [f'--cover-package={self.package_name}', '--with-coverage', '--cover-tests', '--cover-erase'] if timer: @@ -403,9 +403,9 @@ def test(self, label='fast', verbose=1, extra_argv=None, label, verbose, extra_argv, doctests, coverage, timer) if doctests: - print("Running unit tests and doctests for %s" % self.package_name) + print(f'Running unit tests and doctests for {self.package_name}') else: - print("Running unit tests for %s" % self.package_name) + print(f'Running unit tests for {self.package_name}') self._show_system_info() @@ -520,7 +520,7 @@ def bench(self, label='fast', verbose=1, extra_argv=None): """ - print("Running benchmarks for %s" % self.package_name) + print(f'Running benchmarks for {self.package_name}') self._show_system_info() argv = self._test_argv(label, verbose, extra_argv) diff --git a/numpy/testing/_private/parameterized.py b/numpy/testing/_private/parameterized.py index 3bd8ede91fad..ac7db6c4041f 100644 --- a/numpy/testing/_private/parameterized.py +++ b/numpy/testing/_private/parameterized.py @@ -205,7 +205,7 @@ def default_doc_func(func, num, p): all_args_with_values = parameterized_argument_value_pairs(func, p) # Assumes that the function passed is a bound method. - descs = ["%s=%s" %(n, short_repr(v)) for n, v in all_args_with_values] + descs = [f'{n}={short_repr(v)}' for n, v in all_args_with_values] # The documentation might be a multiline string, so split it # and just work with the first string, ignoring the period diff --git a/numpy/testing/_private/utils.py b/numpy/testing/_private/utils.py index 4fc22dcebaec..b81250bbabc1 100644 --- a/numpy/testing/_private/utils.py +++ b/numpy/testing/_private/utils.py @@ -113,14 +113,14 @@ def gisnan(x): def gisfinite(x): - """like isfinite, but always raise an error if type not supported instead of - returning a TypeError object. + """like isfinite, but always raise an error if type not supported instead + of returning a TypeError object. Notes ----- - isfinite and other ufunc sometimes return a NotImplementedType object instead - of raising any exception. This function is a wrapper to make sure an - exception is always raised. + isfinite and other ufunc sometimes return a NotImplementedType object + instead of raising any exception. This function is a wrapper to make sure + an exception is always raised. This should be removed once this problem is solved at the Ufunc level.""" from numpy.core import isfinite, errstate @@ -160,12 +160,13 @@ def GetPerformanceAttributes(object, counter, instance=None, # you should copy this function, but keep the counter open, and call # CollectQueryData() each time you need to know. # See http://msdn.microsoft.com/library/en-us/dnperfmo/html/perfmonpt2.asp (dead link) - # My older explanation for this was that the "AddCounter" process forced - # the CPU to 100%, but the above makes more sense :) + # My older explanation for this was that the "AddCounter" process + # forced the CPU to 100%, but the above makes more sense :) import win32pdh if format is None: format = win32pdh.PDH_FMT_LONG - path = win32pdh.MakeCounterPath( (machine, object, instance, None, inum, counter)) + path = win32pdh.MakeCounterPath( (machine, object, instance, None, + inum, counter)) hq = win32pdh.OpenQuery() try: hc = win32pdh.AddCounter(hq, path) @@ -186,7 +187,7 @@ def memusage(processName="python", instance=0): win32pdh.PDH_FMT_LONG, None) elif sys.platform[:5] == 'linux': - def memusage(_proc_pid_stat='/proc/%s/stat' % (os.getpid())): + def memusage(_proc_pid_stat=f'/proc/{os.getpid()}/stat'): """ Return virtual memory size in bytes of the running python. @@ -207,8 +208,7 @@ def memusage(): if sys.platform[:5] == 'linux': - def jiffies(_proc_pid_stat='/proc/%s/stat' % (os.getpid()), - _load_time=[]): + def jiffies(_proc_pid_stat=f'/proc/{os.getpid()}/stat', _load_time=[]): """ Return number of jiffies elapsed. @@ -263,11 +263,11 @@ def build_err_msg(arrays, err_msg, header='Items are not equal:', try: r = r_func(a) except Exception as exc: - r = '[repr failed for <{}>: {}]'.format(type(a).__name__, exc) + r = f'[repr failed for <{type(a).__name__}>: {exc}]' if r.count('\n') > 3: r = '\n'.join(r.splitlines()[:3]) r += '...' - msg.append(' %s: %s' % (names[i], r)) + msg.append(f' {names[i]}: {r}') return '\n'.join(msg) @@ -329,12 +329,14 @@ def assert_equal(actual, desired, err_msg='', verbose=True): for k, i in desired.items(): if k not in actual: raise AssertionError(repr(k)) - assert_equal(actual[k], desired[k], 'key=%r\n%s' % (k, err_msg), verbose) + assert_equal(actual[k], desired[k], f'key={k!r}\n{err_msg}', + verbose) return if isinstance(desired, (list, tuple)) and isinstance(actual, (list, tuple)): assert_equal(len(actual), len(desired), err_msg, verbose) for k in range(len(desired)): - assert_equal(actual[k], desired[k], 'item=%r\n%s' % (k, err_msg), verbose) + assert_equal(actual[k], desired[k], f'item={k!r}\n{err_msg}', + verbose) return from numpy.core import ndarray, isscalar, signbit from numpy.lib import iscomplexobj, real, imag @@ -694,9 +696,8 @@ def assert_approx_equal(actual,desired,significant=7,err_msg='',verbose=True): raise AssertionError(msg) -def assert_array_compare(comparison, x, y, err_msg='', verbose=True, - header='', precision=6, equal_nan=True, - equal_inf=True): +def assert_array_compare(comparison, x, y, err_msg='', verbose=True, header='', + precision=6, equal_nan=True, equal_inf=True): __tracebackhide__ = True # Hide traceback for py.test from numpy.core import array, array2string, isnan, inf, bool_, errstate, all, max, object_ @@ -754,8 +755,7 @@ def func_assert_same_pos(x, y, func=isnan, hasval='nan'): if not cond: msg = build_err_msg([x, y], err_msg - + '\n(shapes %s, %s mismatch)' % (x.shape, - y.shape), + + f'\n(shapes {x.shape}, {y.shape} mismatch)', verbose=verbose, header=header, names=('x', 'y'), precision=precision) raise AssertionError(msg) @@ -843,7 +843,7 @@ def func_assert_same_pos(x, y, func=isnan, hasval='nan'): except ValueError: import traceback efmt = traceback.format_exc() - header = 'error during assertion:\n\n%s\n\n%s' % (efmt, header) + header = f'error during assertion:\n\n{efmt}\n\n{header}' msg = build_err_msg([x, y], err_msg, verbose=verbose, header=header, names=('x', 'y'), precision=precision) @@ -1170,7 +1170,8 @@ def assert_string_equal(actual, desired): if desired == actual: return - diff = list(difflib.Differ().compare(actual.splitlines(True), desired.splitlines(True))) + diff = list(difflib.Differ().compare(actual.splitlines(True), + desired.splitlines(True))) diff_list = [] while diff: d1 = diff.pop(0) @@ -1198,7 +1199,7 @@ def assert_string_equal(actual, desired): raise AssertionError(repr(d1)) if not diff_list: return - msg = 'Differences in strings:\n%s' % (''.join(diff_list)).rstrip() + msg = f'Differences in strings:\n{''.join(diff_list).rstrip()}' if actual != desired: raise AssertionError(msg) @@ -1434,9 +1435,7 @@ def measure(code_str, times=1, label=None): frame = sys._getframe(1) locs, globs = frame.f_locals, frame.f_globals - code = compile(code_str, - 'Test name: %s ' % label, - 'exec') + code = compile(code_str, f'Test name: {label} ', 'exec') i = 0 elapsed = jiffies() while i < times: @@ -1525,7 +1524,7 @@ def compare(x, y): equal_nan=equal_nan) actual, desired = np.asanyarray(actual), np.asanyarray(desired) - header = 'Not equal to tolerance rtol=%g, atol=%g' % (rtol, atol) + header = f'Not equal to tolerance rtol={rtol:g}, atol={atol:g}' assert_array_compare(compare, actual, desired, err_msg=str(err_msg), verbose=verbose, header=header, equal_nan=equal_nan) @@ -1724,8 +1723,8 @@ def _integer_repr(x, vdt, comp): def integer_repr(x): - """Return the signed-magnitude interpretation of the binary representation of - x.""" + """Return the signed-magnitude interpretation of the binary representation + of x.""" import numpy as np if x.dtype == np.float16: return _integer_repr(x, np.int16, np.int16(-2**15)) @@ -1734,7 +1733,7 @@ def integer_repr(x): elif x.dtype == np.float64: return _integer_repr(x, np.int64, np.int64(-2**63)) else: - raise ValueError("Unsupported dtype %s" % x.dtype) + raise ValueError(f"Unsupported dtype {x.dtype}") @contextlib.contextmanager @@ -1744,7 +1743,7 @@ def _assert_warns_context(warning_class, name=None): l = sup.record(warning_class) yield if not len(l) > 0: - name_str = " when calling %s" % name if name is not None else "" + name_str = f' when calling {name}' if name is not None else '' raise AssertionError("No warning raised" + name_str) @@ -1809,8 +1808,8 @@ def _assert_no_warnings_context(name=None): warnings.simplefilter('always') yield if len(l) > 0: - name_str = " when calling %s" % name if name is not None else "" - raise AssertionError("Got warnings%s: %s" % (name_str, l)) + name_str = f' when calling {name}' if name is not None else '' + raise AssertionError(f'Got warnings{name_str}: {l}') def assert_no_warnings(*args, **kwargs): @@ -2322,8 +2321,8 @@ def _assert_no_gc_cycles_context(name=None): break else: raise RuntimeError( - "Unable to fully collect garbage - perhaps a __del__ method is " - "creating more reference cycles?") + "Unable to fully collect garbage - perhaps a __del__ method " + "is creating more reference cycles?") gc.set_debug(gc.DEBUG_SAVEALL) yield @@ -2337,7 +2336,7 @@ def _assert_no_gc_cycles_context(name=None): gc.enable() if n_objects_in_cycles: - name_str = " when calling %s" % name if name is not None else "" + ame_str = f' when calling {name}' if name is not None else '' raise AssertionError( "Reference cycles were found{}: {} objects were collected, " "of which {} are shown below:{}" @@ -2441,12 +2440,10 @@ def check_free_memory(free_bytes): try: mem_free = _parse_size(env_value) except ValueError as exc: - raise ValueError('Invalid environment variable {}: {!s}'.format( - env_var, exc)) + raise ValueError('Invalid environment variable {env_var}: {str(exc)}') - msg = ('{0} GB memory required, but environment variable ' - 'NPY_AVAILABLE_MEM={1} set'.format( - free_bytes/1e9, env_value)) + msg = ('{free_bytes/1e9} GB memory required, but environment variable ' + 'NPY_AVAILABLE_MEM={env_value} set') else: mem_free = _get_mem_available() @@ -2456,8 +2453,7 @@ def check_free_memory(free_bytes): "the test.") mem_free = -1 else: - msg = '{0} GB memory required, but {1} GB available'.format( - free_bytes/1e9, mem_free/1e9) + msg = f'{free_bytes/1e9} GB memory required, but {mem_free/1e9} GB available' return msg if mem_free < free_bytes else None @@ -2474,7 +2470,7 @@ def _parse_size(size_str): m = size_re.match(size_str.lower()) if not m or m.group(2) not in suffixes: - raise ValueError("value {!r} not a valid size".format(size_str)) + raise ValueError(f'value {repr(size_str)} not a valid size') return int(float(m.group(1)) * suffixes[m.group(2)]) From 7a4588e00bea078a2e0ebdb1540c05d061796ccf Mon Sep 17 00:00:00 2001 From: Jakob Date: Fri, 23 Oct 2020 21:39:39 +0200 Subject: [PATCH 340/409] fix typo --- numpy/testing/_private/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/testing/_private/utils.py b/numpy/testing/_private/utils.py index b81250bbabc1..a8d0f582c273 100644 --- a/numpy/testing/_private/utils.py +++ b/numpy/testing/_private/utils.py @@ -2336,7 +2336,7 @@ def _assert_no_gc_cycles_context(name=None): gc.enable() if n_objects_in_cycles: - ame_str = f' when calling {name}' if name is not None else '' + name_str = f' when calling {name}' if name is not None else '' raise AssertionError( "Reference cycles were found{}: {} objects were collected, " "of which {} are shown below:{}" From ec359c37ba8f542fae97ff4e51a6d23876dcf185 Mon Sep 17 00:00:00 2001 From: Jakob Date: Fri, 23 Oct 2020 21:49:38 +0200 Subject: [PATCH 341/409] fix quotation marks --- numpy/testing/_private/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/testing/_private/utils.py b/numpy/testing/_private/utils.py index a8d0f582c273..e750db58b438 100644 --- a/numpy/testing/_private/utils.py +++ b/numpy/testing/_private/utils.py @@ -1199,7 +1199,7 @@ def assert_string_equal(actual, desired): raise AssertionError(repr(d1)) if not diff_list: return - msg = f'Differences in strings:\n{''.join(diff_list).rstrip()}' + msg = f"Differences in strings:\n{''.join(diff_list).rstrip()}" if actual != desired: raise AssertionError(msg) From 196d4eb031548c37577c086db9658c21f3ea2b81 Mon Sep 17 00:00:00 2001 From: Jakob Date: Fri, 23 Oct 2020 23:45:50 +0200 Subject: [PATCH 342/409] more fixes --- numpy/testing/_private/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/numpy/testing/_private/utils.py b/numpy/testing/_private/utils.py index e750db58b438..ae2617f4f124 100644 --- a/numpy/testing/_private/utils.py +++ b/numpy/testing/_private/utils.py @@ -2440,10 +2440,10 @@ def check_free_memory(free_bytes): try: mem_free = _parse_size(env_value) except ValueError as exc: - raise ValueError('Invalid environment variable {env_var}: {str(exc)}') + raise ValueError(f'Invalid environment variable {env_var}: {exc}') - msg = ('{free_bytes/1e9} GB memory required, but environment variable ' - 'NPY_AVAILABLE_MEM={env_value} set') + msg = (f'{free_bytes/1e9} GB memory required, but environment variable ' + f'NPY_AVAILABLE_MEM={env_value} set') else: mem_free = _get_mem_available() @@ -2470,7 +2470,7 @@ def _parse_size(size_str): m = size_re.match(size_str.lower()) if not m or m.group(2) not in suffixes: - raise ValueError(f'value {repr(size_str)} not a valid size') + raise ValueError(f'value {size_str!r} not a valid size') return int(float(m.group(1)) * suffixes[m.group(2)]) From 14bb26753df4ca4ef42ecd830699433ed781cf9f Mon Sep 17 00:00:00 2001 From: takanori-pskq Date: Sat, 24 Oct 2020 13:49:37 +0900 Subject: [PATCH 343/409] DOC: Fix some references --- doc/source/f2py/distutils.rst | 2 ++ doc/source/reference/arrays.ndarray.rst | 8 ++++---- doc/source/reference/c-api/array.rst | 4 ++-- .../reference/c-api/types-and-structures.rst | 9 +++++---- .../reference/internals.code-explanations.rst | 7 ++++--- doc/source/reference/maskedarray.generic.rst | 18 +++++++++--------- doc/source/reference/random/c-api.rst | 2 +- .../reference/routines.array-manipulation.rst | 1 + doc/source/reference/ufuncs.rst | 4 ++-- doc/source/release/1.16.0-notes.rst | 4 ++-- doc/source/user/basics.rec.rst | 4 ++-- doc/source/user/quickstart.rst | 2 +- 12 files changed, 35 insertions(+), 30 deletions(-) diff --git a/doc/source/f2py/distutils.rst b/doc/source/f2py/distutils.rst index 71f6eab5a96f..4cf30045ec1d 100644 --- a/doc/source/f2py/distutils.rst +++ b/doc/source/f2py/distutils.rst @@ -2,6 +2,8 @@ Using via `numpy.distutils` ============================= +.. currentmodule:: numpy.distutils.core + :mod:`numpy.distutils` is part of NumPy extending standard Python ``distutils`` to deal with Fortran sources and F2PY signature files, e.g. compile Fortran sources, call F2PY to construct extension modules, etc. diff --git a/doc/source/reference/arrays.ndarray.rst b/doc/source/reference/arrays.ndarray.rst index 689240c7d6c5..405ae8f6ac99 100644 --- a/doc/source/reference/arrays.ndarray.rst +++ b/doc/source/reference/arrays.ndarray.rst @@ -469,7 +469,7 @@ Comparison operators: ndarray.__eq__ ndarray.__ne__ -Truth value of an array (:func:`bool()`): +Truth value of an array (:class:`bool() `): .. autosummary:: :toctree: generated/ @@ -604,9 +604,9 @@ Container customization: (see :ref:`Indexing `) ndarray.__setitem__ ndarray.__contains__ -Conversion; the operations :func:`int()`, :func:`float()` and -:func:`complex()`. -. They work only on arrays that have one element in them +Conversion; the operations :class:`int() `, +:class:`float() ` and :class:`complex() `. +They work only on arrays that have one element in them and return the appropriate scalar. .. autosummary:: diff --git a/doc/source/reference/c-api/array.rst b/doc/source/reference/c-api/array.rst index 68547fd45ece..ae39f980f311 100644 --- a/doc/source/reference/c-api/array.rst +++ b/doc/source/reference/c-api/array.rst @@ -1335,7 +1335,7 @@ Special functions for NPY_OBJECT locations in the structure with object data-types. No checking is performed but *arr* must be of data-type :c:type:`NPY_OBJECT` and be single-segment and uninitialized (no previous objects in - position). Use :c:func:`PyArray_DECREF` (*arr*) if you need to + position). Use :c:func:`PyArray_XDECREF` (*arr*) if you need to decrement all the items in the object array prior to calling this function. @@ -1354,7 +1354,7 @@ Special functions for NPY_OBJECT strides, ordering, etc.) Sets the :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` flag and ``arr->base``, and set ``base`` to READONLY. Call :c:func:`PyArray_ResolveWritebackIfCopy` before calling - `Py_DECREF`` in order copy any changes back to ``base`` and + `Py_DECREF` in order copy any changes back to ``base`` and reset the READONLY flag. Returns 0 for success, -1 for failure. diff --git a/doc/source/reference/c-api/types-and-structures.rst b/doc/source/reference/c-api/types-and-structures.rst index fb5753c02df5..c45ebb692f2e 100644 --- a/doc/source/reference/c-api/types-and-structures.rst +++ b/doc/source/reference/c-api/types-and-structures.rst @@ -69,6 +69,7 @@ PyArray_Type and PyArrayObject typeobject. .. c:type:: PyArrayObject + NPY_AO The :c:type:`PyArrayObject` C-structure contains all of the required information for an array. All instances of an ndarray (and its @@ -77,7 +78,7 @@ PyArray_Type and PyArrayObject provided macros. If you need a shorter name, then you can make use of :c:type:`NPY_AO` (deprecated) which is defined to be equivalent to :c:type:`PyArrayObject`. Direct access to the struct fields are - deprecated. Use the `PyArray_*(arr)` form instead. + deprecated. Use the ``PyArray_*(arr)`` form instead. .. code-block:: c @@ -813,7 +814,7 @@ PyUFunc_Type and PyUFuncObject .. c:member:: int identity Either :c:data:`PyUFunc_One`, :c:data:`PyUFunc_Zero`, - :c:data:`PyUFunc_None` or :c:data:`PyUFunc_AllOnes` to indicate + or :c:data:`PyUFunc_None` to indicate the identity for this operation. It is only used for a reduce-like call on an empty array. @@ -1216,8 +1217,8 @@ are ``Py{TYPE}ArrType_Type`` where ``{TYPE}`` can be **Object**. These type names are part of the C-API and can therefore be created in -extension C-code. There is also a :c:data:`PyIntpArrType_Type` and a -:c:data:`PyUIntpArrType_Type` that are simple substitutes for one of the +extension C-code. There is also a ``PyIntpArrType_Type`` and a +``PyUIntpArrType_Type`` that are simple substitutes for one of the integer types that can hold a pointer on the platform. The structure of these scalar objects is not exposed to C-code. The function :c:func:`PyArray_ScalarAsCtype` (..) can be used to extract the C-type diff --git a/doc/source/reference/internals.code-explanations.rst b/doc/source/reference/internals.code-explanations.rst index 65553e07e6b2..e8e428f2ec6b 100644 --- a/doc/source/reference/internals.code-explanations.rst +++ b/doc/source/reference/internals.code-explanations.rst @@ -147,7 +147,8 @@ an iterator for each of the arrays being broadcast. The :c:func:`PyArray_Broadcast` function takes the iterators that have already been defined and uses them to determine the broadcast shape in each dimension (to create the iterators at the same time that broadcasting -occurs then use the :c:func:`PyMultiIter_New` function). Then, the iterators are +occurs then use the :c:func:`PyArray_MultiIterNew` function). +Then, the iterators are adjusted so that each iterator thinks it is iterating over an array with the broadcast size. This is done by adjusting the iterators number of dimensions, and the shape in each dimension. This works @@ -162,7 +163,7 @@ for the extended dimensions. It is done in exactly the same way in NumPy. The big difference is that now the array of strides is kept track of in a :c:type:`PyArrayIterObject`, the iterators involved in a broadcast result are kept track of in a :c:type:`PyArrayMultiIterObject`, -and the :c:func:`PyArray_BroadCast` call implements the broad-casting rules. +and the :c:func:`PyArray_Broadcast` call implements the broad-casting rules. Array Scalars @@ -368,7 +369,7 @@ The output arguments (if any) are then processed and any missing return arrays are constructed. If any provided output array doesn't have the correct type (or is mis-aligned) and is smaller than the buffer size, then a new output array is constructed with the special -:c:data:`WRITEBACKIFCOPY` flag set. At the end of the function, +:c:data:`NPY_ARRAY_WRITEBACKIFCOPY` flag set. At the end of the function, :c:func:`PyArray_ResolveWritebackIfCopy` is called so that its contents will be copied back into the output array. Iterators for the output arguments are then processed. diff --git a/doc/source/reference/maskedarray.generic.rst b/doc/source/reference/maskedarray.generic.rst index 41c3ee564809..d3849c50deec 100644 --- a/doc/source/reference/maskedarray.generic.rst +++ b/doc/source/reference/maskedarray.generic.rst @@ -177,8 +177,8 @@ attribute. We must keep in mind that a ``True`` entry in the mask indicates an *invalid* data. Another possibility is to use the :func:`getmask` and :func:`getmaskarray` -functions. :func:`getmask(x)` outputs the mask of ``x`` if ``x`` is a masked -array, and the special value :data:`nomask` otherwise. :func:`getmaskarray(x)` +functions. ``getmask(x)`` outputs the mask of ``x`` if ``x`` is a masked +array, and the special value :data:`nomask` otherwise. ``getmaskarray(x)`` outputs the mask of ``x`` if ``x`` is a masked array. If ``x`` has no invalid entry or is not a masked array, the function outputs a boolean array of ``False`` with as many elements as ``x``. @@ -296,11 +296,11 @@ new valid values to them:: .. note:: Unmasking an entry by direct assignment will silently fail if the masked - array has a *hard* mask, as shown by the :attr:`hardmask` attribute. This - feature was introduced to prevent overwriting the mask. To force the - unmasking of an entry where the array has a hard mask, the mask must first - to be softened using the :meth:`soften_mask` method before the allocation. - It can be re-hardened with :meth:`harden_mask`:: + array has a *hard* mask, as shown by the :attr:`~MaskedArray.hardmask` + attribute. This feature was introduced to prevent overwriting the mask. + To force the unmasking of an entry where the array has a hard mask, + the mask must first to be softened using the :meth:`soften_mask` method + before the allocation. It can be re-hardened with :meth:`harden_mask`:: >>> x = ma.array([1, 2, 3], mask=[0, 0, 1], hard_mask=True) >>> x @@ -406,8 +406,8 @@ Operations on masked arrays Arithmetic and comparison operations are supported by masked arrays. As much as possible, invalid entries of a masked array are not processed, -meaning that the corresponding :attr:`data` entries *should* be the same -before and after the operation. +meaning that the corresponding :attr:`~MaskedArray.data` entries +*should* be the same before and after the operation. .. warning:: We need to stress that this behavior may not be systematic, that masked diff --git a/doc/source/reference/random/c-api.rst b/doc/source/reference/random/c-api.rst index 63b0fdc2b420..bd119abe3d7c 100644 --- a/doc/source/reference/random/c-api.rst +++ b/doc/source/reference/random/c-api.rst @@ -181,6 +181,6 @@ Generate a single integer Generate random uint64 numbers in closed interval [off, off + rng]. -.. c:function:: npy_uint64 random_bounded_uint64(bitgen_t *bitgen_state, npy_uint64 off, npy_uint64 rng, npy_uint64 mask, bint use_masked) +.. c:function:: npy_uint64 random_bounded_uint64(bitgen_t *bitgen_state, npy_uint64 off, npy_uint64 rng, npy_uint64 mask, int use_masked) diff --git a/doc/source/reference/routines.array-manipulation.rst b/doc/source/reference/routines.array-manipulation.rst index 8d13a1800abf..1c96495d96f7 100644 --- a/doc/source/reference/routines.array-manipulation.rst +++ b/doc/source/reference/routines.array-manipulation.rst @@ -74,6 +74,7 @@ Joining arrays hstack dstack column_stack + row_stack Splitting arrays ================ diff --git a/doc/source/reference/ufuncs.rst b/doc/source/reference/ufuncs.rst index c729f8d37e08..06fbe28ddcba 100644 --- a/doc/source/reference/ufuncs.rst +++ b/doc/source/reference/ufuncs.rst @@ -1,5 +1,7 @@ .. sectionauthor:: adapted from "Guide to NumPy" by Travis E. Oliphant +.. currentmodule:: numpy + .. _ufuncs: ************************************ @@ -8,8 +10,6 @@ Universal functions (:class:`ufunc`) .. note: XXX: section might need to be made more reference-guideish... -.. currentmodule:: numpy - .. index: ufunc, universal function, arithmetic, operation A universal function (or :term:`ufunc` for short) is a function that diff --git a/doc/source/release/1.16.0-notes.rst b/doc/source/release/1.16.0-notes.rst index e78e270f4a55..17d24160adb4 100644 --- a/doc/source/release/1.16.0-notes.rst +++ b/doc/source/release/1.16.0-notes.rst @@ -170,8 +170,8 @@ See the "accessing multiple fields" section of the C API changes ============= -The :c:data:`NPY_API_VERSION` was incremented to 0x0000D, due to the addition -of: +The :c:data:`NPY_FEATURE_VERSION` was incremented to 0x0000D, due to +the addition of: * :c:member:`PyUFuncObject.core_dim_flags` * :c:member:`PyUFuncObject.core_dim_sizes` diff --git a/doc/source/user/basics.rec.rst b/doc/source/user/basics.rec.rst index bb4ed89e999a..6906e4500df1 100644 --- a/doc/source/user/basics.rec.rst +++ b/doc/source/user/basics.rec.rst @@ -132,7 +132,7 @@ summary they are: Offsets may be chosen such that the fields overlap, though this will mean that assigning to one field may clobber any overlapping field's data. As - an exception, fields of :class:`numpy.object` type cannot overlap with + an exception, fields of :class:`object` type cannot overlap with other fields, because of the risk of clobbering the internal object pointer and then dereferencing it. @@ -545,7 +545,7 @@ Viewing Structured Arrays Containing Objects -------------------------------------------- In order to prevent clobbering object pointers in fields of -:class:`numpy.object` type, numpy currently does not allow views of structured +:class:`object` type, numpy currently does not allow views of structured arrays containing objects. Structure Comparison diff --git a/doc/source/user/quickstart.rst b/doc/source/user/quickstart.rst index b675204e0ad5..387a7f7151c5 100644 --- a/doc/source/user/quickstart.rst +++ b/doc/source/user/quickstart.rst @@ -15,7 +15,7 @@ Prerequisites You'll need to know a bit of Python. For a refresher, see the `Python tutorial `__. -To work the examples, you'll need `matplotlib` installed +To work the examples, you'll need ``matplotlib`` installed in addition to NumPy. **Learner profile** From 15bb98cf3ea6a5d8449874acd808c096d27a604c Mon Sep 17 00:00:00 2001 From: takanori-pskq Date: Sat, 24 Oct 2020 18:57:44 +0900 Subject: [PATCH 344/409] DOC: Fixup for PR #17614 --- doc/source/reference/arrays.interface.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/reference/arrays.interface.rst b/doc/source/reference/arrays.interface.rst index aee95ba5bf69..d0a1c06c47f1 100644 --- a/doc/source/reference/arrays.interface.rst +++ b/doc/source/reference/arrays.interface.rst @@ -63,7 +63,7 @@ This approach to the interface consists of the object having an Tuple whose elements are the array size in each dimension. Each entry is an integer (a Python :py:class:`int`). Note that these integers could be larger than the platform ``int`` or ``long`` - could hold (a Python :py:class`int` is a C ``long``). It is up to the code + could hold (a Python :py:class:`int` is a C ``long``). It is up to the code using this attribute to handle this appropriately; either by raising an error when overflow is possible, or by using ``long long`` as the C type for the shapes. From 74aee159583bf4631fbbdd12eef7e2bee394d907 Mon Sep 17 00:00:00 2001 From: Jakob Date: Sat, 24 Oct 2020 13:00:52 +0200 Subject: [PATCH 345/409] minor typographic change --- numpy/testing/_private/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/testing/_private/utils.py b/numpy/testing/_private/utils.py index ae2617f4f124..fb33bdcbd7f7 100644 --- a/numpy/testing/_private/utils.py +++ b/numpy/testing/_private/utils.py @@ -1733,7 +1733,7 @@ def integer_repr(x): elif x.dtype == np.float64: return _integer_repr(x, np.int64, np.int64(-2**63)) else: - raise ValueError(f"Unsupported dtype {x.dtype}") + raise ValueError(f'Unsupported dtype {x.dtype}') @contextlib.contextmanager From b6e5711fd50969fb87d784e835d9ccc62dde716d Mon Sep 17 00:00:00 2001 From: Jakob Date: Sat, 24 Oct 2020 13:12:06 +0200 Subject: [PATCH 346/409] conversion of strings to fstrings --- numpy/random/_examples/numba/extending.py | 4 ++-- numpy/random/tests/test_direct.py | 4 ++-- numpy/random/tests/test_generator_mt19937_regressions.py | 4 ++-- numpy/random/tests/test_randomstate_regression.py | 4 ++-- numpy/random/tests/test_regression.py | 4 ++-- numpy/random/tests/test_smoke.py | 9 ++++----- 6 files changed, 14 insertions(+), 15 deletions(-) diff --git a/numpy/random/_examples/numba/extending.py b/numpy/random/_examples/numba/extending.py index 0d240596b67c..da4d9394461a 100644 --- a/numpy/random/_examples/numba/extending.py +++ b/numpy/random/_examples/numba/extending.py @@ -44,9 +44,9 @@ def numpycall(): assert r1.shape == r2.shape t1 = timeit(numbacall, number=1000) -print('{:.2f} secs for {} PCG64 (Numba/PCG64) gaussian randoms'.format(t1, n)) ++print(f'{t1:.2f} secs for {n} PCG64 (Numba/PCG64) gaussian randoms') t2 = timeit(numpycall, number=1000) -print('{:.2f} secs for {} PCG64 (NumPy/PCG64) gaussian randoms'.format(t2, n)) +print(f'{t2:.2f} secs for {n} PCG64 (NumPy/PCG64) gaussian randoms') # example 2 diff --git a/numpy/random/tests/test_direct.py b/numpy/random/tests/test_direct.py index dad12c8a80ec..d602b36b4169 100644 --- a/numpy/random/tests/test_direct.py +++ b/numpy/random/tests/test_direct.py @@ -230,13 +230,13 @@ def test_uniform_float(self): def test_repr(self): rs = Generator(self.bit_generator(*self.data1['seed'])) assert 'Generator' in repr(rs) - assert '{:#x}'.format(id(rs)).upper().replace('X', 'x') in repr(rs) + assert f'{id(rs):#x}'.upper().replace('X', 'x') in repr(rs) def test_str(self): rs = Generator(self.bit_generator(*self.data1['seed'])) assert 'Generator' in str(rs) assert str(self.bit_generator.__name__) in str(rs) - assert '{:#x}'.format(id(rs)).upper().replace('X', 'x') not in str(rs) + assert f'{id(rs):#x}'.upper().replace('X', 'x') not in str(rs) def test_pickle(self): import pickle diff --git a/numpy/random/tests/test_generator_mt19937_regressions.py b/numpy/random/tests/test_generator_mt19937_regressions.py index 456c932d4cb2..2ef6b0631920 100644 --- a/numpy/random/tests/test_generator_mt19937_regressions.py +++ b/numpy/random/tests/test_generator_mt19937_regressions.py @@ -33,11 +33,11 @@ def test_logseries_convergence(self): # numbers with this large sample # theoretical large N result is 0.49706795 freq = np.sum(rvsn == 1) / float(N) - msg = "Frequency was %f, should be > 0.45" % freq + msg = f'Frequency was {freq:f}, should be > 0.45' assert_(freq > 0.45, msg) # theoretical large N result is 0.19882718 freq = np.sum(rvsn == 2) / float(N) - msg = "Frequency was %f, should be < 0.23" % freq + msg = f'Frequency was {freq:f}, should be < 0.23' assert_(freq < 0.23, msg) def test_shuffle_mixed_dimension(self): diff --git a/numpy/random/tests/test_randomstate_regression.py b/numpy/random/tests/test_randomstate_regression.py index 4eb82fc4ca93..0bf361e5eb46 100644 --- a/numpy/random/tests/test_randomstate_regression.py +++ b/numpy/random/tests/test_randomstate_regression.py @@ -44,11 +44,11 @@ def test_logseries_convergence(self): # numbers with this large sample # theoretical large N result is 0.49706795 freq = np.sum(rvsn == 1) / float(N) - msg = "Frequency was %f, should be > 0.45" % freq + msg = f'Frequency was {freq:f}, should be > 0.45' assert_(freq > 0.45, msg) # theoretical large N result is 0.19882718 freq = np.sum(rvsn == 2) / float(N) - msg = "Frequency was %f, should be < 0.23" % freq + msg = f'Frequency was {freq:f}, should be < 0.23' assert_(freq < 0.23, msg) def test_shuffle_mixed_dimension(self): diff --git a/numpy/random/tests/test_regression.py b/numpy/random/tests/test_regression.py index 278622287173..54d5a3efbdba 100644 --- a/numpy/random/tests/test_regression.py +++ b/numpy/random/tests/test_regression.py @@ -40,11 +40,11 @@ def test_logseries_convergence(self): # numbers with this large sample # theoretical large N result is 0.49706795 freq = np.sum(rvsn == 1) / float(N) - msg = "Frequency was %f, should be > 0.45" % freq + msg = f'Frequency was {freq:f}, should be > 0.45' assert_(freq > 0.45, msg) # theoretical large N result is 0.19882718 freq = np.sum(rvsn == 2) / float(N) - msg = "Frequency was %f, should be < 0.23" % freq + msg = f'Frequency was {freq:f}, should be < 0.23' assert_(freq < 0.23, msg) def test_shuffle_mixed_dimension(self): diff --git a/numpy/random/tests/test_smoke.py b/numpy/random/tests/test_smoke.py index ebfc6825ebb9..909bfaa8dab6 100644 --- a/numpy/random/tests/test_smoke.py +++ b/numpy/random/tests/test_smoke.py @@ -129,7 +129,7 @@ def test_advance(self): assert_(not comp_state(state, self.rg.bit_generator.state)) else: bitgen_name = self.rg.bit_generator.__class__.__name__ - pytest.skip('Advance is not supported by {0}'.format(bitgen_name)) + pytest.skip(f'Advance is not supported by {bitgen_name}') def test_jump(self): state = self.rg.bit_generator.state @@ -145,8 +145,8 @@ def test_jump(self): else: bitgen_name = self.rg.bit_generator.__class__.__name__ if bitgen_name not in ('SFC64',): - raise AttributeError('no "jumped" in %s' % bitgen_name) - pytest.skip('Jump is not supported by {0}'.format(bitgen_name)) + raise AttributeError(f'no "jumped" in {bitgen_name}') + pytest.skip(f'Jump is not supported by {bitgen_name}') def test_uniform(self): r = self.rg.uniform(-1.0, 0.0, size=10) @@ -447,8 +447,7 @@ def test_pickle(self): def test_seed_array(self): if self.seed_vector_bits is None: bitgen_name = self.bit_generator.__name__ - pytest.skip('Vector seeding is not supported by ' - '{0}'.format(bitgen_name)) + pytest.skip(f'Vector seeding is not supported by {bitgen_name}') if self.seed_vector_bits == 32: dtype = np.uint32 From d38d0c92c7a8c14482c2f74623511fb8c10c8774 Mon Sep 17 00:00:00 2001 From: Takanori H Date: Sat, 24 Oct 2020 20:23:35 +0900 Subject: [PATCH 347/409] Update doc/source/user/basics.rec.rst Co-authored-by: Eric Wieser --- doc/source/user/basics.rec.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/user/basics.rec.rst b/doc/source/user/basics.rec.rst index 6906e4500df1..3dba6536111d 100644 --- a/doc/source/user/basics.rec.rst +++ b/doc/source/user/basics.rec.rst @@ -132,7 +132,7 @@ summary they are: Offsets may be chosen such that the fields overlap, though this will mean that assigning to one field may clobber any overlapping field's data. As - an exception, fields of :class:`object` type cannot overlap with + an exception, fields of :class:`numpy.object_` type cannot overlap with other fields, because of the risk of clobbering the internal object pointer and then dereferencing it. From 11c57968e98fb881722dffb1a35bd70bea3b422a Mon Sep 17 00:00:00 2001 From: Takanori H Date: Sat, 24 Oct 2020 20:23:46 +0900 Subject: [PATCH 348/409] Update doc/source/reference/random/c-api.rst Co-authored-by: Eric Wieser --- doc/source/reference/random/c-api.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/source/reference/random/c-api.rst b/doc/source/reference/random/c-api.rst index bd119abe3d7c..a79da7a492f7 100644 --- a/doc/source/reference/random/c-api.rst +++ b/doc/source/reference/random/c-api.rst @@ -181,6 +181,5 @@ Generate a single integer Generate random uint64 numbers in closed interval [off, off + rng]. -.. c:function:: npy_uint64 random_bounded_uint64(bitgen_t *bitgen_state, npy_uint64 off, npy_uint64 rng, npy_uint64 mask, int use_masked) - +.. c:function:: npy_uint64 random_bounded_uint64(bitgen_t *bitgen_state, npy_uint64 off, npy_uint64 rng, npy_uint64 mask, bool use_masked) From becc3bc83421677530abfe54c4757ed6adcc822b Mon Sep 17 00:00:00 2001 From: takanori-pskq Date: Sat, 24 Oct 2020 20:33:50 +0900 Subject: [PATCH 349/409] DOC: Fixup for ``PyUFunc_*`` --- doc/source/reference/c-api/types-and-structures.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/source/reference/c-api/types-and-structures.rst b/doc/source/reference/c-api/types-and-structures.rst index c45ebb692f2e..8c1a679dc9d3 100644 --- a/doc/source/reference/c-api/types-and-structures.rst +++ b/doc/source/reference/c-api/types-and-structures.rst @@ -814,7 +814,9 @@ PyUFunc_Type and PyUFuncObject .. c:member:: int identity Either :c:data:`PyUFunc_One`, :c:data:`PyUFunc_Zero`, - or :c:data:`PyUFunc_None` to indicate + :c:data:`PyUFunc_MinusOne`, :c:data:`PyUFunc_None`, + :c:data:`PyUFunc_ReorderableNone`, or + :c:data:`PyUFunc_IdentityValue` to indicate the identity for this operation. It is only used for a reduce-like call on an empty array. From b0f1cc13a04dd8225f6e6fb90604c7d8379aa3a3 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Sat, 24 Oct 2020 14:00:46 +0100 Subject: [PATCH 350/409] DOC: Tidy up references to str_ / bytes_ This: * Fixes some broken references * Cleans up some use of Python 2 aliases: * Switches references to `np.string_` to describe `np.bytes_` instead * Switches references to `np.unicode_` to describe `np.str_` instead --- doc/source/reference/arrays.classes.rst | 8 ++++---- doc/source/reference/arrays.dtypes.rst | 11 +++++------ doc/source/reference/arrays.scalars.rst | 4 ++-- doc/source/reference/routines.char.rst | 2 +- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/doc/source/reference/arrays.classes.rst b/doc/source/reference/arrays.classes.rst index c5563bdddd45..3a4ed21689e5 100644 --- a/doc/source/reference/arrays.classes.rst +++ b/doc/source/reference/arrays.classes.rst @@ -480,16 +480,16 @@ Character arrays (:mod:`numpy.char`) The `chararray` class exists for backwards compatibility with Numarray, it is not recommended for new development. Starting from numpy 1.4, if one needs arrays of strings, it is recommended to use arrays of - `dtype` `object_`, `string_` or `unicode_`, and use the free functions + `dtype` `object_`, `bytes_` or `str_`, and use the free functions in the `numpy.char` module for fast vectorized string operations. -These are enhanced arrays of either :class:`string_` type or -:class:`unicode_` type. These arrays inherit from the +These are enhanced arrays of either :class:`str_` type or +:class:`bytes_` type. These arrays inherit from the :class:`ndarray`, but specially-define the operations ``+``, ``*``, and ``%`` on a (broadcasting) element-by-element basis. These operations are not available on the standard :class:`ndarray` of character type. In addition, the :class:`chararray` has all of the -standard :class:`string ` (and :class:`unicode`) methods, +standard :class:`str` (and :class:`bytes`) methods, executing them on an element-by-element basis. Perhaps the easiest way to create a chararray is to use :meth:`self.view(chararray) ` where *self* is an ndarray of str or unicode diff --git a/doc/source/reference/arrays.dtypes.rst b/doc/source/reference/arrays.dtypes.rst index 301e26c55cb3..8f664cd1998a 100644 --- a/doc/source/reference/arrays.dtypes.rst +++ b/doc/source/reference/arrays.dtypes.rst @@ -264,9 +264,8 @@ Array-protocol type strings (see :ref:`arrays.interface`) .. admonition:: Note on string types For backward compatibility with Python 2 the ``S`` and ``a`` typestrings - remain zero-terminated bytes and ``np.string_`` continues to map to - ``np.bytes_``. - To use actual strings in Python 3 use ``U`` or ``np.unicode_``. + remain zero-terminated bytes and `numpy.string_` continues to alias + `numpy.bytes_`. To use actual strings in Python 3 use ``U`` or `numpy.str_`. For signed bytes that do not need zero-termination ``b`` or ``i1`` can be used. @@ -392,9 +391,9 @@ Type strings their values must each be lists of the same length as the *names* and *formats* lists. The *offsets* value is a list of byte offsets (limited to `ctypes.c_int`) for each field, while the *titles* value is a - list of titles for each field (None can be used if no title is - desired for that field). The *titles* can be any :class:`string` - or :class:`unicode` object and will add another entry to the + list of titles for each field (``None`` can be used if no title is + desired for that field). The *titles* can be any object, but when a + :class:`str` object will add another entry to the fields dictionary keyed by the title and referencing the same field tuple which will contain the title as an additional tuple member. diff --git a/doc/source/reference/arrays.scalars.rst b/doc/source/reference/arrays.scalars.rst index 13d117af2432..24393f1e7dc3 100644 --- a/doc/source/reference/arrays.scalars.rst +++ b/doc/source/reference/arrays.scalars.rst @@ -30,8 +30,8 @@ present can be determined using other members of the data type hierarchy. Thus, for example ``isinstance(val, np.complexfloating)`` will return :py:data:`True` if *val* is a complex valued type, while ``isinstance(val, np.flexible)`` will return true if *val* is one -of the flexible itemsize array types (:class:`string`, -:class:`unicode`, :class:`void`). +of the flexible itemsize array types (:class:`str_`, +:class:`bytes_`, :class:`void`). .. figure:: figures/dtype-hierarchy.png diff --git a/doc/source/reference/routines.char.rst b/doc/source/reference/routines.char.rst index ed8393855763..90df14125b45 100644 --- a/doc/source/reference/routines.char.rst +++ b/doc/source/reference/routines.char.rst @@ -6,7 +6,7 @@ String operations .. module:: numpy.char The `numpy.char` module provides a set of vectorized string -operations for arrays of type `numpy.string_` or `numpy.unicode_`. +operations for arrays of type `numpy.str_` or `numpy.bytes_`. All of them are based on the string methods in the Python standard library. String operations From f7eed258a49edc9d6a3c2500e8f09ba48f4a8e4b Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Sat, 24 Oct 2020 15:17:09 +0100 Subject: [PATCH 351/409] DOC: Fix doctests --- numpy/lib/polynomial.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/numpy/lib/polynomial.py b/numpy/lib/polynomial.py index 351f2b5730f4..0fd9bbd79c6a 100644 --- a/numpy/lib/polynomial.py +++ b/numpy/lib/polynomial.py @@ -426,7 +426,7 @@ def polyder(p, m=1): >>> np.polyder(p, 3) poly1d([6]) >>> np.polyder(p, 4) - poly1d([0.]) + poly1d([0]) """ m = int(m) @@ -754,11 +754,11 @@ def polyval(p, x): >>> np.polyval([3,0,1], 5) # 3 * 5**2 + 0 * 5**1 + 1 76 >>> np.polyval([3,0,1], np.poly1d(5)) - poly1d([76.]) + poly1d([76]) >>> np.polyval(np.poly1d([3,0,1]), 5) 76 >>> np.polyval(np.poly1d([3,0,1]), np.poly1d(5)) - poly1d([76.]) + poly1d([76]) """ p = NX.asarray(p) From aec05765b07ed925add22120aaf9a9e9f7e51d1a Mon Sep 17 00:00:00 2001 From: Jakob Date: Sat, 24 Oct 2020 18:56:42 +0200 Subject: [PATCH 352/409] Convert some strings to fstrigns --- numpy/linalg/lapack_lite/make_lite.py | 18 +++++++++--------- numpy/linalg/linalg.py | 2 +- numpy/linalg/tests/test_build.py | 4 ++-- numpy/linalg/tests/test_linalg.py | 6 +++--- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/numpy/linalg/lapack_lite/make_lite.py b/numpy/linalg/lapack_lite/make_lite.py index 23921acf4bd5..6cbd11445c8d 100755 --- a/numpy/linalg/lapack_lite/make_lite.py +++ b/numpy/linalg/lapack_lite/make_lite.py @@ -81,7 +81,7 @@ def dependencies(self): return self._dependencies def __repr__(self): - return "FortranRoutine({!r}, filename={!r})".format(self.name, self.filename) + return return f'FortranRoutine({self.name!r}, filename={self.filename!r})' class UnknownFortranRoutine(FortranRoutine): """Wrapper for a Fortran routine for which the corresponding file @@ -193,7 +193,7 @@ def allRoutinesByType(self, typename): def printRoutineNames(desc, routines): print(desc) for r in routines: - print('\t%s' % r.name) + print(f'\t{r.name}') def getLapackRoutines(wrapped_routines, ignores, lapack_dir): blas_src_dir = os.path.join(lapack_dir, 'BLAS', 'SRC') @@ -243,7 +243,7 @@ def dumpRoutineNames(library, output_dir): with open(filename, 'w') as fo: for r in routines: deps = r.dependencies() - fo.write('%s: %s\n' % (r.name, ' '.join(deps))) + fo.write(f"{r.name}: {' '.join(deps)}\n") def concatenateRoutines(routines, output_file): with open(output_file, 'w') as output_fo: @@ -316,13 +316,13 @@ def create_name_header(output_dir): # Rename BLAS/LAPACK symbols for name in sorted(symbols): - f.write("#define %s_ BLAS_FUNC(%s)\n" % (name, name)) + f.write(f'#define {name}_ BLAS_FUNC({name})\n') # Rename also symbols that f2c exports itself f.write("\n" "/* Symbols exported by f2c.c */\n") for name in sorted(f2c_symbols): - f.write("#define %s numpy_lapack_lite_%s\n" % (name, name)) + f.write(f'#define {name} numpy_lapack_lite_{name}\n') def main(): if len(sys.argv) != 3: @@ -348,9 +348,9 @@ def main(): dumpRoutineNames(library, output_dir) for typename in types: - fortran_file = os.path.join(output_dir, 'f2c_%s.f' % typename) + fortran_file = os.path.join(output_dir, f'f2c_{typename}.f') c_file = fortran_file[:-2] + '.c' - print('creating %s ...' % c_file) + print(f'creating {c_file} ...') routines = library.allRoutinesByType(typename) concatenateRoutines(routines, fortran_file) @@ -358,11 +358,11 @@ def main(): patch_file = os.path.basename(fortran_file) + '.patch' if os.path.exists(patch_file): subprocess.check_call(['patch', '-u', fortran_file, patch_file]) - print("Patched {}".format(fortran_file)) + print(f'Patched {fortran_file}') try: runF2C(fortran_file, output_dir) except F2CError: - print('f2c failed on %s' % fortran_file) + print(f'f2c failed on {fortran_file}') break scrubF2CSource(c_file) diff --git a/numpy/linalg/linalg.py b/numpy/linalg/linalg.py index b6d860dfabad..5970a00baca3 100644 --- a/numpy/linalg/linalg.py +++ b/numpy/linalg/linalg.py @@ -901,7 +901,7 @@ def qr(a, mode='reduced'): warnings.warn(msg, DeprecationWarning, stacklevel=3) mode = 'economic' else: - raise ValueError("Unrecognized mode '%s'" % mode) + raise ValueError(f"Unrecognized mode '{mode}'") a, wrap = _makearray(a) _assert_2d(a) diff --git a/numpy/linalg/tests/test_build.py b/numpy/linalg/tests/test_build.py index cbf3089bcf5f..4859226d99a4 100644 --- a/numpy/linalg/tests/test_build.py +++ b/numpy/linalg/tests/test_build.py @@ -16,13 +16,13 @@ def __init__(self): p = Popen(self.cmd, stdout=PIPE, stderr=PIPE) stdout, stderr = p.communicate() except OSError: - raise RuntimeError("command %s cannot be run" % self.cmd) + raise RuntimeError(f'command {self.cmd} cannot be run') def get_dependencies(self, lfile): p = Popen(self.cmd + [lfile], stdout=PIPE, stderr=PIPE) stdout, stderr = p.communicate() if not (p.returncode == 0): - raise RuntimeError("failed dependencies check for %s" % lfile) + raise RuntimeError(f'failed dependencies check for {lfile}') return stdout diff --git a/numpy/linalg/tests/test_linalg.py b/numpy/linalg/tests/test_linalg.py index 3f3bf9f70917..21fab58e1d42 100644 --- a/numpy/linalg/tests/test_linalg.py +++ b/numpy/linalg/tests/test_linalg.py @@ -85,7 +85,7 @@ def check(self, do): do(self.a, self.b, tags=self.tags) def __repr__(self): - return "" % (self.name,) + return f'' def apply_tag(tag, cases): @@ -349,7 +349,7 @@ def check_cases(self, require=set(), exclude=set()): try: case.check(self.do) except Exception: - msg = "In test case: %r\n\n" % case + msg = f'In test case: {case!r}\n\n' msg += traceback.format_exc() raise AssertionError(msg) @@ -1732,7 +1732,7 @@ def test_basic_property(self): b = np.matmul(c, c.transpose(t).conj()) assert_allclose(b, a, - err_msg="{} {}\n{}\n{}".format(shape, dtype, a, c), + err_msg=f'{shape} {dtype}\n{a}\n{c}', atol=500 * a.shape[0] * np.finfo(dtype).eps) def test_0_size(self): From bc90452a095424bf837736a8615407ab4f1dd649 Mon Sep 17 00:00:00 2001 From: takanori-pskq Date: Sun, 25 Oct 2020 10:07:17 +0900 Subject: [PATCH 353/409] DOC: Fix the references for ``__array_*__`` --- doc/source/reference/arrays.dtypes.rst | 2 +- doc/source/reference/arrays.interface.rst | 8 ++++---- doc/source/reference/arrays.ndarray.rst | 8 ++++---- doc/source/reference/arrays.scalars.rst | 2 +- doc/source/reference/c-api/array.rst | 4 ++-- doc/source/reference/c-api/types-and-structures.rst | 6 +++--- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/doc/source/reference/arrays.dtypes.rst b/doc/source/reference/arrays.dtypes.rst index 8f664cd1998a..b5ffa1a8b9c1 100644 --- a/doc/source/reference/arrays.dtypes.rst +++ b/doc/source/reference/arrays.dtypes.rst @@ -344,7 +344,7 @@ Type strings ``[(field_name, field_dtype, field_shape), ...]`` *obj* should be a list of fields where each field is described by a tuple of length 2 or 3. (Equivalent to the ``descr`` item in the - :obj:`__array_interface__` attribute.) + :obj:`~object.__array_interface__` attribute.) The first element, *field_name*, is the field name (if this is ``''`` then a standard field name, ``'f#'``, is assigned). The diff --git a/doc/source/reference/arrays.interface.rst b/doc/source/reference/arrays.interface.rst index d0a1c06c47f1..6a8c5f9c4d09 100644 --- a/doc/source/reference/arrays.interface.rst +++ b/doc/source/reference/arrays.interface.rst @@ -49,9 +49,9 @@ Python side =========== This approach to the interface consists of the object having an -:data:`__array_interface__` attribute. +:data:`~object.__array_interface__` attribute. -.. data:: __array_interface__ +.. data:: object.__array_interface__ A dictionary of items (3 required and 5 optional). The optional keys in the dictionary have implied defaults if they are not @@ -189,7 +189,7 @@ C-struct access This approach to the array interface allows for faster access to an array using only one attribute lookup and a well-defined C-structure. -.. c:var:: __array_struct__ +.. data:: object.__array_struct__ A :c:type:`PyCapsule` whose ``pointer`` member contains a pointer to a filled :c:type:`PyArrayInterface` structure. Memory @@ -248,7 +248,7 @@ Type description examples ========================= For clarity it is useful to provide some examples of the type -description and corresponding :data:`__array_interface__` 'descr' +description and corresponding :data:`~object.__array_interface__` 'descr' entries. Thanks to Scott Gilbert for these examples: In every case, the 'descr' key is optional, but of course provides diff --git a/doc/source/reference/arrays.ndarray.rst b/doc/source/reference/arrays.ndarray.rst index 405ae8f6ac99..6b805904a0e2 100644 --- a/doc/source/reference/arrays.ndarray.rst +++ b/doc/source/reference/arrays.ndarray.rst @@ -259,10 +259,10 @@ Array interface .. seealso:: :ref:`arrays.interface`. -========================== =================================== -:obj:`__array_interface__` Python-side of the array interface -:obj:`__array_struct__` C-side of the array interface -========================== =================================== +================================== =================================== +:obj:`~object.__array_interface__` Python-side of the array interface +:obj:`~object.__array_struct__` C-side of the array interface +================================== =================================== :mod:`ctypes` foreign function interface ---------------------------------------- diff --git a/doc/source/reference/arrays.scalars.rst b/doc/source/reference/arrays.scalars.rst index 24393f1e7dc3..4b5da2e13a5c 100644 --- a/doc/source/reference/arrays.scalars.rst +++ b/doc/source/reference/arrays.scalars.rst @@ -362,7 +362,7 @@ Attributes ========== The array scalar objects have an :obj:`array priority -<__array_priority__>` of :c:data:`NPY_SCALAR_PRIORITY` +` of :c:data:`NPY_SCALAR_PRIORITY` (-1,000,000.0). They also do not (yet) have a :attr:`ctypes ` attribute. Otherwise, they share the same attributes as arrays: diff --git a/doc/source/reference/c-api/array.rst b/doc/source/reference/c-api/array.rst index ae39f980f311..5c495097794b 100644 --- a/doc/source/reference/c-api/array.rst +++ b/doc/source/reference/c-api/array.rst @@ -604,14 +604,14 @@ From other objects .. c:function:: PyObject* PyArray_FromStructInterface(PyObject* op) Returns an ndarray object from a Python object that exposes the - :obj:`__array_struct__` attribute and follows the array interface + :obj:`~object.__array_struct__` attribute and follows the array interface protocol. If the object does not contain this attribute then a borrowed reference to :c:data:`Py_NotImplemented` is returned. .. c:function:: PyObject* PyArray_FromInterface(PyObject* op) Returns an ndarray object from a Python object that exposes the - :obj:`__array_interface__` attribute following the array interface + :obj:`~object.__array_interface__` attribute following the array interface protocol. If the object does not contain this attribute then a borrowed reference to :c:data:`Py_NotImplemented` is returned. diff --git a/doc/source/reference/c-api/types-and-structures.rst b/doc/source/reference/c-api/types-and-structures.rst index 8c1a679dc9d3..b0d2407951b0 100644 --- a/doc/source/reference/c-api/types-and-structures.rst +++ b/doc/source/reference/c-api/types-and-structures.rst @@ -1323,13 +1323,13 @@ PyArrayInterface The :c:type:`PyArrayInterface` structure is defined so that NumPy and other extension modules can use the rapid array interface - protocol. The :obj:`__array_struct__` method of an object that + protocol. The :obj:`~object.__array_struct__` method of an object that supports the rapid array interface protocol should return a :c:type:`PyCObject` that contains a pointer to a :c:type:`PyArrayInterface` structure with the relevant details of the array. After the new array is created, the attribute should be ``DECREF``'d which will free the :c:type:`PyArrayInterface` structure. Remember to ``INCREF`` the - object (whose :obj:`__array_struct__` attribute was retrieved) and + object (whose :obj:`~object.__array_struct__` attribute was retrieved) and point the base member of the new :c:type:`PyArrayObject` to this same object. In this way the memory for the array will be managed correctly. @@ -1398,7 +1398,7 @@ PyArrayInterface .. c:member:: PyObject *descr A Python object describing the data-type in more detail (same - as the *descr* key in :obj:`__array_interface__`). This can be + as the *descr* key in :obj:`~object.__array_interface__`). This can be ``NULL`` if *typekind* and *itemsize* provide enough information. This field is also ignored unless :c:data:`NPY_ARR_HAS_DESCR` flag is on in *flags*. From feefb2b518168f197f042f2d41d5596d9d869f5c Mon Sep 17 00:00:00 2001 From: takanori-pskq Date: Sun, 25 Oct 2020 12:59:56 +0900 Subject: [PATCH 354/409] DOC: Add entries for macros --- doc/source/reference/c-api/array.rst | 32 +++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/doc/source/reference/c-api/array.rst b/doc/source/reference/c-api/array.rst index ae39f980f311..2e7e84946cd1 100644 --- a/doc/source/reference/c-api/array.rst +++ b/doc/source/reference/c-api/array.rst @@ -2810,11 +2810,21 @@ Data-type descriptors Create a new data-type object with the byteorder set according to *newendian*. All referenced data-type objects (in subdescr and fields members of the data-type object) are also changed - (recursively). If a byteorder of :c:data:`NPY_IGNORE` is encountered it + (recursively). + + The value of *newendian* is one of these macros: + + .. c:macro:: NPY_IGNORE + NPY_SWAP + NPY_NATIVE + NPY_LITTLE + NPY_BIG + + If a byteorder of :c:data:`NPY_IGNORE` is encountered it is left alone. If newendian is :c:data:`NPY_SWAP`, then all byte-orders are swapped. Other valid newendian values are :c:data:`NPY_NATIVE`, - :c:data:`NPY_LITTLE`, and :c:data:`NPY_BIG` which all cause the returned - data-typed descriptor (and all it's + :c:data:`NPY_LITTLE`, and :c:data:`NPY_BIG` which all cause + the returned data-typed descriptor (and all it's referenced data-type descriptors) to have the corresponding byte- order. @@ -3625,6 +3635,22 @@ Enumerated Types Wraps an index to the valid range if it is out of bounds. +.. c:type:: NPY_SEARCHSIDE + + A variable type indicating whether the index returned should be that of + the first suitable location (if :c:data:`NPY_SEARCHLEFT`) or of the last + (if :c:data:`NPY_SEARCHRIGHT`). + + .. c:var:: NPY_SEARCHLEFT + + .. c:var:: NPY_SEARCHRIGHT + +.. c:type:: NPY_SELECTKIND + + A variable type indicating the selection algorithm being used. + + .. c:var:: NPY_INTROSELECT + .. c:type:: NPY_CASTING .. versionadded:: 1.6 From 54a61a8afbee6c1b4aab1d7e71c5d0e6e1bbf862 Mon Sep 17 00:00:00 2001 From: takanori-pskq Date: Sun, 25 Oct 2020 13:58:13 +0900 Subject: [PATCH 355/409] DOC: Replace ``PyCObject`` with ``PyCapsule`` --- doc/source/reference/c-api/types-and-structures.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/source/reference/c-api/types-and-structures.rst b/doc/source/reference/c-api/types-and-structures.rst index 8c1a679dc9d3..a9857391f7fb 100644 --- a/doc/source/reference/c-api/types-and-structures.rst +++ b/doc/source/reference/c-api/types-and-structures.rst @@ -611,7 +611,8 @@ PyArrayDescr_Type and PyArray_Descr Either ``NULL`` or a dictionary containing low-level casting functions for user- defined data-types. Each function is - wrapped in a :c:type:`PyCObject *` and keyed by the data-type number. + wrapped in a :c:type:`PyCapsule *` and keyed by + the data-type number. .. c:member:: NPY_SCALARKIND scalarkind(PyArrayObject* arr) @@ -1325,7 +1326,7 @@ PyArrayInterface other extension modules can use the rapid array interface protocol. The :obj:`__array_struct__` method of an object that supports the rapid array interface protocol should return a - :c:type:`PyCObject` that contains a pointer to a :c:type:`PyArrayInterface` + :c:type:`PyCapsule` that contains a pointer to a :c:type:`PyArrayInterface` structure with the relevant details of the array. After the new array is created, the attribute should be ``DECREF``'d which will free the :c:type:`PyArrayInterface` structure. Remember to ``INCREF`` the From 70c3565244ada9edaae1c1f86e48fb42cfde04d3 Mon Sep 17 00:00:00 2001 From: takanori-pskq Date: Sun, 25 Oct 2020 14:01:25 +0900 Subject: [PATCH 356/409] DOC: Add ``identity_value`` to ``PyUFuncObject`` --- doc/source/reference/c-api/types-and-structures.rst | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/doc/source/reference/c-api/types-and-structures.rst b/doc/source/reference/c-api/types-and-structures.rst index 8c1a679dc9d3..d9c48c039723 100644 --- a/doc/source/reference/c-api/types-and-structures.rst +++ b/doc/source/reference/c-api/types-and-structures.rst @@ -790,8 +790,8 @@ PyUFunc_Type and PyUFuncObject npy_uint32 *iter_flags; /* new in API version 0x0000000D */ npy_intp *core_dim_sizes; - npy_intp *core_dim_flags; - + npy_uint32 *core_dim_flags; + PyObject *identity_value; } PyUFuncObject; .. c:macro: PyObject_HEAD @@ -956,7 +956,8 @@ PyUFunc_Type and PyUFuncObject .. c:member:: npy_intp *core_dim_sizes For each distinct core dimension, the possible - :ref:`frozen ` size if :c:data:`UFUNC_CORE_DIM_SIZE_INFERRED` is 0 + :ref:`frozen ` size if + :c:data:`UFUNC_CORE_DIM_SIZE_INFERRED` is ``0`` .. c:member:: npy_uint32 *core_dim_flags @@ -971,6 +972,11 @@ PyUFunc_Type and PyUFuncObject if the dim size will be determined from the operands and not from a :ref:`frozen ` signature + .. c:member:: PyObject *identity_value + + Identity for reduction, when :c:member:`PyUFuncObject.identity` + is equal to :c:data:`PyUFunc_IdentityValue`. + PyArrayIter_Type and PyArrayIterObject -------------------------------------- From 3b557bcc2216bcc7ef239a2706832fc8ebf0065e Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Sun, 25 Oct 2020 11:46:49 +0000 Subject: [PATCH 357/409] DOC: Don't use Python highlighting for non-python code --- doc/neps/nep-0000.rst | 8 ++- doc/neps/nep-0002-warnfix.rst | 10 ++-- doc/neps/nep-0010-new-iterator-ufunc.rst | 48 ++++++++++++---- .../nep-0011-deferred-ufunc-evaluation.rst | 8 ++- doc/neps/nep-0012-missing-data.rst | 56 ++++++++++--------- doc/neps/nep-0013-ufunc-overrides.rst | 2 +- doc/neps/nep-0025-missing-data-3.rst | 4 ++ doc/neps/nep-0043-extensible-ufuncs.rst | 8 ++- doc/neps/nep-0045-c_style_guide.rst | 2 + doc/source/reference/c-api/array.rst | 4 +- doc/source/reference/c-api/deprecations.rst | 4 +- 11 files changed, 104 insertions(+), 50 deletions(-) diff --git a/doc/neps/nep-0000.rst b/doc/neps/nep-0000.rst index dcc7f4cf8b68..590976081723 100644 --- a/doc/neps/nep-0000.rst +++ b/doc/neps/nep-0000.rst @@ -234,7 +234,9 @@ Header Preamble Each NEP must begin with a header preamble. The headers must appear in the following order. Headers marked with ``*`` are -optional. All other headers are required. :: +optional. All other headers are required. + +.. code-block:: rst :Author: :Status: if the email address is included, and just +.. code-block:: rst + Random J. User if the address is not given. If there are multiple authors, each should be on diff --git a/doc/neps/nep-0002-warnfix.rst b/doc/neps/nep-0002-warnfix.rst index dfccd5ab802d..a1138b2f1b83 100644 --- a/doc/neps/nep-0002-warnfix.rst +++ b/doc/neps/nep-0002-warnfix.rst @@ -9,6 +9,8 @@ NEP 2 — A proposal to build numpy without warning with a big set of warning fl :Date: 2008-09-04 :Status: Deferred +.. highlight:: c + Executive summary ================= @@ -23,7 +25,9 @@ Warning flags ============= Each compiler detects a different set of potential errors. The baseline will -be gcc -Wall -W -Wextra. Ideally, a complete set would be nice:: +be gcc -Wall -W -Wextra. Ideally, a complete set would be nice: + +.. code-block:: bash -W -Wall -Wextra -Wstrict-prototypes -Wmissing-prototypes -Waggregate-return -Wcast-align -Wcast-qual -Wnested-externs -Wshadow -Wbad-function-cast @@ -67,9 +71,7 @@ When applied to a variable, one would get:: int foo(int * NPY_UNUSED(dummy)) -expanded to - -:: +expanded to:: int foo(int * __NPY_UNUSED_TAGGEDdummy __COMP_NPY_UNUSED) diff --git a/doc/neps/nep-0010-new-iterator-ufunc.rst b/doc/neps/nep-0010-new-iterator-ufunc.rst index 87617f414ea0..358018c46417 100644 --- a/doc/neps/nep-0010-new-iterator-ufunc.rst +++ b/doc/neps/nep-0010-new-iterator-ufunc.rst @@ -393,7 +393,9 @@ The proposed ‘order=’ flags become as follows: ‘K’ a layout equivalent to ‘C’ followed by some permutation of the axes, as close to the layout of the input(s) as possible (“Keep Layout”) === ===================================================================================== -Or as an enum:: +Or as an enum: + +.. code-block:: c /* For specifying array memory layout or iteration order */ typedef enum { @@ -416,7 +418,9 @@ parameter to control the layout of their output(s). The iterator can do automatic casting, and I have created a sequence of progressively more permissive casting rules. Perhaps for 2.0, NumPy -could adopt this enum as its preferred way of dealing with casting.:: +could adopt this enum as its preferred way of dealing with casting. + +.. code-block:: c /* For specifying allowed casting in operations which support it */ typedef enum { @@ -496,7 +500,9 @@ Proposed Iterator Memory Layout The following struct describes the iterator memory. All items are packed together, which means that different values of the flags, -ndim, and niter will produce slightly different layouts. :: +ndim, and niter will produce slightly different layouts. + +.. code-block:: c struct { /* Flags indicate what optimizations have been applied, and @@ -710,13 +716,17 @@ Construction and Destruction Returns NULL if there is an error, otherwise returns the allocated iterator. - To make an iterator similar to the old iterator, this should work.:: + To make an iterator similar to the old iterator, this should work. + + .. code-block:: c iter = NpyIter_New(op, NPY_ITER_READWRITE, NPY_CORDER, NPY_NO_CASTING, NULL, 0, NULL); If you want to edit an array with aligned ``double`` code, - but the order doesn't matter, you would use this.:: + but the order doesn't matter, you would use this. + + .. code-block:: c dtype = PyArray_DescrFromType(NPY_DOUBLE); iter = NpyIter_New(op, NPY_ITER_READWRITE | @@ -764,7 +774,9 @@ Construction and Destruction In ``op_axes[j][i]`` is stored either a valid axis of ``op[j]``, or -1 which means ``newaxis``. Within each ``op_axes[j]`` array, axes may not be repeated. The following example is how normal broadcasting - applies to a 3-D array, a 2-D array, a 1-D array and a scalar.:: + applies to a 3-D array, a 2-D array, a 1-D array and a scalar. + + .. code-block:: c npy_intp oa_ndim = 3; /* # iteration axes */ npy_intp op0_axes[] = {0, 1, 2}; /* 3-D operand */ @@ -1139,7 +1151,9 @@ Construction and Destruction If you want to reset both the ``iterindex`` range and the base pointers at the same time, you can do the following to avoid extra buffer copying (be sure to add the return code error checks - when you copy this code).:: + when you copy this code). + + .. code-block:: c /* Set to a trivial empty range */ NpyIter_ResetToIterIndexRange(iter, 0, 0); @@ -1190,7 +1204,9 @@ Construction and Destruction is used as the source for ``baseptrs``, it will point into a small buffer instead of the array and the inner iteration will be invalid. - The pattern for using nested iterators is as follows.:: + The pattern for using nested iterators is as follows: + + .. code-block:: c NpyIter *iter1, *iter1; NpyIter_IterNext_Fn iternext1, iternext2; @@ -1412,7 +1428,9 @@ Functions For Iteration non-NULL, the function may be safely called without holding the Python GIL. - The typical looping construct is as follows.:: + The typical looping construct is as follows: + + .. code-block:: c NpyIter_IterNext_Fn iternext = NpyIter_GetIterNext(iter, NULL); char **dataptr = NpyIter_GetDataPtrArray(iter); @@ -1422,7 +1440,9 @@ Functions For Iteration } while(iternext(iter)); When ``NPY_ITER_NO_INNER_ITERATION`` is specified, the typical - inner loop construct is as follows.:: + inner loop construct is as follows: + + .. code-block:: c NpyIter_IterNext_Fn iternext = NpyIter_GetIterNext(iter, NULL); char **dataptr = NpyIter_GetDataPtrArray(iter); @@ -1451,7 +1471,9 @@ Functions For Iteration to become zero when ``iternext()`` returns false, enabling the following loop construct. Note that if you use this construct, you should not pass ``NPY_ITER_GROWINNER`` as a flag, because it - will cause larger sizes under some circumstances.:: + will cause larger sizes under some circumstances: + + .. code-block:: c /* The constructor should have buffersize passed as this value */ #define FIXED_BUFFER_SIZE 1024 @@ -1571,7 +1593,9 @@ result. If the input is a reference type, this function will fail. To fix this, the code must be changed to specially handle writeable -references, and add ``NPY_ITER_WRITEABLE_REFERENCES`` to the flags.:: +references, and add ``NPY_ITER_WRITEABLE_REFERENCES`` to the flags: + +.. code-block:: c /* NOTE: This code has not been compiled/tested */ PyObject *CopyArray(PyObject *arr, NPY_ORDER order) diff --git a/doc/neps/nep-0011-deferred-ufunc-evaluation.rst b/doc/neps/nep-0011-deferred-ufunc-evaluation.rst index c40ca56d78bf..866a774d15b0 100644 --- a/doc/neps/nep-0011-deferred-ufunc-evaluation.rst +++ b/doc/neps/nep-0011-deferred-ufunc-evaluation.rst @@ -41,11 +41,15 @@ For an idea of how to get this kind of behavior in NumPy without changing the Python code, consider the C++ technique of expression templates. These can be used to quite arbitrarily rearrange expressions using -vectors or other data structures, example,:: +vectors or other data structures, example: + +.. code-block:: cpp A = B + C + D; -can be transformed into something equivalent to:: +can be transformed into something equivalent to: + +.. code-block:: cpp for(i = 0; i < A.size; ++i) { A[i] = B[i] + C[i] + D[i]; diff --git a/doc/neps/nep-0012-missing-data.rst b/doc/neps/nep-0012-missing-data.rst index 0adcea69ab2f..f47feadbd031 100644 --- a/doc/neps/nep-0012-missing-data.rst +++ b/doc/neps/nep-0012-missing-data.rst @@ -957,6 +957,8 @@ so the later code will raise exceptions as desired. C Implementation Details ************************ +.. highlight:: c + The first version to implement is the array masks, because it is the more general approach. The mask itself is an array, but since it is intended to never be directly accessible from Python, it won't @@ -1159,32 +1161,32 @@ Acknowledgments In addition to feedback from Travis Oliphant and others at Enthought, this NEP has been revised based on a great deal of feedback from the NumPy-Discussion mailing list. The people participating in -the discussion are:: - - Nathaniel Smith - Robert Kern - Charles Harris - Gael Varoquaux - Eric Firing - Keith Goodman - Pierre GM - Christopher Barker - Josef Perktold - Ben Root - Laurent Gautier - Neal Becker - Bruce Southey - Matthew Brett - Wes McKinney - Lluís - Olivier Delalleau - Alan G Isaac - E. Antero Tammi - Jason Grout - Dag Sverre Seljebotn - Joe Harrington - Gary Strangman - Chris Jordan-Squire - Peter +the discussion are: + +- Nathaniel Smith +- Robert Kern +- Charles Harris +- Gael Varoquaux +- Eric Firing +- Keith Goodman +- Pierre GM +- Christopher Barker +- Josef Perktold +- Ben Root +- Laurent Gautier +- Neal Becker +- Bruce Southey +- Matthew Brett +- Wes McKinney +- Lluís +- Olivier Delalleau +- Alan G Isaac +- E. Antero Tammi +- Jason Grout +- Dag Sverre Seljebotn +- Joe Harrington +- Gary Strangman +- Chris Jordan-Squire +- Peter I apologize if I missed anyone. diff --git a/doc/neps/nep-0013-ufunc-overrides.rst b/doc/neps/nep-0013-ufunc-overrides.rst index 5c28e9446370..698e45738708 100644 --- a/doc/neps/nep-0013-ufunc-overrides.rst +++ b/doc/neps/nep-0013-ufunc-overrides.rst @@ -102,7 +102,7 @@ Take this example of ufuncs interoperability with sparse matrices.:: [ 4, 1, 4]], dtype=int64) In [5]: np.multiply(a, bsp) # Returns NotImplemented to user, bad! - Out[5]: NotImplemted + Out[5]: NotImplemented Returning :obj:`NotImplemented` to user should not happen. Moreover:: diff --git a/doc/neps/nep-0025-missing-data-3.rst b/doc/neps/nep-0025-missing-data-3.rst index 19be3cf1e591..ffe208c98070 100644 --- a/doc/neps/nep-0025-missing-data-3.rst +++ b/doc/neps/nep-0025-missing-data-3.rst @@ -132,6 +132,8 @@ Some example use cases: dtype C-level API extensions ============================ +.. highlight:: c + The `PyArray_Descr`_ struct gains the following new fields:: void * NA_value; @@ -357,6 +359,8 @@ so convenient as that for NaN values now, but then, NaN values don't have their own global singleton.) So for now we stick to scalar indexing just returning ``np.NA``, but this can be revisited if anyone objects. +.. highlight:: python + Python API for generic NA support ================================= diff --git a/doc/neps/nep-0043-extensible-ufuncs.rst b/doc/neps/nep-0043-extensible-ufuncs.rst index 008cfad5f1b6..974a705e61d9 100644 --- a/doc/neps/nep-0043-extensible-ufuncs.rst +++ b/doc/neps/nep-0043-extensible-ufuncs.rst @@ -572,6 +572,8 @@ fast C-functions and NumPy API creates the new ``ArrayMethod`` or ArrayMethod Specifications ========================== +.. highlight:: c + These specifications provide a minimal initial C-API, which shall be expanded in the future, for example to allow specialized inner-loops. @@ -696,6 +698,7 @@ to contain the following (this may extend in the future):: PyType_Slot *slots; } PyArrayMethod_Spec; +.. highlight:: python Discussion and alternatives =========================== @@ -810,10 +813,12 @@ Both are discussed in the next section. Error Handling """""""""""""" +.. highlight:: c + We expect that future inner-loops will generally set Python errors as soon as an error is found. This is complicated when the inner-loop is run without locking the GIL. In this case the function will have to lock the GIL, -set the Python error and return ``-1`` to indicate an error occurred:: +set the Python error and return ``-1`` to indicate an error occurred::: int inner_loop(PyArrayMethod_Context *context, ..., void *userdata) @@ -891,6 +896,7 @@ is not set seems convenient, but I am uncertain about it, since I am not aware of any similar prior art. This "scratch space" could also be part of the ``context`` in principle. +.. highlight:: python Reusing existing Loops/Implementations ====================================== diff --git a/doc/neps/nep-0045-c_style_guide.rst b/doc/neps/nep-0045-c_style_guide.rst index f579e4f69b81..5a2fcf946aa7 100644 --- a/doc/neps/nep-0045-c_style_guide.rst +++ b/doc/neps/nep-0045-c_style_guide.rst @@ -10,6 +10,8 @@ NEP 45 — C Style Guide :Created: 2012-02-26 :Resolution: https://github.com/numpy/numpy/issues/11911 +.. highlight:: c + Abstract -------- diff --git a/doc/source/reference/c-api/array.rst b/doc/source/reference/c-api/array.rst index b22e20818f6e..e9717d59a692 100644 --- a/doc/source/reference/c-api/array.rst +++ b/doc/source/reference/c-api/array.rst @@ -2353,7 +2353,9 @@ it is possible to do this. Defining an :c:type:`NpyAuxData` is similar to defining a class in C++, but the object semantics have to be tracked manually since the API is in C. Here's an example for a function which doubles up an element using -an element copier function as a primitive.:: +an element copier function as a primitive. + +.. code-block:: c typedef struct { NpyAuxData base; diff --git a/doc/source/reference/c-api/deprecations.rst b/doc/source/reference/c-api/deprecations.rst index a382017a2ad7..5b1abc6f2add 100644 --- a/doc/source/reference/c-api/deprecations.rst +++ b/doc/source/reference/c-api/deprecations.rst @@ -48,7 +48,9 @@ warnings). To use the NPY_NO_DEPRECATED_API mechanism, you need to #define it to the target API version of NumPy before #including any NumPy headers. -If you want to confirm that your code is clean against 1.7, use:: +If you want to confirm that your code is clean against 1.7, use: + +.. code-block:: c #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION From 046a327f0bd452e53a4c65117e23d4e343fed233 Mon Sep 17 00:00:00 2001 From: Cameron Blocker Date: Sun, 25 Oct 2020 14:43:51 -0400 Subject: [PATCH 358/409] DOC: Added compatibility release note for dtype of all-zero poly1d --- doc/release/upcoming_changes/17577.compatibility.rst | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 doc/release/upcoming_changes/17577.compatibility.rst diff --git a/doc/release/upcoming_changes/17577.compatibility.rst b/doc/release/upcoming_changes/17577.compatibility.rst new file mode 100644 index 000000000000..d08805607323 --- /dev/null +++ b/doc/release/upcoming_changes/17577.compatibility.rst @@ -0,0 +1,6 @@ +poly1d respects the dtype of all-zero argument +---------------------------------------------- +Previously, constructing an instance of ``poly1d`` with all-zero +coefficients would cast the coefficients to ``np.float64``. +This affected the output dtype of methods which construct +``poly1d`` instances internally, such as ``np.polymul``. \ No newline at end of file From ca39060bd4f98657139f782ff9c02a1f095ed3dc Mon Sep 17 00:00:00 2001 From: jakobjakobson13 <43045863+jakobjakobson13@users.noreply.github.com> Date: Sun, 25 Oct 2020 19:47:15 +0100 Subject: [PATCH 359/409] MAINT: Conversion of some strings to fstrings, part III (#17623) * Conversion of some strings to fstrings * Remove spaces * Update numpy/ma/mrecords.py Co-authored-by: Bas van Beek <43369155+BvB93@users.noreply.github.com> * Update numpy/ma/tests/test_old_ma.py Co-authored-by: Bas van Beek <43369155+BvB93@users.noreply.github.com> * Update numpy/ma/timer_comparison.py Co-authored-by: Bas van Beek <43369155+BvB93@users.noreply.github.com> Co-authored-by: Jakob Co-authored-by: Bas van Beek <43369155+BvB93@users.noreply.github.com> --- numpy/ma/bench.py | 10 +++++----- numpy/ma/mrecords.py | 16 ++++++++-------- numpy/ma/tests/test_core.py | 24 ++++++++++++------------ numpy/ma/tests/test_old_ma.py | 2 +- numpy/ma/tests/test_subclassing.py | 8 ++++---- numpy/ma/testutils.py | 10 +++++----- numpy/ma/timer_comparison.py | 5 ++--- 7 files changed, 37 insertions(+), 38 deletions(-) diff --git a/numpy/ma/bench.py b/numpy/ma/bench.py index 83cc6aea7fb2..e29d54365c33 100644 --- a/numpy/ma/bench.py +++ b/numpy/ma/bench.py @@ -58,7 +58,7 @@ def compare_functions_1v(func, nloop=500, xs=xs, nmxs=nmxs, xl=xl, nmxl=nmxl): funcname = func.__name__ print("-"*50) - print("%s on small arrays" % funcname) + print(f'{funcname} on small arrays') module, data = "numpy.ma", "nmxs" timer("%(module)s.%(funcname)s(%(data)s)" % locals(), v="%11s" % module, nloop=nloop) @@ -70,8 +70,8 @@ def compare_functions_1v(func, nloop=500, def compare_methods(methodname, args, vars='x', nloop=500, test=True, xs=xs, nmxs=nmxs, xl=xl, nmxl=nmxl): print("-"*50) - print("%s on small arrays" % methodname) - data, ver = "nm%ss" % vars, 'numpy.ma' + print(f'{methodname} on small arrays') + data, ver = f'nm{vars}l', 'numpy.ma' timer("%(data)s.%(methodname)s(%(args)s)" % locals(), v=ver, nloop=nloop) print("%s on large arrays" % methodname) @@ -86,11 +86,11 @@ def compare_functions_2v(func, nloop=500, test=True, yl=yl, nmyl=nmyl): funcname = func.__name__ print("-"*50) - print("%s on small arrays" % funcname) + print(f'{funcname} on small arrays') module, data = "numpy.ma", "nmxs,nmys" timer("%(module)s.%(funcname)s(%(data)s)" % locals(), v="%11s" % module, nloop=nloop) - print("%s on large arrays" % funcname) + print(f'{funcname} on large arrays') module, data = "numpy.ma", "nmxl,nmyl" timer("%(module)s.%(funcname)s(%(data)s)" % locals(), v="%11s" % module, nloop=nloop) return diff --git a/numpy/ma/mrecords.py b/numpy/ma/mrecords.py index c017bee95684..70087632e9f6 100644 --- a/numpy/ma/mrecords.py +++ b/numpy/ma/mrecords.py @@ -60,7 +60,7 @@ def _checknames(descr, names=None): elif isinstance(names, str): new_names = names.split(',') else: - raise NameError("illegal input names %s" % repr(names)) + raise NameError(f'illegal input names {names!r}') nnames = len(new_names) if nnames < ndescr: new_names += default_names[nnames:] @@ -199,7 +199,7 @@ def __getattribute__(self, attr): try: res = fielddict[attr][:2] except (TypeError, KeyError) as e: - raise AttributeError("record array has no attribute %s" % attr) from e + raise AttributeError(f'record array has no attribute {attr}') from e # So far, so good _localdict = ndarray.__getattribute__(self, '__dict__') _data = ndarray.view(self, _localdict['_baseclass']) @@ -274,7 +274,7 @@ def __setattr__(self, attr, val): try: res = fielddict[attr][:2] except (TypeError, KeyError): - raise AttributeError("record array has no attribute %s" % attr) + raise AttributeError(f'record array has no attribute {attr}') if val is masked: _fill_value = _localdict['_fill_value'] @@ -337,13 +337,13 @@ def __str__(self): """ if self.size > 1: - mstr = ["(%s)" % ",".join([str(i) for i in s]) + mstr = [f"({','.join([str(i) for i in s])})" for s in zip(*[getattr(self, f) for f in self.dtype.names])] - return "[%s]" % ", ".join(mstr) + return f"[{', '.join(mstr)}]" else: - mstr = ["%s" % ",".join([str(i) for i in s]) + mstr = [f"{','.join([str(i) for i in s])}" for s in zip([getattr(self, f) for f in self.dtype.names])] - return "(%s)" % ", ".join(mstr) + return f"({', '.join(mstr)})" def __repr__(self): """ @@ -657,7 +657,7 @@ def openfile(fname): try: f = open(fname) except IOError: - raise IOError("No such file: '%s'" % fname) + raise IOError(f"No such file: '{fname}'") if f.readline()[:2] != "\\x": f.seek(0, 0) return f diff --git a/numpy/ma/tests/test_core.py b/numpy/ma/tests/test_core.py index 0ed2971e6bd2..6a2ed8ca75ea 100644 --- a/numpy/ma/tests/test_core.py +++ b/numpy/ma/tests/test_core.py @@ -2748,7 +2748,7 @@ def test_inplace_addition_scalar_type(self): xm += t(1) assert_equal(xm, y + t(1)) - assert_equal(len(w), 0, "Failed on type=%s." % t) + assert_equal(len(w), 0, f'Failed on type={t}.') def test_inplace_addition_array_type(self): # Test of inplace additions @@ -2765,7 +2765,7 @@ def test_inplace_addition_array_type(self): assert_equal(xm, y + a) assert_equal(xm.mask, mask_or(m, a.mask)) - assert_equal(len(w), 0, "Failed on type=%s." % t) + assert_equal(len(w), 0, f'Failed on type={t}.') def test_inplace_subtraction_scalar_type(self): # Test of inplace subtractions @@ -2778,7 +2778,7 @@ def test_inplace_subtraction_scalar_type(self): xm -= t(1) assert_equal(xm, y - t(1)) - assert_equal(len(w), 0, "Failed on type=%s." % t) + assert_equal(len(w), 0, f'Failed on type={t}.') def test_inplace_subtraction_array_type(self): # Test of inplace subtractions @@ -2795,7 +2795,7 @@ def test_inplace_subtraction_array_type(self): assert_equal(xm, y - a) assert_equal(xm.mask, mask_or(m, a.mask)) - assert_equal(len(w), 0, "Failed on type=%s." % t) + assert_equal(len(w), 0, f'Failed on type={t}.') def test_inplace_multiplication_scalar_type(self): # Test of inplace multiplication @@ -2808,7 +2808,7 @@ def test_inplace_multiplication_scalar_type(self): xm *= t(2) assert_equal(xm, y * t(2)) - assert_equal(len(w), 0, "Failed on type=%s." % t) + assert_equal(len(w), 0, f'Failed on type={t}.') def test_inplace_multiplication_array_type(self): # Test of inplace multiplication @@ -2825,7 +2825,7 @@ def test_inplace_multiplication_array_type(self): assert_equal(xm, y * a) assert_equal(xm.mask, mask_or(m, a.mask)) - assert_equal(len(w), 0, "Failed on type=%s." % t) + assert_equal(len(w), 0, f'Failed on type={t}.') def test_inplace_floor_division_scalar_type(self): # Test of inplace division @@ -2861,7 +2861,7 @@ def test_inplace_floor_division_array_type(self): mask_or(mask_or(m, a.mask), (a == t(0))) ) - assert_equal(len(w), 0, "Failed on type=%s." % t) + assert_equal(len(w), 0, f'Failed on type={t}.') def test_inplace_division_scalar_type(self): # Test of inplace division @@ -2895,9 +2895,9 @@ def test_inplace_division_scalar_type(self): warnings.warn(str(e), stacklevel=1) if issubclass(t, np.integer): - assert_equal(len(sup.log), 2, "Failed on type=%s." % t) + assert_equal(len(sup.log), 2, f'Failed on type={t}.') else: - assert_equal(len(sup.log), 0, "Failed on type=%s." % t) + assert_equal(len(sup.log), 0, f'Failed on type={t}.') def test_inplace_division_array_type(self): # Test of inplace division @@ -2934,9 +2934,9 @@ def test_inplace_division_array_type(self): warnings.warn(str(e), stacklevel=1) if issubclass(t, np.integer): - assert_equal(len(sup.log), 2, "Failed on type=%s." % t) + assert_equal(len(sup.log), 2, f'Failed on type={t}.') else: - assert_equal(len(sup.log), 0, "Failed on type=%s." % t) + assert_equal(len(sup.log), 0, f'Failed on type={t}.') def test_inplace_pow_type(self): # Test keeping data w/ (inplace) power @@ -2954,7 +2954,7 @@ def test_inplace_pow_type(self): assert_equal(x.data, xx_r.data) assert_equal(x.mask, xx_r.mask) - assert_equal(len(w), 0, "Failed on type=%s." % t) + assert_equal(len(w), 0, f'Failed on type={t}.') class TestMaskedArrayMethods: diff --git a/numpy/ma/tests/test_old_ma.py b/numpy/ma/tests/test_old_ma.py index 96c7e3609795..ab003b94e584 100644 --- a/numpy/ma/tests/test_old_ma.py +++ b/numpy/ma/tests/test_old_ma.py @@ -27,7 +27,7 @@ def eq(v, w, msg=''): result = allclose(v, w) if not result: - print("Not eq:%s\n%s\n----%s" % (msg, str(v), str(w))) + print(f'Not eq:{msg}\n{v}\n----{w}') return result diff --git a/numpy/ma/tests/test_subclassing.py b/numpy/ma/tests/test_subclassing.py index caa746740534..f2623406d183 100644 --- a/numpy/ma/tests/test_subclassing.py +++ b/numpy/ma/tests/test_subclassing.py @@ -109,11 +109,11 @@ def __next__(self): class ComplicatedSubArray(SubArray): def __str__(self): - return 'myprefix {0} mypostfix'.format(self.view(SubArray)) + return f'myprefix {self.view(SubArray)} mypostfix' def __repr__(self): # Return a repr that does not start with 'name(' - return '<{0} {1}>'.format(self.__class__.__name__, self) + return f'<{self.__class__.__name__} {self}>' def _validate_input(self, value): if not isinstance(value, ComplicatedSubArray): @@ -317,8 +317,8 @@ def test_subclass_repr(self): assert_startswith(repr(mx), 'masked_array') xsub = SubArray(x) mxsub = masked_array(xsub, mask=[True, False, True, False, False]) - assert_startswith(repr(mxsub), - 'masked_{0}(data=[--, 1, --, 3, 4]'.format(SubArray.__name__)) + assert_startswith(repr(mxsub), + f'masked_{SubArray.__name__}(data=[--, 1, --, 3, 4]') def test_subclass_str(self): """test str with subclass that has overridden str, setitem""" diff --git a/numpy/ma/testutils.py b/numpy/ma/testutils.py index 51ab03948ebc..8d55e1763da1 100644 --- a/numpy/ma/testutils.py +++ b/numpy/ma/testutils.py @@ -86,7 +86,7 @@ def _assert_equal_on_sequences(actual, desired, err_msg=''): """ assert_equal(len(actual), len(desired), err_msg) for k in range(len(desired)): - assert_equal(actual[k], desired[k], 'item=%r\n%s' % (k, err_msg)) + assert_equal(actual[k], desired[k], f'item={k!r}\n{err_msg}') return @@ -117,8 +117,8 @@ def assert_equal(actual, desired, err_msg=''): assert_equal(len(actual), len(desired), err_msg) for k, i in desired.items(): if k not in actual: - raise AssertionError("%s not in %s" % (k, actual)) - assert_equal(actual[k], desired[k], 'key=%r\n%s' % (k, err_msg)) + raise AssertionError(f"{k} not in {actual}") + assert_equal(actual[k], desired[k], f'key={k!r}\n{err_msg}') return # Case #2: lists ..... if isinstance(desired, (list, tuple)) and isinstance(actual, (list, tuple)): @@ -156,12 +156,12 @@ def fail_if_equal(actual, desired, err_msg='',): for k, i in desired.items(): if k not in actual: raise AssertionError(repr(k)) - fail_if_equal(actual[k], desired[k], 'key=%r\n%s' % (k, err_msg)) + fail_if_equal(actual[k], desired[k], f'key={k!r}\n{err_msg}') return if isinstance(desired, (list, tuple)) and isinstance(actual, (list, tuple)): fail_if_equal(len(actual), len(desired), err_msg) for k in range(len(desired)): - fail_if_equal(actual[k], desired[k], 'item=%r\n%s' % (k, err_msg)) + fail_if_equal(actual[k], desired[k], f'item={k!r}\n{err_msg}') return if isinstance(actual, np.ndarray) or isinstance(desired, np.ndarray): return fail_if_array_equal(actual, desired, err_msg) diff --git a/numpy/ma/timer_comparison.py b/numpy/ma/timer_comparison.py index f5855efcf737..aff72f79b8d2 100644 --- a/numpy/ma/timer_comparison.py +++ b/numpy/ma/timer_comparison.py @@ -77,8 +77,7 @@ def assert_array_compare(self, comparison, x, y, err_msg='', header='', if not cond: msg = build_err_msg([x, y], err_msg - + '\n(shapes %s, %s mismatch)' % (x.shape, - y.shape), + + f'\n(shapes {x.shape}, {y.shape} mismatch)', header=header, names=('x', 'y')) assert cond, msg @@ -434,4 +433,4 @@ def test_A(self): cur = np.sort(cur) print("#%i" % i + 50*'.') print(eval("ModuleTester.test_%i.__doc__" % i)) - print("core_current : %.3f - %.3f" % (cur[0], cur[1])) + print(f'core_current : {cur[0]:.3f} - {cur[1]:.3f}') From d212ee4ceeb0a1ff8dba1c6d9e0cc54360e81a94 Mon Sep 17 00:00:00 2001 From: takanori-pskq Date: Mon, 26 Oct 2020 14:11:44 +0900 Subject: [PATCH 360/409] DOC: Fix some references --- doc/source/reference/arrays.ndarray.rst | 4 ++-- doc/source/reference/arrays.nditer.cython.rst | 2 +- doc/source/reference/c-api/array.rst | 8 ++++---- doc/source/reference/c-api/types-and-structures.rst | 3 ++- doc/source/reference/maskedarray.baseclass.rst | 4 ++-- doc/source/user/basics.rec.rst | 4 ++-- 6 files changed, 13 insertions(+), 12 deletions(-) diff --git a/doc/source/reference/arrays.ndarray.rst b/doc/source/reference/arrays.ndarray.rst index 6b805904a0e2..191367058dba 100644 --- a/doc/source/reference/arrays.ndarray.rst +++ b/doc/source/reference/arrays.ndarray.rst @@ -1,11 +1,11 @@ +.. currentmodule:: numpy + .. _arrays.ndarray: ****************************************** The N-dimensional array (:class:`ndarray`) ****************************************** -.. currentmodule:: numpy - An :class:`ndarray` is a (usually fixed-size) multidimensional container of items of the same type and size. The number of dimensions and items in an array is defined by its :attr:`shape `, diff --git a/doc/source/reference/arrays.nditer.cython.rst b/doc/source/reference/arrays.nditer.cython.rst index 2cc7763edfce..43aad99275c7 100644 --- a/doc/source/reference/arrays.nditer.cython.rst +++ b/doc/source/reference/arrays.nditer.cython.rst @@ -5,7 +5,7 @@ Those who want really good performance out of their low level operations should strongly consider directly using the iteration API provided in C, but for those who are not comfortable with C or C++, Cython is a good middle ground with reasonable performance tradeoffs. For -the :class:`nditer` object, this means letting the iterator take care +the :class:`~numpy.nditer` object, this means letting the iterator take care of broadcasting, dtype conversion, and buffering, while giving the inner loop to Cython. diff --git a/doc/source/reference/c-api/array.rst b/doc/source/reference/c-api/array.rst index e9717d59a692..7a3f0fb7c7f9 100644 --- a/doc/source/reference/c-api/array.rst +++ b/doc/source/reference/c-api/array.rst @@ -2960,9 +2960,9 @@ to. already a buffer object pointing to another object). If you need to hold on to the memory be sure to INCREF the base member. The chunk of memory is pointed to by *buf* ->ptr member and has length - *buf* ->len. The flags member of *buf* is :c:data:`NPY_BEHAVED_RO` with - the :c:data:`NPY_ARRAY_WRITEABLE` flag set if *obj* has a writeable buffer - interface. + *buf* ->len. The flags member of *buf* is :c:data:`NPY_ARRAY_BEHAVED` + with the :c:data:`NPY_ARRAY_WRITEABLE` flag set if *obj* has + a writeable buffer interface. .. c:function:: int PyArray_AxisConverter(PyObject* obj, int* axis) @@ -3136,7 +3136,7 @@ the C-API is needed then some additional steps must be taken. be defined in another compilation unit. * Whenever :c:macro:`PY_ARRAY_UNIQUE_SYMBOL` is #defined, it also changes the name of the variable holding the C-API, which - defaults to :c:data:`PyArray_API`, to whatever the macro is + defaults to ``PyArray_API``, to whatever the macro is #defined to. Checking the API Version diff --git a/doc/source/reference/c-api/types-and-structures.rst b/doc/source/reference/c-api/types-and-structures.rst index 87d4be6f3da9..6a9c4a9cf766 100644 --- a/doc/source/reference/c-api/types-and-structures.rst +++ b/doc/source/reference/c-api/types-and-structures.rst @@ -412,7 +412,8 @@ PyArrayDescr_Type and PyArray_Descr Metadata specific to the C implementation of the particular dtype. Added for NumPy 1.7.0. - .. c:member:: Npy_hash_t *hash + .. c:type:: npy_hash_t + .. c:member:: npy_hash_t *hash Currently unused. Reserved for future use in caching hash values. diff --git a/doc/source/reference/maskedarray.baseclass.rst b/doc/source/reference/maskedarray.baseclass.rst index 5c1bdda239c6..5a0f99651c3f 100644 --- a/doc/source/reference/maskedarray.baseclass.rst +++ b/doc/source/reference/maskedarray.baseclass.rst @@ -242,8 +242,8 @@ Comparison operators: MaskedArray.__eq__ MaskedArray.__ne__ -Truth value of an array (:func:`bool()`): -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Truth value of an array (:class:`bool() `): +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. autosummary:: :toctree: generated/ diff --git a/doc/source/user/basics.rec.rst b/doc/source/user/basics.rec.rst index 3dba6536111d..0524fde8ef23 100644 --- a/doc/source/user/basics.rec.rst +++ b/doc/source/user/basics.rec.rst @@ -619,8 +619,8 @@ appropriate `view `_:: >>> recordarr = arr.view(dtype=np.dtype((np.record, arr.dtype)), ... type=np.recarray) -For convenience, viewing an ndarray as type :class:`np.recarray` will -automatically convert to :class:`np.record` datatype, so the dtype can be left +For convenience, viewing an ndarray as type :class:`numpy.recarray` will +automatically convert to :class:`numpy.record` datatype, so the dtype can be left out of the view:: >>> recordarr = arr.view(np.recarray) From 594e66d1e5a2cb79b974984864138ff38e9b8c2d Mon Sep 17 00:00:00 2001 From: mattip Date: Mon, 26 Oct 2020 09:11:40 +0200 Subject: [PATCH 361/409] MAINT, BLD: update to OpenBLAS v0.3.12 --- tools/openblas_support.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/openblas_support.py b/tools/openblas_support.py index 46803742c986..3d966a8f601a 100644 --- a/tools/openblas_support.py +++ b/tools/openblas_support.py @@ -12,9 +12,9 @@ from urllib.request import urlopen, Request from urllib.error import HTTPError -OPENBLAS_V = '0.3.10' +OPENBLAS_V = '0.3.12' # Temporary build of OpenBLAS to test a fix for dynamic detection of CPU -OPENBLAS_LONG = 'v0.3.9-452-g349b722d' +OPENBLAS_LONG = 'v0.3.12' BASE_LOC = 'https://anaconda.org/multibuild-wheels-staging/openblas-libs' BASEURL = f'{BASE_LOC}/{OPENBLAS_LONG}/download' ARCHITECTURES = ['', 'windows', 'darwin', 'aarch64', 'x86_64', @@ -143,9 +143,9 @@ def download_openblas(target, arch, ilp64, is_32bit): typ = 'tar.gz' elif arch == 'windows': if is_32bit: - suffix = 'win32-gcc_7_1_0.zip' + suffix = 'win32-gcc_8_1_0.zip' else: - suffix = 'win_amd64-gcc_7_1_0.zip' + suffix = 'win_amd64-gcc_8_1_0.zip' filename = f'{BASEURL}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}' typ = 'zip' if not filename: From 58aba098a1b42ef71712ff694fc66799ec311fba Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2020 07:23:03 +0000 Subject: [PATCH 362/409] MAINT: Bump hypothesis from 5.37.3 to 5.38.0 Bumps [hypothesis](https://github.com/HypothesisWorks/hypothesis) from 5.37.3 to 5.38.0. - [Release notes](https://github.com/HypothesisWorks/hypothesis/releases) - [Commits](https://github.com/HypothesisWorks/hypothesis/compare/hypothesis-python-5.37.3...hypothesis-python-5.38.0) Signed-off-by: dependabot-preview[bot] --- test_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index 6d8b94cae630..6e858857cd30 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,7 +1,7 @@ cython==0.29.21 wheel setuptools<49.2.0 -hypothesis==5.37.3 +hypothesis==5.38.0 pytest==6.0.2 pytz==2020.1 pytest-cov==2.10.1 From 75b004254f46992f742b2f128700f8cf7d5b3d6d Mon Sep 17 00:00:00 2001 From: James Myatt Date: Mon, 26 Oct 2020 15:48:31 +0000 Subject: [PATCH 363/409] DOC: Fix reference to atleast_1d --- numpy/core/fromnumeric.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/core/fromnumeric.py b/numpy/core/fromnumeric.py index b1524b8915f4..771dfb66e15b 100644 --- a/numpy/core/fromnumeric.py +++ b/numpy/core/fromnumeric.py @@ -1840,11 +1840,11 @@ def nonzero(a): .. note:: When called on a zero-d array or scalar, ``nonzero(a)`` is treated - as ``nonzero(atleast1d(a))``. + as ``nonzero(atleast_1d(a))``. .. deprecated:: 1.17.0 - Use `atleast1d` explicitly if this behavior is deliberate. + Use `atleast_1d` explicitly if this behavior is deliberate. Parameters ---------- From 2dcbc560320cb70c2664ed442b9083f80b23840b Mon Sep 17 00:00:00 2001 From: Hassan Kibirige Date: Mon, 26 Oct 2020 19:23:40 +0300 Subject: [PATCH 364/409] MAINT: valid doctest for config_py function show() (#17621) numpy.distutils generates a function `__config__.show`. This change makes the docstring of that function complete (with the required imports) and runnable. This prevents numpy.distutils from adding doctests that will fail -- if picked-up by the test runner. --- numpy/distutils/misc_util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/numpy/distutils/misc_util.py b/numpy/distutils/misc_util.py index aa649a23f390..778723bfeeda 100644 --- a/numpy/distutils/misc_util.py +++ b/numpy/distutils/misc_util.py @@ -2356,6 +2356,7 @@ def show(): Examples -------- + >>> import numpy as np >>> np.show_config() blas_opt_info: language = c From 947fea4c2f8db6cd9f8628b1b34c8697b89eaf82 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Wed, 26 Aug 2020 23:34:28 +0200 Subject: [PATCH 365/409] ENH: Add annotations for `np.core.shape_base` --- numpy/__init__.pyi | 17 +++++++++------- numpy/core/shape_base.pyi | 41 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 7 deletions(-) create mode 100644 numpy/core/shape_base.pyi diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index ce4aa1ae6fea..339a651c0583 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -193,6 +193,16 @@ from numpy.core.numerictypes import ( find_common_type as find_common_type, ) +from numpy.core.shape_base import ( + atleast_1d as atleast_1d, + atleast_2d as atleast_2d, + atleast_3d as atleast_3d, + block as block, + hstack as hstack, + stack as stack, + vstack as vstack, +) + # Add an object to `__all__` if their stubs are defined in an external file; # their stubs will not be recognized otherwise. # NOTE: This is redundant for objects defined within this file. @@ -256,15 +266,11 @@ asarray_chkfinite: Any asfarray: Any asmatrix: Any asscalar: Any -atleast_1d: Any -atleast_2d: Any -atleast_3d: Any average: Any bartlett: Any bincount: Any bitwise_not: Any blackman: Any -block: Any bmat: Any bool8: Any broadcast: Any @@ -354,7 +360,6 @@ histogram2d: Any histogram_bin_edges: Any histogramdd: Any hsplit: Any -hstack: Any i0: Any iinfo: Any imag: Any @@ -488,7 +493,6 @@ singlecomplex: Any sort_complex: Any source: Any split: Any -stack: Any string_: Any take_along_axis: Any tile: Any @@ -521,7 +525,6 @@ vdot: Any vectorize: Any void0: Any vsplit: Any -vstack: Any where: Any who: Any diff --git a/numpy/core/shape_base.pyi b/numpy/core/shape_base.pyi new file mode 100644 index 000000000000..2fe945f3bdca --- /dev/null +++ b/numpy/core/shape_base.pyi @@ -0,0 +1,41 @@ +import sys +from typing import TypeVar, overload, List, Sequence, Optional + +from numpy import ndarray +from numpy.typing import ArrayLike + +if sys.version_info >= (3, 8): + from typing import SupportsIndex, Literal +else: + from typing_extensions import Literal, Protocol + class SupportsIndex(Protocol): + def __index__(self) -> int: ... + +_ArrayType = TypeVar("_ArrayType", bound=ndarray) + +@overload +def atleast_1d(__arys: ArrayLike) -> ndarray: ... +@overload +def atleast_1d(*arys: ArrayLike) -> List[ndarray]: ... + +@overload +def atleast_2d(__arys: ArrayLike) -> ndarray: ... +@overload +def atleast_2d(*arys: ArrayLike) -> List[ndarray]: ... + +@overload +def atleast_3d(__arys: ArrayLike) -> ndarray: ... +@overload +def atleast_3d(*arys: ArrayLike) -> List[ndarray]: ... + +def vstack(tup: Sequence[ArrayLike]) -> ndarray: ... +def hstack(tup: Sequence[ArrayLike]) -> ndarray: ... +@overload +def stack( + arrays: Sequence[ArrayLike], axis: SupportsIndex = ..., out: None = ... +) -> ndarray: ... +@overload +def stack( + arrays: Sequence[ArrayLike], axis: SupportsIndex = ..., out: _ArrayType = ... +) -> _ArrayType: ... +def block(arrays: ArrayLike) -> ndarray: ... From 132f667d2be8ae69ad10600b483c4cf4992dd3f8 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Mon, 26 Oct 2020 22:14:10 +0100 Subject: [PATCH 366/409] TST: Added typing tests for `np.core.shape_base` --- .../tests/data/fail/array_constructors.py | 5 ++++ .../tests/data/pass/array_constructors.py | 24 +++++++++++++++++++ .../tests/data/reveal/array_constructors.py | 23 ++++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/numpy/typing/tests/data/fail/array_constructors.py b/numpy/typing/tests/data/fail/array_constructors.py index 5218572a61c2..9cb59fe5f72e 100644 --- a/numpy/typing/tests/data/fail/array_constructors.py +++ b/numpy/typing/tests/data/fail/array_constructors.py @@ -1,6 +1,7 @@ import numpy as np a: np.ndarray +generator = (i for i in range(10)) np.require(a, requirements=1) # E: No overload variant np.require(a, requirements="TEST") # E: incompatible type @@ -24,3 +25,7 @@ np.logspace(0, 2, base=None) # E: Argument "base" np.geomspace(None, 'bob') # E: Argument 1 + +np.stack(generator) # E: No overload variant +np.hstack({1, 2}) # E: incompatible type +np.vstack(1) # E: incompatible type diff --git a/numpy/typing/tests/data/pass/array_constructors.py b/numpy/typing/tests/data/pass/array_constructors.py index 08f4b6211dc9..63208f139c39 100644 --- a/numpy/typing/tests/data/pass/array_constructors.py +++ b/numpy/typing/tests/data/pass/array_constructors.py @@ -11,6 +11,7 @@ class SubClass(np.ndarray): ... A = np.array([1]) B = A.view(SubClass).copy() +B_stack = np.array([[1], [1]]).view(SubClass) C = [1] def func(i: int, j: int, **kwargs: Any) -> SubClass: @@ -102,3 +103,26 @@ def func(i: int, j: int, **kwargs: Any) -> SubClass: np.fromfunction(func, (3, 5)) np.identity(10) + +np.atleast_1d(C) +np.atleast_1d(A) +np.atleast_1d(C, C) +np.atleast_1d(C, A) +np.atleast_1d(A, A) + +np.atleast_2d(C) + +np.atleast_3d(C) + +np.vstack([C, C]) +np.vstack([C, A]) +np.vstack([A, A]) + +np.hstack([C, C]) + +np.stack([C, C]) +np.stack([C, C], axis=0) +np.stack([C, C], out=B_stack) + +np.block([[C, C], [C, C]]) +np.block(A) diff --git a/numpy/typing/tests/data/reveal/array_constructors.py b/numpy/typing/tests/data/reveal/array_constructors.py index 04732ff2ec0c..106174736085 100644 --- a/numpy/typing/tests/data/reveal/array_constructors.py +++ b/numpy/typing/tests/data/reveal/array_constructors.py @@ -77,3 +77,26 @@ def func(i: int, j: int, **kwargs: Any) -> SubClass: ... reveal_type(np.fromfunction(func, (3, 5))) # E: SubClass reveal_type(np.identity(10)) # E: numpy.ndarray + +reveal_type(np.atleast_1d(A)) # E: numpy.ndarray +reveal_type(np.atleast_1d(C)) # E: numpy.ndarray +reveal_type(np.atleast_1d(A, A)) # E: list[numpy.ndarray] +reveal_type(np.atleast_1d(A, C)) # E: list[numpy.ndarray] +reveal_type(np.atleast_1d(C, C)) # E: list[numpy.ndarray] + +reveal_type(np.atleast_2d(A)) # E: numpy.ndarray + +reveal_type(np.atleast_3d(A)) # E: numpy.ndarray + +reveal_type(np.vstack([A, A])) # E: numpy.ndarray +reveal_type(np.vstack([A, C])) # E: numpy.ndarray +reveal_type(np.vstack([C, C])) # E: numpy.ndarray + +reveal_type(np.hstack([A, A])) # E: numpy.ndarray + +reveal_type(np.stack([A, A])) # E: numpy.ndarray +reveal_type(np.stack([A, A], axis=0)) # E: numpy.ndarray +reveal_type(np.stack([A, A], out=B)) # E: SubClass + +reveal_type(np.block([[A, A], [A, A]])) # E: numpy.ndarray +reveal_type(np.block(C)) # E: numpy.ndarray From 061c842994866bbb9786f8316921a3b8ed3e3adc Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Mon, 26 Oct 2020 22:29:22 +0100 Subject: [PATCH 367/409] MAINT: Removed unused imports --- numpy/core/shape_base.pyi | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/numpy/core/shape_base.pyi b/numpy/core/shape_base.pyi index 2fe945f3bdca..b20598b1ab5b 100644 --- a/numpy/core/shape_base.pyi +++ b/numpy/core/shape_base.pyi @@ -1,13 +1,13 @@ import sys -from typing import TypeVar, overload, List, Sequence, Optional +from typing import TypeVar, overload, List, Sequence from numpy import ndarray from numpy.typing import ArrayLike if sys.version_info >= (3, 8): - from typing import SupportsIndex, Literal + from typing import SupportsIndex else: - from typing_extensions import Literal, Protocol + from typing_extensions import Protocol class SupportsIndex(Protocol): def __index__(self) -> int: ... From 688817e760b11b27481c9a9057fbfc633fc00774 Mon Sep 17 00:00:00 2001 From: Takanori H Date: Tue, 27 Oct 2020 08:12:35 +0900 Subject: [PATCH 368/409] Update doc/source/reference/c-api/array.rst Co-authored-by: Eric Wieser --- doc/source/reference/c-api/array.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/reference/c-api/array.rst b/doc/source/reference/c-api/array.rst index 7a3f0fb7c7f9..3aa541b79e64 100644 --- a/doc/source/reference/c-api/array.rst +++ b/doc/source/reference/c-api/array.rst @@ -2960,7 +2960,7 @@ to. already a buffer object pointing to another object). If you need to hold on to the memory be sure to INCREF the base member. The chunk of memory is pointed to by *buf* ->ptr member and has length - *buf* ->len. The flags member of *buf* is :c:data:`NPY_ARRAY_BEHAVED` + *buf* ->len. The flags member of *buf* is :c:data:`NPY_ARRAY_ALIGNED` with the :c:data:`NPY_ARRAY_WRITEABLE` flag set if *obj* has a writeable buffer interface. From cb3efe8e03b53dbab457a99be1a48384312abe16 Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Wed, 8 Jul 2020 09:27:13 +0200 Subject: [PATCH 369/409] ENH: Expose the NumPy C SIMD vectorization interface "NPYV" to Python '_simd' is a new module to bring the NumPy C SIMD vectorization interface "NPYV" The module is designed to be extremely flexible so that it can accommodate any kind intrinsics, also to generate a python interface almost similar to the C interface. The main purpose of this module is to test NPYV intrinsics in python, but still can be used as an effective solution in designing SIMD kernels. Also add a new command-line argument `--simd-test` to control of targeted CPU features for the `_simd` module. Co-authored-by: Matti Picus Co-authored-by: Eric Wieser --- numpy/core/setup.py | 23 ++ numpy/core/src/_simd/_simd.c | 63 ++++ numpy/core/src/_simd/_simd.dispatch.c.src | 351 +++++++++++++++++ numpy/core/src/_simd/_simd.h | 30 ++ numpy/core/src/_simd/_simd_inc.h.src | 395 ++++++++++++++++++++ numpy/core/src/_simd/_simd_inc_arg.h | 108 ++++++ numpy/core/src/_simd/_simd_inc_convert.h | 206 ++++++++++ numpy/core/src/_simd/_simd_inc_data.h.src | 91 +++++ numpy/core/src/_simd/_simd_inc_easyintrin.h | 229 ++++++++++++ numpy/core/src/_simd/_simd_inc_vector.h | 187 +++++++++ numpy/distutils/command/build.py | 12 + numpy/distutils/command/build_ext.py | 8 +- runtests.py | 5 + 13 files changed, 1706 insertions(+), 2 deletions(-) create mode 100644 numpy/core/src/_simd/_simd.c create mode 100644 numpy/core/src/_simd/_simd.dispatch.c.src create mode 100644 numpy/core/src/_simd/_simd.h create mode 100644 numpy/core/src/_simd/_simd_inc.h.src create mode 100644 numpy/core/src/_simd/_simd_inc_arg.h create mode 100644 numpy/core/src/_simd/_simd_inc_convert.h create mode 100644 numpy/core/src/_simd/_simd_inc_data.h.src create mode 100644 numpy/core/src/_simd/_simd_inc_easyintrin.h create mode 100644 numpy/core/src/_simd/_simd_inc_vector.h diff --git a/numpy/core/setup.py b/numpy/core/setup.py index b3e17baed8df..e9a9a4e464ae 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -626,6 +626,7 @@ def generate_api(ext, build_dir): config.add_include_dirs(join('src', 'multiarray')) config.add_include_dirs(join('src', 'umath')) config.add_include_dirs(join('src', 'npysort')) + config.add_include_dirs(join('src', '_simd')) config.add_define_macros([("NPY_INTERNAL_BUILD", "1")]) # this macro indicates that Numpy build is in process config.add_define_macros([("HAVE_NPY_CONFIG_H", "1")]) @@ -974,6 +975,28 @@ def generate_umath_c(ext, build_dir): config.add_extension('_operand_flag_tests', sources=[join('src', 'umath', '_operand_flag_tests.c.src')]) + ####################################################################### + # SIMD module # + ####################################################################### + + config.add_extension('_simd', sources=[ + join('src', 'common', 'npy_cpu_features.c.src'), + join('src', '_simd', '_simd.c'), + join('src', '_simd', '_simd_inc.h.src'), + join('src', '_simd', '_simd_inc_data.h.src'), + join('src', '_simd', '_simd.dispatch.c.src'), + ], depends=[ + join('src', 'common', 'npy_cpu_dispatch.h'), + join('src', 'common', 'simd', 'simd.h'), + join('src', '_simd', '_simd.h'), + join('src', '_simd', '_simd_inc.h.src'), + join('src', '_simd', '_simd_inc_data.h.src'), + join('src', '_simd', '_simd_inc_arg.h'), + join('src', '_simd', '_simd_inc_convert.h'), + join('src', '_simd', '_simd_inc_easyintrin.h'), + join('src', '_simd', '_simd_inc_vector.h'), + ]) + config.add_subpackage('tests') config.add_data_dir('tests/data') config.add_data_dir('tests/examples') diff --git a/numpy/core/src/_simd/_simd.c b/numpy/core/src/_simd/_simd.c new file mode 100644 index 000000000000..e5cb582b3962 --- /dev/null +++ b/numpy/core/src/_simd/_simd.c @@ -0,0 +1,63 @@ +#include "_simd.h" + +PyMODINIT_FUNC PyInit__simd(void) +{ + static struct PyModuleDef defs = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "_simd", + .m_size = -1 + }; + if (npy_cpu_init() < 0) { + return NULL; + } + PyObject *m = PyModule_Create(&defs); + if (m == NULL) { + return NULL; + } + PyObject *targets = PyDict_New(); + if (targets == NULL) { + goto err; + } + if (PyModule_AddObject(m, "targets", targets) < 0) { + Py_DECREF(targets); + goto err; + } + // add keys for non-supported optimizations with None value + #define ATTACH_MODULE(TESTED_FEATURES, TARGET_NAME, MAKE_MSVC_HAPPY) \ + { \ + PyObject *simd_mod; \ + if (!TESTED_FEATURES) { \ + Py_INCREF(Py_None); \ + simd_mod = Py_None; \ + } else { \ + simd_mod = NPY_CAT(simd_create_module_, TARGET_NAME)(); \ + if (simd_mod == NULL) { \ + goto err; \ + } \ + } \ + const char *target_name = NPY_TOSTRING(TARGET_NAME); \ + if (PyDict_SetItemString(targets, target_name, simd_mod) < 0) { \ + Py_DECREF(simd_mod); \ + goto err; \ + } \ + } + + #define ATTACH_BASELINE_MODULE(MAKE_MSVC_HAPPY) \ + { \ + PyObject *simd_mod = simd_create_module(); \ + if (simd_mod == NULL) { \ + goto err; \ + } \ + if (PyDict_SetItemString(targets, "baseline", simd_mod) < 0) { \ + Py_DECREF(simd_mod); \ + goto err; \ + } \ + } + + NPY__CPU_DISPATCH_CALL(NPY_CPU_HAVE, ATTACH_MODULE, MAKE_MSVC_HAPPY) + NPY__CPU_DISPATCH_BASELINE_CALL(ATTACH_BASELINE_MODULE, MAKE_MSVC_HAPPY) + return m; +err: + Py_DECREF(m); + return NULL; +} diff --git a/numpy/core/src/_simd/_simd.dispatch.c.src b/numpy/core/src/_simd/_simd.dispatch.c.src new file mode 100644 index 000000000000..69a9ffd45660 --- /dev/null +++ b/numpy/core/src/_simd/_simd.dispatch.c.src @@ -0,0 +1,351 @@ +/*@targets $werror #simd_test*/ +#include "_simd.h" +#include "_simd_inc.h" + +#if NPY_SIMD +#include "_simd_inc_data.h" +#include "_simd_inc_convert.h" +#include "_simd_inc_vector.h" +#include "_simd_inc_arg.h" +#include "_simd_inc_easyintrin.h" + +/************************************************************************* + * Defining NPYV intrinsics as module functions + *************************************************************************/ +/**begin repeat + * #sfx = u8, s8, u16, s16, u32, s32, u64, s64, f32, f64# + * #bsfx = b8, b8, b16, b16, b32, b32, b64, b64, b32, b64# + * #simd_sup = 1, 1, 1, 1, 1, 1, 1, 1, 1, NPY_SIMD_F64# + * #sat_sup = 1, 1, 1, 1, 0, 0, 0, 0, 0, 0# + * #mul_sup = 1, 1, 1, 1, 1, 1, 0, 0, 1, 1# + * #div_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# + * #shl_imm = 0, 0, 15, 15, 31, 31, 63, 63, 0, 0# + * #shr_imm = 0, 0, 16, 16, 32, 32, 64, 64, 0, 0# + */ +#if @simd_sup@ +/*************************** + * Memory + ***************************/ +/**begin repeat1 + * # intrin = load, loada, loads, loadl# + */ +SIMD_IMPL_INTRIN_1(@intrin@_@sfx@, v@sfx@, q@sfx@) +/**end repeat1**/ +/**begin repeat1 + * # intrin = store, storea, stores, storel, storeh# + */ +// special definition due to the nature of @intrin@ +static PyObject * +simd__intrin_@intrin@_@sfx@(PyObject* NPY_UNUSED(self), PyObject *args) +{ + simd_arg req_args[] = { + {.dtype = simd_data_q@sfx@}, + {.dtype = simd_data_v@sfx@}, + }; + if (simd_args_from_tuple(args, req_args, 2, "@intrin@_@sfx@")) { + return NULL; + } + npyv_@intrin@_@sfx@( + req_args[0].data.q@sfx@, req_args[1].data.v@sfx@ + ); + // write-back + if (simd_sequence_fill_obj(req_args[0].obj, req_args[0].data.q@sfx@, simd_data_q@sfx@)) { + simd_args_sequence_free(req_args, 2); + return NULL; + } + simd_args_sequence_free(req_args, 2); + Py_RETURN_NONE; +} +/**end repeat1**/ + +/*************************** + * Misc + ***************************/ +SIMD_IMPL_INTRIN_0(zero_@sfx@, v@sfx@) +SIMD_IMPL_INTRIN_1(setall_@sfx@, v@sfx@, @sfx@) +SIMD_IMPL_INTRIN_3(select_@sfx@, v@sfx@, v@bsfx@, v@sfx@, v@sfx@) + +/**begin repeat1 + * #sfx_to = u8, s8, u16, s16, u32, s32, u64, s64, f32, f64# + * #simd_sup2 = 1, 1, 1, 1, 1, 1, 1, 1, 1, NPY_SIMD_F64# + */ +#if @simd_sup2@ +SIMD_IMPL_INTRIN_1(reinterpret_@sfx_to@_@sfx@, v@sfx_to@, v@sfx@) +#endif // simd_sup2 +/**end repeat1**/ + +/** + * special definition due to the nature of intrinsics + * npyv_setf_@sfx@ and npy_set_@sfx@. +*/ +/**begin repeat1 + * #intrin = setf, set# + */ +static PyObject * +simd__intrin_@intrin@_@sfx@(PyObject* NPY_UNUSED(self), PyObject *args) +{ + npyv_lanetype_@sfx@ *data = simd_sequence_from_obj(args, simd_data_q@sfx@, npyv_nlanes_@sfx@); + if (data == NULL) { + return NULL; + } + simd_data r = {.v@sfx@ = npyv_@intrin@_@sfx@( + data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7], + data[8], data[9], data[10], data[11], data[12], data[13], data[14], data[15], + data[16], data[17], data[18], data[19], data[20], data[21], data[22], data[23], + data[24], data[25], data[26], data[27], data[28], data[29], data[30], data[31], + data[32], data[33], data[34], data[35], data[36], data[37], data[38], data[39], + data[40], data[41], data[42], data[43], data[44], data[45], data[46], data[47], + data[48], data[49], data[50], data[51], data[52], data[53], data[54], data[55], + data[56], data[57], data[58], data[59], data[60], data[61], data[62], data[63], + data[64] // for setf + )}; + simd_sequence_free(data); + return (PyObject*)simd_vector_to_obj(r, simd_data_v@sfx@); +} +/**end repeat1**/ + +/*************************** + * Reorder + ***************************/ +/**begin repeat1 + * # intrin = combinel, combineh# + */ +SIMD_IMPL_INTRIN_2(@intrin@_@sfx@, v@sfx@, v@sfx@, v@sfx@) +/**end repeat1**/ + +/**begin repeat1 + * # intrin = combine, zip# + */ +SIMD_IMPL_INTRIN_2(@intrin@_@sfx@, v@sfx@x2, v@sfx@, v@sfx@) +/**end repeat1**/ + +/*************************** + * Operators + ***************************/ +#if @shl_imm@ > 0 +SIMD_IMPL_INTRIN_2(shl_@sfx@, v@sfx@, v@sfx@, u8) +SIMD_IMPL_INTRIN_2(shr_@sfx@, v@sfx@, v@sfx@, u8) +// immediate constant +SIMD_IMPL_INTRIN_2IMM(shli_@sfx@, v@sfx@, v@sfx@, @shl_imm@) +SIMD_IMPL_INTRIN_2IMM(shri_@sfx@, v@sfx@, v@sfx@, @shr_imm@) +#endif // shl_imm + +/**begin repeat1 + * #intrin = and, or, xor# + */ +SIMD_IMPL_INTRIN_2(@intrin@_@sfx@, v@sfx@, v@sfx@, v@sfx@) +/**end repeat1**/ + +SIMD_IMPL_INTRIN_1(not_@sfx@, v@sfx@, v@sfx@) + +/**begin repeat1 + * #intrin = cmpeq, cmpneq, cmpgt, cmpge, cmplt, cmple# + */ +SIMD_IMPL_INTRIN_2(@intrin@_@sfx@, v@bsfx@, v@sfx@, v@sfx@) +/**end repeat1**/ + +/*************************** + * Conversion + ***************************/ +SIMD_IMPL_INTRIN_1(cvt_@sfx@_@bsfx@, v@sfx@, v@bsfx@) +SIMD_IMPL_INTRIN_1(cvt_@bsfx@_@sfx@, v@bsfx@, v@sfx@) + +/*************************** + * Arithmetic + ***************************/ +/**begin repeat1 + * #intrin = add, sub# + */ +SIMD_IMPL_INTRIN_2(@intrin@_@sfx@, v@sfx@, v@sfx@, v@sfx@) +/**end repeat1**/ + +#if @sat_sup@ +/**begin repeat1 + * #intrin = adds, subs# + */ +SIMD_IMPL_INTRIN_2(@intrin@_@sfx@, v@sfx@, v@sfx@, v@sfx@) +/**end repeat1**/ +#endif // sat_sup + +#if @mul_sup@ +SIMD_IMPL_INTRIN_2(mul_@sfx@, v@sfx@, v@sfx@, v@sfx@) +#endif // mul_sup + +#if @div_sup@ +SIMD_IMPL_INTRIN_2(div_@sfx@, v@sfx@, v@sfx@, v@sfx@) +#endif // div_sup + +#endif // simd_sup +/**end repeat**/ + +/*************************** + * Variant + ***************************/ +SIMD_IMPL_INTRIN_0N(cleanup) + +/************************************************************************* + * Attach module functions + *************************************************************************/ +static PyMethodDef simd__intrinsics_methods[] = { +/**begin repeat + * #sfx = u8, s8, u16, s16, u32, s32, u64, s64, f32, f64# + * #bsfx = b8, b8, b16, b16, b32, b32, b64, b64, b32, b64# + * #simd_sup = 1, 1, 1, 1, 1, 1, 1, 1, 1, NPY_SIMD_F64# + * #sat_sup = 1, 1, 1, 1, 0, 0, 0, 0, 0, 0# + * #mul_sup = 1, 1, 1, 1, 1, 1, 0, 0, 1, 1# + * #div_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# + * #shl_imm = 0, 0, 15, 15, 31, 31, 63, 63, 0, 0# + * #shr_imm = 0, 0, 16, 16, 32, 32, 64, 64, 0, 0# + */ +#if @simd_sup@ + +/*************************** + * Memory + ***************************/ +/**begin repeat1 + * # intrin = load, loada, loads, loadl, store, storea, stores, storel, storeh# + */ +SIMD_INTRIN_DEF(@intrin@_@sfx@) +/**end repeat1**/ + +/*************************** + * Misc + ***************************/ +/**begin repeat1 + * #sfx_to = u8, s8, u16, s16, u32, s32, u64, s64, f32, f64# + * #simd_sup2 = 1, 1, 1, 1, 1, 1, 1, 1, 1, NPY_SIMD_F64# + */ +#if @simd_sup2@ +SIMD_INTRIN_DEF(reinterpret_@sfx_to@_@sfx@) +#endif // simd_sup2 +/**end repeat1**/ + +/**begin repeat1 + * # intrin = set, setf, setall, zero, select# + */ +SIMD_INTRIN_DEF(@intrin@_@sfx@) +/**end repeat1**/ + +/*************************** + * Reorder + ***************************/ +/**begin repeat1 + * # intrin = combinel, combineh, combine, zip# + */ +SIMD_INTRIN_DEF(@intrin@_@sfx@) +/**end repeat1**/ + +SIMD_INTRIN_DEF(cvt_@sfx@_@bsfx@) +SIMD_INTRIN_DEF(cvt_@bsfx@_@sfx@) + +/*************************** + * Operators + ***************************/ +#if @shl_imm@ > 0 +/**begin repeat1 + * # intrin = shl, shr, shli, shri# + */ +SIMD_INTRIN_DEF(@intrin@_@sfx@) +/**end repeat1**/ +#endif // shl_imm + +/**begin repeat1 + * #intrin = and, or, xor, not, cmpeq, cmpneq, cmpgt, cmpge, cmplt, cmple# + */ +SIMD_INTRIN_DEF(@intrin@_@sfx@) +/**end repeat1**/ + +/*************************** + * Conversion + ***************************/ +SIMD_INTRIN_DEF(cvt_@sfx@_@bsfx@) +SIMD_INTRIN_DEF(cvt_@bsfx@_@sfx@) + +/*************************** + * Arithmetic + ***************************/ +/**begin repeat1 + * #intrin = add, sub# + */ +SIMD_INTRIN_DEF(@intrin@_@sfx@) +/**end repeat1**/ + +#if @sat_sup@ +/**begin repeat1 + * #intrin = adds, subs# + */ +SIMD_INTRIN_DEF(@intrin@_@sfx@) +/**end repeat1**/ +#endif // sat_sup + +#if @mul_sup@ +SIMD_INTRIN_DEF(mul_@sfx@) +#endif // mul_sup + +#if @div_sup@ +SIMD_INTRIN_DEF(div_@sfx@) +#endif // div_sup + +#endif // simd_sup +/**end repeat**/ + +/*************************** + * Variant + ***************************/ +SIMD_INTRIN_DEF(cleanup) +/***************************/ +{NULL, NULL, 0, NULL} +}; // PyMethodDef + +#endif // NPY_SIMD + +/************************************************************************* + * Defining a separate module for each target + *************************************************************************/ +NPY_VISIBILITY_HIDDEN PyObject * +NPY_CPU_DISPATCH_CURFX(simd_create_module)(void) +{ + static struct PyModuleDef defs = { + .m_base = PyModuleDef_HEAD_INIT, + .m_size = -1, + #ifdef NPY__CPU_TARGET_CURRENT + .m_name = "NPYV_" NPY_TOSTRING(NPY__CPU_TARGET_CURRENT), + #else + .m_name = "NPYV_BASELINE", + #endif + #if NPY_SIMD + .m_methods = simd__intrinsics_methods + #else + .m_methods = NULL + #endif + }; + PyObject *m = PyModule_Create(&defs); + if (m == NULL) { + return NULL; + } + if (PyModule_AddIntConstant(m, "simd", NPY_SIMD)) { + goto err; + } + if (PyModule_AddIntConstant(m, "simd_f64", NPY_SIMD_F64)) { + goto err; + } + if (PyModule_AddIntConstant(m, "simd_width", NPY_SIMD_WIDTH)) { + goto err; + } +#if NPY_SIMD + if (simd_vector_register(m)) { + goto err; + } + /**begin repeat + * #sfx = u8, s8, u16, s16, u32, s32, u64, s64, f32, f64# + */ + if (PyModule_AddIntConstant(m, "nlanes_@sfx@", npyv_nlanes_@sfx@)) { + goto err; + } + /**end repeat**/ +#endif // NPY_SIMD + return m; +err: + Py_DECREF(m); + return NULL; +} diff --git a/numpy/core/src/_simd/_simd.h b/numpy/core/src/_simd/_simd.h new file mode 100644 index 000000000000..d9905c80127c --- /dev/null +++ b/numpy/core/src/_simd/_simd.h @@ -0,0 +1,30 @@ +/** + * A module to expose the NumPy C SIMD vectorization interface "NPYV" for testing purposes. + * + * Please keep this module independent from other c-extension modules, + * since NPYV intrinsics may be involved in their functionality, + * which increases the degree of complexity in tracking and detecting errors. + * + * TODO: Add an independent sphinx doc. + * + * Please add any new NPYV intrinsics in '_simd.dispatch.c.src'. + */ +#ifndef _SIMD_SIMD_H_ +#define _SIMD_SIMD_H_ + +#include +#include "numpy/npy_common.h" + +#ifndef NPY_DISABLE_OPTIMIZATION +// autogenerated, required for CPU dispatch macros +#include "_simd.dispatch.h" +#endif +/** + * Create a new module for each required optimization which contains all NPYV intrinsics, + * + * If required optimization is not supported by NPYV, the module will still provides + * access to NPYV constants NPY_SIMD, NPY_SIMD_F64, and NPY_SIMD_WIDTH but without + * any intrinsics. + */ +NPY_CPU_DISPATCH_DECLARE(NPY_VISIBILITY_HIDDEN PyObject *simd_create_module, (void)) +#endif // _SIMD_SIMD_H_ diff --git a/numpy/core/src/_simd/_simd_inc.h.src b/numpy/core/src/_simd/_simd_inc.h.src new file mode 100644 index 000000000000..4261ce14892f --- /dev/null +++ b/numpy/core/src/_simd/_simd_inc.h.src @@ -0,0 +1,395 @@ +/** + * This header works only through '_simd.dispatch.c' + */ +#include +#include "simd/simd.h" + +#if NPY_SIMD +/************************************ + ** Types + ************************************/ +/** + * Gather all data types supported by the module. +*/ +typedef union +{ + // scalars + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32, f64# + */ + npyv_lanetype_@sfx@ @sfx@; + /**end repeat**/ + // sequence + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32, f64# + */ + npyv_lanetype_@sfx@ *q@sfx@; + /**end repeat**/ + // vectors + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32, b8, b16, b32, b64# + */ + npyv_@sfx@ v@sfx@; + /**end repeat**/ + // multi-vectors x2 + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32# + */ + npyv_@sfx@x2 v@sfx@x2; + /**end repeat**/ + // multi-vectors x3 + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32# + */ + npyv_@sfx@x3 v@sfx@x3; + /**end repeat**/ +#if NPY_SIMD_F64 + npyv_f64 vf64; + npyv_f64x2 vf64x2; + npyv_f64x3 vf64x3; +#endif +} simd_data; + +/** + * Data types IDs and suffixes. Must be same data types as the ones + * in union 'simd_data' to fit the macros in '_simd_inc_easyintrin.h'. +*/ +typedef enum +{ + simd_data_none = 0, + // scalars + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32, f64# + */ + simd_data_@sfx@, + /**end repeat**/ + // sequences + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32, f64# + */ + simd_data_q@sfx@, + /**end repeat**/ + // vectors + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32, f64, b8, b16, b32, b64# + */ + simd_data_v@sfx@, + /**end repeat**/ + // multi-vectors x2 + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32, f64# + */ + simd_data_v@sfx@x2, + /**end repeat**/ + // multi-vectors x3 + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32, f64# + */ + simd_data_v@sfx@x3, + /**end repeat**/ + simd_data_end, +} simd_data_type; +/************************************ + ** Declarations (inc_data) + ************************************/ +/** + * simd_data_type information + */ +typedef struct +{ + // type name compatible with python style + const char *pyname; + // returns '1' if the type represent a unsigned integer + int is_unsigned:1; + // returns '1' if the type represent a signed integer + int is_signed:1; + // returns '1' if the type represent a single or double precision + int is_float:1; + // returns '1' if the type represent a boolean + int is_bool:1; + // returns '1' if the type represent a sequence + int is_sequence:1; + // returns '1' if the type represent a scalar + int is_scalar:1; + // returns '1' if the type represent a vector + int is_vector:1; + // returns the len of multi-vector if the type reprsent x2 or x3 vector + // otherwise returns 0, e.g. returns 2 if data type is simd_data_vu8x2 + int is_vectorx; + // returns the equivalent scalar data type e.g. simd_data_vu8 -> simd_data_u8 + simd_data_type to_scalar; + // returns the equivalent scalar data type e.g. simd_data_s8 -> simd_data_vs8 + // NOTE: returns the will equivalent "unsigned" vector type in case of "boolean" vector + // e.g. simd_data_vb8 -> simd_data_vu8 + simd_data_type to_vector; + // number of vector lanes + int nlanes; + // sizeof lane type + int lane_size; +} simd_data_info; + +/** + * Returns data info of certain dtype. + * + * Example: + ** const simd_data_info *info = simd_data_getinfo(simd_data_vu8); + ** if (info->is_vector && info->is_unsigned) { + ** ... + ** } + */ +static const simd_data_info * +simd_data_getinfo(simd_data_type dtype); + +/************************************ + ** Declarations (inc_vector) + ************************************/ +typedef struct +{ + PyObject_HEAD + // vector type id + simd_data_type type; + // vector data, aligned for safe casting + npyv_lanetype_u8 NPY_DECL_ALIGNED(NPY_SIMD_WIDTH) data[NPY_SIMD_WIDTH]; +} simd_vector; +/** + * convert simd_data to PyObject(simd_vector), + * raise Python exception on failure and returns NULL. + */ +static simd_vector * +simd_vector_to_obj(simd_data data, simd_data_type vtype); +/** + * convert PyObject(simd_vector) to simd_data, + * raise Python exception on failure. + */ +static simd_data +simd_vector_from_obj(simd_vector *vec, simd_data_type vtype); +/** + * initialize and register vector type(PyTypeObject) to PyModule, + * vector type can be reached through attribute 'vector_type'. + * return -1 on error, 0 on success. + */ +static int +simd_vector_register(PyObject *module); + +/************************************ + ** Declarations (inc_convert) + ************************************/ +/** + * Return a C scalar(simd_data) representation of `obj` and + * according to the scalar data type `dtype` on range (simd_data_[u8:f64]). + * Raise a Python exception on failure. + * + * Example: + ** simd_data data = simd_scalar_from_obj(obj, simd_data_f32); + ** if (!PyErr_Occurred()) { + ** printf("I have a valid float %d\n", data.f32); + ** } + */ +static simd_data +simd_scalar_from_obj(PyObject *obj, simd_data_type dtype); +/** + * Create a Python scalar from a C scalar based on the contents + * of `data`(simd_data) and according to the scalar data type `dtype` + * on range(simd_data_[u8:f64]). + * Return NULL and a Python exception on failure, otherwise new reference. + * + * Example: + ** simd_data data = {.u32 = 0x7fffffff}; + ** PyObject *obj = simd_scalar_to_obj(data, simd_data_s32); + ** if (obj != NULL) { + ** printf("I have a valid Python integer %d\n", PyLong_AsLong(obj)); + ** Py_DECREF(obj); + ** } + */ +static PyObject * +simd_scalar_to_obj(simd_data data, simd_data_type dtype); +/** + * Allocate a C array in memory according to number of elements `len` + * and sequence data type `dtype` on range(simd_data_[qu8:qf64]). + * + * Return aligned pointer based on `NPY_SIMD_WIDTH` or NULL + * with a Python exception on failure. + * + * Example: + ** npyv_lanetype_f64 *aligned_ptr = simd_sequence_new(npyv_nlanes_f64, simd_data_f64); + ** if (aligned_ptr != NULL) { + ** // aligned store + ** npyv_storea_f64(aligned_ptr, npyv_setall_f64(1.0)); + ** printf("The first element of my array %f\n", aligned_ptr[0]); + ** simd_sequence_free(aligned_ptr); + ** } + */ +static void * +simd_sequence_new(Py_ssize_t len, simd_data_type dtype); +/** + * Return the number of elements of the allocated C array `ptr` + * by `simd_sequence_new()` or `simd_sequence_from_obj()`. + */ +static size_t +simd_sequence_len(const void *ptr); +/** + * Free the allocated C array by `simd_sequence_new()` or + * `simd_sequence_from_obj()`. + */ +static void +simd_sequence_free(void *ptr); +/** + * Return a C array representation of a PyObject sequence `obj` and + * according to the sequence data type `dtype` on range (simd_data_[qu8:qf64]). + * + * Note: parameter `min_size` takes the number of minimum acceptable elements. + * + * Return aligned pointer based on `NPY_SIMD_WIDTH` or NULL + * with a Python exception on failure. + * + * Example: + ** npyv_lanetype_u32 *ptr = simd_sequence_from_obj(seq_obj, simd_data_qu32, npyv_nlanes_u32); + ** if (ptr != NULL) { + ** npyv_u32 a = npyv_load_u32(ptr); + ** ... + ** simd_sequence_free(ptr); + ** } + ** + */ +static void * +simd_sequence_from_obj(PyObject *obj, simd_data_type dtype, unsigned min_size); +/** + * Fill a Python sequence object `obj` with a C array `ptr` allocated by + * `simd_sequence_new()` or `simd_sequence_from_obj()` according to + * to the sequence data type `dtype` on range (simd_data_[qu8:qf64]). + * + * Return 0 on success or -1 with a Python exception on failure. + */ +static int +simd_sequence_fill_obj(PyObject *obj, const void *ptr, simd_data_type dtype); +/** + * Create a Python list from a C array `ptr` allocated by + * `simd_sequence_new()` or `simd_sequence_from_obj()` according to + * to the sequence data type `dtype` on range (simd_data_[qu8:qf64]). + * + * Return NULL and a Python exception on failure, otherwise new reference. + */ +static PyObject * +simd_sequence_to_obj(const void *ptr, simd_data_type dtype); +/** + * Return a SIMD multi-vector(simd_data) representation of Python tuple of + * (simd_vector*,) `obj` according to the scalar data type `dtype` + * on range (simd_data_[vu8x2:vf64x2])-(simd_data_[vu8x3:vf64x3]). + * + * Raise a Python exception on failure. + * + * Example: + ** simd_data data = simd_vectorx_from_obj(tuple_obj, simd_data_vf32x2); + ** if (!PyErr_Occurred()) { + ** npyv_f32 sum = npyv_add_f32(data.vf32x2.val[0], data.vf32x2.val[1]); + ** ... + ** } + ** + */ +static simd_data +simd_vectorx_from_obj(PyObject *obj, simd_data_type dtype); +/** + * Create a Python tuple of 'simd_vector' from a SIMD multi-vector + * based on the contents of `data`(simd_data) and according to + * the multi-vector data type `dtype` on range + * (simd_data_[vu8x2:vf64x2])-(simd_data_[vu8x3:vf64x3]). + * + * Return NULL and a Python exception on failure, otherwise new reference. + */ +static PyObject * +simd_vectorx_to_obj(simd_data data, simd_data_type dtype); + +/************************************ + ** Declarations (inc_arg) + ************************************/ +typedef struct +{ + simd_data_type dtype; + simd_data data; + // set by simd_args_from_tuple() + PyObject *obj; +} simd_arg; +/** + * The following functions gather all conversions between all data types + * and they can used instead of all above functions. + */ +/** + * Convert a Python object `obj` into simd_data `arg->data` according to the + * required data type `arg->dtype`. + * + * Return -1 and raise Python exception on failure, otherwise return 0. + * + * Notes: + * - requires `simd_args_sequence_free()` or `simd_sequence_free()` + * to free allocated C array, in case of sequence data types. + * - the number of minimum acceptable elements for sequence data + * types is the number of lanes of the equivalent vector data type. + * + * Example #1: + ** simd_arg arg = {.dtype = simd_data_qu8}; + ** if (simd_arg_from_obj(seq_obj, &arg) < 0) { + ** // fails to convert a python sequence object to C array of uint8 + ** return; + ** } + ** npyv_u8 v_u8 = npyv_load_u8(arg->data.qu8); + ** ... + ** simd_args_sequence_free(&arg, 1); + * + * Example #2: + ** simd_arg arg = {.dtype = simd_data_vf32}; + ** if (simd_arg_from_obj(vector_obj, &arg) < 0) { + ** // fails to convert a python simd_vector to NPYV vector + ** return; + ** } + ** npyv_f32 add_one = npyv_add_f32(arg->data.vu8, npyv_setall_f32(1)); + ** ... + */ +static int +simd_arg_from_obj(PyObject *obj, simd_arg *arg); +/** + * Convert a simd_data `arg->data` to into a Python object according to the + * required data type `arg->dtype`. + * + * Return NULL and raise Python exception on failure, otherwise return + * new reference. + * + * Example: + ** simd_arg arg = {.dtype = simd_data_u32, .data = {.u32 = 0xffffffff}}; + ** PyObject *obj = simd_arg_to_obj(&arg); + ** if (obj == NULL) { + ** // fails convert C uint32 to Python integer. + ** return; + ** } + ** + */ +static PyObject * +simd_arg_to_obj(const simd_arg *arg); +/** + * Similar to simd_arg_from_obj() but convert multiple objects + * from a Python tuple. + * This function is used to parse method parameters. + * + * Notes: + * - requires `simd_args_sequence_free()` or `simd_sequence_free()` + * to free allocated C array, in case of sequence data types. + * - the number of minimum acceptable elements for sequence data + * types is the number of lanes of the equivalent vector data type. + * - use 'arg->obj' to retrieve the parameter obj. + * + * Parameters: + * - `tuple_obj`: a valid Python tuple + * - `args`: array of 'simd_arg' contain valid data types + * - `args_len`: length of `args`. + * - `method_name`: method name, required for exception message. + * + * Return -1 and raise Python exception on failure, otherwise return 0. + */ +static int +simd_args_from_tuple(PyObject *tuple_obj, simd_arg *args, int args_len, const char *method_name); +/** + * Free the allocated C array for sequence data types. + */ +static void +simd_args_sequence_free(simd_arg *args, int args_len); + +#endif // NPY_SIMD diff --git a/numpy/core/src/_simd/_simd_inc_arg.h b/numpy/core/src/_simd/_simd_inc_arg.h new file mode 100644 index 000000000000..1db1ff202927 --- /dev/null +++ b/numpy/core/src/_simd/_simd_inc_arg.h @@ -0,0 +1,108 @@ +#if !NPY_SIMD + #error "Not a standalone header, only works through 'simd.dispatch.c.src'" +#endif + +/************************************ + ** Protected Definitions + ************************************/ +static int +simd_arg_from_obj(PyObject *obj, simd_arg *arg) +{ + assert(arg->dtype != 0); + const simd_data_info *info = simd_data_getinfo(arg->dtype); + if (info->is_scalar) { + arg->data = simd_scalar_from_obj(obj, arg->dtype); + } + else if (info->is_sequence) { + unsigned min_seq_size = simd_data_getinfo(info->to_vector)->nlanes; + arg->data.qu8 = simd_sequence_from_obj(obj, arg->dtype, min_seq_size); + } + else if (info->is_vectorx) { + arg->data = simd_vectorx_from_obj(obj, arg->dtype); + } + else if (info->is_vector) { + arg->data = simd_vector_from_obj((simd_vector*)obj, arg->dtype); + } else { + arg->data.u64 = 0; + PyErr_Format(PyExc_RuntimeError, + "unhandled arg from obj type id:%d, name:%s", arg->dtype, info->pyname + ); + return -1; + } + if (PyErr_Occurred()) { + return -1; + } + return 0; +} + +static PyObject * +simd_arg_to_obj(const simd_arg *arg) +{ + assert(arg->dtype != 0); + const simd_data_info *info = simd_data_getinfo(arg->dtype); + if (info->is_scalar) { + return simd_scalar_to_obj(arg->data, arg->dtype); + } + if (info->is_sequence) { + return simd_sequence_to_obj(arg->data.qu8, arg->dtype); + } + if (info->is_vectorx) { + return simd_vectorx_to_obj(arg->data, arg->dtype); + } + if (info->is_vector) { + return (PyObject*)simd_vector_to_obj(arg->data, arg->dtype); + } + PyErr_Format(PyExc_RuntimeError, + "unhandled arg to object type id:%d, name:%s", arg->dtype, info->pyname + ); + return NULL; +} + +static void +simd_args_sequence_free(simd_arg *args, int args_len) +{ + assert(args_len > 0); + while (--args_len >= 0) { + simd_arg *arg = &args[args_len]; + const simd_data_info *info = simd_data_getinfo(arg->dtype); + if (!info->is_sequence) { + continue; + } + simd_sequence_free(arg->data.qu8); + } +} + +static int +simd_args_from_tuple(PyObject *tuple_obj, simd_arg *args, int args_len, const char *method_name) +{ + assert(args_len > 0); + assert(PyTuple_Check(tuple_obj)); + + Py_ssize_t obj_arg_len = PyTuple_GET_SIZE(tuple_obj); + if (obj_arg_len != args_len) { + if (args_len == 1) { + PyErr_Format(PyExc_TypeError, + "%s() takes only one argument (%d given)", method_name, obj_arg_len + ); + return -1; + } + PyErr_Format(PyExc_TypeError, + "%s() takes exactly %d arguments (%d given)", method_name, args_len, obj_arg_len + ); + return -1; + } + for (int arg_pos = 0; arg_pos < args_len; ++arg_pos) { + simd_arg *arg = &args[arg_pos]; + arg->obj = PyTuple_GET_ITEM(tuple_obj, arg_pos); + assert(arg->obj != NULL); + if (simd_arg_from_obj(arg->obj, arg) != 0) { + // free previous args + if (arg_pos > 0) { + simd_args_sequence_free(args, arg_pos); + } + // TODO: improve log by add argument number and method name + return -1; + } + } + return 0; +} diff --git a/numpy/core/src/_simd/_simd_inc_convert.h b/numpy/core/src/_simd/_simd_inc_convert.h new file mode 100644 index 000000000000..3601012473b0 --- /dev/null +++ b/numpy/core/src/_simd/_simd_inc_convert.h @@ -0,0 +1,206 @@ +#if !NPY_SIMD + #error "Not a standalone header, only works through 'simd.dispatch.c.src'" +#endif + +/************************************ + ** Protected Definitions + ************************************/ +static simd_data +simd_scalar_from_obj(PyObject *obj, simd_data_type dtype) +{ + const simd_data_info *info = simd_data_getinfo(dtype); + assert(info->is_scalar && info->lane_size > 0); + simd_data data; + if (info->is_float) { + data.f64 = PyFloat_AsDouble(obj); + if (dtype == simd_data_f32){ + data.f32 = (float)data.f64; + } + } else { + data.u64 = PyLong_AsUnsignedLongLongMask(obj); + } + return data; +} + +static PyObject * +simd_scalar_to_obj(simd_data data, simd_data_type dtype) +{ + const simd_data_info *info = simd_data_getinfo(dtype); + assert(info->is_scalar && info->lane_size > 0); + if (info->is_float) { + if (dtype == simd_data_f32) { + return PyFloat_FromDouble(data.f32); + } + return PyFloat_FromDouble(data.f64); + } + int leftb = (sizeof(npyv_lanetype_u64) - info->lane_size) * 8; + data.u64 <<= leftb; + if (info->is_signed) { + return PyLong_FromLongLong(data.s64 >> leftb); + } + return PyLong_FromUnsignedLongLong(data.u64 >> leftb); +} + +static void * +simd_sequence_new(Py_ssize_t len, simd_data_type dtype) +{ + const simd_data_info *info = simd_data_getinfo(dtype); + assert(info->is_sequence && info->lane_size > 0); + + size_t size = NPY_SIMD_WIDTH + sizeof(size_t) + sizeof(size_t*); + size += len * info->lane_size; + + size_t *ptr = malloc(size); + if (ptr == NULL) { + return PyErr_NoMemory(); + } + *(ptr++) = len; + size_t **a_ptr = (size_t**)( + ((size_t)ptr + NPY_SIMD_WIDTH) & ~(size_t)(NPY_SIMD_WIDTH-1) + ); + a_ptr[-1] = ptr; + return a_ptr; +} + +static size_t +simd_sequence_len(const void *ptr) +{ + size_t *ptrz = ((size_t**)ptr)[-1]; + return *(ptrz-1); +} + +static void +simd_sequence_free(void *ptr) +{ + size_t *ptrz = ((size_t**)ptr)[-1]; + free(ptrz-1); +} + +static void * +simd_sequence_from_obj(PyObject *obj, simd_data_type dtype, unsigned min_size) +{ + const simd_data_info *info = simd_data_getinfo(dtype); + assert(info->is_sequence && info->lane_size > 0); + PyObject *seq_obj = PySequence_Fast(obj, "expected a sequence"); + if (seq_obj == NULL) { + return NULL; + } + Py_ssize_t seq_size = PySequence_Fast_GET_SIZE(seq_obj); + if (seq_size < (Py_ssize_t)min_size) { + PyErr_Format(PyExc_ValueError, + "minimum acceptable size of the required sequence is %d, given(%d)", + min_size, seq_size + ); + return NULL; + } + npyv_lanetype_u8 *dst = simd_sequence_new(seq_size, dtype); + if (dst == NULL) { + return NULL; + } + PyObject **seq_items = PySequence_Fast_ITEMS(seq_obj); + for (Py_ssize_t i = 0; i < seq_size; ++i) { + simd_data data = simd_scalar_from_obj(seq_items[i], info->to_scalar); + npyv_lanetype_u8 *sdst = dst + i * info->lane_size; + memcpy(sdst, &data.u64, info->lane_size); + } + Py_DECREF(seq_obj); + + if (PyErr_Occurred()) { + simd_sequence_free(dst); + return NULL; + } + return dst; +} + +static int +simd_sequence_fill_obj(PyObject *obj, const void *ptr, simd_data_type dtype) +{ + const simd_data_info *info = simd_data_getinfo(dtype); + if (!PySequence_Check(obj)) { + PyErr_Format(PyExc_TypeError, + "a sequence object is required to fill %s", info->pyname + ); + return -1; + } + const npyv_lanetype_u8 *src = ptr; + Py_ssize_t seq_len = (Py_ssize_t)simd_sequence_len(ptr); + for (Py_ssize_t i = 0; i < seq_len; ++i) { + const npyv_lanetype_u8 *ssrc = src + i * info->lane_size; + simd_data data; + memcpy(&data.u64, ssrc, info->lane_size); + PyObject *item = simd_scalar_to_obj(data, info->to_scalar); + if (item == NULL) { + return -1; + } + if (PySequence_SetItem(obj, i, item) < 0) { + Py_DECREF(item); + return -1; + } + } + return 0; +} + +static PyObject * +simd_sequence_to_obj(const void *ptr, simd_data_type dtype) +{ + PyObject *list = PyList_New((Py_ssize_t)simd_sequence_len(ptr)); + if (list == NULL) { + return NULL; + } + if (simd_sequence_fill_obj(list, ptr, dtype) < 0) { + Py_DECREF(list); + return NULL; + } + return list; +} + +static simd_data +simd_vectorx_from_obj(PyObject *obj, simd_data_type dtype) +{ + const simd_data_info *info = simd_data_getinfo(dtype); + // NPYV currently only supports x2 and x3 + assert(info->is_vectorx > 1 && info->is_vectorx < 4); + + simd_data data = {.u64 = 0}; + if (!PyTuple_Check(obj) || PyTuple_GET_SIZE(obj) != info->is_vectorx) { + PyErr_Format(PyExc_TypeError, + "a tuple of %d vector type %s is required", + info->is_vectorx, simd_data_getinfo(info->to_vector)->pyname + ); + return data; + } + for (int i = 0; i < info->is_vectorx; ++i) { + PyObject *item = PyTuple_GET_ITEM(obj, i); + // get the max multi-vec and let the compiler do the rest + data.vu64x3.val[i] = simd_vector_from_obj((simd_vector*)item, info->to_vector).vu64; + if (PyErr_Occurred()) { + return data; + } + } + return data; +} + +static PyObject * +simd_vectorx_to_obj(simd_data data, simd_data_type dtype) +{ + const simd_data_info *info = simd_data_getinfo(dtype); + // NPYV currently only supports x2 and x3 + assert(info->is_vectorx > 1 && info->is_vectorx < 4); + + PyObject *tuple = PyTuple_New(info->is_vectorx); + if (tuple == NULL) { + return NULL; + } + for (int i = 0; i < info->is_vectorx; ++i) { + // get the max multi-vector and let the compiler handle the rest + simd_data vdata = {.vu64 = data.vu64x3.val[i]}; + PyObject *item = (PyObject*)simd_vector_to_obj(vdata, info->to_vector); + if (item == NULL) { + // TODO: improve log add item number + Py_DECREF(tuple); + return NULL; + } + PyTuple_SET_ITEM(tuple, i, item); + } + return tuple; +} diff --git a/numpy/core/src/_simd/_simd_inc_data.h.src b/numpy/core/src/_simd/_simd_inc_data.h.src new file mode 100644 index 000000000000..eefac483bf2c --- /dev/null +++ b/numpy/core/src/_simd/_simd_inc_data.h.src @@ -0,0 +1,91 @@ +#if !NPY_SIMD + #error "Not a standalone header, only works through 'simd.dispatch.c.src'" +#endif + +/************************************ + ** Private Definitions + ************************************/ +static simd_data_info simd__data_registry[simd_data_end] = +{ + [simd_data_none] = {.pyname="none"}, + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32, f64# + * #sig = 0*4, 1*4, 0*2# + * #fp = 0*4, 0*4, 1*2# + * #name = int*8, float, float# + */ + [simd_data_@sfx@] = { + .pyname="@name@", .is_unsigned=!@sig@&&!@fp@, .is_signed=@sig@, .is_float=@fp@, + .is_scalar=1, .to_scalar = simd_data_@sfx@, .to_vector = simd_data_v@sfx@, + .lane_size = sizeof(npyv_lanetype_@sfx@) + }, + /**end repeat**/ + // sequences + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32, f64# + * #sig = 0*4, 1*4, 0*2# + * #fp = 0*4, 0*4, 1*2# + * #name = int*8, float, float# + */ + [simd_data_q@sfx@] = { + .pyname="[@name@]", .is_unsigned=!@sig@&&!@fp@, .is_signed=@sig@, .is_float=@fp@, + .is_sequence=1, .to_scalar = simd_data_@sfx@, .to_vector = simd_data_v@sfx@, + .nlanes=npyv_nlanes_@sfx@, .lane_size = sizeof(npyv_lanetype_@sfx@) + }, + /**end repeat**/ + // vectors + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32, f64# + * #sig = 0*4, 1*4, 0*2# + * #fp = 0*4, 0*4, 1*2# + */ + [simd_data_v@sfx@] = { + .pyname="npyv_@sfx@", .is_unsigned=!@sig@&&!@fp@, .is_signed=@sig@, .is_float=@fp@, + .is_vector=1, .to_scalar = simd_data_@sfx@, .to_vector = simd_data_v@sfx@, + .nlanes=npyv_nlanes_@sfx@, .lane_size = sizeof(npyv_lanetype_@sfx@) + }, + /**end repeat**/ + // boolean vectors, treated as unsigned and converted internally + // to add compatibility among all SIMD extensions + /**begin repeat + * #sfx = u8, u16, u32, u64# + * #bsfx = b8, b16, b32, b64# + */ + [simd_data_v@bsfx@] = { + .pyname="npyv_@bsfx@", .is_bool=1, .is_vector=1, + .to_scalar = simd_data_@sfx@, .to_vector = simd_data_v@sfx@, + .nlanes=npyv_nlanes_@sfx@, .lane_size = sizeof(npyv_lanetype_@sfx@) + }, + /**end repeat**/ + // multi-vectors x2 + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32, f64# + * #sig = 0*4, 1*4, 0*2# + * #fp = 0*4, 0*4, 1*2# + */ + [simd_data_v@sfx@x2] = { + .pyname="npyv_@sfx@x2", .is_unsigned=!@sig@&&!@fp@, .is_signed=@sig@, .is_float=@fp@, + .is_vectorx=2, .to_scalar = simd_data_@sfx@, .to_vector = simd_data_v@sfx@, + .nlanes=2, .lane_size = sizeof(npyv_lanetype_@sfx@) + }, + /**end repeat**/ + // multi-vectors x3 + /**begin repeat + * #sfx = u8, u16, u32, u64, s8, s16, s32, s64, f32, f64# + * #sig = 0*4, 1*4, 0*2# + * #fp = 0*4, 0*4, 1*2# + */ + [simd_data_v@sfx@x3] = { + .pyname="npyv_@sfx@x3", .is_unsigned=!@sig@&&!@fp@, .is_signed=@sig@, .is_float=@fp@, + .is_vectorx=3, .to_scalar = simd_data_@sfx@, .to_vector = simd_data_v@sfx@, + .nlanes=3, .lane_size = sizeof(npyv_lanetype_@sfx@) + }, + /**end repeat**/ +}; + +/************************************ + ** Protected Definitions + ************************************/ +static const simd_data_info * +simd_data_getinfo(simd_data_type dtype) +{ return &simd__data_registry[dtype]; } diff --git a/numpy/core/src/_simd/_simd_inc_easyintrin.h b/numpy/core/src/_simd/_simd_inc_easyintrin.h new file mode 100644 index 000000000000..7216b373ad0f --- /dev/null +++ b/numpy/core/src/_simd/_simd_inc_easyintrin.h @@ -0,0 +1,229 @@ +#if !NPY_SIMD + #error "Not a standalone header, only works through 'simd.dispatch.c.src'" +#endif + +#define SIMD_INTRIN_DEF(NAME) \ + { NPY_TOSTRING(NAME), simd__intrin_##NAME, METH_VARARGS, NULL } , // comma + +static int simd__no_arguments(PyObject *args, const char* method_name) +{ + if (args == NULL) { + return 0; + } + assert(PyTuple_Check(args)); + Py_ssize_t obj_arg_len = PyTuple_GET_SIZE(args); + if (obj_arg_len != 0) { + PyErr_Format(PyExc_RuntimeError, + "%s(), takes no arguments, given(%d)", method_name, obj_arg_len + ); + return -1; + } + return 0; +} + +#define SIMD_IMPL_INTRIN_0(NAME, RET) \ + static PyObject *simd__intrin_##NAME \ + (PyObject* NPY_UNUSED(self), PyObject *args) \ + { \ + if (simd__no_arguments( \ + args, NPY_TOSTRING(NAME) \ + )) return NULL; \ + simd_arg a = { \ + .dtype = simd_data_##RET, \ + .data = {.RET = npyv_##NAME()}, \ + }; \ + return simd_arg_to_obj(&a); \ + } + +#define SIMD_IMPL_INTRIN_0N(NAME) \ + static PyObject *simd__intrin_##NAME \ + (PyObject* NPY_UNUSED(self), PyObject *args) \ + { \ + if (simd__no_arguments( \ + args, NPY_TOSTRING(NAME) \ + )) return NULL; \ + npyv_##NAME(); \ + Py_RETURN_NONE; \ + } + +#define SIMD_IMPL_INTRIN_1(NAME, RET, IN0) \ + static PyObject *simd__intrin_##NAME \ + (PyObject* NPY_UNUSED(self), PyObject *args) \ + { \ + simd_arg req_args[] = { \ + {.dtype = simd_data_##IN0}, \ + }; \ + if (simd_args_from_tuple( \ + args, req_args, 1, NPY_TOSTRING(NAME)) \ + ) return NULL; \ + simd_data r = {.RET = npyv_##NAME( \ + req_args[0].data.IN0 \ + )}; \ + simd_args_sequence_free(req_args, 1); \ + req_args[0].data = r; \ + req_args[0].dtype = simd_data_##RET; \ + return simd_arg_to_obj(req_args); \ + } + +#define SIMD_IMPL_INTRIN_2(NAME, RET, IN0, IN1) \ + static PyObject *simd__intrin_##NAME \ + (PyObject* NPY_UNUSED(self), PyObject *args) \ + { \ + simd_arg req_args[] = { \ + {.dtype = simd_data_##IN0}, \ + {.dtype = simd_data_##IN1}, \ + }; \ + if (simd_args_from_tuple( \ + args, req_args, 2, NPY_TOSTRING(NAME)) \ + ) { \ + return NULL; \ + } \ + simd_data r = {.RET = npyv_##NAME( \ + req_args[0].data.IN0, \ + req_args[1].data.IN1 \ + )}; \ + simd_args_sequence_free(req_args, 2); \ + req_args[0].data = r; \ + req_args[0].dtype = simd_data_##RET; \ + return simd_arg_to_obj(req_args); \ + } + +#define SIMD__REPEAT_2IMM(C, NAME, IN0) \ + C == req_args[1].data.u8 ? NPY_CAT(npyv_, NAME)(req_args[0].data.IN0, C) : + +#define SIMD_IMPL_INTRIN_2IMM(NAME, RET, IN0, CONST_RNG) \ + static PyObject *simd__intrin_##NAME \ + (PyObject* NPY_UNUSED(self), PyObject *args) \ + { \ + simd_arg req_args[] = { \ + {.dtype = simd_data_##IN0}, \ + {.dtype = simd_data_u8}, \ + }; \ + if (simd_args_from_tuple( \ + args, req_args, 2, NPY_TOSTRING(NAME)) \ + ) { \ + return NULL; \ + } \ + simd_data r; \ + r.RET = NPY_CAT(SIMD__IMPL_COUNT_, CONST_RNG)( \ + SIMD__REPEAT_2IMM, NAME, IN0 \ + ) npyv_##NAME(req_args[0].data.IN0, 0); \ + simd_args_sequence_free(req_args, 2); \ + req_args[0].data = r; \ + req_args[0].dtype = simd_data_##RET; \ + return simd_arg_to_obj(req_args); \ + } + +#define SIMD_IMPL_INTRIN_3(NAME, RET, IN0, IN1, IN2) \ + static PyObject *simd__intrin_##NAME \ + (PyObject* NPY_UNUSED(self), PyObject *args) \ + { \ + simd_arg req_args[] = { \ + {.dtype = simd_data_##IN0}, \ + {.dtype = simd_data_##IN1}, \ + {.dtype = simd_data_##IN2}, \ + }; \ + if (simd_args_from_tuple( \ + args, req_args, 3, NPY_TOSTRING(NAME)) \ + ) { \ + return NULL; \ + } \ + simd_data r = {.RET = npyv_##NAME( \ + req_args[0].data.IN0, \ + req_args[1].data.IN1, \ + req_args[2].data.IN2 \ + )}; \ + simd_args_sequence_free(req_args, 3); \ + req_args[0].data = r; \ + req_args[0].dtype = simd_data_##RET; \ + return simd_arg_to_obj(req_args); \ + } +/** + * Helper macros for repeating and expand a certain macro. + * Mainly used for converting a scalar to an immediate constant. + */ +#define SIMD__IMPL_COUNT_7(FN, ...) \ + NPY_EXPAND(FN(0, __VA_ARGS__)) \ + SIMD__IMPL_COUNT_7_(FN, __VA_ARGS__) + +#define SIMD__IMPL_COUNT_8(FN, ...) \ + SIMD__IMPL_COUNT_7_(FN, __VA_ARGS__) \ + NPY_EXPAND(FN(8, __VA_ARGS__)) + +#define SIMD__IMPL_COUNT_15(FN, ...) \ + NPY_EXPAND(FN(0, __VA_ARGS__)) \ + SIMD__IMPL_COUNT_15_(FN, __VA_ARGS__) + +#define SIMD__IMPL_COUNT_16(FN, ...) \ + SIMD__IMPL_COUNT_15_(FN, __VA_ARGS__) \ + NPY_EXPAND(FN(16, __VA_ARGS__)) + +#define SIMD__IMPL_COUNT_31(FN, ...) \ + NPY_EXPAND(FN(0, __VA_ARGS__)) \ + SIMD__IMPL_COUNT_31_(FN, __VA_ARGS__) + +#define SIMD__IMPL_COUNT_32(FN, ...) \ + SIMD__IMPL_COUNT_31_(FN, __VA_ARGS__) \ + NPY_EXPAND(FN(32, __VA_ARGS__)) + +#define SIMD__IMPL_COUNT_47(FN, ...) \ + NPY_EXPAND(FN(0, __VA_ARGS__)) \ + SIMD__IMPL_COUNT_47_(FN, __VA_ARGS__) + +#define SIMD__IMPL_COUNT_48(FN, ...) \ + SIMD__IMPL_COUNT_47_(FN, __VA_ARGS__) \ + NPY_EXPAND(FN(48, __VA_ARGS__)) + +#define SIMD__IMPL_COUNT_63(FN, ...) \ + NPY_EXPAND(FN(0, __VA_ARGS__)) \ + SIMD__IMPL_COUNT_63_(FN, __VA_ARGS__) + +#define SIMD__IMPL_COUNT_64(FN, ...) \ + SIMD__IMPL_COUNT_63_(FN, __VA_ARGS__) \ + NPY_EXPAND(FN(64, __VA_ARGS__)) + +#define SIMD__IMPL_COUNT_7_(FN, ...) \ + NPY_EXPAND(FN(1, __VA_ARGS__)) \ + NPY_EXPAND(FN(2, __VA_ARGS__)) NPY_EXPAND(FN(3, __VA_ARGS__)) \ + NPY_EXPAND(FN(4, __VA_ARGS__)) NPY_EXPAND(FN(5, __VA_ARGS__)) \ + NPY_EXPAND(FN(6, __VA_ARGS__)) NPY_EXPAND(FN(7, __VA_ARGS__)) + +#define SIMD__IMPL_COUNT_15_(FN, ...) \ + SIMD__IMPL_COUNT_7_(FN, __VA_ARGS__) \ + NPY_EXPAND(FN(8, __VA_ARGS__)) NPY_EXPAND(FN(9, __VA_ARGS__)) \ + NPY_EXPAND(FN(10, __VA_ARGS__)) NPY_EXPAND(FN(11, __VA_ARGS__)) \ + NPY_EXPAND(FN(12, __VA_ARGS__)) NPY_EXPAND(FN(13, __VA_ARGS__)) \ + NPY_EXPAND(FN(14, __VA_ARGS__)) NPY_EXPAND(FN(15, __VA_ARGS__)) + +#define SIMD__IMPL_COUNT_31_(FN, ...) \ + SIMD__IMPL_COUNT_15_(FN, __VA_ARGS__) \ + NPY_EXPAND(FN(16, __VA_ARGS__)) NPY_EXPAND(FN(17, __VA_ARGS__)) \ + NPY_EXPAND(FN(18, __VA_ARGS__)) NPY_EXPAND(FN(19, __VA_ARGS__)) \ + NPY_EXPAND(FN(20, __VA_ARGS__)) NPY_EXPAND(FN(21, __VA_ARGS__)) \ + NPY_EXPAND(FN(22, __VA_ARGS__)) NPY_EXPAND(FN(23, __VA_ARGS__)) \ + NPY_EXPAND(FN(24, __VA_ARGS__)) NPY_EXPAND(FN(25, __VA_ARGS__)) \ + NPY_EXPAND(FN(26, __VA_ARGS__)) NPY_EXPAND(FN(27, __VA_ARGS__)) \ + NPY_EXPAND(FN(28, __VA_ARGS__)) NPY_EXPAND(FN(29, __VA_ARGS__)) \ + NPY_EXPAND(FN(30, __VA_ARGS__)) NPY_EXPAND(FN(31, __VA_ARGS__)) + +#define SIMD__IMPL_COUNT_47_(FN, ...) \ + SIMD__IMPL_COUNT_31_(FN, __VA_ARGS__) \ + NPY_EXPAND(FN(32, __VA_ARGS__)) NPY_EXPAND(FN(33, __VA_ARGS__)) \ + NPY_EXPAND(FN(34, __VA_ARGS__)) NPY_EXPAND(FN(35, __VA_ARGS__)) \ + NPY_EXPAND(FN(36, __VA_ARGS__)) NPY_EXPAND(FN(37, __VA_ARGS__)) \ + NPY_EXPAND(FN(38, __VA_ARGS__)) NPY_EXPAND(FN(39, __VA_ARGS__)) \ + NPY_EXPAND(FN(40, __VA_ARGS__)) NPY_EXPAND(FN(41, __VA_ARGS__)) \ + NPY_EXPAND(FN(42, __VA_ARGS__)) NPY_EXPAND(FN(43, __VA_ARGS__)) \ + NPY_EXPAND(FN(44, __VA_ARGS__)) NPY_EXPAND(FN(45, __VA_ARGS__)) \ + NPY_EXPAND(FN(46, __VA_ARGS__)) NPY_EXPAND(FN(47, __VA_ARGS__)) + +#define SIMD__IMPL_COUNT_63_(FN, ...) \ + SIMD__IMPL_COUNT_47_(FN, __VA_ARGS__) \ + NPY_EXPAND(FN(48, __VA_ARGS__)) NPY_EXPAND(FN(49, __VA_ARGS__)) \ + NPY_EXPAND(FN(50, __VA_ARGS__)) NPY_EXPAND(FN(51, __VA_ARGS__)) \ + NPY_EXPAND(FN(52, __VA_ARGS__)) NPY_EXPAND(FN(53, __VA_ARGS__)) \ + NPY_EXPAND(FN(54, __VA_ARGS__)) NPY_EXPAND(FN(55, __VA_ARGS__)) \ + NPY_EXPAND(FN(56, __VA_ARGS__)) NPY_EXPAND(FN(57, __VA_ARGS__)) \ + NPY_EXPAND(FN(58, __VA_ARGS__)) NPY_EXPAND(FN(59, __VA_ARGS__)) \ + NPY_EXPAND(FN(60, __VA_ARGS__)) NPY_EXPAND(FN(61, __VA_ARGS__)) \ + NPY_EXPAND(FN(62, __VA_ARGS__)) NPY_EXPAND(FN(63, __VA_ARGS__)) diff --git a/numpy/core/src/_simd/_simd_inc_vector.h b/numpy/core/src/_simd/_simd_inc_vector.h new file mode 100644 index 000000000000..b0fa17b9ab9e --- /dev/null +++ b/numpy/core/src/_simd/_simd_inc_vector.h @@ -0,0 +1,187 @@ +#if !NPY_SIMD + #error "Not a standalone header, only works through 'simd.dispatch.c.src'" +#endif + +/************************************ + ** Private Definitions + ************************************/ +// PySequenceMethods +static Py_ssize_t +simd__vector_length(simd_vector *self) +{ + return simd_data_getinfo(self->type)->nlanes; +} +static PyObject * +simd__vector_item(simd_vector *self, Py_ssize_t i) +{ + const simd_data_info *info = simd_data_getinfo(self->type); + int nlanes = info->nlanes; + if (i >= nlanes) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + npyv_lanetype_u8 *src = self->data + i * info->lane_size; + simd_data data; + memcpy(&data.u64, src, info->lane_size); + return simd_scalar_to_obj(data, info->to_scalar); +} + +static PySequenceMethods simd__vector_as_sequence = { + (lenfunc) simd__vector_length, /* sq_length */ + (binaryfunc) NULL, /* sq_concat */ + (ssizeargfunc) NULL, /* sq_repeat */ + (ssizeargfunc) simd__vector_item, /* sq_item */ + (ssizessizeargfunc) NULL, /* sq_slice */ + (ssizeobjargproc) NULL, /* sq_ass_item */ + (ssizessizeobjargproc) NULL, /* sq_ass_slice */ + (objobjproc) NULL, /* sq_contains */ + (binaryfunc) NULL, /* sq_inplace_concat */ + (ssizeargfunc) NULL, /* sq_inplace_repeat */ +}; + +// PyGetSetDef +static PyObject * +simd__vector_name(simd_vector *self) +{ + return PyUnicode_FromString(simd_data_getinfo(self->type)->pyname); +} +static PyGetSetDef simd__vector_getset[] = { + { "__name__", (getter)simd__vector_name, NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL, NULL } +}; + +// PyTypeObject(simd__vector_type) +static PyObject * +simd__vector_repr(PyObject *self) +{ + // PySequence_Fast returns Tuple in PyPy + PyObject *obj = PySequence_List(self); + if (obj != NULL) { + PyObject *repr = PyObject_Str(obj); + Py_DECREF(obj); + return repr; + } + return obj; +} +static PyObject * +simd__vector_compare(PyObject *self, PyObject *other, int cmp_op) +{ + PyObject *obj; + if (PyTuple_Check(other)) { + obj = PySequence_Tuple(self); + } else if (PyList_Check(other)) { + obj = PySequence_List(self); + } else { + obj = PySequence_Fast(self, "invalid argument, expected a vector"); + } + if (obj != NULL) { + PyObject *rich = PyObject_RichCompare(obj, other, cmp_op); + Py_DECREF(obj); + return rich; + } + return obj; +} +static PyTypeObject simd__vector_type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = NPY_TOSTRING(NPY_CPU_DISPATCH_CURFX(VECTOR)), + .tp_basicsize = sizeof(simd_vector), + .tp_repr = (reprfunc)simd__vector_repr, + .tp_as_sequence = &simd__vector_as_sequence, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_richcompare = simd__vector_compare, + .tp_getset = simd__vector_getset +}; + +/************************************ + ** Protected Definitions + ************************************/ +static simd_vector * +simd_vector_to_obj(simd_data data, simd_data_type vtype) +{ + const simd_data_info *info = simd_data_getinfo(vtype); + assert(info->is_vector && info->nlanes > 0); + + simd_vector *vec = PyObject_New(simd_vector, &simd__vector_type); + if (vec == NULL) { + return (simd_vector*)PyErr_NoMemory(); + } + vec->type = vtype; + if (info->is_bool) { + // boolean vectors are internally treated as unsigned + // vectors to add compatibility among all SIMD extensions + switch(vtype) { + case simd_data_vb8: + data.vu8 = npyv_cvt_u8_b8(data.vb8); + break; + case simd_data_vb16: + data.vu16 = npyv_cvt_u16_b16(data.vb16); + break; + case simd_data_vb32: + data.vu32 = npyv_cvt_u32_b32(data.vb32); + break; + default: + data.vu64 = npyv_cvt_u64_b64(data.vb64); + } + } + npyv_store_u8(vec->data, data.vu8); + return vec; +} + +static simd_data +simd_vector_from_obj(simd_vector *vec, simd_data_type vtype) +{ + const simd_data_info *info = simd_data_getinfo(vtype); + assert(info->is_vector && info->nlanes > 0); + + simd_data data = {.u64 = 0}; + if (!PyObject_IsInstance( + (PyObject *)vec, (PyObject *)&simd__vector_type + )) { + PyErr_Format(PyExc_TypeError, + "a vector type %s is required", info->pyname + ); + return data; + } + if (vec->type != vtype) { + PyErr_Format(PyExc_TypeError, + "a vector type %s is required, got(%s)", + info->pyname, simd_data_getinfo(vec->type)->pyname + ); + return data; + } + + data.vu8 = npyv_load_u8(vec->data); + if (info->is_bool) { + // boolean vectors are internally treated as unsigned + // vectors to add compatibility among all SIMD extensions + switch(vtype) { + case simd_data_vb8: + data.vb8 = npyv_cvt_b8_u8(data.vu8); + break; + case simd_data_vb16: + data.vb16 = npyv_cvt_b16_u16(data.vu16); + break; + case simd_data_vb32: + data.vb32 = npyv_cvt_b32_u32(data.vu32); + break; + default: + data.vb64 = npyv_cvt_b64_u64(data.vu64); + } + } + return data; +} + +static int +simd_vector_register(PyObject *module) +{ + Py_INCREF(&simd__vector_type); + if (PyType_Ready(&simd__vector_type)) { + return -1; + } + if (PyModule_AddObject( + module, "vector_type",(PyObject *)&simd__vector_type + )) { + return -1; + } + return 0; +} diff --git a/numpy/distutils/command/build.py b/numpy/distutils/command/build.py index 60ba4c917169..6025586cd3d0 100644 --- a/numpy/distutils/command/build.py +++ b/numpy/distutils/command/build.py @@ -22,6 +22,8 @@ class build(old_build): "specify a list of dispatched CPU optimizations"), ('disable-optimization', None, "disable CPU optimized code(dispatch,simd,fast...)"), + ('simd-test=', None, + "specify a list of CPU optimizations to be tested against NumPy SIMD interface"), ] help_options = old_build.help_options + [ @@ -36,6 +38,16 @@ def initialize_options(self): self.cpu_baseline = "min" self.cpu_dispatch = "max -xop -fma4" # drop AMD legacy features by default self.disable_optimization = False + """ + the '_simd' module is a very large. Adding more dispatched features + will increase binary size and compile time. By default we minimize + the targeted features to those most commonly used by the NumPy SIMD interface(NPYV), + NOTE: any specified features will be ignored if they're: + - part of the baseline(--cpu-baseline) + - not part of dispatch-able features(--cpu-dispatch) + - not supported by compiler or platform + """ + self.simd_test = "BASELINE SSE2 SSE41 SSE42 XOP (FMA3 AVX2) AVX512F AVX512_SKX VSX VSX2 VSX3 NEON ASIMD" def finalize_options(self): build_scripts = self.build_scripts diff --git a/numpy/distutils/command/build_ext.py b/numpy/distutils/command/build_ext.py index 1a881c56a2aa..ca6f8bcd24d6 100644 --- a/numpy/distutils/command/build_ext.py +++ b/numpy/distutils/command/build_ext.py @@ -19,8 +19,7 @@ has_cxx_sources, has_f_sources, is_sequence ) from numpy.distutils.command.config_compiler import show_fortran_compilers -from numpy.distutils.ccompiler_opt import new_ccompiler_opt - +from numpy.distutils.ccompiler_opt import new_ccompiler_opt, CCompilerOpt class build_ext (old_build_ext): @@ -39,6 +38,8 @@ class build_ext (old_build_ext): "specify a list of dispatched CPU optimizations"), ('disable-optimization', None, "disable CPU optimized code(dispatch,simd,fast...)"), + ('simd-test=', None, + "specify a list of CPU optimizations to be tested against NumPy SIMD interface"), ] help_options = old_build_ext.help_options + [ @@ -56,6 +57,7 @@ def initialize_options(self): self.cpu_baseline = None self.cpu_dispatch = None self.disable_optimization = None + self.simd_test = None def finalize_options(self): if self.parallel: @@ -87,7 +89,9 @@ def finalize_options(self): ('cpu_baseline', 'cpu_baseline'), ('cpu_dispatch', 'cpu_dispatch'), ('disable_optimization', 'disable_optimization'), + ('simd_test', 'simd_test') ) + CCompilerOpt.conf_target_groups["simd_test"] = self.simd_test def run(self): if not self.extensions: diff --git a/runtests.py b/runtests.py index f8b70d9360df..87e26768b351 100755 --- a/runtests.py +++ b/runtests.py @@ -122,6 +122,9 @@ def main(argv): help="Specify a list of dispatched CPU optimizations"), parser.add_argument("--disable-optimization", action="store_true", help="Disable CPU optimized code(dispatch,simd,fast...)"), + parser.add_argument("--simd-test", default=None, + help="Specify a list of CPU optimizations to be " + "tested against NumPy SIMD interface"), parser.add_argument("--show-build-log", action="store_true", help="Show build output rather than using a log file") parser.add_argument("--bench", action="store_true", @@ -439,6 +442,8 @@ def build_project(args): cmd += ["--cpu-dispatch", args.cpu_dispatch] if args.disable_optimization: cmd += ["--disable-optimization"] + if args.simd_test is not None: + cmd += ["--simd-test", args.simd_test] # Install; avoid producing eggs so numpy can be imported from dst_dir. cmd += ['install', '--prefix=' + dst_dir, '--single-version-externally-managed', From c65a559bcd459bacca3e8cf15e129cf2576540b5 Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Wed, 8 Jul 2020 09:27:53 +0200 Subject: [PATCH 370/409] ENH, TST: Add testing unit that covers the current implemented intrinsics. --- numpy/core/tests/test_simd.py | 408 ++++++++++++++++++++++++++++++++++ 1 file changed, 408 insertions(+) create mode 100644 numpy/core/tests/test_simd.py diff --git a/numpy/core/tests/test_simd.py b/numpy/core/tests/test_simd.py new file mode 100644 index 000000000000..26e78d40254c --- /dev/null +++ b/numpy/core/tests/test_simd.py @@ -0,0 +1,408 @@ +# NOTE: Please avoid the use of numpy.testing since NPYV intrinsics +# may be involved in their functionality. +import pytest +from numpy.core._simd import targets + +class _Test_Utility(object): + # submodule of the desired SIMD extention, e.g. targets["AVX512F"] + npyv = None + # the current data type suffix e.g. 's8' + sfx = None + + def __getattr__(self, attr): + """ + To call NPV intrinsics without the prefix 'npyv_' and + auto suffixing intrinsics according to class attribute 'sfx' + """ + nattr = getattr(self.npyv, attr + "_" + self.sfx) + if callable(nattr): + return lambda *args: nattr(*args) + return nattr + + def _data(self, n=None, reverse=False): + """ + Create list of consecutive numbers according to number of vector's lanes. + """ + if n is None: + n = 1 + rng = range(n, n + self.nlanes) + if reverse: + rng = reversed(rng) + if self._is_fp(): + return [x / 1.0 for x in rng] + return list(rng) + + def _is_unsigned(self): + return self.sfx[0] == 'u' + + def _is_signed(self): + return self.sfx[0] == 's' + + def _is_fp(self): + return self.sfx[0] == 'f' + + def _scalar_size(self): + return int(self.sfx[1:]) + + def _int_clip(self, seq): + if self._is_fp(): + return seq + max_int = self._int_max() + min_int = self._int_min() + return [min(max(v, min_int), max_int) for v in seq] + + def _int_max(self): + if self._is_fp(): + return None + max_u = self._to_unsigned(self.setall(-1))[0] + if self._is_signed(): + return max_u // 2 + return max_u + + def _int_min(self): + if self._is_fp(): + return None + if self._is_unsigned(): + return 0 + return -(self._int_max() + 1) + + def _true_mask(self): + max_unsig = getattr(self.npyv, "setall_u" + self.sfx[1:])(-1) + return max_unsig[0] + + def _to_unsigned(self, vector): + if isinstance(vector, (list, tuple)): + return getattr(self.npyv, "load_u" + self.sfx[1:])(vector) + else: + sfx = vector.__name__.replace("npyv_", "") + if sfx[0] == "b": + cvt_intrin = "cvt_u{0}_b{0}" + else: + cvt_intrin = "reinterpret_u{0}_{1}" + return getattr(self.npyv, cvt_intrin.format(sfx[1:], sfx))(vector) + +class _SIMD_INT(_Test_Utility): + """ + To test all integer vector types at once + """ + def test_operators_shift(self): + if self.sfx in ("u8", "s8"): + pytest.skip("there are no shift intrinsics for npyv_" + self.sfx) + + data_a = self._data(self._int_max() - self.nlanes) + data_b = self._data(self._int_min(), reverse=True) + vdata_a, vdata_b = self.load(data_a), self.load(data_b) + + for count in range(self._scalar_size()): + # load to cast + data_shl_a = self.load([a << count for a in data_a]) + # left shift + shl = self.shl(vdata_a, count) + assert shl == data_shl_a + # left shift by an immediate constant + shli = self.shli(vdata_a, count) + assert shli == data_shl_a + # load to cast + data_shr_a = self.load([a >> count for a in data_a]) + # right shift + shr = self.shr(vdata_a, count) + assert shr == data_shr_a + # right shift by an immediate constant + shri = self.shri(vdata_a, count) + assert shri == data_shr_a + + def test_arithmetic_subadd_saturated(self): + if self.sfx in ("u32", "s32", "u64", "s64"): + pytest.skip("there are no saturated add/sub intrinsics for npyv_" + self.sfx) + + data_a = self._data(self._int_max() - self.nlanes) + data_b = self._data(self._int_min(), reverse=True) + vdata_a, vdata_b = self.load(data_a), self.load(data_b) + + data_adds = self._int_clip([a + b for a, b in zip(data_a, data_b)]) + adds = self.adds(vdata_a, vdata_b) + assert adds == data_adds + + data_subs = self._int_clip([a - b for a, b in zip(data_a, data_b)]) + subs = self.subs(vdata_a, vdata_b) + assert subs == data_subs + +class _SIMD_FP(_Test_Utility): + """ + To test all float vector types at once + """ + pass + +class _SIMD_ALL(_Test_Utility): + """ + To test all vector types at once + """ + def test_memory_load(self): + data = self._data() + # unaligned load + load_data = self.load(data) + assert load_data == data + # aligned load + loada_data = self.loada(data) + assert loada_data == data + # stream load + loads_data = self.loads(data) + assert loads_data == data + # load lower part + loadl = self.loadl(data) + loadl_half = list(loadl)[:self.nlanes//2] + data_half = data[:self.nlanes//2] + assert loadl_half == data_half + assert loadl != data # detect overflow + + def test_memory_store(self): + data = self._data() + vdata = self.load(data) + # unaligned store + store = [0] * self.nlanes + self.store(store, vdata) + assert store == data + # aligned store + store_a = [0] * self.nlanes + self.storea(store_a, vdata) + assert store_a == data + # stream store + store_s = [0] * self.nlanes + self.stores(store_s, vdata) + assert store_s == data + # store lower part + store_l = [0] * self.nlanes + self.storel(store_l, vdata) + assert store_l[:self.nlanes//2] == data[:self.nlanes//2] + assert store_l != vdata # detect overflow + # store higher part + store_h = [0] * self.nlanes + self.storeh(store_h, vdata) + assert store_h[:self.nlanes//2] == data[self.nlanes//2:] + assert store_h != vdata # detect overflow + + def test_misc(self): + broadcast_zero = self.zero() + assert broadcast_zero == [0] * self.nlanes + for i in range(1, 10): + broadcasti = self.setall(i) + assert broadcasti == [i] * self.nlanes + + data_a, data_b = self._data(), self._data(reverse=True) + vdata_a, vdata_b = self.load(data_a), self.load(data_b) + + # py level of npyv_set_* don't support ignoring the extra specified lanes or + # fill non-specified lanes with zero. + vset = self.set(*data_a) + assert vset == data_a + # py level of npyv_setf_* don't support ignoring the extra specified lanes or + # fill non-specified lanes with the specified scalar. + vsetf = self.setf(10, *data_a) + assert vsetf == data_a + + # We're testing the sainty of _simd's type-vector, + # reinterpret* intrinsics itself are tested via compiler + # during the build of _simd module + sfxes = ["u8", "s8", "u16", "s16", "u32", "s32", "u64", "s64", "f32"] + if self.npyv.simd_f64: + sfxes.append("f64") + for sfx in sfxes: + vec_name = getattr(self, "reinterpret_" + sfx)(vdata_a).__name__ + assert vec_name == "npyv_" + sfx + + # select & mask operations + select_a = self.select(self.cmpeq(self.zero(), self.zero()), vdata_a, vdata_b) + assert select_a == data_a + select_b = self.select(self.cmpneq(self.zero(), self.zero()), vdata_a, vdata_b) + assert select_b == data_b + + # cleanup intrinsic is only used with AVX for + # zeroing registers to avoid the AVX-SSE transition penalty, + # so nothing to test here + self.npyv.cleanup() + + def test_reorder(self): + data_a, data_b = self._data(), self._data(reverse=True) + vdata_a, vdata_b = self.load(data_a), self.load(data_b) + # lower half part + data_a_lo = data_a[:self.nlanes//2] + data_b_lo = data_b[:self.nlanes//2] + # higher half part + data_a_hi = data_a[self.nlanes//2:] + data_b_hi = data_b[self.nlanes//2:] + # combine two lower parts + combinel = self.combinel(vdata_a, vdata_b) + assert combinel == data_a_lo + data_b_lo + # combine two higher parts + combineh = self.combineh(vdata_a, vdata_b) + assert combineh == data_a_hi + data_b_hi + # combine x2 + combine = self.combine(vdata_a, vdata_b) + assert combine == (data_a_lo + data_b_lo, data_a_hi + data_b_hi) + # zip(interleave) + data_zipl = [v for p in zip(data_a_lo, data_b_lo) for v in p] + data_ziph = [v for p in zip(data_a_hi, data_b_hi) for v in p] + vzip = self.zip(vdata_a, vdata_b) + assert vzip == (data_zipl, data_ziph) + + def test_operators_comparison(self): + if self._is_fp(): + data_a = self._data() + else: + data_a = self._data(self._int_max() - self.nlanes) + data_b = self._data(self._int_min(), reverse=True) + vdata_a, vdata_b = self.load(data_a), self.load(data_b) + + mask_true = self._true_mask() + def to_bool(vector): + return [lane == mask_true for lane in vector] + # equal + data_eq = [a == b for a, b in zip(data_a, data_b)] + cmpeq = to_bool(self.cmpeq(vdata_a, vdata_b)) + assert cmpeq == data_eq + # not equal + data_neq = [a != b for a, b in zip(data_a, data_b)] + cmpneq = to_bool(self.cmpneq(vdata_a, vdata_b)) + assert cmpneq == data_neq + # greater than + data_gt = [a > b for a, b in zip(data_a, data_b)] + cmpgt = to_bool(self.cmpgt(vdata_a, vdata_b)) + assert cmpgt == data_gt + # greater than and equal + data_ge = [a >= b for a, b in zip(data_a, data_b)] + cmpge = to_bool(self.cmpge(vdata_a, vdata_b)) + assert cmpge == data_ge + # less than + data_lt = [a < b for a, b in zip(data_a, data_b)] + cmplt = to_bool(self.cmplt(vdata_a, vdata_b)) + assert cmplt == data_lt + # less than and equal + data_le = [a <= b for a, b in zip(data_a, data_b)] + cmple = to_bool(self.cmple(vdata_a, vdata_b)) + assert cmple == data_le + + def test_operators_logical(self): + if self._is_fp(): + data_a = self._data() + else: + data_a = self._data(self._int_max() - self.nlanes) + data_b = self._data(self._int_min(), reverse=True) + vdata_a, vdata_b = self.load(data_a), self.load(data_b) + + if self._is_fp(): + data_cast_a = self._to_unsigned(vdata_a) + data_cast_b = self._to_unsigned(vdata_b) + cast, cast_data = self._to_unsigned, self._to_unsigned + else: + data_cast_a, data_cast_b = data_a, data_b + cast, cast_data = lambda a: a, self.load + + data_xor = cast_data([a ^ b for a, b in zip(data_cast_a, data_cast_b)]) + vxor = cast(self.xor(vdata_a, vdata_b)) + assert vxor == data_xor + + data_or = cast_data([a | b for a, b in zip(data_cast_a, data_cast_b)]) + vor = cast(getattr(self, "or")(vdata_a, vdata_b)) + assert vor == data_or + + data_and = cast_data([a & b for a, b in zip(data_cast_a, data_cast_b)]) + vand = cast(getattr(self, "and")(vdata_a, vdata_b)) + assert vand == data_and + + data_not = cast_data([~a for a in data_cast_a]) + vnot = cast(getattr(self, "not")(vdata_a)) + assert vnot == data_not + + def test_conversion_boolean(self): + bsfx = "b" + self.sfx[1:] + to_boolean = getattr(self.npyv, "cvt_%s_%s" % (bsfx, self.sfx)) + from_boolean = getattr(self.npyv, "cvt_%s_%s" % (self.sfx, bsfx)) + + false_vb = to_boolean(self.setall(0)) + true_vb = self.cmpeq(self.setall(0), self.setall(0)) + assert false_vb != true_vb + + false_vsfx = from_boolean(false_vb) + true_vsfx = from_boolean(true_vb) + assert false_vsfx != true_vsfx + + def test_arithmetic_subadd(self): + if self._is_fp(): + data_a = self._data() + else: + data_a = self._data(self._int_max() - self.nlanes) + data_b = self._data(self._int_min(), reverse=True) + vdata_a, vdata_b = self.load(data_a), self.load(data_b) + + # non-saturated + data_add = self.load([a + b for a, b in zip(data_a, data_b)]) # load to cast + add = self.add(vdata_a, vdata_b) + assert add == data_add + data_sub = self.load([a - b for a, b in zip(data_a, data_b)]) + sub = self.sub(vdata_a, vdata_b) + assert sub == data_sub + + def test_arithmetic_mul(self): + if self.sfx in ("u64", "s64"): + pytest.skip("there is no multiplication intrinsic for npyv_" + self.sfx) + + if self._is_fp(): + data_a = self._data() + else: + data_a = self._data(self._int_max() - self.nlanes) + data_b = self._data(self._int_min(), reverse=True) + vdata_a, vdata_b = self.load(data_a), self.load(data_b) + + data_mul = self.load([a * b for a, b in zip(data_a, data_b)]) + mul = self.mul(vdata_a, vdata_b) + assert mul == data_mul + + def test_arithmetic_div(self): + if not self._is_fp(): + pytest.skip("there is no division intrinsic for npyv_" + self.sfx) + + data_a, data_b = self._data(), self._data(reverse=True) + vdata_a, vdata_b = self.load(data_a), self.load(data_b) + + # load to truncate f64 to precision of f32 + data_div = self.load([a / b for a, b in zip(data_a, data_b)]) + div = self.div(vdata_a, vdata_b) + assert div == data_div + +int_sfx = ("u8", "s8", "u16", "s16", "u32", "s32", "u64", "s64") +fp_sfx = ("f32", "f64") +all_sfx = int_sfx + fp_sfx +tests_registery = { + int_sfx : "_SIMD_INT", + fp_sfx : "_SIMD_FP", + all_sfx : "_SIMD_ALL" +} +for name, npyv in targets.items(): + simd_width = npyv.simd if npyv else '' + pretty_name = name.split('__') # multi-target separator + if len(pretty_name) > 1: + # multi-target + pretty_name = f"({' '.join(pretty_name)})" + else: + pretty_name = pretty_name[0] + + skip = "" + skip_sfx = dict() + if not npyv: + skip = f"target '{pretty_name}' isn't supported by current machine" + elif not npyv.simd: + skip = f"target '{pretty_name}' isn't supported by NPYV" + elif not npyv.simd_f64: + skip_sfx["f64"] = f"target '{pretty_name}' doesn't support double-precision" + + for sfxes, class_name in tests_registery.items(): + for sfx in sfxes: + skip_m = skip_sfx.get(sfx, skip) + if skip_m: + skip_m = '@pytest.mark.skip(reason="%s")' % skip_m + exec( + f"{skip_m}\n" + f"class Test{class_name}_{simd_width}_{name}_{sfx}({class_name}):\n" + f" npyv = targets['{name}']\n" + f" sfx = '{sfx}'\n" + ) From 1779c2a9287d2242226ee6bac575c75684c3d696 Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Wed, 2 Sep 2020 15:15:28 +0200 Subject: [PATCH 371/409] TST: Add testing unit for checking the sainty of SIMD module --- numpy/core/tests/test_simd_module.py | 104 +++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 numpy/core/tests/test_simd_module.py diff --git a/numpy/core/tests/test_simd_module.py b/numpy/core/tests/test_simd_module.py new file mode 100644 index 000000000000..5bf82c0aa024 --- /dev/null +++ b/numpy/core/tests/test_simd_module.py @@ -0,0 +1,104 @@ +import pytest +from numpy.core._simd import targets + +npyv = None +npyv2 = None +for target_name, npyv_mod in targets.items(): + if npyv: + if npyv_mod and npyv_mod.simd: + npyv2 = npyv_mod + break + continue + if npyv_mod and npyv_mod.simd: + npyv = npyv_mod + +unsigned_sfx = ["u8", "u16", "u32", "u64"] +signed_sfx = ["s8", "s16", "s32", "s64"] +fp_sfx = ["f32"] +if npyv and npyv.simd_f64: + fp_sfx.append("f64") + +int_sfx = unsigned_sfx + signed_sfx +all_sfx = unsigned_sfx + int_sfx + +@pytest.mark.skipif(not npyv, reason="could not find any SIMD extension with NPYV support") +class Test_SIMD_MODULE(object): + def test_num_lanes(self): + for sfx in all_sfx: + nlanes = getattr(npyv, "nlanes_" + sfx) + vector = getattr(npyv, "setall_" + sfx)(1) + assert len(vector) == nlanes + + def test_type_name(self): + for sfx in all_sfx: + vector = getattr(npyv, "setall_" + sfx)(1) + assert vector.__name__ == "npyv_" + sfx + + def test_raises(self): + def assert_raises(e, callback, *args): + __tracebackhide__ = True # Hide traceback for py.test + try: + callback(*args) + raise AssertionError("expected to raise " + e.__name__) + except e: + pass + + a, b = [npyv.setall_u32(1)]*2 + for sfx in all_sfx: + vcb = lambda intrin: getattr(npyv, f"{intrin}_{sfx}") + assert_raises(TypeError, vcb("add"), a) + assert_raises(TypeError, vcb("add"), a, b, a) + assert_raises(TypeError, vcb("setall")) + assert_raises(TypeError, vcb("setall"), [1]) + assert_raises(TypeError, vcb("load"), 1) + assert_raises(ValueError, vcb("load"), [1]) + assert_raises(ValueError, vcb("store"), [1], getattr(npyv, f"reinterpret_{sfx}_u32")(a)) + + # mix among submodules isn't allowed + if not npyv2: + return + a2 = npyv2.setall_u32(1) + assert_raises(TypeError, npyv.add_u32, a2, a2) + assert_raises(TypeError, npyv2.add_u32, a, a) + + def test_unsigned_overflow(self): + for sfx in unsigned_sfx: + nlanes = getattr(npyv, "nlanes_" + sfx) + hfbyte_len = int(sfx[1:])//4 + maxu = int(f"0x{'f'*hfbyte_len}", 16) + maxu_72 = 0xfffffffffffffffff + lane = getattr(npyv, "setall_" + sfx)(maxu_72)[0] + assert lane == maxu + lanes = getattr(npyv, "load_" + sfx)([maxu_72] * nlanes) + assert lanes == [maxu] * nlanes + lane = getattr(npyv, "setall_" + sfx)(-1)[0] + assert lane == maxu + lanes = getattr(npyv, "load_" + sfx)([-1] * nlanes) + assert lanes == [maxu] * nlanes + + def test_signed_overflow(self): + for sfx in signed_sfx: + nlanes = getattr(npyv, "nlanes_" + sfx) + maxs_72 = 0x7fffffffffffffffff + lane = getattr(npyv, "setall_" + sfx)(maxs_72)[0] + assert lane == -1 + lanes = getattr(npyv, "load_" + sfx)([maxs_72] * nlanes) + assert lanes == [-1] * nlanes + mins_72 = -0x80000000000000000 + lane = getattr(npyv, "setall_" + sfx)(mins_72)[0] + assert lane == 0 + lanes = getattr(npyv, "load_" + sfx)([mins_72] * nlanes) + assert lanes == [0] * nlanes + + def test_truncate_f32(self): + f32 = npyv.setall_f32(0.1)[0] + assert f32 != 0.1 + assert round(f32, 1) == 0.1 + + def test_compare(self): + data_range = range(0, npyv.nlanes_u32) + vdata = npyv.load_u32(data_range) + assert vdata == list(data_range) + assert vdata == tuple(data_range) + for i in data_range: + assert vdata[i] == data_range[i] From 92a4693cb0df392003cf99a2f1abe8927cdd58b4 Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Mon, 14 Sep 2020 15:52:27 +0200 Subject: [PATCH 372/409] TST: Add testing unit for fused NPYV intrinsics --- numpy/core/src/_simd/_simd.dispatch.c.src | 19 ++++++++++++++++++- numpy/core/tests/test_simd.py | 20 +++++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/numpy/core/src/_simd/_simd.dispatch.c.src b/numpy/core/src/_simd/_simd.dispatch.c.src index 69a9ffd45660..506ef93f08e1 100644 --- a/numpy/core/src/_simd/_simd.dispatch.c.src +++ b/numpy/core/src/_simd/_simd.dispatch.c.src @@ -19,6 +19,7 @@ * #sat_sup = 1, 1, 1, 1, 0, 0, 0, 0, 0, 0# * #mul_sup = 1, 1, 1, 1, 1, 1, 0, 0, 1, 1# * #div_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# + * #fused_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# * #shl_imm = 0, 0, 15, 15, 31, 31, 63, 63, 0, 0# * #shr_imm = 0, 0, 16, 16, 32, 32, 64, 64, 0, 0# */ @@ -175,9 +176,16 @@ SIMD_IMPL_INTRIN_2(mul_@sfx@, v@sfx@, v@sfx@, v@sfx@) SIMD_IMPL_INTRIN_2(div_@sfx@, v@sfx@, v@sfx@, v@sfx@) #endif // div_sup +#if @fused_sup@ +/**begin repeat1 + * #intrin = muladd, mulsub, nmuladd, nmulsub# + */ +SIMD_IMPL_INTRIN_3(@intrin@_@sfx@, v@sfx@, v@sfx@, v@sfx@, v@sfx@) +/**end repeat1**/ +#endif // fused_sup + #endif // simd_sup /**end repeat**/ - /*************************** * Variant ***************************/ @@ -194,6 +202,7 @@ static PyMethodDef simd__intrinsics_methods[] = { * #sat_sup = 1, 1, 1, 1, 0, 0, 0, 0, 0, 0# * #mul_sup = 1, 1, 1, 1, 1, 1, 0, 0, 1, 1# * #div_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# + * #fused_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# * #shl_imm = 0, 0, 15, 15, 31, 31, 63, 63, 0, 0# * #shr_imm = 0, 0, 16, 16, 32, 32, 64, 64, 0, 0# */ @@ -286,6 +295,14 @@ SIMD_INTRIN_DEF(mul_@sfx@) SIMD_INTRIN_DEF(div_@sfx@) #endif // div_sup +#if @fused_sup@ +/**begin repeat1 + * #intrin = muladd, mulsub, nmuladd, nmulsub# + */ +SIMD_INTRIN_DEF(@intrin@_@sfx@) +/**end repeat1**/ +#endif // fused_sup + #endif // simd_sup /**end repeat**/ diff --git a/numpy/core/tests/test_simd.py b/numpy/core/tests/test_simd.py index 26e78d40254c..3ca6b068dea8 100644 --- a/numpy/core/tests/test_simd.py +++ b/numpy/core/tests/test_simd.py @@ -131,7 +131,25 @@ class _SIMD_FP(_Test_Utility): """ To test all float vector types at once """ - pass + def test_arithmetic_fused(self): + vdata_a, vdata_b, vdata_c = [self.load(self._data())]*3 + vdata_cx2 = self.add(vdata_c, vdata_c) + # multiply and add, a*b + c + data_fma = self.load([a * b + c for a, b, c in zip(vdata_a, vdata_b, vdata_c)]) + fma = self.muladd(vdata_a, vdata_b, vdata_c) + assert fma == data_fma + # multiply and subtract, a*b - c + fms = self.mulsub(vdata_a, vdata_b, vdata_c) + data_fms = self.sub(data_fma, vdata_cx2) + assert fms == data_fms + # negate multiply and add, -(a*b) + c + nfma = self.nmuladd(vdata_a, vdata_b, vdata_c) + data_nfma = self.sub(vdata_cx2, data_fma) + assert nfma == data_nfma + # negate multiply and subtract, -(a*b) - c + nfms = self.nmulsub(vdata_a, vdata_b, vdata_c) + data_nfms = self.mul(data_fma, self.setall(-1)) + assert nfms == data_nfms class _SIMD_ALL(_Test_Utility): """ From e19f7a8cfe59226fed6cc7a7da9e881218364d49 Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Tue, 15 Sep 2020 15:04:28 +0200 Subject: [PATCH 373/409] MAINT, TST: use PyArg_ParseTuple() instead of iterate Python tuple directly Co-authored-by: Eric Wieser --- numpy/core/src/_simd/_simd.dispatch.c.src | 6 ++- numpy/core/src/_simd/_simd_inc.h.src | 33 ++++++++----- numpy/core/src/_simd/_simd_inc_arg.h | 38 ++++----------- numpy/core/src/_simd/_simd_inc_easyintrin.h | 52 ++++++++++----------- 4 files changed, 61 insertions(+), 68 deletions(-) diff --git a/numpy/core/src/_simd/_simd.dispatch.c.src b/numpy/core/src/_simd/_simd.dispatch.c.src index 506ef93f08e1..a776ba37b7e1 100644 --- a/numpy/core/src/_simd/_simd.dispatch.c.src +++ b/numpy/core/src/_simd/_simd.dispatch.c.src @@ -43,7 +43,11 @@ simd__intrin_@intrin@_@sfx@(PyObject* NPY_UNUSED(self), PyObject *args) {.dtype = simd_data_q@sfx@}, {.dtype = simd_data_v@sfx@}, }; - if (simd_args_from_tuple(args, req_args, 2, "@intrin@_@sfx@")) { + if (!PyArg_ParseTuple( + args, "O&O&:@intrin@_@sfx@", + simd_arg_converter, &req_args[0], + simd_arg_converter, &req_args[1] + )) { return NULL; } npyv_@intrin@_@sfx@( diff --git a/numpy/core/src/_simd/_simd_inc.h.src b/numpy/core/src/_simd/_simd_inc.h.src index 4261ce14892f..530c7a736294 100644 --- a/numpy/core/src/_simd/_simd_inc.h.src +++ b/numpy/core/src/_simd/_simd_inc.h.src @@ -306,7 +306,7 @@ typedef struct { simd_data_type dtype; simd_data data; - // set by simd_args_from_tuple() + // set by simd_arg_converter() PyObject *obj; } simd_arg; /** @@ -365,9 +365,8 @@ simd_arg_from_obj(PyObject *obj, simd_arg *arg); static PyObject * simd_arg_to_obj(const simd_arg *arg); /** - * Similar to simd_arg_from_obj() but convert multiple objects - * from a Python tuple. - * This function is used to parse method parameters. + * Converter function used similar to simd_arg_from_obj() but + * used with PyArg_Parse*(). * * Notes: * - requires `simd_args_sequence_free()` or `simd_sequence_free()` @@ -376,16 +375,26 @@ simd_arg_to_obj(const simd_arg *arg); * types is the number of lanes of the equivalent vector data type. * - use 'arg->obj' to retrieve the parameter obj. * - * Parameters: - * - `tuple_obj`: a valid Python tuple - * - `args`: array of 'simd_arg' contain valid data types - * - `args_len`: length of `args`. - * - `method_name`: method name, required for exception message. - * - * Return -1 and raise Python exception on failure, otherwise return 0. + * Example: + ** simd_arg req_args[] = { + ** {.dtype = simd_data_qf32}, + ** {.dtype = simd_data_vf32}, + ** }; + ** if (!PyArg_ParseTuple( + ** args, "O&O&:add_sum_f32", + ** simd_arg_converter, &req_args[0], + ** simd_arg_converter, &req_args[1] + ** )) { + ** // fail + ** return; + ** } + ** npyv_f32 load_a = npyv_load_f32(req_args[0].data.qf32); + ** npyv_f32 sum = npyv_add_f32(load_a, req_args[1].data.vf32); + ** ... + ** simd_args_sequence_free(req_args, 2); */ static int -simd_args_from_tuple(PyObject *tuple_obj, simd_arg *args, int args_len, const char *method_name); +simd_arg_converter(PyObject *obj, simd_arg *arg); /** * Free the allocated C array for sequence data types. */ diff --git a/numpy/core/src/_simd/_simd_inc_arg.h b/numpy/core/src/_simd/_simd_inc_arg.h index 1db1ff202927..eabf49c5f157 100644 --- a/numpy/core/src/_simd/_simd_inc_arg.h +++ b/numpy/core/src/_simd/_simd_inc_arg.h @@ -73,36 +73,16 @@ simd_args_sequence_free(simd_arg *args, int args_len) } static int -simd_args_from_tuple(PyObject *tuple_obj, simd_arg *args, int args_len, const char *method_name) +simd_arg_converter(PyObject *obj, simd_arg *arg) { - assert(args_len > 0); - assert(PyTuple_Check(tuple_obj)); - - Py_ssize_t obj_arg_len = PyTuple_GET_SIZE(tuple_obj); - if (obj_arg_len != args_len) { - if (args_len == 1) { - PyErr_Format(PyExc_TypeError, - "%s() takes only one argument (%d given)", method_name, obj_arg_len - ); - return -1; - } - PyErr_Format(PyExc_TypeError, - "%s() takes exactly %d arguments (%d given)", method_name, args_len, obj_arg_len - ); - return -1; - } - for (int arg_pos = 0; arg_pos < args_len; ++arg_pos) { - simd_arg *arg = &args[arg_pos]; - arg->obj = PyTuple_GET_ITEM(tuple_obj, arg_pos); - assert(arg->obj != NULL); - if (simd_arg_from_obj(arg->obj, arg) != 0) { - // free previous args - if (arg_pos > 0) { - simd_args_sequence_free(args, arg_pos); - } - // TODO: improve log by add argument number and method name - return -1; + if (obj != NULL) { + if (simd_arg_from_obj(obj, arg) < 0) { + return 0; } + arg->obj = obj; + return Py_CLEANUP_SUPPORTED; + } else { + simd_args_sequence_free(arg, 1); } - return 0; + return 1; } diff --git a/numpy/core/src/_simd/_simd_inc_easyintrin.h b/numpy/core/src/_simd/_simd_inc_easyintrin.h index 7216b373ad0f..9ff22799956f 100644 --- a/numpy/core/src/_simd/_simd_inc_easyintrin.h +++ b/numpy/core/src/_simd/_simd_inc_easyintrin.h @@ -50,19 +50,18 @@ static int simd__no_arguments(PyObject *args, const char* method_name) static PyObject *simd__intrin_##NAME \ (PyObject* NPY_UNUSED(self), PyObject *args) \ { \ - simd_arg req_args[] = { \ - {.dtype = simd_data_##IN0}, \ - }; \ - if (simd_args_from_tuple( \ - args, req_args, 1, NPY_TOSTRING(NAME)) \ - ) return NULL; \ + simd_arg arg = {.dtype = simd_data_##IN0}; \ + if (!PyArg_ParseTuple( \ + args, "O&:"NPY_TOSTRING(NAME), \ + simd_arg_converter, &arg \ + )) return NULL; \ simd_data r = {.RET = npyv_##NAME( \ - req_args[0].data.IN0 \ + arg.data.IN0 \ )}; \ - simd_args_sequence_free(req_args, 1); \ - req_args[0].data = r; \ - req_args[0].dtype = simd_data_##RET; \ - return simd_arg_to_obj(req_args); \ + simd_args_sequence_free(&arg, 1); \ + arg.data = r; \ + arg.dtype = simd_data_##RET; \ + return simd_arg_to_obj(&arg); \ } #define SIMD_IMPL_INTRIN_2(NAME, RET, IN0, IN1) \ @@ -73,11 +72,11 @@ static int simd__no_arguments(PyObject *args, const char* method_name) {.dtype = simd_data_##IN0}, \ {.dtype = simd_data_##IN1}, \ }; \ - if (simd_args_from_tuple( \ - args, req_args, 2, NPY_TOSTRING(NAME)) \ - ) { \ - return NULL; \ - } \ + if (!PyArg_ParseTuple( \ + args, "O&O&:"NPY_TOSTRING(NAME), \ + simd_arg_converter, &req_args[0], \ + simd_arg_converter, &req_args[1] \ + )) return NULL; \ simd_data r = {.RET = npyv_##NAME( \ req_args[0].data.IN0, \ req_args[1].data.IN1 \ @@ -99,11 +98,11 @@ static int simd__no_arguments(PyObject *args, const char* method_name) {.dtype = simd_data_##IN0}, \ {.dtype = simd_data_u8}, \ }; \ - if (simd_args_from_tuple( \ - args, req_args, 2, NPY_TOSTRING(NAME)) \ - ) { \ - return NULL; \ - } \ + if (!PyArg_ParseTuple( \ + args, "O&O&:"NPY_TOSTRING(NAME), \ + simd_arg_converter, &req_args[0], \ + simd_arg_converter, &req_args[1] \ + )) return NULL; \ simd_data r; \ r.RET = NPY_CAT(SIMD__IMPL_COUNT_, CONST_RNG)( \ SIMD__REPEAT_2IMM, NAME, IN0 \ @@ -123,11 +122,12 @@ static int simd__no_arguments(PyObject *args, const char* method_name) {.dtype = simd_data_##IN1}, \ {.dtype = simd_data_##IN2}, \ }; \ - if (simd_args_from_tuple( \ - args, req_args, 3, NPY_TOSTRING(NAME)) \ - ) { \ - return NULL; \ - } \ + if (!PyArg_ParseTuple( \ + args, "O&O&O&:"NPY_TOSTRING(NAME), \ + simd_arg_converter, &req_args[0], \ + simd_arg_converter, &req_args[1], \ + simd_arg_converter, &req_args[2] \ + )) return NULL; \ simd_data r = {.RET = npyv_##NAME( \ req_args[0].data.IN0, \ req_args[1].data.IN1, \ From 7d125fb70cb149207171b7181312f9679dd4d451 Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Tue, 15 Sep 2020 16:06:14 +0200 Subject: [PATCH 374/409] MAINT, TST: Serveral imporvments to _SIMD module - use plain variables - clean up aligned allocate - use `PyArg_ParseTuple` for empty args - use `Py_ssize_t` instead of `unsigned` and `size_t` - improve coding style - no need for a custom raises assertions - use parametrize instead of inner loops - leave a comment about nature of mode testing unit - shift to get max/min of int72 - add more info to repr of vector object - get ride of exec() and use type() instead - use `.inc` as extension for sub-headers instead of `.h` - add `FMA4` and drop `SSE41` from _SIMD targets Co-authored-by: Eric Wieser --- numpy/core/setup.py | 12 +- numpy/core/src/_simd/_simd.dispatch.c.src | 34 +++-- .../_simd/{_simd_inc_arg.h => _simd_arg.inc} | 39 +++-- ...{_simd_inc_convert.h => _simd_convert.inc} | 73 +++++----- ...simd_inc_data.h.src => _simd_data.inc.src} | 10 +- ..._inc_easyintrin.h => _simd_easyintrin.inc} | 133 ++++++++---------- numpy/core/src/_simd/_simd_inc.h.src | 107 ++++++++------ .../{_simd_inc_vector.h => _simd_vector.inc} | 87 +++++------- numpy/core/tests/test_simd.py | 40 +++--- numpy/core/tests/test_simd_module.py | 127 ++++++++--------- numpy/distutils/command/build.py | 2 +- 11 files changed, 323 insertions(+), 341 deletions(-) rename numpy/core/src/_simd/{_simd_inc_arg.h => _simd_arg.inc} (58%) rename numpy/core/src/_simd/{_simd_inc_convert.h => _simd_convert.inc} (69%) rename numpy/core/src/_simd/{_simd_inc_data.h.src => _simd_data.inc.src} (89%) rename numpy/core/src/_simd/{_simd_inc_easyintrin.h => _simd_easyintrin.inc} (68%) rename numpy/core/src/_simd/{_simd_inc_vector.h => _simd_vector.inc} (60%) diff --git a/numpy/core/setup.py b/numpy/core/setup.py index e9a9a4e464ae..68aa0a8513fb 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -983,18 +983,18 @@ def generate_umath_c(ext, build_dir): join('src', 'common', 'npy_cpu_features.c.src'), join('src', '_simd', '_simd.c'), join('src', '_simd', '_simd_inc.h.src'), - join('src', '_simd', '_simd_inc_data.h.src'), + join('src', '_simd', '_simd_data.inc.src'), join('src', '_simd', '_simd.dispatch.c.src'), ], depends=[ join('src', 'common', 'npy_cpu_dispatch.h'), join('src', 'common', 'simd', 'simd.h'), join('src', '_simd', '_simd.h'), join('src', '_simd', '_simd_inc.h.src'), - join('src', '_simd', '_simd_inc_data.h.src'), - join('src', '_simd', '_simd_inc_arg.h'), - join('src', '_simd', '_simd_inc_convert.h'), - join('src', '_simd', '_simd_inc_easyintrin.h'), - join('src', '_simd', '_simd_inc_vector.h'), + join('src', '_simd', '_simd_data.inc.src'), + join('src', '_simd', '_simd_arg.inc'), + join('src', '_simd', '_simd_convert.inc'), + join('src', '_simd', '_simd_easyintrin.inc'), + join('src', '_simd', '_simd_vector.inc'), ]) config.add_subpackage('tests') diff --git a/numpy/core/src/_simd/_simd.dispatch.c.src b/numpy/core/src/_simd/_simd.dispatch.c.src index a776ba37b7e1..1989be7e355b 100644 --- a/numpy/core/src/_simd/_simd.dispatch.c.src +++ b/numpy/core/src/_simd/_simd.dispatch.c.src @@ -3,11 +3,11 @@ #include "_simd_inc.h" #if NPY_SIMD -#include "_simd_inc_data.h" -#include "_simd_inc_convert.h" -#include "_simd_inc_vector.h" -#include "_simd_inc_arg.h" -#include "_simd_inc_easyintrin.h" +#include "_simd_data.inc" +#include "_simd_convert.inc" +#include "_simd_vector.inc" +#include "_simd_arg.inc" +#include "_simd_easyintrin.inc" /************************************************************************* * Defining NPYV intrinsics as module functions @@ -39,26 +39,24 @@ SIMD_IMPL_INTRIN_1(@intrin@_@sfx@, v@sfx@, q@sfx@) static PyObject * simd__intrin_@intrin@_@sfx@(PyObject* NPY_UNUSED(self), PyObject *args) { - simd_arg req_args[] = { - {.dtype = simd_data_q@sfx@}, - {.dtype = simd_data_v@sfx@}, - }; + simd_arg seq_arg = {.dtype = simd_data_q@sfx@}; + simd_arg vec_arg = {.dtype = simd_data_v@sfx@}; if (!PyArg_ParseTuple( args, "O&O&:@intrin@_@sfx@", - simd_arg_converter, &req_args[0], - simd_arg_converter, &req_args[1] + simd_arg_converter, &seq_arg, + simd_arg_converter, &vec_arg )) { return NULL; } npyv_@intrin@_@sfx@( - req_args[0].data.q@sfx@, req_args[1].data.v@sfx@ + seq_arg.data.q@sfx@, vec_arg.data.v@sfx@ ); // write-back - if (simd_sequence_fill_obj(req_args[0].obj, req_args[0].data.q@sfx@, simd_data_q@sfx@)) { - simd_args_sequence_free(req_args, 2); + if (simd_sequence_fill_iterable(seq_arg.obj, seq_arg.data.q@sfx@, simd_data_q@sfx@)) { + simd_arg_free(&seq_arg); return NULL; } - simd_args_sequence_free(req_args, 2); + simd_arg_free(&seq_arg); Py_RETURN_NONE; } /**end repeat1**/ @@ -89,7 +87,7 @@ SIMD_IMPL_INTRIN_1(reinterpret_@sfx_to@_@sfx@, v@sfx_to@, v@sfx@) static PyObject * simd__intrin_@intrin@_@sfx@(PyObject* NPY_UNUSED(self), PyObject *args) { - npyv_lanetype_@sfx@ *data = simd_sequence_from_obj(args, simd_data_q@sfx@, npyv_nlanes_@sfx@); + npyv_lanetype_@sfx@ *data = simd_sequence_from_iterable(args, simd_data_q@sfx@, npyv_nlanes_@sfx@); if (data == NULL) { return NULL; } @@ -105,7 +103,7 @@ simd__intrin_@intrin@_@sfx@(PyObject* NPY_UNUSED(self), PyObject *args) data[64] // for setf )}; simd_sequence_free(data); - return (PyObject*)simd_vector_to_obj(r, simd_data_v@sfx@); + return (PyObject*)PySIMDVector_FromData(r, simd_data_v@sfx@); } /**end repeat1**/ @@ -354,7 +352,7 @@ NPY_CPU_DISPATCH_CURFX(simd_create_module)(void) goto err; } #if NPY_SIMD - if (simd_vector_register(m)) { + if (PySIMDVectorType_Init(m)) { goto err; } /**begin repeat diff --git a/numpy/core/src/_simd/_simd_inc_arg.h b/numpy/core/src/_simd/_simd_arg.inc similarity index 58% rename from numpy/core/src/_simd/_simd_inc_arg.h rename to numpy/core/src/_simd/_simd_arg.inc index eabf49c5f157..f5bcf5487c65 100644 --- a/numpy/core/src/_simd/_simd_inc_arg.h +++ b/numpy/core/src/_simd/_simd_arg.inc @@ -1,7 +1,9 @@ -#if !NPY_SIMD - #error "Not a standalone header, only works through 'simd.dispatch.c.src'" -#endif - +/** + * This file is included by `_simd.dispatch.c.src`. Its contents are affected by the simd configuration, and + * therefore must be built multiple times. Making it a standalone `.c` file with `NPY_VISIBILITY_HIDDEN` + * symbols would require judicious use of `NPY_CPU_DISPATCH_DECLARE` and `NPY_CPU_DISPATCH_CURFX`, which was + * deemed too harmful to readability. + */ /************************************ ** Protected Definitions ************************************/ @@ -11,17 +13,17 @@ simd_arg_from_obj(PyObject *obj, simd_arg *arg) assert(arg->dtype != 0); const simd_data_info *info = simd_data_getinfo(arg->dtype); if (info->is_scalar) { - arg->data = simd_scalar_from_obj(obj, arg->dtype); + arg->data = simd_scalar_from_number(obj, arg->dtype); } else if (info->is_sequence) { unsigned min_seq_size = simd_data_getinfo(info->to_vector)->nlanes; - arg->data.qu8 = simd_sequence_from_obj(obj, arg->dtype, min_seq_size); + arg->data.qu8 = simd_sequence_from_iterable(obj, arg->dtype, min_seq_size); } else if (info->is_vectorx) { - arg->data = simd_vectorx_from_obj(obj, arg->dtype); + arg->data = simd_vectorx_from_tuple(obj, arg->dtype); } else if (info->is_vector) { - arg->data = simd_vector_from_obj((simd_vector*)obj, arg->dtype); + arg->data = PySIMDVector_AsData((PySIMDVectorObject*)obj, arg->dtype); } else { arg->data.u64 = 0; PyErr_Format(PyExc_RuntimeError, @@ -41,16 +43,16 @@ simd_arg_to_obj(const simd_arg *arg) assert(arg->dtype != 0); const simd_data_info *info = simd_data_getinfo(arg->dtype); if (info->is_scalar) { - return simd_scalar_to_obj(arg->data, arg->dtype); + return simd_scalar_to_number(arg->data, arg->dtype); } if (info->is_sequence) { - return simd_sequence_to_obj(arg->data.qu8, arg->dtype); + return simd_sequence_to_list(arg->data.qu8, arg->dtype); } if (info->is_vectorx) { - return simd_vectorx_to_obj(arg->data, arg->dtype); + return simd_vectorx_to_tuple(arg->data, arg->dtype); } if (info->is_vector) { - return (PyObject*)simd_vector_to_obj(arg->data, arg->dtype); + return (PyObject*)PySIMDVector_FromData(arg->data, arg->dtype); } PyErr_Format(PyExc_RuntimeError, "unhandled arg to object type id:%d, name:%s", arg->dtype, info->pyname @@ -59,15 +61,10 @@ simd_arg_to_obj(const simd_arg *arg) } static void -simd_args_sequence_free(simd_arg *args, int args_len) +simd_arg_free(simd_arg *arg) { - assert(args_len > 0); - while (--args_len >= 0) { - simd_arg *arg = &args[args_len]; - const simd_data_info *info = simd_data_getinfo(arg->dtype); - if (!info->is_sequence) { - continue; - } + const simd_data_info *info = simd_data_getinfo(arg->dtype); + if (info->is_sequence) { simd_sequence_free(arg->data.qu8); } } @@ -82,7 +79,7 @@ simd_arg_converter(PyObject *obj, simd_arg *arg) arg->obj = obj; return Py_CLEANUP_SUPPORTED; } else { - simd_args_sequence_free(arg, 1); + simd_arg_free(arg); } return 1; } diff --git a/numpy/core/src/_simd/_simd_inc_convert.h b/numpy/core/src/_simd/_simd_convert.inc similarity index 69% rename from numpy/core/src/_simd/_simd_inc_convert.h rename to numpy/core/src/_simd/_simd_convert.inc index 3601012473b0..f5bfc3f50409 100644 --- a/numpy/core/src/_simd/_simd_inc_convert.h +++ b/numpy/core/src/_simd/_simd_convert.inc @@ -1,12 +1,14 @@ -#if !NPY_SIMD - #error "Not a standalone header, only works through 'simd.dispatch.c.src'" -#endif - +/** + * This file is included by `_simd.dispatch.c.src`. Its contents are affected by the simd configuration, and + * therefore must be built multiple times. Making it a standalone `.c` file with `NPY_VISIBILITY_HIDDEN` + * symbols would require judicious use of `NPY_CPU_DISPATCH_DECLARE` and `NPY_CPU_DISPATCH_CURFX`, which was + * deemed too harmful to readability. + */ /************************************ ** Protected Definitions ************************************/ static simd_data -simd_scalar_from_obj(PyObject *obj, simd_data_type dtype) +simd_scalar_from_number(PyObject *obj, simd_data_type dtype) { const simd_data_info *info = simd_data_getinfo(dtype); assert(info->is_scalar && info->lane_size > 0); @@ -23,7 +25,7 @@ simd_scalar_from_obj(PyObject *obj, simd_data_type dtype) } static PyObject * -simd_scalar_to_obj(simd_data data, simd_data_type dtype) +simd_scalar_to_number(simd_data data, simd_data_type dtype) { const simd_data_info *info = simd_data_getinfo(dtype); assert(info->is_scalar && info->lane_size > 0); @@ -41,43 +43,44 @@ simd_scalar_to_obj(simd_data data, simd_data_type dtype) return PyLong_FromUnsignedLongLong(data.u64 >> leftb); } +typedef struct { + Py_ssize_t len; + void *ptr; +} simd__alloc_data; + static void * simd_sequence_new(Py_ssize_t len, simd_data_type dtype) { const simd_data_info *info = simd_data_getinfo(dtype); - assert(info->is_sequence && info->lane_size > 0); - - size_t size = NPY_SIMD_WIDTH + sizeof(size_t) + sizeof(size_t*); - size += len * info->lane_size; - - size_t *ptr = malloc(size); + assert(len > 0 && info->is_sequence && info->lane_size > 0); + size_t size = sizeof(simd__alloc_data) + len * info->lane_size + NPY_SIMD_WIDTH; + void *ptr = malloc(size); if (ptr == NULL) { return PyErr_NoMemory(); } - *(ptr++) = len; - size_t **a_ptr = (size_t**)( - ((size_t)ptr + NPY_SIMD_WIDTH) & ~(size_t)(NPY_SIMD_WIDTH-1) + // align the pointer + simd__alloc_data *a_ptr = (simd__alloc_data *)( + ((uintptr_t)ptr + sizeof(simd__alloc_data) + NPY_SIMD_WIDTH) & ~(uintptr_t)(NPY_SIMD_WIDTH-1) ); - a_ptr[-1] = ptr; + a_ptr[-1].len = len; + a_ptr[-1].ptr = ptr; return a_ptr; } -static size_t -simd_sequence_len(const void *ptr) +static Py_ssize_t +simd_sequence_len(void const *ptr) { - size_t *ptrz = ((size_t**)ptr)[-1]; - return *(ptrz-1); + return ((simd__alloc_data const*)ptr)[-1].len; } static void simd_sequence_free(void *ptr) { - size_t *ptrz = ((size_t**)ptr)[-1]; - free(ptrz-1); + free(((simd__alloc_data *)ptr)[-1].ptr); } static void * -simd_sequence_from_obj(PyObject *obj, simd_data_type dtype, unsigned min_size) +simd_sequence_from_iterable(PyObject *obj, simd_data_type dtype, Py_ssize_t min_size) { const simd_data_info *info = simd_data_getinfo(dtype); assert(info->is_sequence && info->lane_size > 0); @@ -86,7 +89,7 @@ simd_sequence_from_obj(PyObject *obj, simd_data_type dtype, unsigned min_size) return NULL; } Py_ssize_t seq_size = PySequence_Fast_GET_SIZE(seq_obj); - if (seq_size < (Py_ssize_t)min_size) { + if (seq_size < min_size) { PyErr_Format(PyExc_ValueError, "minimum acceptable size of the required sequence is %d, given(%d)", min_size, seq_size @@ -99,7 +102,7 @@ simd_sequence_from_obj(PyObject *obj, simd_data_type dtype, unsigned min_size) } PyObject **seq_items = PySequence_Fast_ITEMS(seq_obj); for (Py_ssize_t i = 0; i < seq_size; ++i) { - simd_data data = simd_scalar_from_obj(seq_items[i], info->to_scalar); + simd_data data = simd_scalar_from_number(seq_items[i], info->to_scalar); npyv_lanetype_u8 *sdst = dst + i * info->lane_size; memcpy(sdst, &data.u64, info->lane_size); } @@ -113,7 +116,7 @@ simd_sequence_from_obj(PyObject *obj, simd_data_type dtype, unsigned min_size) } static int -simd_sequence_fill_obj(PyObject *obj, const void *ptr, simd_data_type dtype) +simd_sequence_fill_iterable(PyObject *obj, const void *ptr, simd_data_type dtype) { const simd_data_info *info = simd_data_getinfo(dtype); if (!PySequence_Check(obj)) { @@ -123,12 +126,12 @@ simd_sequence_fill_obj(PyObject *obj, const void *ptr, simd_data_type dtype) return -1; } const npyv_lanetype_u8 *src = ptr; - Py_ssize_t seq_len = (Py_ssize_t)simd_sequence_len(ptr); + Py_ssize_t seq_len = simd_sequence_len(ptr); for (Py_ssize_t i = 0; i < seq_len; ++i) { const npyv_lanetype_u8 *ssrc = src + i * info->lane_size; simd_data data; memcpy(&data.u64, ssrc, info->lane_size); - PyObject *item = simd_scalar_to_obj(data, info->to_scalar); + PyObject *item = simd_scalar_to_number(data, info->to_scalar); if (item == NULL) { return -1; } @@ -141,13 +144,13 @@ simd_sequence_fill_obj(PyObject *obj, const void *ptr, simd_data_type dtype) } static PyObject * -simd_sequence_to_obj(const void *ptr, simd_data_type dtype) +simd_sequence_to_list(const void *ptr, simd_data_type dtype) { - PyObject *list = PyList_New((Py_ssize_t)simd_sequence_len(ptr)); + PyObject *list = PyList_New(simd_sequence_len(ptr)); if (list == NULL) { return NULL; } - if (simd_sequence_fill_obj(list, ptr, dtype) < 0) { + if (simd_sequence_fill_iterable(list, ptr, dtype) < 0) { Py_DECREF(list); return NULL; } @@ -155,7 +158,7 @@ simd_sequence_to_obj(const void *ptr, simd_data_type dtype) } static simd_data -simd_vectorx_from_obj(PyObject *obj, simd_data_type dtype) +simd_vectorx_from_tuple(PyObject *obj, simd_data_type dtype) { const simd_data_info *info = simd_data_getinfo(dtype); // NPYV currently only supports x2 and x3 @@ -172,7 +175,7 @@ simd_vectorx_from_obj(PyObject *obj, simd_data_type dtype) for (int i = 0; i < info->is_vectorx; ++i) { PyObject *item = PyTuple_GET_ITEM(obj, i); // get the max multi-vec and let the compiler do the rest - data.vu64x3.val[i] = simd_vector_from_obj((simd_vector*)item, info->to_vector).vu64; + data.vu64x3.val[i] = PySIMDVector_AsData((PySIMDVectorObject*)item, info->to_vector).vu64; if (PyErr_Occurred()) { return data; } @@ -181,7 +184,7 @@ simd_vectorx_from_obj(PyObject *obj, simd_data_type dtype) } static PyObject * -simd_vectorx_to_obj(simd_data data, simd_data_type dtype) +simd_vectorx_to_tuple(simd_data data, simd_data_type dtype) { const simd_data_info *info = simd_data_getinfo(dtype); // NPYV currently only supports x2 and x3 @@ -194,7 +197,7 @@ simd_vectorx_to_obj(simd_data data, simd_data_type dtype) for (int i = 0; i < info->is_vectorx; ++i) { // get the max multi-vector and let the compiler handle the rest simd_data vdata = {.vu64 = data.vu64x3.val[i]}; - PyObject *item = (PyObject*)simd_vector_to_obj(vdata, info->to_vector); + PyObject *item = (PyObject*)PySIMDVector_FromData(vdata, info->to_vector); if (item == NULL) { // TODO: improve log add item number Py_DECREF(tuple); diff --git a/numpy/core/src/_simd/_simd_inc_data.h.src b/numpy/core/src/_simd/_simd_data.inc.src similarity index 89% rename from numpy/core/src/_simd/_simd_inc_data.h.src rename to numpy/core/src/_simd/_simd_data.inc.src index eefac483bf2c..5c796487c923 100644 --- a/numpy/core/src/_simd/_simd_inc_data.h.src +++ b/numpy/core/src/_simd/_simd_data.inc.src @@ -1,7 +1,9 @@ -#if !NPY_SIMD - #error "Not a standalone header, only works through 'simd.dispatch.c.src'" -#endif - +/** + * This file is included by `_simd.dispatch.c.src`. Its contents are affected by the simd configuration, and + * therefore must be built multiple times. Making it a standalone `.c` file with `NPY_VISIBILITY_HIDDEN` + * symbols would require judicious use of `NPY_CPU_DISPATCH_DECLARE` and `NPY_CPU_DISPATCH_CURFX`, which was + * deemed too harmful to readability. + */ /************************************ ** Private Definitions ************************************/ diff --git a/numpy/core/src/_simd/_simd_inc_easyintrin.h b/numpy/core/src/_simd/_simd_easyintrin.inc similarity index 68% rename from numpy/core/src/_simd/_simd_inc_easyintrin.h rename to numpy/core/src/_simd/_simd_easyintrin.inc index 9ff22799956f..54e7ccf01f50 100644 --- a/numpy/core/src/_simd/_simd_inc_easyintrin.h +++ b/numpy/core/src/_simd/_simd_easyintrin.inc @@ -1,33 +1,19 @@ -#if !NPY_SIMD - #error "Not a standalone header, only works through 'simd.dispatch.c.src'" -#endif - +/** + * This file is included by `_simd.dispatch.c.src`. Its contents are affected by the simd configuration, and + * therefore must be built multiple times. Making it a standalone `.c` file with `NPY_VISIBILITY_HIDDEN` + * symbols would require judicious use of `NPY_CPU_DISPATCH_DECLARE` and `NPY_CPU_DISPATCH_CURFX`, which was + * deemed too harmful to readability. + */ #define SIMD_INTRIN_DEF(NAME) \ { NPY_TOSTRING(NAME), simd__intrin_##NAME, METH_VARARGS, NULL } , // comma -static int simd__no_arguments(PyObject *args, const char* method_name) -{ - if (args == NULL) { - return 0; - } - assert(PyTuple_Check(args)); - Py_ssize_t obj_arg_len = PyTuple_GET_SIZE(args); - if (obj_arg_len != 0) { - PyErr_Format(PyExc_RuntimeError, - "%s(), takes no arguments, given(%d)", method_name, obj_arg_len - ); - return -1; - } - return 0; -} - #define SIMD_IMPL_INTRIN_0(NAME, RET) \ static PyObject *simd__intrin_##NAME \ (PyObject* NPY_UNUSED(self), PyObject *args) \ { \ - if (simd__no_arguments( \ - args, NPY_TOSTRING(NAME) \ - )) return NULL; \ + if (!PyArg_ParseTuple( \ + args, ":" NPY_TOSTRING(NAME)) \ + ) return NULL; \ simd_arg a = { \ .dtype = simd_data_##RET, \ .data = {.RET = npyv_##NAME()}, \ @@ -39,9 +25,9 @@ static int simd__no_arguments(PyObject *args, const char* method_name) static PyObject *simd__intrin_##NAME \ (PyObject* NPY_UNUSED(self), PyObject *args) \ { \ - if (simd__no_arguments( \ - args, NPY_TOSTRING(NAME) \ - )) return NULL; \ + if (!PyArg_ParseTuple( \ + args, ":" NPY_TOSTRING(NAME)) \ + ) return NULL; \ npyv_##NAME(); \ Py_RETURN_NONE; \ } @@ -55,88 +41,87 @@ static int simd__no_arguments(PyObject *args, const char* method_name) args, "O&:"NPY_TOSTRING(NAME), \ simd_arg_converter, &arg \ )) return NULL; \ - simd_data r = {.RET = npyv_##NAME( \ + simd_data data = {.RET = npyv_##NAME( \ arg.data.IN0 \ )}; \ - simd_args_sequence_free(&arg, 1); \ - arg.data = r; \ - arg.dtype = simd_data_##RET; \ - return simd_arg_to_obj(&arg); \ + simd_arg_free(&arg); \ + simd_arg ret = { \ + .data = data, .dtype = simd_data_##RET \ + }; \ + return simd_arg_to_obj(&ret); \ } #define SIMD_IMPL_INTRIN_2(NAME, RET, IN0, IN1) \ static PyObject *simd__intrin_##NAME \ (PyObject* NPY_UNUSED(self), PyObject *args) \ { \ - simd_arg req_args[] = { \ - {.dtype = simd_data_##IN0}, \ - {.dtype = simd_data_##IN1}, \ - }; \ + simd_arg arg1 = {.dtype = simd_data_##IN0}; \ + simd_arg arg2 = {.dtype = simd_data_##IN1}; \ if (!PyArg_ParseTuple( \ args, "O&O&:"NPY_TOSTRING(NAME), \ - simd_arg_converter, &req_args[0], \ - simd_arg_converter, &req_args[1] \ + simd_arg_converter, &arg1, \ + simd_arg_converter, &arg2 \ )) return NULL; \ - simd_data r = {.RET = npyv_##NAME( \ - req_args[0].data.IN0, \ - req_args[1].data.IN1 \ + simd_data data = {.RET = npyv_##NAME( \ + arg1.data.IN0, arg2.data.IN1 \ )}; \ - simd_args_sequence_free(req_args, 2); \ - req_args[0].data = r; \ - req_args[0].dtype = simd_data_##RET; \ - return simd_arg_to_obj(req_args); \ + simd_arg_free(&arg1); \ + simd_arg_free(&arg2); \ + simd_arg ret = { \ + .data = data, .dtype = simd_data_##RET \ + }; \ + return simd_arg_to_obj(&ret); \ } #define SIMD__REPEAT_2IMM(C, NAME, IN0) \ - C == req_args[1].data.u8 ? NPY_CAT(npyv_, NAME)(req_args[0].data.IN0, C) : + C == arg2.data.u8 ? NPY_CAT(npyv_, NAME)(arg1.data.IN0, C) : #define SIMD_IMPL_INTRIN_2IMM(NAME, RET, IN0, CONST_RNG) \ static PyObject *simd__intrin_##NAME \ (PyObject* NPY_UNUSED(self), PyObject *args) \ { \ - simd_arg req_args[] = { \ - {.dtype = simd_data_##IN0}, \ - {.dtype = simd_data_u8}, \ - }; \ + simd_arg arg1 = {.dtype = simd_data_##IN0}; \ + simd_arg arg2 = {.dtype = simd_data_u8}; \ if (!PyArg_ParseTuple( \ args, "O&O&:"NPY_TOSTRING(NAME), \ - simd_arg_converter, &req_args[0], \ - simd_arg_converter, &req_args[1] \ + simd_arg_converter, &arg1, \ + simd_arg_converter, &arg2 \ )) return NULL; \ - simd_data r; \ - r.RET = NPY_CAT(SIMD__IMPL_COUNT_, CONST_RNG)( \ + simd_data data; \ + data.RET = NPY_CAT(SIMD__IMPL_COUNT_, CONST_RNG)( \ SIMD__REPEAT_2IMM, NAME, IN0 \ - ) npyv_##NAME(req_args[0].data.IN0, 0); \ - simd_args_sequence_free(req_args, 2); \ - req_args[0].data = r; \ - req_args[0].dtype = simd_data_##RET; \ - return simd_arg_to_obj(req_args); \ + ) npyv_##NAME(arg1.data.IN0, 0); \ + simd_arg_free(&arg1); \ + simd_arg ret = { \ + .data = data, .dtype = simd_data_##RET \ + }; \ + return simd_arg_to_obj(&ret); \ } #define SIMD_IMPL_INTRIN_3(NAME, RET, IN0, IN1, IN2) \ static PyObject *simd__intrin_##NAME \ (PyObject* NPY_UNUSED(self), PyObject *args) \ { \ - simd_arg req_args[] = { \ - {.dtype = simd_data_##IN0}, \ - {.dtype = simd_data_##IN1}, \ - {.dtype = simd_data_##IN2}, \ - }; \ + simd_arg arg1 = {.dtype = simd_data_##IN0}; \ + simd_arg arg2 = {.dtype = simd_data_##IN1}; \ + simd_arg arg3 = {.dtype = simd_data_##IN2}; \ if (!PyArg_ParseTuple( \ args, "O&O&O&:"NPY_TOSTRING(NAME), \ - simd_arg_converter, &req_args[0], \ - simd_arg_converter, &req_args[1], \ - simd_arg_converter, &req_args[2] \ + simd_arg_converter, &arg1, \ + simd_arg_converter, &arg2, \ + simd_arg_converter, &arg3 \ )) return NULL; \ - simd_data r = {.RET = npyv_##NAME( \ - req_args[0].data.IN0, \ - req_args[1].data.IN1, \ - req_args[2].data.IN2 \ + simd_data data = {.RET = npyv_##NAME( \ + arg1.data.IN0, arg2.data.IN1, \ + arg3.data.IN2 \ )}; \ - simd_args_sequence_free(req_args, 3); \ - req_args[0].data = r; \ - req_args[0].dtype = simd_data_##RET; \ - return simd_arg_to_obj(req_args); \ + simd_arg_free(&arg1); \ + simd_arg_free(&arg2); \ + simd_arg_free(&arg3); \ + simd_arg ret = { \ + .data = data, .dtype = simd_data_##RET \ + }; \ + return simd_arg_to_obj(&ret); \ } /** * Helper macros for repeating and expand a certain macro. diff --git a/numpy/core/src/_simd/_simd_inc.h.src b/numpy/core/src/_simd/_simd_inc.h.src index 530c7a736294..9858fc0dc624 100644 --- a/numpy/core/src/_simd/_simd_inc.h.src +++ b/numpy/core/src/_simd/_simd_inc.h.src @@ -1,6 +1,6 @@ -/** - * This header works only through '_simd.dispatch.c' - */ +#ifndef _SIMD_SIMD_INC_H_ +#define _SIMD_SIMD_INC_H_ + #include #include "simd/simd.h" @@ -147,29 +147,47 @@ typedef struct { PyObject_HEAD // vector type id - simd_data_type type; + simd_data_type dtype; // vector data, aligned for safe casting npyv_lanetype_u8 NPY_DECL_ALIGNED(NPY_SIMD_WIDTH) data[NPY_SIMD_WIDTH]; -} simd_vector; +} PySIMDVectorObject; /** - * convert simd_data to PyObject(simd_vector), - * raise Python exception on failure and returns NULL. + * Create a Python obj(PySIMDVectorObject) from a NPYV vector based on the contents + * of `data`(simd_data) and according to the vector data type `dtype` + * on range(simd_data_[vu8:vf64]). + * Return NULL and a Python exception on failure, otherwise new reference. + * + * Example: + ** simd_data data = {.vu8 = npyv_setall_u8(0xff)}; + ** PySIMDVectorObject *obj = PySIMDVector_FromData(data, simd_data_vu8); + ** if (obj != NULL) { + ** printf("I have a valid vector obj and first element is \n", obj->data[0]); + ** Py_DECREF(obj); + ** } */ -static simd_vector * -simd_vector_to_obj(simd_data data, simd_data_type vtype); +static PySIMDVectorObject * +PySIMDVector_FromData(simd_data data, simd_data_type dtype); /** - * convert PyObject(simd_vector) to simd_data, - * raise Python exception on failure. + * Return a NPYV vector(simd_data) representation of `obj`(PySIMDVectorObject) and + * according to the vector data type `dtype` on range (simd_data_[vu8:vf64]). + * Raise a Python exception on failure. + * + * Example: + ** simd_data data = PySIMDVector_AsData(vec_obj, simd_data_vf32); + ** if (!PyErr_Occurred()) { + ** npyv_f32 add_1 = npyv_add_f32(data.vf32, npyv_setall_f32(1)); + ** ... + ** } */ static simd_data -simd_vector_from_obj(simd_vector *vec, simd_data_type vtype); +PySIMDVector_AsData(PySIMDVectorObject *obj, simd_data_type dtype); /** - * initialize and register vector type(PyTypeObject) to PyModule, - * vector type can be reached through attribute 'vector_type'. + * initialize and register PySIMDVectorType to certain PyModule, + * PySIMDVectorType can be reached through attribute 'vector_type'. * return -1 on error, 0 on success. */ static int -simd_vector_register(PyObject *module); +PySIMDVectorType_Init(PyObject *module); /************************************ ** Declarations (inc_convert) @@ -180,13 +198,13 @@ simd_vector_register(PyObject *module); * Raise a Python exception on failure. * * Example: - ** simd_data data = simd_scalar_from_obj(obj, simd_data_f32); + ** simd_data data = simd_scalar_from_number(obj, simd_data_f32); ** if (!PyErr_Occurred()) { ** printf("I have a valid float %d\n", data.f32); ** } */ static simd_data -simd_scalar_from_obj(PyObject *obj, simd_data_type dtype); +simd_scalar_from_number(PyObject *obj, simd_data_type dtype); /** * Create a Python scalar from a C scalar based on the contents * of `data`(simd_data) and according to the scalar data type `dtype` @@ -195,14 +213,14 @@ simd_scalar_from_obj(PyObject *obj, simd_data_type dtype); * * Example: ** simd_data data = {.u32 = 0x7fffffff}; - ** PyObject *obj = simd_scalar_to_obj(data, simd_data_s32); + ** PyObject *obj = simd_scalar_to_number(data, simd_data_s32); ** if (obj != NULL) { ** printf("I have a valid Python integer %d\n", PyLong_AsLong(obj)); ** Py_DECREF(obj); ** } */ static PyObject * -simd_scalar_to_obj(simd_data data, simd_data_type dtype); +simd_scalar_to_number(simd_data data, simd_data_type dtype); /** * Allocate a C array in memory according to number of elements `len` * and sequence data type `dtype` on range(simd_data_[qu8:qf64]). @@ -223,13 +241,13 @@ static void * simd_sequence_new(Py_ssize_t len, simd_data_type dtype); /** * Return the number of elements of the allocated C array `ptr` - * by `simd_sequence_new()` or `simd_sequence_from_obj()`. + * by `simd_sequence_new()` or `simd_sequence_from_iterable()`. */ -static size_t +static Py_ssize_t simd_sequence_len(const void *ptr); /** * Free the allocated C array by `simd_sequence_new()` or - * `simd_sequence_from_obj()`. + * `simd_sequence_from_iterable()`. */ static void simd_sequence_free(void *ptr); @@ -243,7 +261,7 @@ simd_sequence_free(void *ptr); * with a Python exception on failure. * * Example: - ** npyv_lanetype_u32 *ptr = simd_sequence_from_obj(seq_obj, simd_data_qu32, npyv_nlanes_u32); + ** npyv_lanetype_u32 *ptr = simd_sequence_from_iterable(seq_obj, simd_data_qu32, npyv_nlanes_u32); ** if (ptr != NULL) { ** npyv_u32 a = npyv_load_u32(ptr); ** ... @@ -252,25 +270,25 @@ simd_sequence_free(void *ptr); ** */ static void * -simd_sequence_from_obj(PyObject *obj, simd_data_type dtype, unsigned min_size); +simd_sequence_from_iterable(PyObject *obj, simd_data_type dtype, Py_ssize_t min_size); /** * Fill a Python sequence object `obj` with a C array `ptr` allocated by - * `simd_sequence_new()` or `simd_sequence_from_obj()` according to + * `simd_sequence_new()` or `simd_sequence_from_iterable()` according to * to the sequence data type `dtype` on range (simd_data_[qu8:qf64]). * * Return 0 on success or -1 with a Python exception on failure. */ static int -simd_sequence_fill_obj(PyObject *obj, const void *ptr, simd_data_type dtype); +simd_sequence_fill_iterable(PyObject *obj, const void *ptr, simd_data_type dtype); /** * Create a Python list from a C array `ptr` allocated by - * `simd_sequence_new()` or `simd_sequence_from_obj()` according to + * `simd_sequence_new()` or `simd_sequence_from_iterable()` according to * to the sequence data type `dtype` on range (simd_data_[qu8:qf64]). * * Return NULL and a Python exception on failure, otherwise new reference. */ static PyObject * -simd_sequence_to_obj(const void *ptr, simd_data_type dtype); +simd_sequence_to_list(const void *ptr, simd_data_type dtype); /** * Return a SIMD multi-vector(simd_data) representation of Python tuple of * (simd_vector*,) `obj` according to the scalar data type `dtype` @@ -279,7 +297,7 @@ simd_sequence_to_obj(const void *ptr, simd_data_type dtype); * Raise a Python exception on failure. * * Example: - ** simd_data data = simd_vectorx_from_obj(tuple_obj, simd_data_vf32x2); + ** simd_data data = simd_vectorx_from_tuple(tuple_obj, simd_data_vf32x2); ** if (!PyErr_Occurred()) { ** npyv_f32 sum = npyv_add_f32(data.vf32x2.val[0], data.vf32x2.val[1]); ** ... @@ -287,7 +305,7 @@ simd_sequence_to_obj(const void *ptr, simd_data_type dtype); ** */ static simd_data -simd_vectorx_from_obj(PyObject *obj, simd_data_type dtype); +simd_vectorx_from_tuple(PyObject *obj, simd_data_type dtype); /** * Create a Python tuple of 'simd_vector' from a SIMD multi-vector * based on the contents of `data`(simd_data) and according to @@ -297,7 +315,7 @@ simd_vectorx_from_obj(PyObject *obj, simd_data_type dtype); * Return NULL and a Python exception on failure, otherwise new reference. */ static PyObject * -simd_vectorx_to_obj(simd_data data, simd_data_type dtype); +simd_vectorx_to_tuple(simd_data data, simd_data_type dtype); /************************************ ** Declarations (inc_arg) @@ -320,7 +338,7 @@ typedef struct * Return -1 and raise Python exception on failure, otherwise return 0. * * Notes: - * - requires `simd_args_sequence_free()` or `simd_sequence_free()` + * - requires `simd_arg_free()` or `simd_sequence_free()` * to free allocated C array, in case of sequence data types. * - the number of minimum acceptable elements for sequence data * types is the number of lanes of the equivalent vector data type. @@ -333,7 +351,7 @@ typedef struct ** } ** npyv_u8 v_u8 = npyv_load_u8(arg->data.qu8); ** ... - ** simd_args_sequence_free(&arg, 1); + ** simd_arg_free(&arg); * * Example #2: ** simd_arg arg = {.dtype = simd_data_vf32}; @@ -369,36 +387,35 @@ simd_arg_to_obj(const simd_arg *arg); * used with PyArg_Parse*(). * * Notes: - * - requires `simd_args_sequence_free()` or `simd_sequence_free()` + * - requires `simd_arg_free()` or `simd_sequence_free()` * to free allocated C array, in case of sequence data types. * - the number of minimum acceptable elements for sequence data * types is the number of lanes of the equivalent vector data type. * - use 'arg->obj' to retrieve the parameter obj. * * Example: - ** simd_arg req_args[] = { - ** {.dtype = simd_data_qf32}, - ** {.dtype = simd_data_vf32}, - ** }; + ** simd_arg seq_f32 = {.dtype = simd_data_qf32}; + ** simd_arg vec_f32 = {.dtype = simd_data_vf32}; ** if (!PyArg_ParseTuple( ** args, "O&O&:add_sum_f32", - ** simd_arg_converter, &req_args[0], - ** simd_arg_converter, &req_args[1] + ** simd_arg_converter, &seq_f32, + ** simd_arg_converter, &vec_f32 ** )) { ** // fail ** return; ** } - ** npyv_f32 load_a = npyv_load_f32(req_args[0].data.qf32); - ** npyv_f32 sum = npyv_add_f32(load_a, req_args[1].data.vf32); + ** npyv_f32 load_a = npyv_load_f32(seq_f32.data.qf32); + ** npyv_f32 sum = npyv_add_f32(load_a, vec_f32.data.vf32); ** ... - ** simd_args_sequence_free(req_args, 2); + ** simd_arg_free(&seq_f32); */ static int simd_arg_converter(PyObject *obj, simd_arg *arg); /** - * Free the allocated C array for sequence data types. + * Free the allocated C array, if the arg hold sequence data type. */ static void -simd_args_sequence_free(simd_arg *args, int args_len); +simd_arg_free(simd_arg *arg); #endif // NPY_SIMD +#endif // _SIMD_SIMD_INC_H_ diff --git a/numpy/core/src/_simd/_simd_inc_vector.h b/numpy/core/src/_simd/_simd_vector.inc similarity index 60% rename from numpy/core/src/_simd/_simd_inc_vector.h rename to numpy/core/src/_simd/_simd_vector.inc index b0fa17b9ab9e..2a1378f22100 100644 --- a/numpy/core/src/_simd/_simd_inc_vector.h +++ b/numpy/core/src/_simd/_simd_vector.inc @@ -1,63 +1,54 @@ -#if !NPY_SIMD - #error "Not a standalone header, only works through 'simd.dispatch.c.src'" -#endif - +/** + * This file is included by `_simd.dispatch.c.src`. Its contents are affected by the simd configuration, and + * therefore must be built multiple times. Making it a standalone `.c` file with `NPY_VISIBILITY_HIDDEN` + * symbols would require judicious use of `NPY_CPU_DISPATCH_DECLARE` and `NPY_CPU_DISPATCH_CURFX`, which was + * deemed too harmful to readability. + */ /************************************ ** Private Definitions ************************************/ -// PySequenceMethods static Py_ssize_t -simd__vector_length(simd_vector *self) +simd__vector_length(PySIMDVectorObject *self) { - return simd_data_getinfo(self->type)->nlanes; + return simd_data_getinfo(self->dtype)->nlanes; } static PyObject * -simd__vector_item(simd_vector *self, Py_ssize_t i) +simd__vector_item(PySIMDVectorObject *self, Py_ssize_t i) { - const simd_data_info *info = simd_data_getinfo(self->type); + const simd_data_info *info = simd_data_getinfo(self->dtype); int nlanes = info->nlanes; if (i >= nlanes) { - PyErr_SetString(PyExc_IndexError, "list index out of range"); + PyErr_SetString(PyExc_IndexError, "vector index out of range"); return NULL; } npyv_lanetype_u8 *src = self->data + i * info->lane_size; simd_data data; memcpy(&data.u64, src, info->lane_size); - return simd_scalar_to_obj(data, info->to_scalar); + return simd_scalar_to_number(data, info->to_scalar); } static PySequenceMethods simd__vector_as_sequence = { - (lenfunc) simd__vector_length, /* sq_length */ - (binaryfunc) NULL, /* sq_concat */ - (ssizeargfunc) NULL, /* sq_repeat */ - (ssizeargfunc) simd__vector_item, /* sq_item */ - (ssizessizeargfunc) NULL, /* sq_slice */ - (ssizeobjargproc) NULL, /* sq_ass_item */ - (ssizessizeobjargproc) NULL, /* sq_ass_slice */ - (objobjproc) NULL, /* sq_contains */ - (binaryfunc) NULL, /* sq_inplace_concat */ - (ssizeargfunc) NULL, /* sq_inplace_repeat */ + .sq_length = (lenfunc) simd__vector_length, + .sq_item = (ssizeargfunc) simd__vector_item }; -// PyGetSetDef static PyObject * -simd__vector_name(simd_vector *self) +simd__vector_name(PySIMDVectorObject *self) { - return PyUnicode_FromString(simd_data_getinfo(self->type)->pyname); + return PyUnicode_FromString(simd_data_getinfo(self->dtype)->pyname); } static PyGetSetDef simd__vector_getset[] = { { "__name__", (getter)simd__vector_name, NULL, NULL, NULL }, { NULL, NULL, NULL, NULL, NULL } }; -// PyTypeObject(simd__vector_type) static PyObject * -simd__vector_repr(PyObject *self) +simd__vector_repr(PySIMDVectorObject *self) { - // PySequence_Fast returns Tuple in PyPy - PyObject *obj = PySequence_List(self); + PyObject *obj = PySequence_List((PyObject*)self); if (obj != NULL) { - PyObject *repr = PyObject_Str(obj); + const char *type_name = simd_data_getinfo(self->dtype)->pyname; + PyObject *repr = PyUnicode_FromFormat("<%s of %R>", type_name, obj); Py_DECREF(obj); return repr; } @@ -81,10 +72,10 @@ simd__vector_compare(PyObject *self, PyObject *other, int cmp_op) } return obj; } -static PyTypeObject simd__vector_type = { +static PyTypeObject PySIMDVectorType = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = NPY_TOSTRING(NPY_CPU_DISPATCH_CURFX(VECTOR)), - .tp_basicsize = sizeof(simd_vector), + .tp_basicsize = sizeof(PySIMDVectorObject), .tp_repr = (reprfunc)simd__vector_repr, .tp_as_sequence = &simd__vector_as_sequence, .tp_flags = Py_TPFLAGS_DEFAULT, @@ -95,21 +86,21 @@ static PyTypeObject simd__vector_type = { /************************************ ** Protected Definitions ************************************/ -static simd_vector * -simd_vector_to_obj(simd_data data, simd_data_type vtype) +static PySIMDVectorObject * +PySIMDVector_FromData(simd_data data, simd_data_type dtype) { - const simd_data_info *info = simd_data_getinfo(vtype); + const simd_data_info *info = simd_data_getinfo(dtype); assert(info->is_vector && info->nlanes > 0); - simd_vector *vec = PyObject_New(simd_vector, &simd__vector_type); + PySIMDVectorObject *vec = PyObject_New(PySIMDVectorObject, &PySIMDVectorType); if (vec == NULL) { - return (simd_vector*)PyErr_NoMemory(); + return (PySIMDVectorObject*)PyErr_NoMemory(); } - vec->type = vtype; + vec->dtype = dtype; if (info->is_bool) { // boolean vectors are internally treated as unsigned // vectors to add compatibility among all SIMD extensions - switch(vtype) { + switch(dtype) { case simd_data_vb8: data.vu8 = npyv_cvt_u8_b8(data.vb8); break; @@ -128,24 +119,24 @@ simd_vector_to_obj(simd_data data, simd_data_type vtype) } static simd_data -simd_vector_from_obj(simd_vector *vec, simd_data_type vtype) +PySIMDVector_AsData(PySIMDVectorObject *vec, simd_data_type dtype) { - const simd_data_info *info = simd_data_getinfo(vtype); + const simd_data_info *info = simd_data_getinfo(dtype); assert(info->is_vector && info->nlanes > 0); simd_data data = {.u64 = 0}; if (!PyObject_IsInstance( - (PyObject *)vec, (PyObject *)&simd__vector_type + (PyObject *)vec, (PyObject *)&PySIMDVectorType )) { PyErr_Format(PyExc_TypeError, "a vector type %s is required", info->pyname ); return data; } - if (vec->type != vtype) { + if (vec->dtype != dtype) { PyErr_Format(PyExc_TypeError, "a vector type %s is required, got(%s)", - info->pyname, simd_data_getinfo(vec->type)->pyname + info->pyname, simd_data_getinfo(vec->dtype)->pyname ); return data; } @@ -154,7 +145,7 @@ simd_vector_from_obj(simd_vector *vec, simd_data_type vtype) if (info->is_bool) { // boolean vectors are internally treated as unsigned // vectors to add compatibility among all SIMD extensions - switch(vtype) { + switch(dtype) { case simd_data_vb8: data.vb8 = npyv_cvt_b8_u8(data.vu8); break; @@ -172,14 +163,14 @@ simd_vector_from_obj(simd_vector *vec, simd_data_type vtype) } static int -simd_vector_register(PyObject *module) +PySIMDVectorType_Init(PyObject *module) { - Py_INCREF(&simd__vector_type); - if (PyType_Ready(&simd__vector_type)) { + Py_INCREF(&PySIMDVectorType); + if (PyType_Ready(&PySIMDVectorType)) { return -1; } if (PyModule_AddObject( - module, "vector_type",(PyObject *)&simd__vector_type + module, "vector_type",(PyObject *)&PySIMDVectorType )) { return -1; } diff --git a/numpy/core/tests/test_simd.py b/numpy/core/tests/test_simd.py index 3ca6b068dea8..77a636491172 100644 --- a/numpy/core/tests/test_simd.py +++ b/numpy/core/tests/test_simd.py @@ -3,7 +3,7 @@ import pytest from numpy.core._simd import targets -class _Test_Utility(object): +class _Test_Utility: # submodule of the desired SIMD extention, e.g. targets["AVX512F"] npyv = None # the current data type suffix e.g. 's8' @@ -14,10 +14,7 @@ def __getattr__(self, attr): To call NPV intrinsics without the prefix 'npyv_' and auto suffixing intrinsics according to class attribute 'sfx' """ - nattr = getattr(self.npyv, attr + "_" + self.sfx) - if callable(nattr): - return lambda *args: nattr(*args) - return nattr + return getattr(self.npyv, attr + "_" + self.sfx) def _data(self, n=None, reverse=False): """ @@ -87,7 +84,7 @@ class _SIMD_INT(_Test_Utility): """ def test_operators_shift(self): if self.sfx in ("u8", "s8"): - pytest.skip("there are no shift intrinsics for npyv_" + self.sfx) + return data_a = self._data(self._int_max() - self.nlanes) data_b = self._data(self._int_min(), reverse=True) @@ -113,7 +110,7 @@ def test_operators_shift(self): def test_arithmetic_subadd_saturated(self): if self.sfx in ("u32", "s32", "u64", "s64"): - pytest.skip("there are no saturated add/sub intrinsics for npyv_" + self.sfx) + return data_a = self._data(self._int_max() - self.nlanes) data_b = self._data(self._int_min(), reverse=True) @@ -362,7 +359,7 @@ def test_arithmetic_subadd(self): def test_arithmetic_mul(self): if self.sfx in ("u64", "s64"): - pytest.skip("there is no multiplication intrinsic for npyv_" + self.sfx) + return if self._is_fp(): data_a = self._data() @@ -377,7 +374,7 @@ def test_arithmetic_mul(self): def test_arithmetic_div(self): if not self._is_fp(): - pytest.skip("there is no division intrinsic for npyv_" + self.sfx) + return data_a, data_b = self._data(), self._data(reverse=True) vdata_a, vdata_b = self.load(data_a), self.load(data_b) @@ -387,17 +384,18 @@ def test_arithmetic_div(self): div = self.div(vdata_a, vdata_b) assert div == data_div + int_sfx = ("u8", "s8", "u16", "s16", "u32", "s32", "u64", "s64") fp_sfx = ("f32", "f64") all_sfx = int_sfx + fp_sfx tests_registery = { - int_sfx : "_SIMD_INT", - fp_sfx : "_SIMD_FP", - all_sfx : "_SIMD_ALL" + int_sfx : _SIMD_INT, + fp_sfx : _SIMD_FP, + all_sfx : _SIMD_ALL } -for name, npyv in targets.items(): +for target_name, npyv in targets.items(): simd_width = npyv.simd if npyv else '' - pretty_name = name.split('__') # multi-target separator + pretty_name = target_name.split('__') # multi-target separator if len(pretty_name) > 1: # multi-target pretty_name = f"({' '.join(pretty_name)})" @@ -413,14 +411,12 @@ def test_arithmetic_div(self): elif not npyv.simd_f64: skip_sfx["f64"] = f"target '{pretty_name}' doesn't support double-precision" - for sfxes, class_name in tests_registery.items(): + for sfxes, cls in tests_registery.items(): for sfx in sfxes: skip_m = skip_sfx.get(sfx, skip) + inhr = (cls,) + attr = dict(npyv=targets[target_name], sfx=sfx) + tcls = type(f"Test{cls.__name__}_{simd_width}_{target_name}_{sfx}", inhr, attr) if skip_m: - skip_m = '@pytest.mark.skip(reason="%s")' % skip_m - exec( - f"{skip_m}\n" - f"class Test{class_name}_{simd_width}_{name}_{sfx}({class_name}):\n" - f" npyv = targets['{name}']\n" - f" sfx = '{sfx}'\n" - ) + pytest.mark.skip(reason=skip_m)(tcls) + globals()[tcls.__name__] = tcls diff --git a/numpy/core/tests/test_simd_module.py b/numpy/core/tests/test_simd_module.py index 5bf82c0aa024..3d710884ab09 100644 --- a/numpy/core/tests/test_simd_module.py +++ b/numpy/core/tests/test_simd_module.py @@ -1,16 +1,14 @@ import pytest from numpy.core._simd import targets - -npyv = None -npyv2 = None -for target_name, npyv_mod in targets.items(): - if npyv: - if npyv_mod and npyv_mod.simd: - npyv2 = npyv_mod - break - continue - if npyv_mod and npyv_mod.simd: - npyv = npyv_mod +""" +This testing unit only for checking the sanity of common functionality, +therefore all we need is just to take one submodule that represents any +of enabled SIMD extensions to run the test on it and the second submodule +required to run only one check related to the possibility of mixing +the data types among each submodule. +""" +npyvs = [npyv_mod for npyv_mod in targets.values() if npyv_mod and npyv_mod.simd] +npyv, npyv2 = (npyvs + [None, None])[:2] unsigned_sfx = ["u8", "u16", "u32", "u64"] signed_sfx = ["s8", "s16", "s32", "s64"] @@ -22,73 +20,68 @@ all_sfx = unsigned_sfx + int_sfx @pytest.mark.skipif(not npyv, reason="could not find any SIMD extension with NPYV support") -class Test_SIMD_MODULE(object): - def test_num_lanes(self): - for sfx in all_sfx: - nlanes = getattr(npyv, "nlanes_" + sfx) - vector = getattr(npyv, "setall_" + sfx)(1) - assert len(vector) == nlanes +class Test_SIMD_MODULE: - def test_type_name(self): - for sfx in all_sfx: - vector = getattr(npyv, "setall_" + sfx)(1) - assert vector.__name__ == "npyv_" + sfx + @pytest.mark.parametrize('sfx', all_sfx) + def test_num_lanes(self, sfx): + nlanes = getattr(npyv, "nlanes_" + sfx) + vector = getattr(npyv, "setall_" + sfx)(1) + assert len(vector) == nlanes - def test_raises(self): - def assert_raises(e, callback, *args): - __tracebackhide__ = True # Hide traceback for py.test - try: - callback(*args) - raise AssertionError("expected to raise " + e.__name__) - except e: - pass + @pytest.mark.parametrize('sfx', all_sfx) + def test_type_name(self, sfx): + vector = getattr(npyv, "setall_" + sfx)(1) + assert vector.__name__ == "npyv_" + sfx + def test_raises(self): a, b = [npyv.setall_u32(1)]*2 for sfx in all_sfx: vcb = lambda intrin: getattr(npyv, f"{intrin}_{sfx}") - assert_raises(TypeError, vcb("add"), a) - assert_raises(TypeError, vcb("add"), a, b, a) - assert_raises(TypeError, vcb("setall")) - assert_raises(TypeError, vcb("setall"), [1]) - assert_raises(TypeError, vcb("load"), 1) - assert_raises(ValueError, vcb("load"), [1]) - assert_raises(ValueError, vcb("store"), [1], getattr(npyv, f"reinterpret_{sfx}_u32")(a)) + pytest.raises(TypeError, vcb("add"), a) + pytest.raises(TypeError, vcb("add"), a, b, a) + pytest.raises(TypeError, vcb("setall")) + pytest.raises(TypeError, vcb("setall"), [1]) + pytest.raises(TypeError, vcb("load"), 1) + pytest.raises(ValueError, vcb("load"), [1]) + pytest.raises(ValueError, vcb("store"), [1], getattr(npyv, f"reinterpret_{sfx}_u32")(a)) + @pytest.mark.skipif(not npyv2, reason=( + "could not find a second SIMD extension with NPYV support" + )) + def test_nomix(self): # mix among submodules isn't allowed - if not npyv2: - return + a = npyv.setall_u32(1) a2 = npyv2.setall_u32(1) - assert_raises(TypeError, npyv.add_u32, a2, a2) - assert_raises(TypeError, npyv2.add_u32, a, a) + pytest.raises(TypeError, npyv.add_u32, a2, a2) + pytest.raises(TypeError, npyv2.add_u32, a, a) - def test_unsigned_overflow(self): - for sfx in unsigned_sfx: - nlanes = getattr(npyv, "nlanes_" + sfx) - hfbyte_len = int(sfx[1:])//4 - maxu = int(f"0x{'f'*hfbyte_len}", 16) - maxu_72 = 0xfffffffffffffffff - lane = getattr(npyv, "setall_" + sfx)(maxu_72)[0] - assert lane == maxu - lanes = getattr(npyv, "load_" + sfx)([maxu_72] * nlanes) - assert lanes == [maxu] * nlanes - lane = getattr(npyv, "setall_" + sfx)(-1)[0] - assert lane == maxu - lanes = getattr(npyv, "load_" + sfx)([-1] * nlanes) - assert lanes == [maxu] * nlanes + @pytest.mark.parametrize('sfx', unsigned_sfx) + def test_unsigned_overflow(self, sfx): + nlanes = getattr(npyv, "nlanes_" + sfx) + maxu = (1 << int(sfx[1:])) - 1 + maxu_72 = (1 << 72) - 1 + lane = getattr(npyv, "setall_" + sfx)(maxu_72)[0] + assert lane == maxu + lanes = getattr(npyv, "load_" + sfx)([maxu_72] * nlanes) + assert lanes == [maxu] * nlanes + lane = getattr(npyv, "setall_" + sfx)(-1)[0] + assert lane == maxu + lanes = getattr(npyv, "load_" + sfx)([-1] * nlanes) + assert lanes == [maxu] * nlanes - def test_signed_overflow(self): - for sfx in signed_sfx: - nlanes = getattr(npyv, "nlanes_" + sfx) - maxs_72 = 0x7fffffffffffffffff - lane = getattr(npyv, "setall_" + sfx)(maxs_72)[0] - assert lane == -1 - lanes = getattr(npyv, "load_" + sfx)([maxs_72] * nlanes) - assert lanes == [-1] * nlanes - mins_72 = -0x80000000000000000 - lane = getattr(npyv, "setall_" + sfx)(mins_72)[0] - assert lane == 0 - lanes = getattr(npyv, "load_" + sfx)([mins_72] * nlanes) - assert lanes == [0] * nlanes + @pytest.mark.parametrize('sfx', signed_sfx) + def test_signed_overflow(self, sfx): + nlanes = getattr(npyv, "nlanes_" + sfx) + maxs_72 = (1 << 71) - 1 + lane = getattr(npyv, "setall_" + sfx)(maxs_72)[0] + assert lane == -1 + lanes = getattr(npyv, "load_" + sfx)([maxs_72] * nlanes) + assert lanes == [-1] * nlanes + mins_72 = -1 << 71 + lane = getattr(npyv, "setall_" + sfx)(mins_72)[0] + assert lane == 0 + lanes = getattr(npyv, "load_" + sfx)([mins_72] * nlanes) + assert lanes == [0] * nlanes def test_truncate_f32(self): f32 = npyv.setall_f32(0.1)[0] diff --git a/numpy/distutils/command/build.py b/numpy/distutils/command/build.py index 6025586cd3d0..a4fda537d5dc 100644 --- a/numpy/distutils/command/build.py +++ b/numpy/distutils/command/build.py @@ -47,7 +47,7 @@ def initialize_options(self): - not part of dispatch-able features(--cpu-dispatch) - not supported by compiler or platform """ - self.simd_test = "BASELINE SSE2 SSE41 SSE42 XOP (FMA3 AVX2) AVX512F AVX512_SKX VSX VSX2 VSX3 NEON ASIMD" + self.simd_test = "BASELINE SSE2 SSE42 XOP FMA4 (FMA3 AVX2) AVX512F AVX512_SKX VSX VSX2 VSX3 NEON ASIMD" def finalize_options(self): build_scripts = self.build_scripts From 018e0cabcf11196eefa7aae778076062b3560ec7 Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Thu, 15 Oct 2020 20:12:22 +0200 Subject: [PATCH 375/409] MAINT: Syntax highlighting for *.inc.src on github --- .gitattributes | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitattributes b/.gitattributes index ad7d3b227108..bce3dbe6daad 100644 --- a/.gitattributes +++ b/.gitattributes @@ -7,6 +7,7 @@ doc/release/*.rst merge=union # Highlight our custom templating language as C, since it's hopefully better # than nothing. This also affects repo statistics. *.c.src linguist-language=C +*.inc.src linguist-language=C *.h.src linguist-language=C # Mark some files as vendored From d54a45b28fdaad24eedf8a2ffee19d9afc7f2e69 Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Thu, 15 Oct 2020 22:13:41 +0200 Subject: [PATCH 376/409] BUG, BLD: fix multi-target name in `CCompilerOpt`'s report --- numpy/distutils/ccompiler_opt.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/numpy/distutils/ccompiler_opt.py b/numpy/distutils/ccompiler_opt.py index 72ea0c3888c5..3eba6e32af91 100644 --- a/numpy/distutils/ccompiler_opt.py +++ b/numpy/distutils/ccompiler_opt.py @@ -2372,19 +2372,18 @@ def report(self, full=False): else: dispatch_rows.append(("Generated", '')) for tar in self.feature_sorted(target_sources): - tar_as_seq = [tar] if isinstance(tar, str) else tar sources = target_sources[tar] - name = tar if isinstance(tar, str) else '(%s)' % ' '.join(tar) + pretty_name = tar if isinstance(tar, str) else '(%s)' % ' '.join(tar) flags = ' '.join(self.feature_flags(tar)) implies = ' '.join(self.feature_sorted(self.feature_implies(tar))) detect = ' '.join(self.feature_detect(tar)) extra_checks = [] - for name in tar_as_seq: + for name in ((tar,) if isinstance(tar, str) else tar): extra_checks += self.feature_extra_checks(name) extra_checks = (' '.join(extra_checks) if extra_checks else "none") dispatch_rows.append(('', '')) - dispatch_rows.append((name, implies)) + dispatch_rows.append((pretty_name, implies)) dispatch_rows.append(("Flags", flags)) dispatch_rows.append(("Extra checks", extra_checks)) dispatch_rows.append(("Detect", detect)) From 8cc5009857391ab2a11fb1af1042c0b83544e97d Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Tue, 27 Oct 2020 00:04:20 +0000 Subject: [PATCH 377/409] MAINT, TST: Add _SIMD attribute for each enabled SIMD extension --- numpy/core/src/_simd/_simd.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/numpy/core/src/_simd/_simd.c b/numpy/core/src/_simd/_simd.c index e5cb582b3962..d2e26d82125d 100644 --- a/numpy/core/src/_simd/_simd.c +++ b/numpy/core/src/_simd/_simd.c @@ -40,6 +40,11 @@ PyMODINIT_FUNC PyInit__simd(void) Py_DECREF(simd_mod); \ goto err; \ } \ + Py_INCREF(simd_mod); \ + if (PyModule_AddObject(m, target_name, simd_mod) < 0) { \ + Py_DECREF(simd_mod); \ + goto err; \ + } \ } #define ATTACH_BASELINE_MODULE(MAKE_MSVC_HAPPY) \ @@ -52,6 +57,11 @@ PyMODINIT_FUNC PyInit__simd(void) Py_DECREF(simd_mod); \ goto err; \ } \ + Py_INCREF(simd_mod); \ + if (PyModule_AddObject(m, "baseline", simd_mod) < 0) { \ + Py_DECREF(simd_mod); \ + goto err; \ + } \ } NPY__CPU_DISPATCH_CALL(NPY_CPU_HAVE, ATTACH_MODULE, MAKE_MSVC_HAPPY) From 5d8c3be00dcc2b4a64dcd1ff108474b992edbc53 Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Tue, 27 Oct 2020 11:43:09 +0000 Subject: [PATCH 378/409] MAINT, TST: Add testing cases for partial/non-contig load and store --- numpy/core/src/_simd/_simd.dispatch.c.src | 181 ++++++++++++++++++++++ numpy/core/tests/test_simd.py | 142 ++++++++++++++++- 2 files changed, 316 insertions(+), 7 deletions(-) diff --git a/numpy/core/src/_simd/_simd.dispatch.c.src b/numpy/core/src/_simd/_simd.dispatch.c.src index 1989be7e355b..45e1762f4732 100644 --- a/numpy/core/src/_simd/_simd.dispatch.c.src +++ b/numpy/core/src/_simd/_simd.dispatch.c.src @@ -20,6 +20,7 @@ * #mul_sup = 1, 1, 1, 1, 1, 1, 0, 0, 1, 1# * #div_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# * #fused_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# + * #ncont_sup = 0, 0, 0, 0, 1, 1, 1, 1, 1, 1# * #shl_imm = 0, 0, 15, 15, 31, 31, 63, 63, 0, 0# * #shr_imm = 0, 0, 16, 16, 32, 32, 64, 64, 0, 0# */ @@ -61,6 +62,172 @@ simd__intrin_@intrin@_@sfx@(PyObject* NPY_UNUSED(self), PyObject *args) } /**end repeat1**/ +/**************************************** + * Non-contiguous/Partial Memory access + ****************************************/ +#if @ncont_sup@ +// Partial Load +SIMD_IMPL_INTRIN_3(load_till_@sfx@, v@sfx@, q@sfx@, u32, @sfx@) +SIMD_IMPL_INTRIN_2(load_tillz_@sfx@, v@sfx@, q@sfx@, u32) + +// Partial Store +static PyObject * +simd__intrin_store_till_@sfx@(PyObject* NPY_UNUSED(self), PyObject *args) +{ + simd_arg seq_arg = {.dtype = simd_data_q@sfx@}; + simd_arg nlane_arg = {.dtype = simd_data_u32}; + simd_arg vec_arg = {.dtype = simd_data_v@sfx@}; + if (!PyArg_ParseTuple( + args, "O&O&O&:store_till_@sfx@", + simd_arg_converter, &seq_arg, + simd_arg_converter, &nlane_arg, + simd_arg_converter, &vec_arg + )) { + return NULL; + } + npyv_store_till_@sfx@( + seq_arg.data.q@sfx@, nlane_arg.data.u32, vec_arg.data.v@sfx@ + ); + // write-back + if (simd_sequence_fill_iterable(seq_arg.obj, seq_arg.data.q@sfx@, simd_data_q@sfx@)) { + simd_arg_free(&seq_arg); + return NULL; + } + simd_arg_free(&seq_arg); + Py_RETURN_NONE; +} + +// Non-contiguous Load +/**begin repeat1 + * #intrin = loadn, loadn_till, loadn_tillz# + * #till = 0, 1, 1# + * #fill = 0, 1, 0# + * #format = , O&O&, O&# + */ +static PyObject * +simd__intrin_@intrin@_@sfx@(PyObject* NPY_UNUSED(self), PyObject *args) +{ + simd_arg seq_arg = {.dtype = simd_data_q@sfx@}; + simd_arg stride_arg = {.dtype = simd_data_s64}; +#if @till@ + simd_arg nlane_arg = {.dtype = simd_data_u32}; +#endif // till +#if @fill@ + simd_arg fill_arg = {.dtype = simd_data_@sfx@}; +#endif + if (!PyArg_ParseTuple( + args, "@format@O&O&:@intrin@_@sfx@", + simd_arg_converter, &seq_arg, + simd_arg_converter, &stride_arg +#if @till@ + ,simd_arg_converter, &nlane_arg +#endif +#if @fill@ + ,simd_arg_converter, &fill_arg +#endif + )) { + return NULL; + } + npyv_lanetype_@sfx@ *seq_ptr = seq_arg.data.q@sfx@; + npy_intp stride = (npy_intp)stride_arg.data.s64; + Py_ssize_t cur_seq_len = simd_sequence_len(seq_ptr); + Py_ssize_t min_seq_len = stride * npyv_nlanes_@sfx@; + if (stride < 0) { + seq_ptr += cur_seq_len -1; + min_seq_len = -min_seq_len; + } + if (cur_seq_len < min_seq_len) { + PyErr_Format(PyExc_ValueError, + "@intrin@_@sfx@(), according to provided stride %d, the " + "minimum acceptable size of the required sequence is %d, given(%d)", + stride, min_seq_len, cur_seq_len + ); + goto err; + } + npyv_@sfx@ rvec = npyv_@intrin@_@sfx@( + seq_ptr, stride + #if @till@ + , nlane_arg.data.u32 + #endif + #if @fill@ + , fill_arg.data.@sfx@ + #endif + ); + simd_arg ret = { + .dtype = simd_data_v@sfx@, .data = {.v@sfx@=rvec} + }; + simd_arg_free(&seq_arg); + return simd_arg_to_obj(&ret); +err: + simd_arg_free(&seq_arg); + return NULL; +} +/**end repeat1**/ + +// Non-contiguous Store +/**begin repeat1 + * #intrin = storen, storen_till# + * #till = 0, 1# + * #format = , O&# + */ +static PyObject * +simd__intrin_@intrin@_@sfx@(PyObject* NPY_UNUSED(self), PyObject *args) +{ + simd_arg seq_arg = {.dtype = simd_data_q@sfx@}; + simd_arg stride_arg = {.dtype = simd_data_s64}; + simd_arg vec_arg = {.dtype = simd_data_v@sfx@}; +#if @till@ + simd_arg nlane_arg = {.dtype = simd_data_u32}; +#endif + if (!PyArg_ParseTuple( + args, "@format@O&O&O&:storen_@sfx@", + simd_arg_converter, &seq_arg, + simd_arg_converter, &stride_arg +#if @till@ + ,simd_arg_converter, &nlane_arg +#endif + ,simd_arg_converter, &vec_arg + )) { + return NULL; + } + npyv_lanetype_@sfx@ *seq_ptr = seq_arg.data.q@sfx@; + npy_intp stride = (npy_intp)stride_arg.data.s64; + Py_ssize_t cur_seq_len = simd_sequence_len(seq_ptr); + Py_ssize_t min_seq_len = stride * npyv_nlanes_@sfx@; + if (stride < 0) { + seq_ptr += cur_seq_len -1; + min_seq_len = -min_seq_len; + } + // overflow guard + if (cur_seq_len < min_seq_len) { + PyErr_Format(PyExc_ValueError, + "@intrin@_@sfx@(), according to provided stride %d, the" + "minimum acceptable size of the required sequence is %d, given(%d)", + stride, min_seq_len, cur_seq_len + ); + goto err; + } + npyv_@intrin@_@sfx@( + seq_ptr, stride + #if @till@ + ,nlane_arg.data.u32 + #endif + ,vec_arg.data.v@sfx@ + ); + // write-back + if (simd_sequence_fill_iterable(seq_arg.obj, seq_arg.data.q@sfx@, simd_data_q@sfx@)) { + goto err; + } + simd_arg_free(&seq_arg); + Py_RETURN_NONE; +err: + simd_arg_free(&seq_arg); + return NULL; +} +/**end repeat1**/ +#endif // @ncont_sup@ + + /*************************** * Misc ***************************/ @@ -205,6 +372,7 @@ static PyMethodDef simd__intrinsics_methods[] = { * #mul_sup = 1, 1, 1, 1, 1, 1, 0, 0, 1, 1# * #div_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# * #fused_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# + * #ncont_sup = 0, 0, 0, 0, 1, 1, 1, 1, 1, 1# * #shl_imm = 0, 0, 15, 15, 31, 31, 63, 63, 0, 0# * #shr_imm = 0, 0, 16, 16, 32, 32, 64, 64, 0, 0# */ @@ -219,6 +387,19 @@ static PyMethodDef simd__intrinsics_methods[] = { SIMD_INTRIN_DEF(@intrin@_@sfx@) /**end repeat1**/ +/**************************************** + * Non-contiguous/Partial Memory access + ****************************************/ +#if @ncont_sup@ +/**begin repeat1 + * #intrin = load_till, load_tillz, loadn, loadn_till, loadn_tillz, + * store_till, storen, storen_till# + */ +SIMD_INTRIN_DEF(@intrin@_@sfx@) +/**end repeat1**/ +#endif // ncont_sup + + /*************************** * Misc ***************************/ diff --git a/numpy/core/tests/test_simd.py b/numpy/core/tests/test_simd.py index 77a636491172..50e77a4b8c04 100644 --- a/numpy/core/tests/test_simd.py +++ b/numpy/core/tests/test_simd.py @@ -11,18 +11,20 @@ class _Test_Utility: def __getattr__(self, attr): """ - To call NPV intrinsics without the prefix 'npyv_' and + To call NPV intrinsics without the attribute 'npyv' and auto suffixing intrinsics according to class attribute 'sfx' """ return getattr(self.npyv, attr + "_" + self.sfx) - def _data(self, n=None, reverse=False): + def _data(self, start=None, count=None, reverse=False): """ Create list of consecutive numbers according to number of vector's lanes. """ - if n is None: - n = 1 - rng = range(n, n + self.nlanes) + if start is None: + start = 1 + if count is None: + count = self.nlanes + rng = range(start, start + count) if reverse: rng = reversed(rng) if self._is_fp(): @@ -196,6 +198,132 @@ def test_memory_store(self): assert store_h[:self.nlanes//2] == data[self.nlanes//2:] assert store_h != vdata # detect overflow + def test_memory_partial_load(self): + if self.sfx in ("u8", "s8", "u16", "s16"): + return + + data = self._data() + lanes = list(range(1, self.nlanes + 1)) + lanes += [self.nlanes**2, self.nlanes**4] # test out of range + for n in lanes: + load_till = self.load_till(data, n, 15) + data_till = data[:n] + [15] * (self.nlanes-n) + assert load_till == data_till + load_tillz = self.load_tillz(data, n) + data_tillz = data[:n] + [0] * (self.nlanes-n) + assert load_tillz == data_tillz + + def test_memory_partial_store(self): + if self.sfx in ("u8", "s8", "u16", "s16"): + return + + data = self._data() + data_rev = self._data(reverse=True) + vdata = self.load(data) + lanes = list(range(1, self.nlanes + 1)) + lanes += [self.nlanes**2, self.nlanes**4] + for n in lanes: + data_till = data_rev.copy() + data_till[:n] = data[:n] + store_till = self._data(reverse=True) + self.store_till(store_till, n, vdata) + assert store_till == data_till + + def test_memory_noncont_load(self): + if self.sfx in ("u8", "s8", "u16", "s16"): + return + + for stride in range(1, 64): + data = self._data(count=stride*self.nlanes) + data_stride = data[::stride] + loadn = self.loadn(data, stride) + assert loadn == data_stride + + for stride in range(-64, 0): + data = self._data(stride, -stride*self.nlanes) + data_stride = self.load(data[::stride]) # cast unsigned + loadn = self.loadn(data, stride) + assert loadn == data_stride + + def test_memory_noncont_partial_load(self): + if self.sfx in ("u8", "s8", "u16", "s16"): + return + + lanes = list(range(1, self.nlanes + 1)) + lanes += [self.nlanes**2, self.nlanes**4] + for stride in range(1, 64): + data = self._data(count=stride*self.nlanes) + data_stride = data[::stride] + for n in lanes: + data_stride_till = data_stride[:n] + [15] * (self.nlanes-n) + loadn_till = self.loadn_till(data, stride, n, 15) + assert loadn_till == data_stride_till + data_stride_tillz = data_stride[:n] + [0] * (self.nlanes-n) + loadn_tillz = self.loadn_tillz(data, stride, n) + assert loadn_tillz == data_stride_tillz + + for stride in range(-64, 0): + data = self._data(stride, -stride*self.nlanes) + data_stride = list(self.load(data[::stride])) # cast unsigned + for n in lanes: + data_stride_till = data_stride[:n] + [15] * (self.nlanes-n) + loadn_till = self.loadn_till(data, stride, n, 15) + assert loadn_till == data_stride_till + data_stride_tillz = data_stride[:n] + [0] * (self.nlanes-n) + loadn_tillz = self.loadn_tillz(data, stride, n) + assert loadn_tillz == data_stride_tillz + + def test_memory_noncont_store(self): + if self.sfx in ("u8", "s8", "u16", "s16"): + return + + vdata = self.load(self._data()) + for stride in range(1, 64): + data = [15] * stride * self.nlanes + data[::stride] = vdata + storen = [15] * stride * self.nlanes + storen += [127]*64 + self.storen(storen, stride, vdata) + assert storen[:-64] == data + assert storen[-64:] == [127]*64 # detect overflow + + for stride in range(-64, 0): + data = [15] * -stride * self.nlanes + data[::stride] = vdata + storen = [127]*64 + storen += [15] * -stride * self.nlanes + self.storen(storen, stride, vdata) + assert storen[64:] == data + assert storen[:64] == [127]*64 # detect overflow + + def test_memory_noncont_partial_store(self): + if self.sfx in ("u8", "s8", "u16", "s16"): + return + + data = self._data() + vdata = self.load(data) + lanes = list(range(1, self.nlanes + 1)) + lanes += [self.nlanes**2, self.nlanes**4] + for stride in range(1, 64): + for n in lanes: + data_till = [15] * stride * self.nlanes + data_till[::stride] = data[:n] + [15] * (self.nlanes-n) + storen_till = [15] * stride * self.nlanes + storen_till += [127]*64 + self.storen_till(storen_till, stride, n, vdata) + assert storen_till[:-64] == data_till + assert storen_till[-64:] == [127]*64 # detect overflow + + for stride in range(-64, 0): + for n in lanes: + data_till = [15] * -stride * self.nlanes + data_till[::stride] = data[:n] + [15] * (self.nlanes-n) + storen_till = [127]*64 + storen_till += [15] * -stride * self.nlanes + self.storen_till(storen_till, stride, n, vdata) + assert storen_till[64:] == data_till + assert storen_till[:64] == [127]*64 # detect overflow + def test_misc(self): broadcast_zero = self.zero() assert broadcast_zero == [0] * self.nlanes @@ -388,7 +516,7 @@ def test_arithmetic_div(self): int_sfx = ("u8", "s8", "u16", "s16", "u32", "s32", "u64", "s64") fp_sfx = ("f32", "f64") all_sfx = int_sfx + fp_sfx -tests_registery = { +tests_registry = { int_sfx : _SIMD_INT, fp_sfx : _SIMD_FP, all_sfx : _SIMD_ALL @@ -411,7 +539,7 @@ def test_arithmetic_div(self): elif not npyv.simd_f64: skip_sfx["f64"] = f"target '{pretty_name}' doesn't support double-precision" - for sfxes, cls in tests_registery.items(): + for sfxes, cls in tests_registry.items(): for sfx in sfxes: skip_m = skip_sfx.get(sfx, skip) inhr = (cls,) From 2a7d75712cdbbe9969b385df748e0c024ea7a1b7 Mon Sep 17 00:00:00 2001 From: Ian Thomas Date: Tue, 27 Oct 2020 14:34:47 +0000 Subject: [PATCH 379/409] BUG: f2py incorrectly translates dimension declarations. In fortran functions passed to f2py, calculations that occur in the dimensions of array declarations are interpreted as floats. Valid fortran requires integers. Problem only occurs with fortran functions not subroutines as only for the former does f2py generate fortran wrapper code that fails to compile. Relevant code is in numpy.f2py.crackfortran.getlincoef(), which calculates and returns the coefficients to getarrlen() for writing to the fortran wrapper code. Fixes gh-8062. --- numpy/f2py/crackfortran.py | 9 +++++++++ numpy/f2py/tests/test_crackfortran.py | 22 ++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/numpy/f2py/crackfortran.py b/numpy/f2py/crackfortran.py index 3d2f97a5626e..2614d3dca7a9 100755 --- a/numpy/f2py/crackfortran.py +++ b/numpy/f2py/crackfortran.py @@ -2156,6 +2156,15 @@ def getlincoef(e, xset): # e = a*x+b ; x in xset m1 = re_1.match(ee) c2 = myeval(ee, {}, {}) if (a * 0.5 + b == c and a * 1.5 + b == c2): + # gh-8062: return integers instead of floats if possible. + try: + a = int(a) + except: + pass + try: + b = int(b) + except: + pass return a, b, x except Exception: pass diff --git a/numpy/f2py/tests/test_crackfortran.py b/numpy/f2py/tests/test_crackfortran.py index 735804024c9e..96adffc727b5 100644 --- a/numpy/f2py/tests/test_crackfortran.py +++ b/numpy/f2py/tests/test_crackfortran.py @@ -86,3 +86,25 @@ def test_defaultPublic(self, tmp_path): assert 'public' not in mod['vars']['a']['attrspec'] assert 'private' not in mod['vars']['seta']['attrspec'] assert 'public' in mod['vars']['seta']['attrspec'] + + +class TestArrayDimCalculation(util.F2PyTest): + # Issue gh-8062. Calculations that occur in the dimensions of fortran + # array declarations should be interpreted by f2py as integers not floats. + # Prior to fix, test fails as generated fortran wrapper does not compile. + code = """ + function test(n, a) + integer, intent(in) :: n + real(8), intent(out) :: a(0:2*n/2) + integer :: test + a(:) = n + test = 1 + endfunction + """ + + def test_issue_8062(self): + for n in (5, 11): + _, a = self.module.test(n) + assert(a.shape == (n+1,)) + assert_array_equal(a, n) + From 7b48608e9c71d53d5e936a5d4bab224d76ff6434 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 27 Oct 2020 12:50:14 -0500 Subject: [PATCH 380/409] NEP: Ensure inner loop signature is complete everywhere This makes sure the signature is complete everywhere and for now renames `userdata` to `innerloop_data`, which may be less confusing with regards to the previous (but different) userdata. --- doc/neps/nep-0043-extensible-ufuncs.rst | 38 +++++++++++++------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/doc/neps/nep-0043-extensible-ufuncs.rst b/doc/neps/nep-0043-extensible-ufuncs.rst index 974a705e61d9..96d4794f3774 100644 --- a/doc/neps/nep-0043-extensible-ufuncs.rst +++ b/doc/neps/nep-0043-extensible-ufuncs.rst @@ -250,13 +250,14 @@ to define string equality, will be added to a ufunc. # The operation is always "safe" casting (most ufuncs are) return (given_descrs[0], given_descrs[1], context.DTypes[2]()), "safe" - def strided_loop(context, n, data, strides): + def strided_loop(context, dimensions, data, strides, innerloop_data): """The 1-D strided loop, similar to those used in current ufuncs""" - # n: Number of elements in the one dimensional loop + # dimensions: Number of loop items and core dimensions # data: Pointers to the array data. # strides: strides to iterate all elements + n = dimensions[0] # number of items to loop over num_chars1 = context.descriptors[0].itemsize - num_chars2 = context.descriptors[0].itemsize + num_chars2 = context.descriptors[1].itemsize # C code using the above information to compare the strings in # both arrays. In particular, this loop requires the `num_chars1` @@ -446,7 +447,8 @@ a new ``ArrayMethod`` object: raise NotImplementedError @staticmethod - def strided_inner_loop(Context : context, data, strides,...): + def strided_inner_loop( + Context : context, data, dimensions, strides, innerloop_data): """The inner-loop (equivalent to the current ufunc loop) which is returned by the default `get_loop()` implementation.""" raise NotImplementedError @@ -659,7 +661,7 @@ definitions (see also :ref:`NEP 42 ` ``CastingImpl``): int aligned, int move_references, npy_intp *strides, PyArray_StridedUnaryOp **out_loop, - NpyAuxData **userdata, + NpyAuxData **innerloop_data, NPY_ARRAYMETHOD_FLAGS *flags); The ``NPY_ARRAYMETHOD_FLAGS`` can indicate whether the Python API is required @@ -667,7 +669,7 @@ definitions (see also :ref:`NEP 42 ` ``CastingImpl``): * The inner-loop function:: - int inner_loop(PyArrayMethod_Context *context, ..., void *userdata); + int inner_loop(PyArrayMethod_Context *context, ..., void *innerloop_data); Will have the identical signature to current inner-loops with the following changes: @@ -677,12 +679,12 @@ definitions (see also :ref:`NEP 42 ` ``CastingImpl``): * The new first argument ``PyArrayMethod_Context *`` is used to pass in potentially required information about the ufunc or descriptors in a convenient way. - * The ``void *userdata`` will be the ``NpyAuxData **userdata`` as set by - ``get_loop``. If ``get_loop`` does not set ``userdata`` an ``npy_intp *`` + * The ``void *innerloop_data`` will be the ``NpyAuxData **innerloop_data`` as set by + ``get_loop``. If ``get_loop`` does not set ``innerloop_data`` an ``npy_intp *`` is passed instead (see `Error Handling`_ below for the motivation). *Note:* Since ``get_loop`` is expected to be private, the exact implementation - of ``userdata`` can be modified until final exposure. + of ``innerloop_data`` can be modified until final exposure. Creation of a new ``BoundArrayMethod`` will use a ``PyArrayMethod_FromSpec()`` function. A shorthand will allow direct registration to a ufunc using @@ -784,7 +786,7 @@ The ``Context`` can also hold potentially useful information such as the the original ``ufunc``, which can be helpful when reporting errors. In principle passing in Context is not necessary, as all information could be -included in ``userdata`` and set up in the ``get_loop`` function. +included in ``innerloop_data`` and set up in the ``get_loop`` function. In this NEP we propose passing the struct to simplify creation of loops for parametric DTypes. @@ -798,10 +800,10 @@ provides a simple solution, which is already used in NumPy and public API. ``NpyAyxData *`` is a light weight, allocated structure and since it already exists in NumPy for this purpose, it seems a natural choice. To simplify some use-cases (see "Error Handling" below), we will pass a -``npy_intp *userdata = 0`` instead when ``userdata`` is not provided. +``npy_intp *innerloop_data = 0`` instead when ``innerloop_data`` is not provided. *Note: Since ``get_loop`` is expected to be private initially we can gain -experience with ``userdata`` before exposing it as public API.* +experience with ``innerloop_data`` before exposing it as public API.* **Return value:** @@ -821,7 +823,7 @@ locking the GIL. In this case the function will have to lock the GIL, set the Python error and return ``-1`` to indicate an error occurred::: int - inner_loop(PyArrayMethod_Context *context, ..., void *userdata) + inner_loop(PyArrayMethod_Context *context, ..., void *innerloop_data) { NPY_ALLOW_C_API_DEF @@ -864,13 +866,13 @@ solution and more complex solution may be possible future extensions. Handling *warnings* is slightly more complex: A warning should be given exactly once for each function call (i.e. for the whole array) even if naively it would be given many times. -To simplify such a use case, we will pass in ``npy_intp *userdata = 0`` +To simplify such a use case, we will pass in ``npy_intp *innerloop_data = 0`` by default which can be used to store flags (or other simple persistent data). For instance, we could imagine an integer multiplication loop which warns when an overflow occurred:: int - integer_multiply(PyArrayMethod_Context *context, ..., npy_intp *userdata) + integer_multiply(PyArrayMethod_Context *context, ..., npy_intp *innerloop_data) { int overflow; NPY_ALLOW_C_API_DEF @@ -878,20 +880,20 @@ when an overflow occurred:: for (npy_intp i = 0; i < N; i++) { *out = multiply_integers(*in1, *in2, &overflow); - if (overflow && !*userdata) { + if (overflow && !*innerloop_data) { NPY_ALLOW_C_API; if (PyErr_Warn(PyExc_UserWarning, "Integer overflow detected.") < 0) { NPY_DISABLE_C_API return -1; } - *userdata = 1; + *innerloop_data = 1; NPY_DISABLE_C_API } return 0; } -*TODO:* The idea of passing an ``npy_intp`` scratch space when ``userdata`` +*TODO:* The idea of passing an ``npy_intp`` scratch space when ``innerloop_data`` is not set seems convenient, but I am uncertain about it, since I am not aware of any similar prior art. This "scratch space" could also be part of the ``context`` in principle. From ac4ffe1d39d9cc845948079a24facf7057effb24 Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Tue, 27 Oct 2020 21:09:27 +0200 Subject: [PATCH 381/409] MAINT, TST: update name of _SIMD submodule to match import path --- numpy/core/src/_simd/_simd.c | 2 +- numpy/core/src/_simd/_simd.dispatch.c.src | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/numpy/core/src/_simd/_simd.c b/numpy/core/src/_simd/_simd.c index d2e26d82125d..b1fdd4478d9d 100644 --- a/numpy/core/src/_simd/_simd.c +++ b/numpy/core/src/_simd/_simd.c @@ -4,7 +4,7 @@ PyMODINIT_FUNC PyInit__simd(void) { static struct PyModuleDef defs = { .m_base = PyModuleDef_HEAD_INIT, - .m_name = "_simd", + .m_name = "numpy.core._simd", .m_size = -1 }; if (npy_cpu_init() < 0) { diff --git a/numpy/core/src/_simd/_simd.dispatch.c.src b/numpy/core/src/_simd/_simd.dispatch.c.src index 45e1762f4732..2d89b9df00f9 100644 --- a/numpy/core/src/_simd/_simd.dispatch.c.src +++ b/numpy/core/src/_simd/_simd.dispatch.c.src @@ -49,9 +49,7 @@ simd__intrin_@intrin@_@sfx@(PyObject* NPY_UNUSED(self), PyObject *args) )) { return NULL; } - npyv_@intrin@_@sfx@( - seq_arg.data.q@sfx@, vec_arg.data.v@sfx@ - ); + npyv_@intrin@_@sfx@(seq_arg.data.q@sfx@, vec_arg.data.v@sfx@); // write-back if (simd_sequence_fill_iterable(seq_arg.obj, seq_arg.data.q@sfx@, simd_data_q@sfx@)) { simd_arg_free(&seq_arg); @@ -509,9 +507,9 @@ NPY_CPU_DISPATCH_CURFX(simd_create_module)(void) .m_base = PyModuleDef_HEAD_INIT, .m_size = -1, #ifdef NPY__CPU_TARGET_CURRENT - .m_name = "NPYV_" NPY_TOSTRING(NPY__CPU_TARGET_CURRENT), + .m_name = "numpy.core._simd." NPY_TOSTRING(NPY__CPU_TARGET_CURRENT), #else - .m_name = "NPYV_BASELINE", + .m_name = "numpy.core._simd.baseline", #endif #if NPY_SIMD .m_methods = simd__intrinsics_methods From 76d12d3fcabad0371b24753ba7d36ca5b1d61bcf Mon Sep 17 00:00:00 2001 From: mattip Date: Tue, 27 Oct 2020 22:43:37 +0200 Subject: [PATCH 382/409] TST: simplify source path names in compilation test --- numpy/core/tests/examples/setup.py | 3 +-- numpy/core/tests/test_cython.py | 8 +++----- numpy/random/_examples/cython/setup.py | 4 ++-- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/numpy/core/tests/examples/setup.py b/numpy/core/tests/examples/setup.py index 9860bf5f7c75..6e34aa7787ad 100644 --- a/numpy/core/tests/examples/setup.py +++ b/numpy/core/tests/examples/setup.py @@ -9,12 +9,11 @@ from setuptools.extension import Extension import os -here = os.path.dirname(__file__) macros = [("NPY_NO_DEPRECATED_API", 0)] checks = Extension( "checks", - sources=[os.path.join(here, "checks.pyx")], + sources=[os.path.join('.', "checks.pyx")], include_dirs=[np.get_include()], define_macros=macros, ) diff --git a/numpy/core/tests/test_cython.py b/numpy/core/tests/test_cython.py index bfdb692d764d..a1f09d0fef12 100644 --- a/numpy/core/tests/test_cython.py +++ b/numpy/core/tests/test_cython.py @@ -34,21 +34,19 @@ def install_temp(request, tmp_path): here = os.path.dirname(__file__) ext_dir = os.path.join(here, "examples") - tmp_path = tmp_path._str - cytest = os.path.join(tmp_path, "cytest") + cytest = str(tmp_path / "cytest") shutil.copytree(ext_dir, cytest) # build the examples and "install" them into a temporary directory - install_log = os.path.join(tmp_path, "tmp_install_log.txt") + install_log = str(tmp_path / "tmp_install_log.txt") subprocess.check_call( [ sys.executable, "setup.py", "build", "install", - "--prefix", - os.path.join(tmp_path, "installdir"), + "--prefix", str(tmp_path / "installdir"), "--single-version-externally-managed", "--record", install_log, diff --git a/numpy/random/_examples/cython/setup.py b/numpy/random/_examples/cython/setup.py index 42425c2c199f..83f06fde8581 100644 --- a/numpy/random/_examples/cython/setup.py +++ b/numpy/random/_examples/cython/setup.py @@ -19,7 +19,7 @@ lib_path = join(np.get_include(), '..', '..', 'random', 'lib') extending = Extension("extending", - sources=[join(path, 'extending.pyx')], + sources=[join('.', 'extending.pyx')], include_dirs=[ np.get_include(), join(path, '..', '..') @@ -27,7 +27,7 @@ define_macros=defs, ) distributions = Extension("extending_distributions", - sources=[join(path, 'extending_distributions.pyx')], + sources=[join('.', 'extending_distributions.pyx')], include_dirs=[inc_path], library_dirs=[lib_path], libraries=['npyrandom'], From 26463e73afe573d9b2769541a6b6a5d9687e1a96 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Tue, 27 Oct 2020 20:29:25 -0600 Subject: [PATCH 383/409] TST: Add 64 bit Python 3.9 test to Windows --- azure-pipelines.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9382ac83c129..7a8196dce14a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -217,6 +217,13 @@ stages: BITS: 64 NPY_USE_BLAS_ILP64: '1' OPENBLAS_SUFFIX: '64_' + Python39-64bit-full: + PYTHON_VERSION: '3.9' + PYTHON_ARCH: 'x64' + TEST_MODE: full + BITS: 64 + NPY_USE_BLAS_ILP64: '1' + OPENBLAS_SUFFIX: '64_' #PyPy36-32bit: #PYTHON_VERSION: 'PyPy3.6' #PYTHON_ARCH: 'x32' From ff26a55e7855f24f065714253fdc5de1c0df9eab Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Wed, 28 Oct 2020 11:44:23 +0000 Subject: [PATCH 384/409] TST: f2py: Add a doctest for `getlincoef` --- numpy/f2py/crackfortran.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/numpy/f2py/crackfortran.py b/numpy/f2py/crackfortran.py index 3d2f97a5626e..1fe2ca037d68 100755 --- a/numpy/f2py/crackfortran.py +++ b/numpy/f2py/crackfortran.py @@ -2103,8 +2103,9 @@ def buildimplicitrules(block): def myeval(e, g=None, l=None): + """ Like `eval` but returns only integers and floats """ r = eval(e, g, l) - if type(r) in [type(0), type(0.0)]: + if type(r) in [int, float]: return r raise ValueError('r=%r' % (r)) @@ -2112,6 +2113,26 @@ def myeval(e, g=None, l=None): def getlincoef(e, xset): # e = a*x+b ; x in xset + """ + Obtain ``a`` and ``b`` when ``e == "a*x+b"``, where ``x`` is a symbol in + xset. + + >>> getlincoef('2*x + 1', {'x'}) + (2, 1, 'x') + >>> getlincoef('3*x + x*2 + 2 + 1', {'x'}) + (5, 3, 'x') + >>> getlincoef('0', {'x'}) + (0, 0, None) + >>> getlincoef('0*x', {'x'}) + (0, 0, 'x') + >>> getlincoef('x*x', {'x'}) + (None, None, None) + + This can be tricked by sufficiently complex expressions + + >>> getlincoef('(x - 0.5)*(x - 1.5)*(x - 1)*x + 2*x + 3', {'x'}) + (2.0, 3.0, 'x') + """ try: c = int(myeval(e, {}, {})) return 0, c, None From 0ae2811bde4daaa4bb68ed818871eb2011a75888 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Wed, 28 Oct 2020 20:23:14 -0600 Subject: [PATCH 385/409] REL: Update master after 1.19.3 release. - Forward port 1.19.3-notes.rst - Forward port 1.19.3-changelog.rst - Update release.rst --- doc/changelog/1.19.3-changelog.rst | 31 +++++++++++++++++++ doc/source/release.rst | 1 + doc/source/release/1.19.3-notes.rst | 46 +++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+) create mode 100644 doc/changelog/1.19.3-changelog.rst create mode 100644 doc/source/release/1.19.3-notes.rst diff --git a/doc/changelog/1.19.3-changelog.rst b/doc/changelog/1.19.3-changelog.rst new file mode 100644 index 000000000000..5e8dfa10b6ba --- /dev/null +++ b/doc/changelog/1.19.3-changelog.rst @@ -0,0 +1,31 @@ + +Contributors +============ + +A total of 8 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Charles Harris +* Chris Brown + +* Daniel Vanzo + +* E. Madison Bray + +* Hugo van Kemenade + +* Ralf Gommers +* Sebastian Berg +* @danbeibei + + +Pull requests merged +==================== + +A total of 10 pull requests were merged for this release. + +* `#17298 `__: BLD: set upper versions for build dependencies +* `#17336 `__: BUG: Set deprecated fields to null in PyArray_InitArrFuncs +* `#17446 `__: ENH: Warn on unsupported Python 3.10+ +* `#17450 `__: MAINT: Update test_requirements.txt. +* `#17522 `__: ENH: Support for the NVIDIA HPC SDK nvfortran compiler +* `#17568 `__: BUG: Cygwin Workaround for #14787 on affected platforms +* `#17647 `__: BUG: Fix memory leak of buffer-info cache due to relaxed strides +* `#17652 `__: MAINT: Backport openblas_support from master. +* `#17653 `__: TST: Add Python 3.9 to the CI testing on Windows, Mac. +* `#17660 `__: TST: Simplify source path names in test_extending. diff --git a/doc/source/release.rst b/doc/source/release.rst index 1c77755fd1af..c7bc5e1b9d39 100644 --- a/doc/source/release.rst +++ b/doc/source/release.rst @@ -6,6 +6,7 @@ Release Notes :maxdepth: 3 1.20.0 + 1.19.3 1.19.2 1.19.1 1.19.0 diff --git a/doc/source/release/1.19.3-notes.rst b/doc/source/release/1.19.3-notes.rst new file mode 100644 index 000000000000..54a4883893ff --- /dev/null +++ b/doc/source/release/1.19.3-notes.rst @@ -0,0 +1,46 @@ +.. currentmodule:: numpy + +========================== +NumPy 1.19.3 Release Notes +========================== + +NumPy 1.19.3 is a small maintenance release with two major improvements: + +- Python 3.9 binary wheels on all supported platforms. +- OpenBLAS fixes for the Windows 10 version 2004 fmod bug. + +This release supports Python 3.6-3.9 and is linked with OpenBLAS 3.7 to avoid +some of the fmod problems on Windows 10 version 2004. Microsoft is aware of the +problem and users should upgrade when the fix becomes available, the fix here +is limited in scope. + +Contributors +============ + +A total of 8 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Charles Harris +* Chris Brown + +* Daniel Vanzo + +* E. Madison Bray + +* Hugo van Kemenade + +* Ralf Gommers +* Sebastian Berg +* @danbeibei + + +Pull requests merged +==================== + +A total of 10 pull requests were merged for this release. + +* `#17298 `__: BLD: set upper versions for build dependencies +* `#17336 `__: BUG: Set deprecated fields to null in PyArray_InitArrFuncs +* `#17446 `__: ENH: Warn on unsupported Python 3.10+ +* `#17450 `__: MAINT: Update test_requirements.txt. +* `#17522 `__: ENH: Support for the NVIDIA HPC SDK nvfortran compiler +* `#17568 `__: BUG: Cygwin Workaround for #14787 on affected platforms +* `#17647 `__: BUG: Fix memory leak of buffer-info cache due to relaxed strides +* `#17652 `__: MAINT: Backport openblas_support from master. +* `#17653 `__: TST: Add Python 3.9 to the CI testing on Windows, Mac. +* `#17660 `__: TST: Simplify source path names in test_extending. From 14c6e451edcf65d412443c7fb045ab706daf3b73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nikola=20Forr=C3=B3?= Date: Mon, 26 Oct 2020 18:54:22 +0100 Subject: [PATCH 386/409] TST: Make test suite work in FIPS (140-2) Mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests using MD5 algorithms fail in FIPS Mode because MD5 is not FIPS compliant. Replace MD5 with SHA256 to overcome that. Signed-off-by: Nikola Forró --- numpy/core/tests/test_regression.py | 4 +- numpy/f2py/tests/util.py | 2 - numpy/random/tests/test_generator_mt19937.py | 50 +++++++-------- numpy/random/tests/test_random.py | 24 ++++---- numpy/random/tests/test_randomstate.py | 64 ++++++++++---------- 5 files changed, 71 insertions(+), 73 deletions(-) diff --git a/numpy/core/tests/test_regression.py b/numpy/core/tests/test_regression.py index 2e731d4faa4d..831e48e8b529 100644 --- a/numpy/core/tests/test_regression.py +++ b/numpy/core/tests/test_regression.py @@ -1506,10 +1506,10 @@ def test_type(t): test_type(t) def test_buffer_hashlib(self): - from hashlib import md5 + from hashlib import sha256 x = np.array([1, 2, 3], dtype=np.dtype('= 0) def test_repeatability(self): - # We use a md5 hash of generated sequences of 1000 samples + # We use a sha256 hash of generated sequences of 1000 samples # in the range [0, 6) for all but bool, where the range # is [0, 2). Hashes are for little endian numbers. - tgt = {'bool': '7dd3170d7aa461d201a65f8bcf3944b0', - 'int16': '1b7741b80964bb190c50d541dca1cac1', - 'int32': '4dc9fcc2b395577ebb51793e58ed1a05', - 'int64': '17db902806f448331b5a758d7d2ee672', - 'int8': '27dd30c4e08a797063dffac2490b0be6', - 'uint16': '1b7741b80964bb190c50d541dca1cac1', - 'uint32': '4dc9fcc2b395577ebb51793e58ed1a05', - 'uint64': '17db902806f448331b5a758d7d2ee672', - 'uint8': '27dd30c4e08a797063dffac2490b0be6'} + tgt = {'bool': '509aea74d792fb931784c4b0135392c65aec64beee12b0cc167548a2c3d31e71', + 'int16': '7b07f1a920e46f6d0fe02314155a2330bcfd7635e708da50e536c5ebb631a7d4', + 'int32': 'e577bfed6c935de944424667e3da285012e741892dcb7051a8f1ce68ab05c92f', + 'int64': '0fbead0b06759df2cfb55e43148822d4a1ff953c7eb19a5b08445a63bb64fa9e', + 'int8': '001aac3a5acb935a9b186cbe14a1ca064b8bb2dd0b045d48abeacf74d0203404', + 'uint16': '7b07f1a920e46f6d0fe02314155a2330bcfd7635e708da50e536c5ebb631a7d4', + 'uint32': 'e577bfed6c935de944424667e3da285012e741892dcb7051a8f1ce68ab05c92f', + 'uint64': '0fbead0b06759df2cfb55e43148822d4a1ff953c7eb19a5b08445a63bb64fa9e', + 'uint8': '001aac3a5acb935a9b186cbe14a1ca064b8bb2dd0b045d48abeacf74d0203404'} for dt in self.itype[1:]: random.seed(1234) @@ -341,13 +341,13 @@ def test_repeatability(self): else: val = self.rfunc(0, 6, size=1000, dtype=dt).byteswap() - res = hashlib.md5(val.view(np.int8)).hexdigest() + res = hashlib.sha256(val.view(np.int8)).hexdigest() assert_(tgt[np.dtype(dt).name] == res) # bools do not depend on endianness random.seed(1234) val = self.rfunc(0, 2, size=1000, dtype=bool).view(np.int8) - res = hashlib.md5(val).hexdigest() + res = hashlib.sha256(val).hexdigest() assert_(tgt[np.dtype(bool).name] == res) @pytest.mark.skipif(np.iinfo('l').max < 2**32, @@ -1974,7 +1974,7 @@ def test_three_arg_funcs(self): # Ensure returned array dtype is correct for platform def test_integer_dtype(int_func): random.seed(123456789) - fname, args, md5 = int_func + fname, args, sha256 = int_func f = getattr(random, fname) actual = f(*args, size=2) assert_(actual.dtype == np.dtype('l')) @@ -1982,13 +1982,13 @@ def test_integer_dtype(int_func): def test_integer_repeat(int_func): random.seed(123456789) - fname, args, md5 = int_func + fname, args, sha256 = int_func f = getattr(random, fname) val = f(*args, size=1000000) if sys.byteorder != 'little': val = val.byteswap() - res = hashlib.md5(val.view(np.int8)).hexdigest() - assert_(res == md5) + res = hashlib.sha256(val.view(np.int8)).hexdigest() + assert_(res == sha256) def test_broadcast_size_error(): From 48a9a519cc7a4d52ff85577efee0048350d07254 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Thu, 29 Oct 2020 10:11:06 +0000 Subject: [PATCH 387/409] DOC: f2py: Add a docstring for getarrlen The API of this function is awful, and I still have no idea what it does. Maybe someone reading the docstring will be able to work it out. --- numpy/f2py/crackfortran.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/numpy/f2py/crackfortran.py b/numpy/f2py/crackfortran.py index 1fe2ca037d68..2e95e45969e5 100755 --- a/numpy/f2py/crackfortran.py +++ b/numpy/f2py/crackfortran.py @@ -2187,6 +2187,37 @@ def getlincoef(e, xset): # e = a*x+b ; x in xset def getarrlen(dl, args, star='*'): + """ + Parameters + ---------- + dl : sequence of two str objects + dimensions of the array + args : Iterable[str] + symbols used in the expression + star : Any + unused + + Returns + ------- + expr : str + Some numeric expression as a string + arg : Optional[str] + If understood, the argument from `args` present in `expr` + expr2 : Optional[str] + If understood, an expression fragment that should be used as + ``"(%s%s".format(something, expr2)``. + + Examples + -------- + >>> getarrlen(['10*x + 20', '40*x'], {'x'}) + ('30 * x - 19', 'x', '+19)/(30)') + >>> getarrlen(['1', '10*x + 20'], {'x'}) + ('10 * x + 20', 'x', '-20)/(10)') + >>> getarrlen(['10*x + 20', '1'], {'x'}) + ('-10 * x - 18', 'x', '+18)/(-10)') + >>> getarrlen(['20', '1'], {'x'}) + ('-18', None, None) + """ edl = [] try: edl.append(myeval(dl[0], {}, {})) From 6e85f750d590eface81e753dad202d266d8656a9 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 29 Oct 2020 15:31:55 +0200 Subject: [PATCH 388/409] DOC: Update README badge for travis-ci.com [CI skip] --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6b2d327f6ef2..9271a5d28872 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # NumPy -[![Travis](https://img.shields.io/travis/numpy/numpy/master.svg?label=Travis%20CI)]( - https://travis-ci.org/numpy/numpy) +[![Travis CI](https://img.shields.io/travis/com/numpy/numpy/master?label=Travis%20CI)]( + https://travis-ci.com/github/numpy/numpy) [![Azure](https://dev.azure.com/numpy/numpy/_apis/build/status/azure-pipeline%20numpy.numpy)]( https://dev.azure.com/numpy/numpy/_build/latest?definitionId=5) [![codecov](https://codecov.io/gh/numpy/numpy/branch/master/graph/badge.svg)]( From e8669044e029829896b66d0b9918330aa4f777c5 Mon Sep 17 00:00:00 2001 From: Qiyu8 Date: Fri, 30 Oct 2020 11:40:33 +0800 Subject: [PATCH 389/409] SIMD: Add sum intrinsics for float/double. --- numpy/core/src/common/simd/avx2/arithmetic.h | 20 +++++++++ .../core/src/common/simd/avx512/arithmetic.h | 43 +++++++++++++++++++ numpy/core/src/common/simd/neon/arithmetic.h | 13 ++++++ numpy/core/src/common/simd/sse/arithmetic.h | 24 +++++++++++ numpy/core/src/common/simd/vsx/arithmetic.h | 12 ++++++ 5 files changed, 112 insertions(+) diff --git a/numpy/core/src/common/simd/avx2/arithmetic.h b/numpy/core/src/common/simd/avx2/arithmetic.h index 4af9e4d1748a..e43b8e7a3b04 100644 --- a/numpy/core/src/common/simd/avx2/arithmetic.h +++ b/numpy/core/src/common/simd/avx2/arithmetic.h @@ -117,3 +117,23 @@ } #endif // !NPY_HAVE_FMA3 #endif // _NPY_SIMD_AVX2_ARITHMETIC_H + +// Horizontal add: Calculates the sum of all vector elements. +NPY_FINLINE float npyv_sum_f32(__m256 a) +{ + __m256 sum_halves = _mm256_hadd_ps(a, a); + sum_halves = _mm256_hadd_ps(sum_halves, sum_halves); + __m128 lo = _mm256_castps256_ps128(sum_halves); + __m128 hi = _mm256_extractf128_ps(sum_halves, 1); + __m128 sum = _mm_add_ps(lo, hi); + return _mm_cvtss_f32(sum); +} + +NPY_FINLINE double npyv_sum_f64(__m256d a) +{ + __m256d sum_halves = _mm256_hadd_pd(a, a); + __m128d lo = _mm256_castpd256_pd128(sum_halves); + __m128d hi = _mm256_extractf128_pd(sum_halves, 1); + __m128d sum = _mm_add_pd(lo, hi); + return _mm_cvtsd_f64(sum); +} \ No newline at end of file diff --git a/numpy/core/src/common/simd/avx512/arithmetic.h b/numpy/core/src/common/simd/avx512/arithmetic.h index 824ae818ee3a..f31051aaf12b 100644 --- a/numpy/core/src/common/simd/avx512/arithmetic.h +++ b/numpy/core/src/common/simd/avx512/arithmetic.h @@ -130,3 +130,46 @@ NPY_FINLINE __m512i npyv_mul_u8(__m512i a, __m512i b) #define npyv_nmulsub_f64 _mm512_fnmsub_pd #endif // _NPY_SIMD_AVX512_ARITHMETIC_H + +/*************************** + * Reduce Sum + * there are three ways to implement reduce sum for AVX512: + * 1- split(256) /add /split(128) /add /hadd /hadd /extract + * 2- shuff(cross) /add /shuff(cross) /add /shuff /add /shuff /add /extract + * 3- _mm512_reduce_add_ps/pd + * The first one is been widely used by many projects + * + * the second one is used by Intel Compiler, maybe because the + * latency of hadd increased by (2-3) starting from Skylake-X which makes two + * extra shuffles(non-cross) cheaper. check https://godbolt.org/z/s3G9Er for more info. + * + * The third one is almost the same as the second one but only works for + * intel compiler/GCC 7.1/Clang 4, we still need to support older GCC. + ***************************/ +#ifdef NPY_HAVE_AVX512F_REDUCE + #define npyv_sum_f32 _mm512_reduce_add_ps + #define npyv_sum_f64 _mm512_reduce_add_pd +#else + NPY_FINLINE float npyv_sum_f32(npyv_f32 a) + { + __m512 h64 = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(3, 2, 3, 2)); + __m512 sum32 = _mm512_add_ps(a, h64); + __m512 h32 = _mm512_shuffle_f32x4(sum32, sum32, _MM_SHUFFLE(1, 0, 3, 2)); + __m512 sum16 = _mm512_add_ps(sum32, h32); + __m512 h16 = _mm512_permute_ps(sum16, _MM_SHUFFLE(1, 0, 3, 2)); + __m512 sum8 = _mm512_add_ps(sum16, h16); + __m512 h4 = _mm512_permute_ps(sum8, _MM_SHUFFLE(2, 3, 0, 1)); + __m512 sum4 = _mm512_add_ps(sum8, h4); + return _mm_cvtss_f32(_mm512_castps512_ps128(sum4)); + } + NPY_FINLINE double npyv_sum_f64(npyv_f64 a) + { + __m512d h64 = _mm512_shuffle_f64x2(a, a, _MM_SHUFFLE(3, 2, 3, 2)); + __m512d sum32 = _mm512_add_pd(a, h64); + __m512d h32 = _mm512_permutex_pd(sum32, _MM_SHUFFLE(1, 0, 3, 2)); + __m512d sum16 = _mm512_add_pd(sum32, h32); + __m512d h16 = _mm512_permute_pd(sum16, _MM_SHUFFLE(2, 3, 0, 1)); + __m512d sum8 = _mm512_add_pd(sum16, h16); + return _mm_cvtsd_f64(_mm512_castpd512_pd128(sum8)); + } +#endif \ No newline at end of file diff --git a/numpy/core/src/common/simd/neon/arithmetic.h b/numpy/core/src/common/simd/neon/arithmetic.h index 5eeee1bb6d02..3bea143846a6 100644 --- a/numpy/core/src/common/simd/neon/arithmetic.h +++ b/numpy/core/src/common/simd/neon/arithmetic.h @@ -119,3 +119,16 @@ { return vfmsq_f64(vnegq_f64(c), a, b); } #endif // NPY_SIMD_F64 #endif // _NPY_SIMD_NEON_ARITHMETIC_H + +// Horizontal add: Calculates the sum of all vector elements. +NPY_FINLINE float npyv_sum_f32(float32x4_t a) +{ + float32x2_t r = vadd_f32(vget_high_f32(a), vget_low_f32(a)); + return vget_lane_f32(vpadd_f32(r, r), 0); +} +#ifdef __aarch64__ + NPY_FINLINE double npyv_sum_f64(float64x2_t a) + { + return vget_lane_f64(vget_low_f64(a) + vget_high_f64(a), 0); + } +#endif \ No newline at end of file diff --git a/numpy/core/src/common/simd/sse/arithmetic.h b/numpy/core/src/common/simd/sse/arithmetic.h index 717dacd39f8d..a92c39bcce4f 100644 --- a/numpy/core/src/common/simd/sse/arithmetic.h +++ b/numpy/core/src/common/simd/sse/arithmetic.h @@ -148,3 +148,27 @@ NPY_FINLINE __m128i npyv_mul_u8(__m128i a, __m128i b) } #endif // !NPY_HAVE_FMA3 #endif // _NPY_SIMD_SSE_ARITHMETIC_H + +// Horizontal add: Calculates the sum of all vector elements. +NPY_FINLINE float npyv_sum_f32(__m128 a) +{ +#ifdef NPY_HAVE_SSE3 + __m128 sum_halves = _mm_hadd_ps(a, a); + return _mm_cvtss_f32(_mm_hadd_ps(sum_halves, sum_halves)); +#else + __m128 t1 = _mm_movehl_ps(a, a); + __m128 t2 = _mm_add_ps(a, t1); + __m128 t3 = _mm_shuffle_ps(t2, t2, 1); + __m128 t4 = _mm_add_ss(t2, t3); + return _mm_cvtss_f32(t4); +#endif +} + +NPY_FINLINE double npyv_sum_f64(__m128d a) +{ +#ifdef NPY_HAVE_SSE3 + return _mm_cvtsd_f64(_mm_hadd_pd(a, a)); +#else + return _mm_cvtsd_f64(_mm_add_pd(a, _mm_unpackhi_pd(a, a))); +#endif +} \ No newline at end of file diff --git a/numpy/core/src/common/simd/vsx/arithmetic.h b/numpy/core/src/common/simd/vsx/arithmetic.h index 6ef007676d03..9b323d7d1512 100644 --- a/numpy/core/src/common/simd/vsx/arithmetic.h +++ b/numpy/core/src/common/simd/vsx/arithmetic.h @@ -117,3 +117,15 @@ #define npyv_nmulsub_f64 vec_nmadd #endif // _NPY_SIMD_VSX_ARITHMETIC_H + +// Horizontal add: Calculates the sum of all vector elements. +NPY_FINLINE float npyv_sum_f32(npyv_f32 a) +{ + return vec_extract(a, 0) + vec_extract(a, 1) + + vec_extract(a, 2) + vec_extract(a, 3); +} + +NPY_FINLINE double npyv_sum_f64(npyv_f64 a) +{ + return vec_extract(a, 0) + vec_extract(a, 1); +} \ No newline at end of file From 5da3833b82c251d3d5cdc15d92d5a7e41356a247 Mon Sep 17 00:00:00 2001 From: vorfol <39548292+vorfol@users.noreply.github.com> Date: Fri, 30 Oct 2020 10:24:12 +0500 Subject: [PATCH 390/409] Update nditer_impl.h operator '+' in line 272 have to advance char** or npy_intp*? --- numpy/core/src/multiarray/nditer_impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/src/multiarray/nditer_impl.h b/numpy/core/src/multiarray/nditer_impl.h index bb483bb1f554..378d6f7117b3 100644 --- a/numpy/core/src/multiarray/nditer_impl.h +++ b/numpy/core/src/multiarray/nditer_impl.h @@ -269,7 +269,7 @@ struct NpyIter_AD { #define NAD_STRIDES(axisdata) ( \ &(axisdata)->ad_flexdata + 0) #define NAD_PTRS(axisdata) ((char **) \ - &(axisdata)->ad_flexdata + 1*(nop+1)) + (&(axisdata)->ad_flexdata + 1*(nop+1))) #define NAD_NSTRIDES() \ ((nop) + ((itflags&NPY_ITFLAG_HASINDEX) ? 1 : 0)) From 505ae178178bac36929bf8afe5d523a4d659531c Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Sun, 31 May 2020 19:07:57 -0600 Subject: [PATCH 391/409] DOC: Improve documentation of download-wheels - List dependences - Give examples --- tools/download-wheels.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/tools/download-wheels.py b/tools/download-wheels.py index 941440ca9afc..28b3fc7ad6a9 100644 --- a/tools/download-wheels.py +++ b/tools/download-wheels.py @@ -1,9 +1,29 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +# -*- encoding:utf-8 -*- """ -Download NumPy wheels from Anaconda staging area. +Script to download NumPy wheels from the Anaconda staging area. + +Usage:: + + $ ./tools/download-wheels.py -w + +The default wheelhouse is ``release/installers``. + +Dependencies +------------ + +- beautifulsoup4 +- urllib3 + +Examples +-------- + +While in the repository root:: + + $ python tools/download-wheels.py 1.19.0 + $ python tools/download-wheels.py 1.19.0 -w ~/wheelhouse """ -import sys import os import re import shutil @@ -18,6 +38,7 @@ STAGING_URL = 'https://anaconda.org/multibuild-wheels-staging/numpy' PREFIX = 'numpy' + def get_wheel_names(version): """ Get wheel names from Anaconda HTML directory. @@ -69,7 +90,7 @@ def download_wheels(version, wheelhouse): parser = argparse.ArgumentParser() parser.add_argument( "version", - help="NumPy version to download.") + help="NumPy version to download.") parser.add_argument( "-w", "--wheelhouse", default=os.path.join(os.getcwd(), "release", "installers"), From 93e589352632206d034344a213fd1dcf5db24d9a Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Sun, 31 May 2020 19:33:11 -0600 Subject: [PATCH 392/409] ENH: Document packages needed for release. This may be useful for new folks setting up to make a release. I don't know that specific versions are required for any of these packages. --- release_requirements.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 release_requirements.txt diff --git a/release_requirements.txt b/release_requirements.txt new file mode 100644 index 000000000000..dc840b9359ec --- /dev/null +++ b/release_requirements.txt @@ -0,0 +1,10 @@ +# These packages are needed for a release in addition to those needed +# for building, testing, and the creation of documentation. + +# download-wheels.py +urllib3 +beatifulsoup4 + +# changelog.py +pygithub +gitpython From f7518be6309a661085ec1f87f0db7b3fe23ba240 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Wed, 22 Jul 2020 11:34:44 -0600 Subject: [PATCH 393/409] MAINT: Update RELEASE_WALKTHROUGH.rst.txt. Update the walkthrough to document changes in procedures and software. --- doc/RELEASE_WALKTHROUGH.rst.txt | 173 +++++++++++++++----------------- 1 file changed, 82 insertions(+), 91 deletions(-) diff --git a/doc/RELEASE_WALKTHROUGH.rst.txt b/doc/RELEASE_WALKTHROUGH.rst.txt index 0ff9ff93352a..733f681af704 100644 --- a/doc/RELEASE_WALKTHROUGH.rst.txt +++ b/doc/RELEASE_WALKTHROUGH.rst.txt @@ -1,41 +1,37 @@ -This file contains a walkthrough of the NumPy 1.14.5 release on Linux, modified +This file contains a walkthrough of the NumPy 1.19.0 release on Linux, modified for building on azure and uploading to anaconda.org The commands can be copied into the command line, but be sure to -replace 1.14.5 by the correct version. +replace 1.19.0 by the correct version. This should be read together with the general directions in `releasing`. -Release Walkthrough -==================== - -Note that in the code snippets below, ``upstream`` refers to the root repository on -github and ``origin`` to a fork in your personal account. You may need to make adjustments -if you have not forked the repository but simply cloned it locally. You can -also edit ``.git/config`` and add ``upstream`` if it isn't already present. - +Release Preparation +=================== Backport Pull Requests ---------------------- Changes that have been marked for this release must be backported to the -maintenance/1.14.x branch. +maintenance/1.19.x branch. Update Release documentation ---------------------------- -The file ``doc/changelog/1.14.5-changelog.rst`` should be updated to reflect +The file ``doc/changelog/1.19.0-changelog.rst`` should be updated to reflect the final list of changes and contributors. This text can be generated by:: - $ python tools/changelog.py $GITHUB v1.14.4..maintenance/1.14.x > doc/changelog/1.14.5-changelog.rst + $ python tools/changelog.py $GITHUB v1.18.0..maintenance/1.19.x > doc/changelog/1.19.0-changelog.rst where ``GITHUB`` contains your github access token. This text may also be -appended to ``doc/release/1.14.5-notes.rst`` for release updates, though not -for new releases like ``1.14.0``, as the changelogs for ``*.0`` releases tend to be -excessively long. The ``doc/source/release.rst`` file should also be -updated with a link to the new release notes. These changes should be committed -to the maintenance branch, and later will be forward ported to master. +appended to ``doc/release/1.19.0-notes.rst`` for patch release, though not for +new releases like ``1.19.0``, as the changelogs for ``*.0`` releases tend to be +excessively long. The ``doc/source/release.rst`` file should also be updated +with a link to the new release notes. These changes should be committed to the +maintenance branch, and later will be forward ported to master. The changelog +should be reviewed for name duplicates or short names and the ``.mailmap`` file +updated if needed. Finish the Release Note @@ -46,25 +42,33 @@ Finish the Release Note This has changed now that we use ``towncrier``. See the instructions for creating the release note in ``doc/release/upcoming_changes/README.rst``. -Fill out the release note ``doc/release/1.14.5-notes.rst`` calling out +Fill out the release note ``doc/release/1.19.0-notes.rst`` calling out significant changes. +Release Walkthrough +==================== + +Note that in the code snippets below, ``upstream`` refers to the root repository on +github and ``origin`` to a fork in your personal account. You may need to make adjustments +if you have not forked the repository but simply cloned it locally. You can +also edit ``.git/config`` and add ``upstream`` if it isn't already present. + Prepare the release commit -------------------------- Checkout the branch for the release, make sure it is up to date, and clean the repository:: - $ git checkout maintenance/1.14.x - $ git pull upstream maintenance/1.14.x + $ git checkout maintenance/1.19.x + $ git pull upstream maintenance/1.19.x $ git submodule update $ git clean -xdfq Edit pavement.py and setup.py as detailed in HOWTO_RELEASE:: - $ gvim pavement.py setup.py - $ git commit -a -m"REL: NumPy 1.14.5 release." + $ gvim pavement.py setup.py # Generally only setup.py needs updating + $ git commit -a -m"REL: NumPy 1.19.0 release." Sanity check:: @@ -73,9 +77,7 @@ Sanity check:: Push this release directly onto the end of the maintenance branch. This requires write permission to the numpy repository:: - $ git push upstream maintenance/1.14.x - -As an example, see the 1.14.3 REL commit: ``_. + $ git push upstream HEAD Build source releases @@ -86,7 +88,7 @@ Paver is used to build the source releases. It will create the ``release`` and source releases in the latter. :: $ python3 -m cython --version # check for correct cython version - $ paver sdist # sdist will do a git clean -xdf, so we omit that + $ paver sdist # sdist will do a git clean -xdfq, so we omit that Build wheels @@ -94,55 +96,46 @@ Build wheels Trigger the wheels build by pointing the numpy-wheels repository at this commit. This can take up to an hour. The numpy-wheels repository is cloned from -``_. Start with a pull as the repo -may have been accessed and changed by someone else and a push will fail:: +``_. If this is the first release in +a series, start with a pull as the repo may have been accessed and changed by +someone else, then create a new branch for the series. If the branch already +exists skip this:: $ cd ../numpy-wheels + $ git co master $ git pull upstream master - $ git branch # only when starting new numpy version - $ git checkout v1.14.x # v1.14.x already existed for the 1.14.4 release + $ git branch v1.19.x -Edit the ``azure/posix.yml`` and ``azure/windows.yml`` files to make sure they -have the correct version, and put in the commit hash for the ``REL`` commit -created above for ``BUILD_COMMIT``, see an _example:: +Checkout the new branch and edit the ``azure-pipelines.yml`` and +``.travis.yml`` files to make sure they have the correct version, and put in +the commit hash for the ``REL`` commit created above for ``BUILD_COMMIT``. The +``azure/posix.yml`` and ``.travis.yml`` files may also need the Cython versions +updated to keep up with Python releases, but generally just do:: - $ gvim azure/posix.yml azure/windows.yml - $ git commit -a + $ git checkout v1.19.x + $ gvim azure-pipelines .travis.yml + $ git commit -a -m"NumPy 1.19.0 release." $ git push upstream HEAD Now wait. If you get nervous at the amount of time taken -- the builds can take a while -- you can check the build progress by following the links -provided at ``_ to check the +provided at ``_ to check the build status. Check if all the needed wheels have been built and -uploaded before proceeding. There should currently be 21 of them at -``_, 3 for Mac, 6 -for Windows, and 12 for Linux. +uploaded to the staging repository before proceeding. -.. example_: https://github.com/MacPython/numpy-wheels/pull/80/commits/cbf4af4 - -Note that sometimes builds, like tests, fail for unrelated reasons and -you will need to restart them. +Note that sometimes builds, like tests, fail for unrelated reasons and you will +need to rerun them. You will need to be logged in under 'numpy' to do this +on azure. Download wheels --------------- -When the wheels have all been successfully built, download them using the ``wheel-uploader`` -in the ``terryfy`` repository. The terryfy repository may be cloned from -``_ if you don't already have it. The -wheels can also be uploaded using the ``wheel-uploader``, but we prefer to -download all the wheels to the ``../numpy/release/installers`` directory and -upload later using ``twine``:: +When the wheels have all been successfully built and staged, download them from the +Anaconda staging directory using the ``tools/download-wheels.py`` script:: - $ cd ../terryfy - $ git pull upstream master - $ CDN_URL=https://anaconda.org/multibuild-wheels-staging/numpy/files - $ NPY_WHLS=../numpy/release/installers - $ ./wheel-uploader -u $CDN_URL -n -v -w $NPY_WHLS -t win numpy 1.14.5 - $ ./wheel-uploader -u $CDN_URL -n -v -w $NPY_WHLS -t manylinux1 numpy 1.14.5 - $ ./wheel-uploader -u $CDN_URL -n -v -w $NPY_WHLS -t macosx numpy 1.14.5 + $ cd ../numpy + $ python3 tools/download-wheels.py 1.19.0 -If you do this often, consider making CDN_URL and NPY_WHLS part of your default -environment. Generate the README files ------------------------- @@ -150,18 +143,16 @@ Generate the README files This needs to be done after all installers are downloaded, but before the pavement file is updated for continued development:: - $ cd ../numpy $ paver write_release Tag the release --------------- -Once the wheels have been built and downloaded without errors, go back to your -numpy repository in the maintenance branch and tag the ``REL`` commit, signing +Once the wheels have been built and downloaded without errors tag the ``REL`` commit, signing it with your gpg key:: - $ git tag -s v1.14.5 + $ git tag -s -m"NumPy 1.19.0 release" v1.19.0 You should upload your public gpg key to github, so that the tag will appear "verified" there. @@ -169,7 +160,7 @@ You should upload your public gpg key to github, so that the tag will appear Check that the files in ``release/installers`` have the correct versions, then push the tag upstream:: - $ git push upstream v1.14.5 + $ git push upstream v1.19.0 We wait until this point to push the tag because it is public and should not be changed after it has been pushed. @@ -185,9 +176,9 @@ Add another ``REL`` commit to the numpy maintenance branch, which resets the Create release notes for next release and edit them to set the version:: - $ cp doc/source/release/template.rst doc/source/release/1.14.6-notes.rst - $ gvim doc/source/release/1.14.6-notes.rst - $ git add doc/source/release/1.14.6-notes.rst + $ cp doc/source/release/template.rst doc/source/release/1.19.1-notes.rst + $ gvim doc/source/release/1.19.1-notes.rst + $ git add doc/source/release/1.19.1-notes.rst Add new release notes to the documentation release list:: @@ -195,26 +186,24 @@ Add new release notes to the documentation release list:: Commit the result:: - $ git commit -a -m"REL: prepare 1.14.x for further development" - $ git push upstream maintenance/1.14.x + $ git commit -a -m"REL: prepare 1.19.x for further development" + $ git push upstream HEAD Upload to PyPI -------------- Upload to PyPI using ``twine``. A recent version of ``twine`` of is needed -after recent PyPI changes, version ``1.11.0`` was used here. - -.. code-block:: sh +after recent PyPI changes, version ``3.1.1`` was used here:: $ cd ../numpy $ twine upload release/installers/*.whl - $ twine upload release/installers/numpy-1.14.5.zip # Upload last. + $ twine upload release/installers/numpy-1.19.0.zip # Upload last. -If one of the commands breaks in the middle, which is not uncommon, you may -need to selectively upload the remaining files because PyPI does not allow the -same file to be uploaded twice. The source file should be uploaded last to -avoid synchronization problems if pip users access the files while this is in +If one of the commands breaks in the middle, you may need to selectively upload +the remaining files because PyPI does not allow the same file to be uploaded +twice. The source file should be uploaded last to avoid synchronization +problems that might occur if pip users access the files while this is in process. Note that PyPI only allows a single source distribution, here we have chosen the zip archive. @@ -222,15 +211,16 @@ chosen the zip archive. Upload files to github ---------------------- -Go to ``_, there should be a ``v1.14.5 +Go to ``_, there should be a ``v1.19.0 tag``, click on it and hit the edit button for that tag. There are two ways to -add files, using an editable text window and as binary uploads. +add files, using an editable text window and as binary uploads. Cut and paste +the ``release/README.md`` file contents into the text window. You will probably +need to make some edits to get it to look right. Then -- Cut and paste the ``release/README.md`` file contents into the text window. -- Upload ``release/installers/numpy-1.14.5.tar.gz`` as a binary file. -- Upload ``release/installers/numpy-1.14.5.zip`` as a binary file. +- Upload ``release/installers/numpy-1.19.0.tar.gz`` as a binary file. +- Upload ``release/installers/numpy-1.19.0.zip`` as a binary file. - Upload ``release/README.rst`` as a binary file. -- Upload ``doc/changelog/1.14.5-changelog.rst`` as a binary file. +- Upload ``doc/changelog/1.19.0-changelog.rst`` as a binary file. - Check the pre-release button if this is a pre-releases. - Hit the ``{Publish,Update} release`` button at the bottom. @@ -255,7 +245,7 @@ If the release series is a new one, you will need to add a new section to the Otherwise, only the ``zip`` and ``pdf`` links should be updated with the new tag name:: - $ gvim doc/build/merge/index.html +/'tag v1.14' + $ gvim doc/build/merge/index.html +/'tag v1.19' You can "test run" the new documentation in a browser to make sure the links work:: @@ -265,7 +255,7 @@ work:: Once everything seems satisfactory, commit and upload the changes:: $ pushd doc/build/merge - $ git commit -am"Add documentation for v1.14.5" + $ git commit -am"Add documentation for v1.19.0" $ git push $ popd @@ -277,7 +267,7 @@ This assumes that you have forked ``_:: $ cd ../scipy.org $ git checkout master $ git pull upstream master - $ git checkout -b numpy-1.14.5 + $ git checkout -b numpy-1.19.0 $ gvim www/index.rst # edit the News section $ git commit -a $ git push origin HEAD @@ -300,13 +290,14 @@ Post-Release Tasks Checkout master and forward port the documentation changes:: - $ git checkout -b update-after-1.14.5-release - $ git checkout maintenance/1.14.x doc/source/release/1.14.5-notes.rst - $ git checkout maintenance/1.14.x doc/changelog/1.14.5-changelog.rst + $ git checkout -b post-1.19.0-release-update + $ git checkout maintenance/1.19.x doc/source/release/1.19.0-notes.rst + $ git checkout maintenance/1.19.x doc/changelog/1.19.0-changelog.rst + $ git checkout maintenance/1.19.x .mailmap # only if updated for release. $ gvim doc/source/release.rst # Add link to new notes - $ git add doc/changelog/1.14.5-changelog.rst doc/source/release/1.14.5-notes.rst + $ git add doc/changelog/1.19.0-changelog.rst doc/source/release/1.19.0-notes.rst $ git status # check status before commit - $ git commit -a -m"REL: Update master after 1.14.5 release." + $ git commit -a -m"REL: Update master after 1.19.0 release." $ git push origin HEAD Go to github and make a PR. From d5441eee24753fe8de829b10cbdc88b32ea04213 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Fri, 30 Oct 2020 08:09:29 -0600 Subject: [PATCH 394/409] STY: Minor style fixups to changelog.py --- tools/changelog.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/changelog.py b/tools/changelog.py index 92f33af24453..920f5b87faa3 100755 --- a/tools/changelog.py +++ b/tools/changelog.py @@ -45,16 +45,17 @@ this_repo = Repo(os.path.join(os.path.dirname(__file__), "..")) author_msg =\ -u""" +""" A total of %d people contributed to this release. People with a "+" by their names contributed a patch for the first time. """ pull_request_msg =\ -u""" +""" A total of %d pull requests were merged for this release. """ + def get_authors(revision_range): pat = u'^.*\\t(.*)$' lst_release, cur_release = [r.strip() for r in revision_range.split('..')] From 3052bbcf1ec4de331154f920dd9bba0bbc200b95 Mon Sep 17 00:00:00 2001 From: Stefan van der Walt Date: Sun, 1 Nov 2020 04:51:13 -0800 Subject: [PATCH 395/409] NEP36: include additional feedback (#17578) Include some last feedback from Ralf and Ben Co-authored-by: Ralf Gommers --- doc/neps/nep-0036-fair-play.rst | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/doc/neps/nep-0036-fair-play.rst b/doc/neps/nep-0036-fair-play.rst index b8cf88788383..32e5e1e75c03 100644 --- a/doc/neps/nep-0036-fair-play.rst +++ b/doc/neps/nep-0036-fair-play.rst @@ -56,7 +56,7 @@ Scope ----- This document aims to define a minimal set of rules that, when -followed, will be considered good faith efforts in line with the +followed, will be considered good-faith efforts in line with the expectations of the NumPy developers. Our hope is that developers who feel they need to modify NumPy will @@ -65,8 +65,8 @@ mechanisms for extending our APIs and for operating on externally defined array objects. When in doubt, please `talk to us first -`__. We may suggest an -alternative; at minimum, we'll be prepared. +`__. We may suggest an alternative; at +minimum, we'll be prepared. Fair play rules --------------- @@ -74,11 +74,15 @@ Fair play rules 1. Do not reuse the NumPy name for projects not developed by the NumPy community. - At time of writing, there are only a handful of `numpy`-named - packages developed by the community, including `numpy`, - `numpy-financial`, and `unumpy`. We ask that external packages not - include the phrase `numpy`, i.e., avoid names such as - `mycompany_numpy`. + At time of writing, there are only a handful of ``numpy``-named + packages developed by the community, including ``numpy``, + ``numpy-financial``, and ``unumpy``. We ask that external packages not + include the phrase ``numpy``, i.e., avoid names such as + ``mycompany_numpy``. + + To be clear, this rule only applies to modules (package names); it + is perfectly acceptable to have a *submodule* of your own library + named ``mylibrary.numpy``. NumPy is a trademark owned by NumFOCUS. @@ -89,7 +93,7 @@ Fair play rules which parts of NumPy have been modified. If you have to break this rule (and we implore you not - to!), then make it clear in the `__version__` tag that + to!), then make it clear in the ``__version__`` tag that you have modified NumPy, e.g.:: >>> print(np.__version__) From dbe23fd71261940d0919071d72d579aa2c831603 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Tue, 27 Oct 2020 19:19:45 -0600 Subject: [PATCH 396/409] BUG: Fix small bug in make_lite.py. Flagged by LGTM. --- numpy/linalg/lapack_lite/make_lite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/linalg/lapack_lite/make_lite.py b/numpy/linalg/lapack_lite/make_lite.py index 6cbd11445c8d..cf15b2541361 100755 --- a/numpy/linalg/lapack_lite/make_lite.py +++ b/numpy/linalg/lapack_lite/make_lite.py @@ -81,7 +81,7 @@ def dependencies(self): return self._dependencies def __repr__(self): - return return f'FortranRoutine({self.name!r}, filename={self.filename!r})' + return f'FortranRoutine({self.name!r}, filename={self.filename!r})' class UnknownFortranRoutine(FortranRoutine): """Wrapper for a Fortran routine for which the corresponding file From e17cdf56c4d40bcebb0a272af573be94e7286fa0 Mon Sep 17 00:00:00 2001 From: Qiyu8 Date: Mon, 2 Nov 2020 10:27:49 +0800 Subject: [PATCH 397/409] add a newline at the end of the file. --- numpy/core/src/common/simd/avx2/arithmetic.h | 2 +- numpy/core/src/common/simd/avx512/arithmetic.h | 2 +- numpy/core/src/common/simd/neon/arithmetic.h | 2 +- numpy/core/src/common/simd/sse/arithmetic.h | 2 +- numpy/core/src/common/simd/vsx/arithmetic.h | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/numpy/core/src/common/simd/avx2/arithmetic.h b/numpy/core/src/common/simd/avx2/arithmetic.h index e43b8e7a3b04..8b98394e0931 100644 --- a/numpy/core/src/common/simd/avx2/arithmetic.h +++ b/numpy/core/src/common/simd/avx2/arithmetic.h @@ -136,4 +136,4 @@ NPY_FINLINE double npyv_sum_f64(__m256d a) __m128d hi = _mm256_extractf128_pd(sum_halves, 1); __m128d sum = _mm_add_pd(lo, hi); return _mm_cvtsd_f64(sum); -} \ No newline at end of file +} diff --git a/numpy/core/src/common/simd/avx512/arithmetic.h b/numpy/core/src/common/simd/avx512/arithmetic.h index f31051aaf12b..989e10f39bf8 100644 --- a/numpy/core/src/common/simd/avx512/arithmetic.h +++ b/numpy/core/src/common/simd/avx512/arithmetic.h @@ -172,4 +172,4 @@ NPY_FINLINE __m512i npyv_mul_u8(__m512i a, __m512i b) __m512d sum8 = _mm512_add_pd(sum16, h16); return _mm_cvtsd_f64(_mm512_castpd512_pd128(sum8)); } -#endif \ No newline at end of file +#endif diff --git a/numpy/core/src/common/simd/neon/arithmetic.h b/numpy/core/src/common/simd/neon/arithmetic.h index 3bea143846a6..2d56adfbd589 100644 --- a/numpy/core/src/common/simd/neon/arithmetic.h +++ b/numpy/core/src/common/simd/neon/arithmetic.h @@ -131,4 +131,4 @@ NPY_FINLINE float npyv_sum_f32(float32x4_t a) { return vget_lane_f64(vget_low_f64(a) + vget_high_f64(a), 0); } -#endif \ No newline at end of file +#endif diff --git a/numpy/core/src/common/simd/sse/arithmetic.h b/numpy/core/src/common/simd/sse/arithmetic.h index a92c39bcce4f..cf33349fc1ad 100644 --- a/numpy/core/src/common/simd/sse/arithmetic.h +++ b/numpy/core/src/common/simd/sse/arithmetic.h @@ -171,4 +171,4 @@ NPY_FINLINE double npyv_sum_f64(__m128d a) #else return _mm_cvtsd_f64(_mm_add_pd(a, _mm_unpackhi_pd(a, a))); #endif -} \ No newline at end of file +} diff --git a/numpy/core/src/common/simd/vsx/arithmetic.h b/numpy/core/src/common/simd/vsx/arithmetic.h index 9b323d7d1512..755aa1ca3169 100644 --- a/numpy/core/src/common/simd/vsx/arithmetic.h +++ b/numpy/core/src/common/simd/vsx/arithmetic.h @@ -128,4 +128,4 @@ NPY_FINLINE float npyv_sum_f32(npyv_f32 a) NPY_FINLINE double npyv_sum_f64(npyv_f64 a) { return vec_extract(a, 0) + vec_extract(a, 1); -} \ No newline at end of file +} From d34ed7dbb4cf58389e278ffd5be6ca2bcb081482 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2020 07:29:08 +0000 Subject: [PATCH 398/409] MAINT: Bump hypothesis from 5.38.0 to 5.41.0 Bumps [hypothesis](https://github.com/HypothesisWorks/hypothesis) from 5.38.0 to 5.41.0. - [Release notes](https://github.com/HypothesisWorks/hypothesis/releases) - [Commits](https://github.com/HypothesisWorks/hypothesis/compare/hypothesis-python-5.38.0...hypothesis-python-5.41.0) Signed-off-by: dependabot-preview[bot] --- test_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index 6e858857cd30..10e4fee83354 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,7 +1,7 @@ cython==0.29.21 wheel setuptools<49.2.0 -hypothesis==5.38.0 +hypothesis==5.41.0 pytest==6.0.2 pytz==2020.1 pytest-cov==2.10.1 From cefad82eaf08b9d905a710a751cc445df6b2a8e7 Mon Sep 17 00:00:00 2001 From: mattip Date: Mon, 2 Nov 2020 10:25:54 +0200 Subject: [PATCH 399/409] TST: use a more standard workflow for PyPy --- azure-pipelines.yml | 24 ++++++++++++++++++++++-- tools/pypy-test.sh | 4 ++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7a8196dce14a..02b305ada112 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -235,8 +235,28 @@ stages: pool: vmIMage: 'ubuntu-18.04' steps: - - script: source tools/pypy-test.sh - displayName: 'Run PyPy3 Build / Tests' + - task: UsePythonVersion@0 + inputs: + versionSpec: pypy3 + - script: | + python -c "import sys; print(sys.version)" + python -m pip install --upgrade "setuptools<49.2.0" + python -m pip install urllib3 + basedir = $(python numpy/tools/openblas_support.py) + cp -r $basedir/lib/* /usr/local/lib + cp $basedir/include/* /usr/local/include + displayName: 'Get OpenBLAS' + - script: | + python setup.py build_src --verbose-cfg bdist_wheel + pushd dist + python -mpip install ./numpy* + python ../tools/openblas_support.py --check_version + popd + displayName: 'Build, check OpenBLAS version' + - script: | + python runtests.py --debug-info --show-build-log -v -- -rsx \ + --junitxml=junit/test-results.xml --durations 10 + displayName: 'Test' - task: PublishTestResults@2 condition: succeededOrFailed() inputs: diff --git a/tools/pypy-test.sh b/tools/pypy-test.sh index 32b7968d8e9a..e6c6ae719c91 100755 --- a/tools/pypy-test.sh +++ b/tools/pypy-test.sh @@ -28,8 +28,8 @@ include_dirs = $target/lib:$LIB runtime_library_dirs = $target/lib EOF -echo getting PyPy 3.6-v7.3.1 -wget -q https://downloads.python.org/pypy/pypy3.6-v7.3.1-linux64.tar.bz2 -O pypy.tar.bz2 +echo getting PyPy 3.6-v7.3.2 +wget -q https://downloads.python.org/pypy/pypy3.6-v7.3.2-linux64.tar.bz2 -O pypy.tar.bz2 mkdir -p pypy3 (cd pypy3; tar --strip-components=1 -xf ../pypy.tar.bz2) pypy3/bin/pypy3 -mensurepip From 313d1070c04455aa4426509b9341961f3c433556 Mon Sep 17 00:00:00 2001 From: mattip Date: Mon, 2 Nov 2020 11:24:02 +0200 Subject: [PATCH 400/409] fix getting OpenBLAS --- azure-pipelines.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 02b305ada112..12d99156d966 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -242,12 +242,15 @@ stages: python -c "import sys; print(sys.version)" python -m pip install --upgrade "setuptools<49.2.0" python -m pip install urllib3 - basedir = $(python numpy/tools/openblas_support.py) - cp -r $basedir/lib/* /usr/local/lib - cp $basedir/include/* /usr/local/include + python -m pip install -r test_requirements.txt + basedir=$(python ./tools/openblas_support.py) + sudo cp -r $basedir/lib/* /usr/local/lib + sudo cp $basedir/include/* /usr/local/include + sudo ldconfig /usr/local/lib displayName: 'Get OpenBLAS' - script: | python setup.py build_src --verbose-cfg bdist_wheel + # do the rest in a subdirectory so 'import numpy' works pushd dist python -mpip install ./numpy* python ../tools/openblas_support.py --check_version From 1da42c20683243088404c78cc4959f87654f9565 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2020 14:58:23 +0000 Subject: [PATCH 401/409] MAINT: Bump pytz from 2020.1 to 2020.4 Bumps [pytz](https://github.com/stub42/pytz) from 2020.1 to 2020.4. - [Release notes](https://github.com/stub42/pytz/releases) - [Commits](https://github.com/stub42/pytz/compare/release_2020.1...release_2020.4) Signed-off-by: dependabot-preview[bot] --- test_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index 10e4fee83354..8b17ccc133f1 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -3,7 +3,7 @@ wheel setuptools<49.2.0 hypothesis==5.41.0 pytest==6.0.2 -pytz==2020.1 +pytz==2020.4 pytest-cov==2.10.1 pickle5; python_version == '3.7' pickle5; python_version == '3.6' and platform_python_implementation != 'PyPy' From 220f3fee717994c07275482e06f06259e055d033 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Mon, 2 Nov 2020 09:44:36 -0700 Subject: [PATCH 402/409] REL: Update master after 1.19.4 release. --- doc/changelog/1.19.4-changelog.rst | 16 +++++++++++++++ doc/source/release.rst | 1 + doc/source/release/1.19.3-notes.rst | 4 ++-- doc/source/release/1.19.4-notes.rst | 30 +++++++++++++++++++++++++++++ 4 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 doc/changelog/1.19.4-changelog.rst create mode 100644 doc/source/release/1.19.4-notes.rst diff --git a/doc/changelog/1.19.4-changelog.rst b/doc/changelog/1.19.4-changelog.rst new file mode 100644 index 000000000000..82632b990855 --- /dev/null +++ b/doc/changelog/1.19.4-changelog.rst @@ -0,0 +1,16 @@ + +Contributors +============ + +A total of 1 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Charles Harris + +Pull requests merged +==================== + +A total of 2 pull requests were merged for this release. + +* `#17679 `__: MAINT: Add check for Windows 10 version 2004 bug. +* `#17680 `__: REV: Revert OpenBLAS to 1.19.2 version for 1.19.4 diff --git a/doc/source/release.rst b/doc/source/release.rst index c7bc5e1b9d39..3ef1b06bd34a 100644 --- a/doc/source/release.rst +++ b/doc/source/release.rst @@ -6,6 +6,7 @@ Release Notes :maxdepth: 3 1.20.0 + 1.19.4 1.19.3 1.19.2 1.19.1 diff --git a/doc/source/release/1.19.3-notes.rst b/doc/source/release/1.19.3-notes.rst index 54a4883893ff..f1f1fd2b3e2f 100644 --- a/doc/source/release/1.19.3-notes.rst +++ b/doc/source/release/1.19.3-notes.rst @@ -7,9 +7,9 @@ NumPy 1.19.3 Release Notes NumPy 1.19.3 is a small maintenance release with two major improvements: - Python 3.9 binary wheels on all supported platforms. -- OpenBLAS fixes for the Windows 10 version 2004 fmod bug. +- OpenBLAS fixes for Windows 10 version 2004 fmod bug. -This release supports Python 3.6-3.9 and is linked with OpenBLAS 3.7 to avoid +This release supports Python 3.6-3.9 and is linked with OpenBLAS 0.3.12 to avoid some of the fmod problems on Windows 10 version 2004. Microsoft is aware of the problem and users should upgrade when the fix becomes available, the fix here is limited in scope. diff --git a/doc/source/release/1.19.4-notes.rst b/doc/source/release/1.19.4-notes.rst new file mode 100644 index 000000000000..e7c0863f4510 --- /dev/null +++ b/doc/source/release/1.19.4-notes.rst @@ -0,0 +1,30 @@ +.. currentmodule:: numpy + +========================== +NumPy 1.19.4 Release Notes +========================== + +NumPy 1.19.4 is a quick release to revert the OpenBLAS library version. It was +hoped that the 0.3.12 OpenBLAS version used in 1.19.3 would work around the +Microsoft fmod bug, but problems in some docker environments turned up. Instead, +1.19.4 will use the older library and run a sanity check on import, raising an +error if the problem is detected. Microsoft is aware of the problem and has +promised a fix, users should upgrade when it becomes available. + +This release supports Python 3.6-3.9 + +Contributors +============ + +A total of 1 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Charles Harris + +Pull requests merged +==================== + +A total of 2 pull requests were merged for this release. + +* `#17679 `__: MAINT: Add check for Windows 10 version 2004 bug. +* `#17680 `__: REV: Revert OpenBLAS to 1.19.2 version for 1.19.4 From 1f0298d62853e5233b0b829b08a11c160f0b6597 Mon Sep 17 00:00:00 2001 From: Qiyu8 Date: Tue, 3 Nov 2020 17:04:35 +0800 Subject: [PATCH 403/409] improve intrinsics and add sum intrinsic test --- numpy/core/src/_simd/_simd.dispatch.c.src | 10 ++++++++++ numpy/core/src/common/simd/avx2/arithmetic.h | 5 ++++- numpy/core/src/common/simd/avx512/arithmetic.h | 4 ++-- numpy/core/src/common/simd/neon/arithmetic.h | 18 +++++++++--------- numpy/core/src/common/simd/sse/arithmetic.h | 5 ++++- numpy/core/src/common/simd/vsx/arithmetic.h | 8 ++++---- numpy/core/tests/test_simd.py | 10 ++++++++++ 7 files changed, 43 insertions(+), 17 deletions(-) diff --git a/numpy/core/src/_simd/_simd.dispatch.c.src b/numpy/core/src/_simd/_simd.dispatch.c.src index 2d89b9df00f9..3d7af23331b7 100644 --- a/numpy/core/src/_simd/_simd.dispatch.c.src +++ b/numpy/core/src/_simd/_simd.dispatch.c.src @@ -20,6 +20,7 @@ * #mul_sup = 1, 1, 1, 1, 1, 1, 0, 0, 1, 1# * #div_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# * #fused_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# + * #sum_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# * #ncont_sup = 0, 0, 0, 0, 1, 1, 1, 1, 1, 1# * #shl_imm = 0, 0, 15, 15, 31, 31, 63, 63, 0, 0# * #shr_imm = 0, 0, 16, 16, 32, 32, 64, 64, 0, 0# @@ -351,6 +352,10 @@ SIMD_IMPL_INTRIN_3(@intrin@_@sfx@, v@sfx@, v@sfx@, v@sfx@, v@sfx@) /**end repeat1**/ #endif // fused_sup +#if @sum_sup@ +SIMD_IMPL_INTRIN_1(sum_@sfx@, @sfx@, v@sfx@) +#endif // sum_sup + #endif // simd_sup /**end repeat**/ /*************************** @@ -370,6 +375,7 @@ static PyMethodDef simd__intrinsics_methods[] = { * #mul_sup = 1, 1, 1, 1, 1, 1, 0, 0, 1, 1# * #div_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# * #fused_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# + * #sum_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# * #ncont_sup = 0, 0, 0, 0, 1, 1, 1, 1, 1, 1# * #shl_imm = 0, 0, 15, 15, 31, 31, 63, 63, 0, 0# * #shr_imm = 0, 0, 16, 16, 32, 32, 64, 64, 0, 0# @@ -484,6 +490,10 @@ SIMD_INTRIN_DEF(@intrin@_@sfx@) /**end repeat1**/ #endif // fused_sup +#if @sum_sup@ +SIMD_INTRIN_DEF(sum_@sfx@) +#endif // sum_sup + #endif // simd_sup /**end repeat**/ diff --git a/numpy/core/src/common/simd/avx2/arithmetic.h b/numpy/core/src/common/simd/avx2/arithmetic.h index 8b98394e0931..3a6dc953534e 100644 --- a/numpy/core/src/common/simd/avx2/arithmetic.h +++ b/numpy/core/src/common/simd/avx2/arithmetic.h @@ -116,7 +116,6 @@ return npyv_sub_f64(npyv_mul_f64(neg_a, b), c); } #endif // !NPY_HAVE_FMA3 -#endif // _NPY_SIMD_AVX2_ARITHMETIC_H // Horizontal add: Calculates the sum of all vector elements. NPY_FINLINE float npyv_sum_f32(__m256 a) @@ -137,3 +136,7 @@ NPY_FINLINE double npyv_sum_f64(__m256d a) __m128d sum = _mm_add_pd(lo, hi); return _mm_cvtsd_f64(sum); } + +#endif // _NPY_SIMD_AVX2_ARITHMETIC_H + + diff --git a/numpy/core/src/common/simd/avx512/arithmetic.h b/numpy/core/src/common/simd/avx512/arithmetic.h index 989e10f39bf8..7372ca29e40d 100644 --- a/numpy/core/src/common/simd/avx512/arithmetic.h +++ b/numpy/core/src/common/simd/avx512/arithmetic.h @@ -129,8 +129,6 @@ NPY_FINLINE __m512i npyv_mul_u8(__m512i a, __m512i b) #define npyv_nmulsub_f32 _mm512_fnmsub_ps #define npyv_nmulsub_f64 _mm512_fnmsub_pd -#endif // _NPY_SIMD_AVX512_ARITHMETIC_H - /*************************** * Reduce Sum * there are three ways to implement reduce sum for AVX512: @@ -173,3 +171,5 @@ NPY_FINLINE __m512i npyv_mul_u8(__m512i a, __m512i b) return _mm_cvtsd_f64(_mm512_castpd512_pd128(sum8)); } #endif + +#endif // _NPY_SIMD_AVX512_ARITHMETIC_H diff --git a/numpy/core/src/common/simd/neon/arithmetic.h b/numpy/core/src/common/simd/neon/arithmetic.h index 2d56adfbd589..bc14ffb75574 100644 --- a/numpy/core/src/common/simd/neon/arithmetic.h +++ b/numpy/core/src/common/simd/neon/arithmetic.h @@ -118,17 +118,17 @@ NPY_FINLINE npyv_f64 npyv_nmulsub_f64(npyv_f64 a, npyv_f64 b, npyv_f64 c) { return vfmsq_f64(vnegq_f64(c), a, b); } #endif // NPY_SIMD_F64 -#endif // _NPY_SIMD_NEON_ARITHMETIC_H // Horizontal add: Calculates the sum of all vector elements. -NPY_FINLINE float npyv_sum_f32(float32x4_t a) -{ - float32x2_t r = vadd_f32(vget_high_f32(a), vget_low_f32(a)); - return vget_lane_f32(vpadd_f32(r, r), 0); -} -#ifdef __aarch64__ - NPY_FINLINE double npyv_sum_f64(float64x2_t a) +#if NPY_SIMD_F64 + #define npyv_sum_f32 vaddvq_f32 + #define npyv_sum_f64 vaddvq_f64 +#else + NPY_FINLINE float npyv_sum_f32(npyv_f32 a) { - return vget_lane_f64(vget_low_f64(a) + vget_high_f64(a), 0); + float32x2_t r = vadd_f32(vget_high_f32(a), vget_low_f32(a)); + return vget_lane_f32(vpadd_f32(r, r), 0); } #endif + +#endif // _NPY_SIMD_NEON_ARITHMETIC_H diff --git a/numpy/core/src/common/simd/sse/arithmetic.h b/numpy/core/src/common/simd/sse/arithmetic.h index cf33349fc1ad..8440cc52e5c8 100644 --- a/numpy/core/src/common/simd/sse/arithmetic.h +++ b/numpy/core/src/common/simd/sse/arithmetic.h @@ -147,7 +147,6 @@ NPY_FINLINE __m128i npyv_mul_u8(__m128i a, __m128i b) return npyv_sub_f64(npyv_mul_f64(neg_a, b), c); } #endif // !NPY_HAVE_FMA3 -#endif // _NPY_SIMD_SSE_ARITHMETIC_H // Horizontal add: Calculates the sum of all vector elements. NPY_FINLINE float npyv_sum_f32(__m128 a) @@ -172,3 +171,7 @@ NPY_FINLINE double npyv_sum_f64(__m128d a) return _mm_cvtsd_f64(_mm_add_pd(a, _mm_unpackhi_pd(a, a))); #endif } + +#endif // _NPY_SIMD_SSE_ARITHMETIC_H + + diff --git a/numpy/core/src/common/simd/vsx/arithmetic.h b/numpy/core/src/common/simd/vsx/arithmetic.h index 755aa1ca3169..2f6762e636b4 100644 --- a/numpy/core/src/common/simd/vsx/arithmetic.h +++ b/numpy/core/src/common/simd/vsx/arithmetic.h @@ -116,16 +116,16 @@ #define npyv_nmulsub_f32 vec_nmadd // equivalent to -(a*b + c) #define npyv_nmulsub_f64 vec_nmadd -#endif // _NPY_SIMD_VSX_ARITHMETIC_H - // Horizontal add: Calculates the sum of all vector elements. NPY_FINLINE float npyv_sum_f32(npyv_f32 a) { - return vec_extract(a, 0) + vec_extract(a, 1) + - vec_extract(a, 2) + vec_extract(a, 3); + npyv_f32 sum = vec_add(a, npyv_combineh_f32(a, a)); + return vec_extract(sum, 0) + vec_extract(sum, 1); } NPY_FINLINE double npyv_sum_f64(npyv_f64 a) { return vec_extract(a, 0) + vec_extract(a, 1); } + +#endif // _NPY_SIMD_VSX_ARITHMETIC_H diff --git a/numpy/core/tests/test_simd.py b/numpy/core/tests/test_simd.py index 50e77a4b8c04..061d0e34f9b2 100644 --- a/numpy/core/tests/test_simd.py +++ b/numpy/core/tests/test_simd.py @@ -512,6 +512,16 @@ def test_arithmetic_div(self): div = self.div(vdata_a, vdata_b) assert div == data_div + def test_arithmetic_reduce_sum(self): + if not self._is_fp(): + return + # reduce sum + data = self._data() + vdata = self.load(data) + + data_sum = sum(data) + vsum = self.sum(vdata) + assert vsum == data_sum int_sfx = ("u8", "s8", "u16", "s16", "u32", "s32", "u64", "s64") fp_sfx = ("f32", "f64") From c4684bb542db2690e678d22bf32745e484ef04e3 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Tue, 3 Nov 2020 13:08:58 +0100 Subject: [PATCH 404/409] MAINT: Rename `DtypeLike` to `DTypeLike` --- numpy/__init__.pyi | 98 +++++++++++----------- numpy/core/_asarray.pyi | 14 ++-- numpy/core/fromnumeric.pyi | 30 +++---- numpy/core/function_base.pyi | 10 +-- numpy/core/numeric.pyi | 22 ++--- numpy/core/numerictypes.pyi | 10 +-- numpy/typing/__init__.py | 6 +- numpy/typing/_array_like.py | 6 +- numpy/typing/_dtype_like.py | 34 ++++---- numpy/typing/tests/data/fail/flatiter.py | 2 +- numpy/typing/tests/data/pass/array_like.py | 4 +- 11 files changed, 118 insertions(+), 118 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 612e56084941..75021634289a 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -6,7 +6,7 @@ from abc import abstractmethod from numpy.core._internal import _ctypes from numpy.typing import ( ArrayLike, - DtypeLike, + DTypeLike, _Shape, _ShapeLike, _CharLike, @@ -15,8 +15,8 @@ from numpy.typing import ( _FloatLike, _ComplexLike, _NumberLike, - _SupportsDtype, - _VoidDtypeLike, + _SupportsDType, + _VoidDTypeLike, NBitBase, _64Bit, _32Bit, @@ -827,11 +827,11 @@ class dtype(Generic[_DTypeScalar]): align: bool = ..., copy: bool = ..., ) -> dtype[_DTypeScalar]: ... - # TODO: handle _SupportsDtype better + # TODO: handle _SupportsDType better @overload def __new__( cls, - dtype: _SupportsDtype, + dtype: _SupportsDType, align: bool = ..., copy: bool = ..., ) -> dtype[Any]: ... @@ -847,16 +847,16 @@ class dtype(Generic[_DTypeScalar]): @overload def __new__( cls, - dtype: _VoidDtypeLike, + dtype: _VoidDTypeLike, align: bool = ..., copy: bool = ..., ) -> dtype[void]: ... - def __eq__(self, other: DtypeLike) -> bool: ... - def __ne__(self, other: DtypeLike) -> bool: ... - def __gt__(self, other: DtypeLike) -> bool: ... - def __ge__(self, other: DtypeLike) -> bool: ... - def __lt__(self, other: DtypeLike) -> bool: ... - def __le__(self, other: DtypeLike) -> bool: ... + def __eq__(self, other: DTypeLike) -> bool: ... + def __ne__(self, other: DTypeLike) -> bool: ... + def __gt__(self, other: DTypeLike) -> bool: ... + def __ge__(self, other: DTypeLike) -> bool: ... + def __lt__(self, other: DTypeLike) -> bool: ... + def __le__(self, other: DTypeLike) -> bool: ... @property def alignment(self) -> int: ... @property @@ -905,7 +905,7 @@ class dtype(Generic[_DTypeScalar]): @property def type(self) -> Type[generic]: ... -_Dtype = dtype # to avoid name conflicts with ndarray.dtype +_DType = dtype # to avoid name conflicts with ndarray.dtype class _flagsobj: aligned: bool @@ -964,7 +964,7 @@ class flatiter(Generic[_ArraySelf]): def __getitem__( self, key: Union[_ArrayLikeInt, slice, ellipsis], ) -> _ArraySelf: ... - def __array__(self, __dtype: DtypeLike = ...) -> ndarray: ... + def __array__(self, __dtype: DTypeLike = ...) -> ndarray: ... _OrderKACF = Optional[Literal["K", "A", "C", "F"]] _OrderACF = Optional[Literal["A", "C", "F"]] @@ -993,7 +993,7 @@ class _ArrayOrScalarCommon: @property def base(self) -> Optional[ndarray]: ... @property - def dtype(self) -> _Dtype: ... + def dtype(self) -> _DType: ... @property def data(self) -> memoryview: ... @property @@ -1010,7 +1010,7 @@ class _ArrayOrScalarCommon: def shape(self) -> _Shape: ... @property def strides(self) -> _Shape: ... - def __array__(self, __dtype: DtypeLike = ...) -> ndarray: ... + def __array__(self, __dtype: DTypeLike = ...) -> ndarray: ... def __bool__(self) -> bool: ... def __bytes__(self) -> bytes: ... def __str__(self) -> str: ... @@ -1025,7 +1025,7 @@ class _ArrayOrScalarCommon: def __ge__(self, other): ... def astype( self: _ArraySelf, - dtype: DtypeLike, + dtype: DTypeLike, order: _OrderKACF = ..., casting: _Casting = ..., subok: bool = ..., @@ -1040,7 +1040,7 @@ class _ArrayOrScalarCommon: def flat(self: _ArraySelf) -> flatiter[_ArraySelf]: ... def flatten(self: _ArraySelf, order: _OrderKACF = ...) -> _ArraySelf: ... def getfield( - self: _ArraySelf, dtype: DtypeLike, offset: int = ... + self: _ArraySelf, dtype: DTypeLike, offset: int = ... ) -> _ArraySelf: ... @overload def item(self, *args: int) -> Any: ... @@ -1085,10 +1085,10 @@ class _ArrayOrScalarCommon: @overload def view(self, type: Type[_NdArraySubClass]) -> _NdArraySubClass: ... @overload - def view(self: _ArraySelf, dtype: DtypeLike = ...) -> _ArraySelf: ... + def view(self: _ArraySelf, dtype: DTypeLike = ...) -> _ArraySelf: ... @overload def view( - self, dtype: DtypeLike, type: Type[_NdArraySubClass] + self, dtype: DTypeLike, type: Type[_NdArraySubClass] ) -> _NdArraySubClass: ... # TODO: Add proper signatures @@ -1210,24 +1210,24 @@ class _ArrayOrScalarCommon: def conjugate(self: _ArraySelf) -> _ArraySelf: ... @overload def cumprod( - self, axis: Optional[int] = ..., dtype: DtypeLike = ..., out: None = ..., + self, axis: Optional[int] = ..., dtype: DTypeLike = ..., out: None = ..., ) -> ndarray: ... @overload def cumprod( self, axis: Optional[int] = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: _NdArraySubClass = ..., ) -> _NdArraySubClass: ... @overload def cumsum( - self, axis: Optional[int] = ..., dtype: DtypeLike = ..., out: None = ..., + self, axis: Optional[int] = ..., dtype: DTypeLike = ..., out: None = ..., ) -> ndarray: ... @overload def cumsum( self, axis: Optional[int] = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: _NdArraySubClass = ..., ) -> _NdArraySubClass: ... @overload @@ -1261,7 +1261,7 @@ class _ArrayOrScalarCommon: def mean( self, axis: None = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: None = ..., keepdims: Literal[False] = ..., ) -> number: ... @@ -1269,7 +1269,7 @@ class _ArrayOrScalarCommon: def mean( self, axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: None = ..., keepdims: bool = ..., ) -> Union[number, ndarray]: ... @@ -1277,7 +1277,7 @@ class _ArrayOrScalarCommon: def mean( self, axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: _NdArraySubClass = ..., keepdims: bool = ..., ) -> _NdArraySubClass: ... @@ -1313,7 +1313,7 @@ class _ArrayOrScalarCommon: def prod( self, axis: None = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: None = ..., keepdims: Literal[False] = ..., initial: _NumberLike = ..., @@ -1323,7 +1323,7 @@ class _ArrayOrScalarCommon: def prod( self, axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: None = ..., keepdims: bool = ..., initial: _NumberLike = ..., @@ -1333,7 +1333,7 @@ class _ArrayOrScalarCommon: def prod( self, axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: _NdArraySubClass = ..., keepdims: bool = ..., initial: _NumberLike = ..., @@ -1367,7 +1367,7 @@ class _ArrayOrScalarCommon: def std( self, axis: None = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: None = ..., ddof: int = ..., keepdims: Literal[False] = ..., @@ -1376,7 +1376,7 @@ class _ArrayOrScalarCommon: def std( self, axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: None = ..., ddof: int = ..., keepdims: bool = ..., @@ -1385,7 +1385,7 @@ class _ArrayOrScalarCommon: def std( self, axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: _NdArraySubClass = ..., ddof: int = ..., keepdims: bool = ..., @@ -1394,7 +1394,7 @@ class _ArrayOrScalarCommon: def sum( self, axis: None = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: None = ..., keepdims: Literal[False] = ..., initial: _NumberLike = ..., @@ -1404,7 +1404,7 @@ class _ArrayOrScalarCommon: def sum( self, axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: None = ..., keepdims: bool = ..., initial: _NumberLike = ..., @@ -1414,7 +1414,7 @@ class _ArrayOrScalarCommon: def sum( self, axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: _NdArraySubClass = ..., keepdims: bool = ..., initial: _NumberLike = ..., @@ -1448,7 +1448,7 @@ class _ArrayOrScalarCommon: def var( self, axis: None = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: None = ..., ddof: int = ..., keepdims: Literal[False] = ..., @@ -1457,7 +1457,7 @@ class _ArrayOrScalarCommon: def var( self, axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: None = ..., ddof: int = ..., keepdims: bool = ..., @@ -1466,7 +1466,7 @@ class _ArrayOrScalarCommon: def var( self, axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: _NdArraySubClass = ..., ddof: int = ..., keepdims: bool = ..., @@ -1487,14 +1487,14 @@ class ndarray(_ArrayOrScalarCommon, Iterable, Sized, Container): def __new__( cls: Type[_ArraySelf], shape: Sequence[int], - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., buffer: _BufferType = ..., offset: int = ..., strides: _ShapeLike = ..., order: _OrderKACF = ..., ) -> _ArraySelf: ... @property - def dtype(self) -> _Dtype: ... + def dtype(self) -> _DType: ... @property def ctypes(self) -> _ctypes: ... @property @@ -1540,7 +1540,7 @@ class ndarray(_ArrayOrScalarCommon, Iterable, Sized, Container): sorter: Optional[_ArrayLikeIntOrBool] = ..., # 1D int array ) -> ndarray: ... def setfield( - self, val: ArrayLike, dtype: DtypeLike, offset: int = ... + self, val: ArrayLike, dtype: DTypeLike, offset: int = ... ) -> None: ... def sort( self, @@ -1554,7 +1554,7 @@ class ndarray(_ArrayOrScalarCommon, Iterable, Sized, Container): offset: int = ..., axis1: int = ..., axis2: int = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: None = ..., ) -> Union[number, ndarray]: ... @overload @@ -1563,7 +1563,7 @@ class ndarray(_ArrayOrScalarCommon, Iterable, Sized, Container): offset: int = ..., axis1: int = ..., axis2: int = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: _NdArraySubClass = ..., ) -> _NdArraySubClass: ... # Many of these special methods are irrelevant currently, since protocols @@ -1923,7 +1923,7 @@ class void(flexible): @property def imag(self: _ArraySelf) -> _ArraySelf: ... def setfield( - self, val: ArrayLike, dtype: DtypeLike, offset: int = ... + self, val: ArrayLike, dtype: DTypeLike, offset: int = ... ) -> None: ... class character(flexible): # type: ignore @@ -1961,7 +1961,7 @@ unicode_ = str0 = str_ def array( object: object, - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., *, copy: bool = ..., order: _OrderKACF = ..., @@ -1971,14 +1971,14 @@ def array( ) -> ndarray: ... def zeros( shape: _ShapeLike, - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., order: _OrderCF = ..., *, like: ArrayLike = ..., ) -> ndarray: ... def empty( shape: _ShapeLike, - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., order: _OrderCF = ..., *, like: ArrayLike = ..., @@ -2055,7 +2055,7 @@ class ufunc: keepdims: bool = ..., casting: _Casting = ..., order: _OrderKACF = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., subok: bool = ..., signature: Union[str, Tuple[str]] = ..., # In reality this should be a length of list 3 containing an diff --git a/numpy/core/_asarray.pyi b/numpy/core/_asarray.pyi index e074d69d2c23..8c200ba22ae2 100644 --- a/numpy/core/_asarray.pyi +++ b/numpy/core/_asarray.pyi @@ -2,7 +2,7 @@ import sys from typing import TypeVar, Union, Iterable, overload from numpy import ndarray, _OrderKACF -from numpy.typing import ArrayLike, DtypeLike +from numpy.typing import ArrayLike, DTypeLike if sys.version_info >= (3, 8): from typing import Literal @@ -13,7 +13,7 @@ _ArrayType = TypeVar("_ArrayType", bound=ndarray) def asarray( a: object, - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., order: _OrderKACF = ..., *, like: ArrayLike = ... @@ -29,16 +29,16 @@ def asanyarray( @overload def asanyarray( a: object, - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., order: _OrderKACF = ..., *, like: ArrayLike = ... ) -> ndarray: ... def ascontiguousarray( - a: object, dtype: DtypeLike = ..., *, like: ArrayLike = ... + a: object, dtype: DTypeLike = ..., *, like: ArrayLike = ... ) -> ndarray: ... def asfortranarray( - a: object, dtype: DtypeLike = ..., *, like: ArrayLike = ... + a: object, dtype: DTypeLike = ..., *, like: ArrayLike = ... ) -> ndarray: ... _Requirements = Literal[ @@ -62,7 +62,7 @@ def require( @overload def require( a: object, - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., requirements: Union[_E, Iterable[_RequirementsWithE]] = ..., *, like: ArrayLike = ... @@ -70,7 +70,7 @@ def require( @overload def require( a: object, - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., requirements: Union[None, _Requirements, Iterable[_Requirements]] = ..., *, like: ArrayLike = ... diff --git a/numpy/core/fromnumeric.pyi b/numpy/core/fromnumeric.pyi index 3167d12b98c6..66eb3bfb892f 100644 --- a/numpy/core/fromnumeric.pyi +++ b/numpy/core/fromnumeric.pyi @@ -18,7 +18,7 @@ from numpy import ( _SortSide, ) from numpy.typing import ( - DtypeLike, + DTypeLike, ArrayLike, _ShapeLike, _Shape, @@ -198,7 +198,7 @@ def trace( offset: int = ..., axis1: int = ..., axis2: int = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: Optional[ndarray] = ..., ) -> Union[number, ndarray]: ... def ravel(a: ArrayLike, order: _OrderKACF = ...) -> ndarray: ... @@ -246,7 +246,7 @@ def clip( def sum( a: _Number, axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: Optional[ndarray] = ..., keepdims: bool = ..., initial: _NumberLike = ..., @@ -256,7 +256,7 @@ def sum( def sum( a: ArrayLike, axis: _ShapeLike = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: Optional[ndarray] = ..., keepdims: bool = ..., initial: _NumberLike = ..., @@ -293,7 +293,7 @@ def any( def cumsum( a: ArrayLike, axis: Optional[int] = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: Optional[ndarray] = ..., ) -> ndarray: ... @overload @@ -383,7 +383,7 @@ def amin( def prod( a: _Number, axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: None = ..., keepdims: bool = ..., initial: _NumberLike = ..., @@ -393,7 +393,7 @@ def prod( def prod( a: ArrayLike, axis: None = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: None = ..., keepdims: Literal[False] = ..., initial: _NumberLike = ..., @@ -403,7 +403,7 @@ def prod( def prod( a: ArrayLike, axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: Optional[ndarray] = ..., keepdims: bool = ..., initial: _NumberLike = ..., @@ -412,7 +412,7 @@ def prod( def cumprod( a: ArrayLike, axis: Optional[int] = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: Optional[ndarray] = ..., ) -> ndarray: ... def ndim(a: ArrayLike) -> int: ... @@ -433,7 +433,7 @@ def around( def mean( a: ArrayLike, axis: None = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: None = ..., keepdims: Literal[False] = ..., ) -> number: ... @@ -441,7 +441,7 @@ def mean( def mean( a: ArrayLike, axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: Optional[ndarray] = ..., keepdims: bool = ..., ) -> Union[number, ndarray]: ... @@ -449,7 +449,7 @@ def mean( def std( a: ArrayLike, axis: None = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: None = ..., ddof: int = ..., keepdims: Literal[False] = ..., @@ -458,7 +458,7 @@ def std( def std( a: ArrayLike, axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: Optional[ndarray] = ..., ddof: int = ..., keepdims: bool = ..., @@ -467,7 +467,7 @@ def std( def var( a: ArrayLike, axis: None = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: None = ..., ddof: int = ..., keepdims: Literal[False] = ..., @@ -476,7 +476,7 @@ def var( def var( a: ArrayLike, axis: Optional[_ShapeLike] = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., out: Optional[ndarray] = ..., ddof: int = ..., keepdims: bool = ..., diff --git a/numpy/core/function_base.pyi b/numpy/core/function_base.pyi index 7057122531fd..1490bed4aff3 100644 --- a/numpy/core/function_base.pyi +++ b/numpy/core/function_base.pyi @@ -2,7 +2,7 @@ import sys from typing import overload, Tuple, Union, Sequence, Any from numpy import ndarray, inexact -from numpy.typing import ArrayLike, DtypeLike, _SupportsArray, _NumberLike +from numpy.typing import ArrayLike, DTypeLike, _SupportsArray, _NumberLike if sys.version_info >= (3, 8): from typing import SupportsIndex, Literal @@ -24,7 +24,7 @@ def linspace( num: SupportsIndex = ..., endpoint: bool = ..., retstep: Literal[False] = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., axis: SupportsIndex = ..., ) -> ndarray: ... @overload @@ -34,7 +34,7 @@ def linspace( num: SupportsIndex = ..., endpoint: bool = ..., retstep: Literal[True] = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., axis: SupportsIndex = ..., ) -> Tuple[ndarray, inexact]: ... def logspace( @@ -43,7 +43,7 @@ def logspace( num: SupportsIndex = ..., endpoint: bool = ..., base: _ArrayLikeNumber = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., axis: SupportsIndex = ..., ) -> ndarray: ... def geomspace( @@ -51,6 +51,6 @@ def geomspace( stop: _ArrayLikeNumber, num: SupportsIndex = ..., endpoint: bool = ..., - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., axis: SupportsIndex = ..., ) -> ndarray: ... diff --git a/numpy/core/numeric.pyi b/numpy/core/numeric.pyi index de80cfe08c31..d91cb31c2031 100644 --- a/numpy/core/numeric.pyi +++ b/numpy/core/numeric.pyi @@ -13,7 +13,7 @@ from typing import ( ) from numpy import ndarray, generic, dtype, bool_, signedinteger, _OrderKACF, _OrderCF -from numpy.typing import ArrayLike, DtypeLike, _ShapeLike +from numpy.typing import ArrayLike, DTypeLike, _ShapeLike if sys.version_info >= (3, 8): from typing import Literal @@ -36,14 +36,14 @@ def zeros_like( @overload def zeros_like( a: ArrayLike, - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., order: _OrderKACF = ..., subok: bool = ..., shape: Optional[_ShapeLike] = ..., ) -> ndarray: ... def ones( shape: _ShapeLike, - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., order: _OrderCF = ..., *, like: ArrayLike = ..., @@ -59,7 +59,7 @@ def ones_like( @overload def ones_like( a: ArrayLike, - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., order: _OrderKACF = ..., subok: bool = ..., shape: Optional[_ShapeLike] = ..., @@ -75,7 +75,7 @@ def empty_like( @overload def empty_like( a: ArrayLike, - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., order: _OrderKACF = ..., subok: bool = ..., shape: Optional[_ShapeLike] = ..., @@ -83,7 +83,7 @@ def empty_like( def full( shape: _ShapeLike, fill_value: Any, - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., order: _OrderCF = ..., *, like: ArrayLike = ..., @@ -101,7 +101,7 @@ def full_like( def full_like( a: ArrayLike, fill_value: Any, - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., order: _OrderKACF = ..., subok: bool = ..., shape: Optional[_ShapeLike] = ..., @@ -150,27 +150,27 @@ def cross( @overload def indices( dimensions: Sequence[int], - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., sparse: Literal[False] = ..., ) -> ndarray: ... @overload def indices( dimensions: Sequence[int], - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., sparse: Literal[True] = ..., ) -> Tuple[ndarray, ...]: ... def fromfunction( function: Callable[..., _T], shape: Sequence[int], *, - dtype: DtypeLike = ..., + dtype: DTypeLike = ..., like: ArrayLike = ..., **kwargs: Any, ) -> _T: ... def isscalar(element: Any) -> bool: ... def binary_repr(num: int, width: Optional[int] = ...) -> str: ... def base_repr(number: int, base: int = ..., padding: int = ...) -> str: ... -def identity(n: int, dtype: DtypeLike = ..., *, like: ArrayLike = ...) -> ndarray: ... +def identity(n: int, dtype: DTypeLike = ..., *, like: ArrayLike = ...) -> ndarray: ... def allclose( a: ArrayLike, b: ArrayLike, diff --git a/numpy/core/numerictypes.pyi b/numpy/core/numerictypes.pyi index 0aba5f1d1519..192015ff13b4 100644 --- a/numpy/core/numerictypes.pyi +++ b/numpy/core/numerictypes.pyi @@ -1,11 +1,11 @@ from typing import TypeVar, Optional, Type, Union, Tuple, Sequence, overload, Any from numpy import generic, ndarray, dtype -from numpy.typing import DtypeLike +from numpy.typing import DTypeLike _DefaultType = TypeVar("_DefaultType") -def maximum_sctype(t: DtypeLike) -> dtype: ... +def maximum_sctype(t: DTypeLike) -> dtype: ... def issctype(rep: object) -> bool: ... @overload def obj2sctype(rep: object) -> Optional[generic]: ... @@ -17,12 +17,12 @@ def obj2sctype( ) -> Union[generic, Type[_DefaultType]]: ... def issubclass_(arg1: object, arg2: Union[object, Tuple[object, ...]]) -> bool: ... def issubsctype( - arg1: Union[ndarray, DtypeLike], arg2: Union[ndarray, DtypeLike] + arg1: Union[ndarray, DTypeLike], arg2: Union[ndarray, DTypeLike] ) -> bool: ... -def issubdtype(arg1: DtypeLike, arg2: DtypeLike) -> bool: ... +def issubdtype(arg1: DTypeLike, arg2: DTypeLike) -> bool: ... def sctype2char(sctype: object) -> str: ... def find_common_type( - array_types: Sequence[DtypeLike], scalar_types: Sequence[DtypeLike] + array_types: Sequence[DTypeLike], scalar_types: Sequence[DTypeLike] ) -> dtype: ... # TODO: Add annotations for the following objects: diff --git a/numpy/typing/__init__.py b/numpy/typing/__init__.py index 2738da071198..cbde75462332 100644 --- a/numpy/typing/__init__.py +++ b/numpy/typing/__init__.py @@ -14,10 +14,10 @@ addition, the following type aliases are available for users. - ``typing.ArrayLike``: objects that can be converted to arrays -- ``typing.DtypeLike``: objects that can be converted to dtypes +- ``typing.DTypeLike``: objects that can be converted to dtypes Roughly speaking, ``typing.ArrayLike`` is "objects that can be used as -inputs to ``np.array``" and ``typing.DtypeLike`` is "objects that can +inputs to ``np.array``" and ``typing.DTypeLike`` is "objects that can be used as inputs to ``np.dtype``". .. _typing-extensions: https://pypi.org/project/typing-extensions/ @@ -184,7 +184,7 @@ class _8Bit(_16Bit): ... # type: ignore[misc] ) from ._array_like import _SupportsArray, ArrayLike from ._shape import _Shape, _ShapeLike -from ._dtype_like import _SupportsDtype, _VoidDtypeLike, DtypeLike +from ._dtype_like import _SupportsDType, _VoidDTypeLike, DTypeLike from numpy._pytesttester import PytestTester test = PytestTester(__name__) diff --git a/numpy/typing/_array_like.py b/numpy/typing/_array_like.py index 1c00b200f4ed..a1a604239390 100644 --- a/numpy/typing/_array_like.py +++ b/numpy/typing/_array_like.py @@ -3,7 +3,7 @@ from numpy import ndarray from ._scalars import _ScalarLike -from ._dtype_like import DtypeLike +from ._dtype_like import DTypeLike if sys.version_info >= (3, 8): from typing import Protocol @@ -19,9 +19,9 @@ if TYPE_CHECKING or HAVE_PROTOCOL: class _SupportsArray(Protocol): @overload - def __array__(self, __dtype: DtypeLike = ...) -> ndarray: ... + def __array__(self, __dtype: DTypeLike = ...) -> ndarray: ... @overload - def __array__(self, dtype: DtypeLike = ...) -> ndarray: ... + def __array__(self, dtype: DTypeLike = ...) -> ndarray: ... else: _SupportsArray = Any diff --git a/numpy/typing/_dtype_like.py b/numpy/typing/_dtype_like.py index 5bfd8ffdc0d6..1953bd5fcfcc 100644 --- a/numpy/typing/_dtype_like.py +++ b/numpy/typing/_dtype_like.py @@ -15,36 +15,36 @@ else: HAVE_PROTOCOL = True -_DtypeLikeNested = Any # TODO: wait for support for recursive types +_DTypeLikeNested = Any # TODO: wait for support for recursive types if TYPE_CHECKING or HAVE_PROTOCOL: # Mandatory keys - class _DtypeDictBase(TypedDict): + class _DTypeDictBase(TypedDict): names: Sequence[str] - formats: Sequence[_DtypeLikeNested] + formats: Sequence[_DTypeLikeNested] # Mandatory + optional keys - class _DtypeDict(_DtypeDictBase, total=False): + class _DTypeDict(_DTypeDictBase, total=False): offsets: Sequence[int] titles: Sequence[Any] # Only `str` elements are usable as indexing aliases, but all objects are legal itemsize: int aligned: bool # A protocol for anything with the dtype attribute - class _SupportsDtype(Protocol): - dtype: _DtypeLikeNested + class _SupportsDType(Protocol): + dtype: _DTypeLikeNested else: - _DtypeDict = Any - _SupportsDtype = Any + _DTypeDict = Any + _SupportsDType = Any # Would create a dtype[np.void] -_VoidDtypeLike = Union[ +_VoidDTypeLike = Union[ # (flexible_dtype, itemsize) - Tuple[_DtypeLikeNested, int], + Tuple[_DTypeLikeNested, int], # (fixed_dtype, shape) - Tuple[_DtypeLikeNested, _ShapeLike], + Tuple[_DTypeLikeNested, _ShapeLike], # [(field_name, field_dtype, field_shape), ...] # # The type here is quite broad because NumPy accepts quite a wide @@ -53,29 +53,29 @@ class _SupportsDtype(Protocol): List[Any], # {'names': ..., 'formats': ..., 'offsets': ..., 'titles': ..., # 'itemsize': ...} - _DtypeDict, + _DTypeDict, # (base_dtype, new_dtype) - Tuple[_DtypeLikeNested, _DtypeLikeNested], + Tuple[_DTypeLikeNested, _DTypeLikeNested], ] # Anything that can be coerced into numpy.dtype. # Reference: https://docs.scipy.org/doc/numpy/reference/arrays.dtypes.html -DtypeLike = Union[ +DTypeLike = Union[ dtype, # default data type (float64) None, # array-scalar types and generic types type, # TODO: enumerate these when we add type hints for numpy scalars # anything with a dtype attribute - _SupportsDtype, + _SupportsDType, # character codes, type strings or comma-separated fields, e.g., 'float64' str, - _VoidDtypeLike, + _VoidDTypeLike, ] # NOTE: while it is possible to provide the dtype as a dict of # dtype-like objects (e.g. `{'field1': ..., 'field2': ..., ...}`), # this syntax is officially discourged and -# therefore not included in the Union defining `DtypeLike`. +# therefore not included in the Union defining `DTypeLike`. # # See https://github.com/numpy/numpy/issues/16891 for more details. diff --git a/numpy/typing/tests/data/fail/flatiter.py b/numpy/typing/tests/data/fail/flatiter.py index e8a82344f622..544ffbe4a7db 100644 --- a/numpy/typing/tests/data/fail/flatiter.py +++ b/numpy/typing/tests/data/fail/flatiter.py @@ -1,7 +1,7 @@ from typing import Any import numpy as np -from numpy.typing import DtypeLike, _SupportsArray +from numpy.typing import _SupportsArray class Index: diff --git a/numpy/typing/tests/data/pass/array_like.py b/numpy/typing/tests/data/pass/array_like.py index 6b823ca7e1af..f85724267448 100644 --- a/numpy/typing/tests/data/pass/array_like.py +++ b/numpy/typing/tests/data/pass/array_like.py @@ -1,7 +1,7 @@ from typing import Any, List, Optional import numpy as np -from numpy.typing import ArrayLike, DtypeLike, _SupportsArray +from numpy.typing import ArrayLike, DTypeLike, _SupportsArray x1: ArrayLike = True x2: ArrayLike = 5 @@ -18,7 +18,7 @@ class A: - def __array__(self, dtype: DtypeLike = None) -> np.ndarray: + def __array__(self, dtype: DTypeLike = None) -> np.ndarray: return np.array([1, 2, 3]) From 779d306fb6d851e7948bf6a97f4d2034db7d6f25 Mon Sep 17 00:00:00 2001 From: Qiyu8 Date: Tue, 3 Nov 2020 20:30:49 +0800 Subject: [PATCH 405/409] fix small typos. --- numpy/core/src/common/npy_cpu_dispatch.h | 12 ++++++------ setup.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/numpy/core/src/common/npy_cpu_dispatch.h b/numpy/core/src/common/npy_cpu_dispatch.h index 274520852569..a0f82fa3da05 100644 --- a/numpy/core/src/common/npy_cpu_dispatch.h +++ b/numpy/core/src/common/npy_cpu_dispatch.h @@ -17,7 +17,7 @@ * NumPy module's attributes `__cpu_baseline__` and `__cpu_dispaٍtch__`. */ /** - * Note: Always gaurd the genreated headers within 'NPY_DISABLE_OPTIMIZATION', + * Note: Always guard the generated headers within 'NPY_DISABLE_OPTIMIZATION', * due the nature of command argument '--disable-optimization', * which is explicitly disabling the module ccompiler_opt. */ @@ -29,7 +29,7 @@ * It's better anyway to take them off and use built-in types(__vector, __pixel, __bool) instead, * since c99 supports bool variables which may lead to ambiguous errors. */ - // backup 'bool' before including '_cpu_dispatch.h', since it may not defiend as a compiler token. + // backup 'bool' before including '_cpu_dispatch.h', since it may not defined as a compiler token. #define NPY__DISPATCH_DEFBOOL typedef bool npy__dispatch_bkbool; #endif @@ -134,10 +134,10 @@ * NPY_CPU_DISPATCH_DECLARE(void dispatch_me, (const int*, int*)) * NPY_CPU_DISPATCH_DECLARE(extern cb_type callback_tab, [TAB_SIZE]) * - * By assuming the provided config header drived from a dispatch-able source, + * By assuming the provided config header derived from a dispatch-able source, * that configured with "@targets baseline sse41 vsx3 asimdhp", * they supported by the compiler and enabled via '--cpu-dspatch', - * then the prototype declrations at the above example will equlivent to the follows: + * then the prototype declrations at the above example will equivalent to the follows: * * - x86: * void dispatch_me(const int*, int*); // baseline @@ -179,7 +179,7 @@ /** * Macro NPY_CPU_DISPATCH_DECLARE_XB(LEFT, ...) * - * Same as `NPY_CPU_DISPATCH_DECLARE` but exclude the baseline declration even + * Same as `NPY_CPU_DISPATCH_DECLARE` but exclude the baseline declaration even * if it was provided within the configration statments. */ #define NPY_CPU_DISPATCH_DECLARE_XB(...) \ @@ -206,7 +206,7 @@ * In order to call or to assign the pointer of it from outside the dispatch-able source, * you have to use this Macro as follows: * - * // bring the genreated config header of the dispatch-abel source + * // bring the generated config header of the dispatch-able source * #ifndef NPY_DISABLE_OPTIMIZATION * #include "dispatchable_source_name.dispatch.h" * #endif diff --git a/setup.py b/setup.py index c64a5c491617..6d7e90c6a15a 100755 --- a/setup.py +++ b/setup.py @@ -423,7 +423,7 @@ def get_docs_url(): if not ISRELEASED: return "https://numpy.org/devdocs" else: - # For releaeses, this URL ends up on pypi. + # For releases, this URL ends up on pypi. # By pinning the version, users looking at old PyPI releases can get # to the associated docs easily. return "https://numpy.org/doc/{}.{}".format(MAJOR, MINOR) From 99051b40de3b77c166a24250df3fd38689aa412e Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Tue, 3 Nov 2020 15:08:38 +0100 Subject: [PATCH 406/409] BUG: Fixed an issue where `.pyi` weren't picked up by numpy sub-packages --- numpy/distutils/setup.py | 1 + numpy/f2py/setup.py | 1 + numpy/fft/setup.py | 1 + numpy/lib/setup.py | 1 + numpy/linalg/setup.py | 1 + numpy/ma/setup.py | 1 + numpy/matrixlib/setup.py | 1 + numpy/polynomial/setup.py | 1 + numpy/random/setup.py | 1 + numpy/testing/setup.py | 1 + numpy/typing/setup.py | 1 + 11 files changed, 11 insertions(+) diff --git a/numpy/distutils/setup.py b/numpy/distutils/setup.py index 798c3686f46d..522756fc9db3 100644 --- a/numpy/distutils/setup.py +++ b/numpy/distutils/setup.py @@ -8,6 +8,7 @@ def configuration(parent_package='',top_path=None): config.add_data_files('site.cfg') config.add_data_files('mingw/gfortran_vs2003_hack.c') config.add_data_dir('checks') + config.add_data_files('*.pyi') config.make_config_py() return config diff --git a/numpy/f2py/setup.py b/numpy/f2py/setup.py index 80b47e527aab..0a35db477494 100644 --- a/numpy/f2py/setup.py +++ b/numpy/f2py/setup.py @@ -30,6 +30,7 @@ def configuration(parent_package='', top_path=None): config.add_data_files( 'src/fortranobject.c', 'src/fortranobject.h') + config.add_data_files('*.pyi') return config diff --git a/numpy/fft/setup.py b/numpy/fft/setup.py index 9ed824e4f8ab..477948a0986b 100644 --- a/numpy/fft/setup.py +++ b/numpy/fft/setup.py @@ -14,6 +14,7 @@ def configuration(parent_package='',top_path=None): define_macros=defs, ) + config.add_data_files('*.pyi') return config if __name__ == '__main__': diff --git a/numpy/lib/setup.py b/numpy/lib/setup.py index b3f441f38d3b..7520b72d7ac0 100644 --- a/numpy/lib/setup.py +++ b/numpy/lib/setup.py @@ -4,6 +4,7 @@ def configuration(parent_package='',top_path=None): config = Configuration('lib', parent_package, top_path) config.add_subpackage('tests') config.add_data_dir('tests/data') + config.add_data_files('*.pyi') return config if __name__ == '__main__': diff --git a/numpy/linalg/setup.py b/numpy/linalg/setup.py index bb070ed9d892..5c9f2a4cb56c 100644 --- a/numpy/linalg/setup.py +++ b/numpy/linalg/setup.py @@ -80,6 +80,7 @@ def get_lapack_lite_sources(ext, build_dir): extra_info=lapack_info, libraries=['npymath'], ) + config.add_data_files('*.pyi') return config if __name__ == '__main__': diff --git a/numpy/ma/setup.py b/numpy/ma/setup.py index d3f34c874607..018d38cdd500 100644 --- a/numpy/ma/setup.py +++ b/numpy/ma/setup.py @@ -3,6 +3,7 @@ def configuration(parent_package='',top_path=None): from numpy.distutils.misc_util import Configuration config = Configuration('ma', parent_package, top_path) config.add_subpackage('tests') + config.add_data_files('*.pyi') return config if __name__ == "__main__": diff --git a/numpy/matrixlib/setup.py b/numpy/matrixlib/setup.py index 19b3bb2de7c6..4fed75de1cbc 100644 --- a/numpy/matrixlib/setup.py +++ b/numpy/matrixlib/setup.py @@ -3,6 +3,7 @@ def configuration(parent_package='', top_path=None): from numpy.distutils.misc_util import Configuration config = Configuration('matrixlib', parent_package, top_path) config.add_subpackage('tests') + config.add_data_files('*.pyi') return config if __name__ == "__main__": diff --git a/numpy/polynomial/setup.py b/numpy/polynomial/setup.py index 6414645186fa..b58e867a133f 100644 --- a/numpy/polynomial/setup.py +++ b/numpy/polynomial/setup.py @@ -2,6 +2,7 @@ def configuration(parent_package='',top_path=None): from numpy.distutils.misc_util import Configuration config = Configuration('polynomial', parent_package, top_path) config.add_subpackage('tests') + config.add_data_files('*.pyi') return config if __name__ == '__main__': diff --git a/numpy/random/setup.py b/numpy/random/setup.py index 88ddb12687ea..bfd08e4691aa 100644 --- a/numpy/random/setup.py +++ b/numpy/random/setup.py @@ -137,6 +137,7 @@ def generate_libraries(ext, build_dir): define_macros=defs + LEGACY_DEFS, ) config.add_data_files(*depends) + config.add_data_files('*.pyi') return config diff --git a/numpy/testing/setup.py b/numpy/testing/setup.py index 13191f13fe0c..7652a94a2660 100755 --- a/numpy/testing/setup.py +++ b/numpy/testing/setup.py @@ -6,6 +6,7 @@ def configuration(parent_package='',top_path=None): config.add_subpackage('_private') config.add_subpackage('tests') + config.add_data_files('*.pyi') return config if __name__ == '__main__': diff --git a/numpy/typing/setup.py b/numpy/typing/setup.py index c444e769fb6d..694a756dc5ab 100644 --- a/numpy/typing/setup.py +++ b/numpy/typing/setup.py @@ -3,6 +3,7 @@ def configuration(parent_package='', top_path=None): config = Configuration('typing', parent_package, top_path) config.add_subpackage('tests') config.add_data_dir('tests/data') + config.add_data_files('*.pyi') return config From 66b17fbc9666b150c71bb94f2492fd880b2641e4 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Tue, 3 Nov 2020 15:09:12 +0100 Subject: [PATCH 407/409] TST: Validate the existence of `.pyi` stub files --- numpy/typing/tests/test_isfile.py | 34 +++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 numpy/typing/tests/test_isfile.py diff --git a/numpy/typing/tests/test_isfile.py b/numpy/typing/tests/test_isfile.py new file mode 100644 index 000000000000..e6b80f995919 --- /dev/null +++ b/numpy/typing/tests/test_isfile.py @@ -0,0 +1,34 @@ +import os +from pathlib import Path + +import numpy as np +from numpy.testing import assert_ + +ROOT = Path(np.__file__).parents[0] +FILES = [ + ROOT / "py.typed", + ROOT / "__init__.pyi", + ROOT / "char.pyi", + ROOT / "ctypeslib.pyi", + ROOT / "emath.pyi", + ROOT / "rec.pyi", + ROOT / "version.pyi", + ROOT / "core" / "__init__.pyi", + ROOT / "distutils" / "__init__.pyi", + ROOT / "f2py" / "__init__.pyi", + ROOT / "fft" / "__init__.pyi", + ROOT / "lib" / "__init__.pyi", + ROOT / "linalg" / "__init__.pyi", + ROOT / "ma" / "__init__.pyi", + ROOT / "matrixlib" / "__init__.pyi", + ROOT / "polynomial" / "__init__.pyi", + ROOT / "random" / "__init__.pyi", + ROOT / "testing" / "__init__.pyi", +] + + +class TestIsFile: + def test_isfile(self): + """Test if all ``.pyi`` files are properly installed.""" + for file in FILES: + assert_(os.path.isfile(file)) From f8c608cc7a8abcfaea1cd655ddaab47065c5b1e5 Mon Sep 17 00:00:00 2001 From: prateek arora Date: Wed, 4 Nov 2020 01:37:33 +0530 Subject: [PATCH 408/409] Add info on transopose to absolute beginners tutorial (#17703) Adds a blurb about the transpose property of ndarray and updates formatting of some rst inline literals. Co-authored-by: Ross Barnowski --- doc/source/user/absolute_beginners.rst | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/doc/source/user/absolute_beginners.rst b/doc/source/user/absolute_beginners.rst index 5873eb10862e..126f5f2a395e 100644 --- a/doc/source/user/absolute_beginners.rst +++ b/doc/source/user/absolute_beginners.rst @@ -1090,7 +1090,7 @@ To learn more about finding the unique elements in an array, see `unique`. Transposing and reshaping a matrix ---------------------------------- -*This section covers* ``arr.reshape()``, ``arr.transpose()``, ``arr.T()`` +*This section covers* ``arr.reshape()``, ``arr.transpose()``, ``arr.T`` ----- @@ -1114,7 +1114,7 @@ You simply need to pass in the new dimensions that you want for the matrix. :: .. image:: images/np_reshape.png -You can also use ``.transpose`` to reverse or change the axes of an array +You can also use ``.transpose()`` to reverse or change the axes of an array according to the values you specify. If you start with this array:: @@ -1131,6 +1131,13 @@ You can transpose your array with ``arr.transpose()``. :: [1, 4], [2, 5]]) +You can also use ``arr.T``:: + + >>> arr.T + array([[0, 3], + [1, 4], + [2, 5]]) + To learn more about transposing and reshaping arrays, see `transpose` and `reshape`. @@ -1138,12 +1145,12 @@ To learn more about transposing and reshaping arrays, see `transpose` and How to reverse an array ----------------------- -*This section covers* ``np.flip`` +*This section covers* ``np.flip()`` ----- NumPy's ``np.flip()`` function allows you to flip, or reverse, the contents of -an array along an axis. When using ``np.flip``, specify the array you would like +an array along an axis. When using ``np.flip()``, specify the array you would like to reverse and the axis. If you don't specify the axis, NumPy will reverse the contents along all of the axes of your input array. From 403812616c98ecd304b22d81258e9674542ae931 Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Tue, 3 Nov 2020 23:38:26 +0000 Subject: [PATCH 409/409] Experiment with MT --- azure-pipelines.yml | 166 -------------------------------------------- numpy/core/setup.py | 9 ++- 2 files changed, 6 insertions(+), 169 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 12d99156d966..e5c0c25569be 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -29,177 +29,12 @@ stages: - stage: ComprehensiveTests jobs: - - job: Linux_Python_38_32bit_full_with_asserts - pool: - vmImage: 'ubuntu-18.04' - steps: - - script: | - docker pull quay.io/pypa/manylinux2010_i686 - docker run -v $(pwd):/numpy -e CFLAGS="-msse2 -std=c99 -UNDEBUG" \ - -e F77=gfortran-5 -e F90=gfortran-5 quay.io/pypa/manylinux2010_i686 \ - /bin/bash -xc "cd numpy && \ - /opt/python/cp38-cp38/bin/python -mvenv venv &&\ - source venv/bin/activate && \ - target=\$(python3 tools/openblas_support.py) && \ - cp -r \$target/lib/* /usr/lib && \ - cp \$target/include/* /usr/include && \ - python3 -m pip install -r test_requirements.txt && \ - echo CFLAGS \$CFLAGS && \ - python3 -m pip install -v . && \ - python3 runtests.py -n --debug-info --mode=full -- -rsx --junitxml=junit/test-results.xml && \ - python3 tools/openblas_support.py --check_version" - displayName: 'Run 32-bit manylinux2010 Docker Build / Tests' - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-*.xml' - failTaskOnFailedTests: true - testRunTitle: 'Publish test results for Python 3.8-32 bit full Linux' - - job: macOS - pool: - # NOTE: at time of writing, there is a danger - # that using an invalid vmIMage string for macOS - # image silently redirects to a Windows build on Azure; - # for now, use the only image name officially present in - # the docs even though i.e., numba uses another in their - # azure config for mac os -- Microsoft has indicated - # they will patch this issue - vmImage: macOS-10.14 - strategy: - maxParallel: 3 - matrix: - Python36: - PYTHON_VERSION: '3.6' - USE_OPENBLAS: '1' - Python36-ILP64: - PYTHON_VERSION: '3.6' - NPY_USE_BLAS_ILP64: '1' - USE_OPENBLAS: '1' - # Disable this job: the azure images do not create the problematic - # symlink from Accelerate to OpenBLAS. We still have the test - # at import to detect a buggy Accelerate, just cannot easily trigger - # it with azure. - # Accelerate: - # PYTHON_VERSION: '3.6' - # USE_OPENBLAS: '0' - - steps: - # the @0 refers to the (major) version of the *task* on Microsoft's - # end, not the order in the build matrix nor anything to do - # with version of Python selected - - task: UsePythonVersion@0 - inputs: - versionSpec: $(PYTHON_VERSION) - addToPath: true - architecture: 'x64' - - script: | - set -xe - [ -n "$USE_XCODE_10" ] && /bin/bash -c "sudo xcode-select -s /Applications/Xcode_10.app/Contents/Developer" - clang --version - displayName: 'report clang version' - # NOTE: might be better if we could avoid installing - # two C compilers, but with homebrew looks like we're - # now stuck getting the full gcc toolchain instead of - # just pulling in gfortran - - script: | - set -xe - # same version of gfortran as the open-libs and numpy-wheel builds - curl -L https://github.com/MacPython/gfortran-install/raw/master/archives/gfortran-4.9.0-Mavericks.dmg -o gfortran.dmg - GFORTRAN_SHA256=$(shasum -a 256 gfortran.dmg) - KNOWN_SHA256="d2d5ca5ba8332d63bbe23a07201c4a0a5d7e09ee56f0298a96775f928c3c4b30 gfortran.dmg" - if [ "$GFORTRAN_SHA256" != "$KNOWN_SHA256" ]; then - echo sha256 mismatch - exit 1 - fi - hdiutil attach -mountpoint /Volumes/gfortran gfortran.dmg - sudo installer -pkg /Volumes/gfortran/gfortran.pkg -target / - otool -L /usr/local/gfortran/lib/libgfortran.3.dylib - # Manually symlink gfortran-4.9 to plain gfortran for f2py. - # No longer needed after Feb 13 2020 as gfortran is already present - # and the attempted link errors. Keep this for future reference. - # ln -s /usr/local/bin/gfortran-4.9 /usr/local/bin/gfortran - displayName: 'make libgfortran available on mac os for openblas' - # use the pre-built openblas binary that most closely - # matches our MacOS wheel builds -- currently based - # primarily on file size / name details - - script: | - set -xe - target=$(python tools/openblas_support.py) - ls -lR $target - # manually link to appropriate system paths - cp $target/lib/lib* /usr/local/lib/ - cp $target/include/* /usr/local/include/ - otool -L /usr/local/lib/libopenblas* - displayName: 'install pre-built openblas' - condition: eq(variables['USE_OPENBLAS'], '1') - - script: python -m pip install --upgrade pip 'setuptools<49.2.0' wheel - displayName: 'Install tools' - - script: | - python -m pip install -r test_requirements.txt - python -m pip install vulture docutils sphinx==2.2.0 numpydoc - displayName: 'Install dependencies; some are optional to avoid test skips' - - script: /bin/bash -c "! vulture . --min-confidence 100 --exclude doc/,numpy/distutils/ | grep 'unreachable'" - displayName: 'Check for unreachable code paths in Python modules' - # prefer usage of clang over gcc proper - # to match likely scenario on many user mac machines - - script: python setup.py build -j 4 build_src --verbose-cfg install - displayName: 'Build NumPy' - env: - BLAS: None - LAPACK: None - ATLAS: None - CC: /usr/bin/clang - condition: eq(variables['USE_OPENBLAS'], '1') - - script: python setup.py build -j 4 build_ext --inplace install - displayName: 'Build NumPy without OpenBLAS' - env: - BLAS: None - LAPACK: None - ATLAS: None - CC: /usr/bin/clang - condition: eq(variables['USE_OPENBLAS'], '0') - # wait until after dev build of NumPy to pip - # install matplotlib to avoid pip install of older numpy - - script: python -m pip install matplotlib - displayName: 'Install matplotlib before refguide run' - - script: python runtests.py -g --refguide-check - displayName: 'Run Refuide Check' - condition: eq(variables['USE_OPENBLAS'], '1') - - script: python runtests.py -n --mode=full -- -rsx --junitxml=junit/test-results.xml - displayName: 'Run Full NumPy Test Suite' - condition: eq(variables['USE_OPENBLAS'], '1') - - bash: python tools/openblas_support.py --check_version - displayName: 'Verify OpenBLAS version' - condition: eq(variables['USE_OPENBLAS'], '1') - # import doesn't work when in numpy src directory , so do a pip dev install of build lib to test - - script: | - #!/bin/bash -v - set +e - python -c "import numpy as np" > test_output.log 2>&1 - check_output_code=$? - cat test_output.log - grep "buggy Accelerate backend" test_output.log - check_message=$? - if [ $check_output_code == 1 ] && [ $check_message == 0 ]; then exit 0; else exit 1;fi - displayName: "Check if numpy import fails with accelerate" - condition: eq(variables['USE_OPENBLAS'], '0') - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-*.xml' - failTaskOnFailedTests: true - testRunTitle: 'Publish test results for Python 3.6 64-bit full Mac OS' - job: Windows pool: vmImage: 'VS2017-Win2016' strategy: maxParallel: 6 matrix: - Python38-32bit-fast: - PYTHON_VERSION: '3.8' - PYTHON_ARCH: 'x86' - TEST_MODE: fast - BITS: 32 Python36-64bit-full: PYTHON_VERSION: '3.6' PYTHON_ARCH: 'x64' @@ -288,4 +123,3 @@ stages: testResultsFiles: '**/test-*.xml' failTaskOnFailedTests: true testRunTitle: 'Publish test results for gcc 4.8' - diff --git a/numpy/core/setup.py b/numpy/core/setup.py index 68aa0a8513fb..775c60d5ed2d 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -688,7 +688,7 @@ def get_mathlib_info(*args): install_dir='lib', build_info={ 'include_dirs' : [], # empty list required for creating npy_math_internal.h - 'extra_compiler_args' : (['/GL-'] if is_msvc else []), + 'extra_compiler_args' : (['/GL-','/MT'] if is_msvc else []), }) config.add_npy_pkg_config("npymath.ini.in", "lib/npy-pkg-config", subst_dict) @@ -940,8 +940,11 @@ def generate_umath_c(ext, build_dir): ], depends=deps + multiarray_deps + umath_deps + common_deps, - libraries=['npymath'], - extra_info=extra_info) + libraries=['npymath','libucrt'], + extra_info=extra_info, + extra_compile_args=["/MT"], + #extra_link_args=["/MT","libucrt.lib"] + ) ####################################################################### # umath_tests module #